def make_master_calibration_frame(self, images, image_config, logging_tags):
dark_data = np.zeros((images[0].ny, images[0].nx, len(images)), dtype=np.float32)
dark_mask = np.zeros((images[0].ny, images[0].nx, len(images)), dtype=np.uint8)
master_dark_filename = self.get_calibration_filename(images[0])
logs.add_tag(logging_tags, 'master_dark', os.path.basename(master_dark_filename))
for i, image in enumerate(images):
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
self.logger.debug('Combining dark', extra=logging_tags)
dark_data[:, :, i] = image.data[:, :]
dark_data[:, :, i] /= image.exptime
dark_mask[:, :, i] = image.bpm[:, :]
master_dark = stats.sigma_clipped_mean(dark_data, 3.0, axis=2, mask=dark_mask, inplace=True)
# Memory cleanup
del dark_data
del dark_mask
master_bpm = np.array(master_dark == 0.0, dtype=np.uint8)
master_dark[master_bpm] = 0.0
# Save the master dark image with all of the combined images in the header
master_dark_header = fits_utils.create_master_calibration_header(images)
master_dark_image = Image(self.pipeline_context, data=master_dark,
header=master_dark_header)
master_dark_image.filename = master_dark_filename
master_dark_image.bpm = master_bpm
logs.pop_tag(logging_tags, 'master_dark')
logs.add_tag(logging_tags, 'filename', os.path.basename(master_dark_image.filename))
self.logger.info('Created master dark', extra=logging_tags)
return [master_dark_image]
def make_master_calibration_frame(self, images, image_config,
logging_tags):
flat_data = np.zeros((images[0].ny, images[0].nx, len(images)),
dtype=np.float32)
flat_mask = np.zeros((images[0].ny, images[0].nx, len(images)),
dtype=np.uint8)
quarter_nx = images[0].nx // 4
quarter_ny = images[0].ny // 4
master_flat_filename = self.get_calibration_filename(images[0])
logs.add_tag(logging_tags, 'master_flat',
os.path.basename(master_flat_filename))
for i, image in enumerate(images):
# Get the sigma clipped mean of the central 25% of the image
flat_normalization = stats.sigma_clipped_mean(
image.data[quarter_ny:-quarter_ny, quarter_nx:-quarter_nx],
3.5)
flat_data[:, :, i] = image.data[:, :]
flat_data[:, :, i] /= flat_normalization
flat_mask[:, :, i] = image.bpm[:, :]
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
logs.add_tag(logging_tags, 'flat_normalization',
flat_normalization)
self.logger.debug('Calculating flat normalization',
extra=logging_tags)
logs.pop_tag(logging_tags, 'flat_normalization')
master_flat = stats.sigma_clipped_mean(flat_data,
3.0,
axis=2,
mask=flat_mask,
fill_value=1.0,
inplace=True)
master_bpm = np.array(master_flat == 1.0, dtype=np.uint8)
master_bpm = np.logical_and(master_bpm, master_flat < 0.2)
master_flat[master_flat < 0.2] = 1.0
master_flat_header = fits_utils.create_master_calibration_header(
images)
master_flat_image = Image(data=master_flat, header=master_flat_header)
master_flat_image.filename = master_flat_filename
master_flat_image.bpm = master_bpm
logs.pop_tag(logging_tags, 'master_flat')
logs.add_tag(logging_tags, 'filename',
os.path.basename(master_flat_image.filename))
self.logger.info('Created master flat', extra=logging_tags)
return [master_flat_image]
def make_master_calibration_frame(self, images, image_config,
logging_tags):
bias_data = np.zeros((image_config.ny, image_config.nx, len(images)),
dtype=np.float32)
bias_mask = np.zeros((image_config.ny, image_config.nx, len(images)),
dtype=np.uint8)
bias_level_array = np.zeros(len(images), dtype=np.float32)
master_bias_filename = self.get_calibration_filename(image_config)
logs.add_tag(logging_tags, 'master_bias',
os.path.basename(master_bias_filename))
for i, image in enumerate(images):
bias_level_array[i] = stats.sigma_clipped_mean(image.data,
3.5,
mask=image.bpm)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
logs.add_tag(logging_tags, 'BIASLVL', float(bias_level_array[i]))
self.logger.debug('Calculating bias level', extra=logging_tags)
# Subtract the bias level for each image
bias_data[:, :, i] = image.data[:, :] - bias_level_array[i]
bias_mask[:, :, i] = image.bpm[:, :]
mean_bias_level = stats.sigma_clipped_mean(bias_level_array, 3.0)
master_bias = stats.sigma_clipped_mean(bias_data,
3.0,
axis=2,
mask=bias_mask,
inplace=True)
del bias_data
del bias_mask
master_bpm = np.array(master_bias == 0.0, dtype=np.uint8)
header = fits_utils.create_master_calibration_header(images)
header['BIASLVL'] = (mean_bias_level, 'Mean bias level of master bias')
master_bias_image = Image(self.pipeline_context,
data=master_bias,
header=header)
master_bias_image.filename = master_bias_filename
master_bias_image.bpm = master_bpm
logs.pop_tag(logging_tags, 'master_bias')
logs.add_tag(logging_tags, 'filename',
os.path.basename(master_bias_image.filename))
logs.add_tag(logging_tags, 'BIASLVL', mean_bias_level)
self.logger.debug('Average bias level in ADU', extra=logging_tags)
return [master_bias_image]
def read_images(image_list, pipeline_context):
images = []
for filename in image_list:
try:
image = Image(filename=filename)
if image.bpm is None:
image.bpm = get_bpm(image, pipeline_context).astype(np.uint8)
images.append(image)
except Exception as e:
logger.error('Error loading {0}'.format(filename))
logger.error(e)
continue
return images
def make_master_calibration_frame(self, images, image_config,
logging_tags):
dark_data = np.zeros((images[0].ny, images[0].nx, len(images)),
dtype=np.float32)
dark_mask = np.zeros((images[0].ny, images[0].nx, len(images)),
dtype=np.uint8)
master_dark_filename = self.get_calibration_filename(images[0])
logs.add_tag(logging_tags, 'master_dark',
os.path.basename(master_dark_filename))
for i, image in enumerate(images):
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
self.logger.debug('Combining dark', extra=logging_tags)
dark_data[:, :, i] = image.data[:, :]
dark_data[:, :, i] /= image.exptime
dark_mask[:, :, i] = image.bpm[:, :]
master_dark = stats.sigma_clipped_mean(dark_data,
3.0,
axis=2,
mask=dark_mask,
inplace=True)
# Memory cleanup
del dark_data
del dark_mask
master_bpm = np.array(master_dark == 0.0, dtype=np.uint8)
master_dark[master_bpm] = 0.0
# Save the master dark image with all of the combined images in the header
master_dark_header = fits_utils.create_master_calibration_header(
images)
master_dark_image = Image(self.pipeline_context,
data=master_dark,
header=master_dark_header)
master_dark_image.filename = master_dark_filename
master_dark_image.bpm = master_bpm
logs.pop_tag(logging_tags, 'master_dark')
logs.add_tag(logging_tags, 'filename',
os.path.basename(master_dark_image.filename))
self.logger.info('Created master dark', extra=logging_tags)
return [master_dark_image]
def make_master_calibration_frame(self, images, image_config, logging_tags):
flat_data = np.zeros((images[0].ny, images[0].nx, len(images)), dtype=np.float32)
flat_mask = np.zeros((images[0].ny, images[0].nx, len(images)), dtype=np.uint8)
quarter_nx = images[0].nx // 4
quarter_ny = images[0].ny // 4
master_flat_filename = self.get_calibration_filename(images[0])
logs.add_tag(logging_tags, 'master_flat', os.path.basename(master_flat_filename))
for i, image in enumerate(images):
# Get the sigma clipped mean of the central 25% of the image
flat_normalization = stats.sigma_clipped_mean(image.data[quarter_ny: -quarter_ny,
quarter_nx:-quarter_nx], 3.5)
flat_data[:, :, i] = image.data[:, :]
flat_data[:, :, i] /= flat_normalization
flat_mask[:, :, i] = image.bpm[:, :]
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
logs.add_tag(logging_tags, 'flat_normalization', flat_normalization)
self.logger.debug('Calculating flat normalization', extra=logging_tags)
logs.pop_tag(logging_tags, 'flat_normalization')
master_flat = stats.sigma_clipped_mean(flat_data, 3.0, axis=2, mask=flat_mask,
fill_value=1.0, inplace=True)
master_bpm = np.array(master_flat == 1.0, dtype=np.uint8)
master_bpm = np.logical_and(master_bpm, master_flat < 0.2)
master_flat[master_flat < 0.2] = 1.0
master_flat_header = fits_utils.create_master_calibration_header(images)
master_flat_image = Image(self.pipeline_context, data=master_flat,
header=master_flat_header)
master_flat_image.filename = master_flat_filename
master_flat_image.bpm = master_bpm
logs.pop_tag(logging_tags, 'master_flat')
logs.add_tag(logging_tags, 'filename', os.path.basename(master_flat_image.filename))
self.logger.info('Created master flat', extra=logging_tags)
return [master_flat_image]
def make_master_calibration_frame(self, images, image_config, logging_tags):
bias_data = np.zeros((image_config.ny, image_config.nx, len(images)), dtype=np.float32)
bias_mask = np.zeros((image_config.ny, image_config.nx, len(images)), dtype=np.uint8)
bias_level_array = np.zeros(len(images), dtype=np.float32)
master_bias_filename = self.get_calibration_filename(image_config)
logs.add_tag(logging_tags, 'master_bias', os.path.basename(master_bias_filename))
for i, image in enumerate(images):
bias_level_array[i] = stats.sigma_clipped_mean(image.data, 3.5, mask=image.bpm)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
logs.add_tag(logging_tags, 'BIASLVL', float(bias_level_array[i]))
self.logger.debug('Calculating bias level', extra=logging_tags)
# Subtract the bias level for each image
bias_data[:, :, i] = image.data[:, :] - bias_level_array[i]
bias_mask[:, :, i] = image.bpm[:, :]
mean_bias_level = stats.sigma_clipped_mean(bias_level_array, 3.0)
master_bias = stats.sigma_clipped_mean(bias_data, 3.0, axis=2, mask=bias_mask, inplace=True)
del bias_data
del bias_mask
master_bpm = np.array(master_bias == 0.0, dtype=np.uint8)
header = fits_utils.create_master_calibration_header(images)
header['BIASLVL'] = (mean_bias_level, 'Mean bias level of master bias')
master_bias_image = Image(self.pipeline_context, data=master_bias, header=header)
master_bias_image.filename = master_bias_filename
master_bias_image.bpm = master_bpm
logs.pop_tag(logging_tags, 'master_bias')
logs.add_tag(logging_tags, 'filename', os.path.basename(master_bias_image.filename))
logs.add_tag(logging_tags, 'BIASLVL', mean_bias_level)
self.logger.debug('Average bias level in ADU', extra=logging_tags)
return [master_bias_image]
def test_image_creates_and_loads_tables_correctly():
"""
Tests that add_data_tables_to_hdu_list and regenerate_data_table_from_fits_hdu_list
create fits.HDUList objects correctly from astropy tables with single element entries
and for astropy tables with columns where each element is a list.
"""
test_image = Image(FakeContext(), filename=None)
table_name = 'test'
a = np.arange(3)
array_1 = [a, a]
array_2 = [a, np.vstack((a, a)).T]
for test_array in [array_1, array_2]:
test_table = Table(test_array, names=('1', '2'), meta={'name': table_name})
test_table['1'].description = 'test_description'
test_table['1'].unit = 'pixel'
test_image.data_tables[table_name] = DataTable(data_table=test_table, name=table_name)
hdu_list = []
hdu_list = test_image._add_data_tables_to_hdu_list(hdu_list)
fits_hdu_list = fits.HDUList(hdu_list)
test_table_dict = regenerate_data_table_from_fits_hdu_list(fits_hdu_list, table_extension_name=table_name)
test_table_recreated = test_table_dict[table_name]
assert (test_table_recreated == test_table).all()
def test_image_creates_and_loads_tables_correctly():
"""
Tests that add_data_tables_to_hdu_list and regenerate_data_table_from_fits_hdu_list
create fits.HDUList objects correctly from astropy tables with single element entries
and for astropy tables with columns where each element is a list.
"""
test_image = Image(FakeContext(), filename=None)
table_name = 'test'
a = np.arange(3)
array_1 = [a, a]
array_2 = [a, np.vstack((a, a)).T]
for test_array in [array_1, array_2]:
test_table = Table(test_array, names=('1', '2'), meta={'name': table_name})
test_table['1'].description = 'test_description'
test_table['1'].unit = 'pixel'
test_image.data_tables[table_name] = DataTable(data_table=test_table, name=table_name)
hdu_list = []
hdu_list = test_image._add_data_tables_to_hdu_list(hdu_list)
fits_hdu_list = fits.HDUList(hdu_list)
test_table_dict = regenerate_data_table_from_fits_hdu_list(fits_hdu_list, table_extension_name=table_name)
test_table_recreated = test_table_dict[table_name]
assert (test_table_recreated == test_table).all()
def do_stage(self, images):
if len(images) == 0:
# Abort!
return []
else:
image_config = check_image_homogeneity(images)
logging_tags = logs.image_config_to_tags(image_config, self.group_by_keywords)
master_calibration_filename = self.get_calibration_filename(images[0])
if master_calibration_filename is None:
self.logger.error('Master Calibration file does not exist for {stage}'.format(stage=self.stage_name),
extra=logging_tags)
raise MasterCalibrationDoesNotExist
master_calibration_image = Image(master_calibration_filename)
return self.apply_master_calibration(images, master_calibration_image, logging_tags)
def test_null_filename():
test_image = Image(FakeContext(), filename=None)
assert test_image.data is None