def get_bpm(image, pipeline_context):
bpm_filename = dbs.get_bpm(image.telescope_id, image.ccdsum,
db_address=pipeline_context.db_address)
if bpm_filename is None:
bpm = None
image.header['L1IDMASK'] = ('', 'Id. of mask file used')
else:
bpm_hdu = fits_utils.open_fits_file(bpm_filename)
bpm_extensions = fits_utils.get_extensions_by_name(bpm_hdu, 'BPM')
if len(bpm_extensions) > 1:
extension_shape = bpm_extensions[0].data.shape
bpm_shape = (len(bpm_extensions), extension_shape[0], extension_shape[1])
bpm = np.zeros(bpm_shape, dtype=np.uint8)
for i, extension in enumerate(bpm_extensions):
bpm[i, :, :] = extension.data[:, :]
elif len(bpm_extensions) == 1:
bpm = np.array(bpm_extensions[0].data, dtype=np.uint8)
else:
bpm = np.array(bpm_hdu[0].data, dtype=np.uint8)
if not bpm_has_valid_size(bpm, image):
tags = logs.image_config_to_tags(image, None)
logs.add_tag(tags, 'filename', image.filename)
logger.error('BPM shape mismatch', extra=tags)
err_msg = 'BPM shape mismatch for {filename} ' \
'{site}/{instrument}'.format(filename=image.filename, site=image.site,
instrument=image.instrument)
raise ValueError(err_msg)
image.header['L1IDMASK'] = (os.path.basename(bpm_filename), 'Id. of mask file used')
return bpm
def get_bpm(image, pipeline_context):
bpm_filename = dbs.get_bpm(image.telescope_id,
image.ccdsum,
db_address=pipeline_context.db_address)
if bpm_filename is None:
bpm = None
image.header['L1IDMASK'] = ('', 'Id. of mask file used')
else:
bpm_hdu = fits_utils.open_fits_file(bpm_filename)
bpm_extensions = fits_utils.get_extensions_by_name(bpm_hdu, 'BPM')
if len(bpm_extensions) > 1:
extension_shape = bpm_extensions[0].data.shape
bpm_shape = (len(bpm_extensions), extension_shape[0],
extension_shape[1])
bpm = np.zeros(bpm_shape, dtype=np.uint8)
for i, extension in enumerate(bpm_extensions):
bpm[i, :, :] = extension.data[:, :]
elif len(bpm_extensions) == 1:
bpm = np.array(bpm_extensions[0].data, dtype=np.uint8)
else:
bpm = np.array(bpm_hdu[0].data, dtype=np.uint8)
if not bpm_has_valid_size(bpm, image):
tags = logs.image_config_to_tags(image, None)
logs.add_tag(tags, 'filename', image.filename)
logger.error('BPM shape mismatch', extra=tags)
err_msg = 'BPM shape mismatch for {filename} ' \
'{site}/{instrument}'.format(filename=image.filename, site=image.site,
instrument=image.instrument)
raise ValueError(err_msg)
image.header['L1IDMASK'] = (os.path.basename(bpm_filename),
'Id. of mask file used')
return bpm
def get_expected_number_of_calibrations(raw_filenames, calibration_type):
number_of_stacks_that_should_have_been_created = 0
for day_obs in DAYS_OBS:
raw_filenames_for_this_dayobs = glob(os.path.join(DATA_ROOT, day_obs, 'raw', raw_filenames))
if calibration_type.lower() == 'skyflat':
# Group by filter
observed_filters = []
for raw_filename in raw_filenames_for_this_dayobs:
skyflat_hdu = fits_utils.open_fits_file(raw_filename)
observed_filters.append(skyflat_hdu[0].header.get('FILTER'))
observed_filters = set(observed_filters)
number_of_stacks_that_should_have_been_created += len(observed_filters)
else:
# Just one calibration per night
if len(raw_filenames_for_this_dayobs) > 0:
number_of_stacks_that_should_have_been_created += 1
return number_of_stacks_that_should_have_been_created
def get_expected_number_of_calibrations(raw_filenames, calibration_type):
number_of_stacks_that_should_have_been_created = 0
for day_obs in DAYS_OBS:
raw_filenames_for_this_dayobs = glob(os.path.join(DATA_ROOT, day_obs, 'raw', raw_filenames))
if calibration_type.lower() == 'skyflat':
# Group by filter
observed_filters = []
for raw_filename in raw_filenames_for_this_dayobs:
skyflat_hdu = fits_utils.open_fits_file(raw_filename)
observed_filters.append(skyflat_hdu[0].header.get('FILTER'))
observed_filters = set(observed_filters)
number_of_stacks_that_should_have_been_created += len(observed_filters)
else:
# Just one calibration per night
if len(raw_filenames_for_this_dayobs) > 0:
number_of_stacks_that_should_have_been_created += 1
return number_of_stacks_that_should_have_been_created
def populate_calibration_table_with_bpms(directory, db_address=_DEFAULT_DB):
with get_session(db_address=db_address) as db_session:
bpm_filenames = glob(os.path.join(directory, '*bpm*.fits*'))
for bpm_filename in bpm_filenames:
hdu = fits_utils.open_fits_file(bpm_filename)
header = hdu[0].header
ccdsum = header.get('CCDSUM')
configuration_mode = fits_utils.get_configuration_mode(header)
dateobs = date_utils.parse_date_obs(header.get('DATE-OBS'))
try:
instrument = get_instrument(header, db_address=db_address)
except ValueError:
logger.error('Instrument is missing from database',
extra_tags={
'site': header['SITEID'],
'camera': header['INSTRUME']
})
continue
bpm_attributes = {
'type': 'BPM',
'filename': os.path.basename(bpm_filename),
'filepath': os.path.abspath(directory),
'dateobs': dateobs,
'datecreated': dateobs,
'instrument_id': instrument.id,
'is_master': True,
'is_bad': False,
'attributes': {
'ccdsum': ccdsum,
'configuration_mode': configuration_mode
}
}
add_or_update_record(db_session, CalibrationImage,
{'filename': bpm_attributes['filename']},
bpm_attributes)
db_session.commit()
def load_bpm(bpm_filename):
bpm_hdu = fits_utils.open_fits_file(bpm_filename)
bpm_extensions = fits_utils.get_extensions_by_name(bpm_hdu, 'BPM')
# Filter out BPM extensions without data
bpm_extensions = [
extension for extension in bpm_extensions if extension.data is not None
]
if len(bpm_extensions) > 1:
extension_shape = bpm_extensions[0].data.shape
bpm_shape = (len(bpm_extensions), extension_shape[0],
extension_shape[1])
bpm = np.zeros(bpm_shape, dtype=np.uint8)
for i, extension in enumerate(bpm_extensions):
bpm[i, :, :] = extension.data[:, :]
elif len(bpm_extensions) == 1:
bpm = np.array(bpm_extensions[0].data, dtype=np.uint8)
else:
bpm = np.array(bpm_hdu[0].data, dtype=np.uint8)
return bpm