def get_bpm(image, pipeline_context): bpm_filename = dbs.get_bpm(image.telescope_id, image.ccdsum, db_address=pipeline_context.db_address) if bpm_filename is None: bpm = None image.header['L1IDMASK'] = ('', 'Id. of mask file used') else: bpm_hdu = fits_utils.open_fits_file(bpm_filename) bpm_extensions = fits_utils.get_extensions_by_name(bpm_hdu, 'BPM') if len(bpm_extensions) > 1: extension_shape = bpm_extensions[0].data.shape bpm_shape = (len(bpm_extensions), extension_shape[0], extension_shape[1]) bpm = np.zeros(bpm_shape, dtype=np.uint8) for i, extension in enumerate(bpm_extensions): bpm[i, :, :] = extension.data[:, :] elif len(bpm_extensions) == 1: bpm = np.array(bpm_extensions[0].data, dtype=np.uint8) else: bpm = np.array(bpm_hdu[0].data, dtype=np.uint8) if not bpm_has_valid_size(bpm, image): tags = logs.image_config_to_tags(image, None) logs.add_tag(tags, 'filename', image.filename) logger.error('BPM shape mismatch', extra=tags) err_msg = 'BPM shape mismatch for {filename} ' \ '{site}/{instrument}'.format(filename=image.filename, site=image.site, instrument=image.instrument) raise ValueError(err_msg) image.header['L1IDMASK'] = (os.path.basename(bpm_filename), 'Id. of mask file used') return bpm
def get_expected_number_of_calibrations(raw_filenames, calibration_type): number_of_stacks_that_should_have_been_created = 0 for day_obs in DAYS_OBS: raw_filenames_for_this_dayobs = glob(os.path.join(DATA_ROOT, day_obs, 'raw', raw_filenames)) if calibration_type.lower() == 'skyflat': # Group by filter observed_filters = [] for raw_filename in raw_filenames_for_this_dayobs: skyflat_hdu = fits_utils.open_fits_file(raw_filename) observed_filters.append(skyflat_hdu[0].header.get('FILTER')) observed_filters = set(observed_filters) number_of_stacks_that_should_have_been_created += len(observed_filters) else: # Just one calibration per night if len(raw_filenames_for_this_dayobs) > 0: number_of_stacks_that_should_have_been_created += 1 return number_of_stacks_that_should_have_been_created
def populate_calibration_table_with_bpms(directory, db_address=_DEFAULT_DB): with get_session(db_address=db_address) as db_session: bpm_filenames = glob(os.path.join(directory, '*bpm*.fits*')) for bpm_filename in bpm_filenames: hdu = fits_utils.open_fits_file(bpm_filename) header = hdu[0].header ccdsum = header.get('CCDSUM') configuration_mode = fits_utils.get_configuration_mode(header) dateobs = date_utils.parse_date_obs(header.get('DATE-OBS')) try: instrument = get_instrument(header, db_address=db_address) except ValueError: logger.error('Instrument is missing from database', extra_tags={ 'site': header['SITEID'], 'camera': header['INSTRUME'] }) continue bpm_attributes = { 'type': 'BPM', 'filename': os.path.basename(bpm_filename), 'filepath': os.path.abspath(directory), 'dateobs': dateobs, 'datecreated': dateobs, 'instrument_id': instrument.id, 'is_master': True, 'is_bad': False, 'attributes': { 'ccdsum': ccdsum, 'configuration_mode': configuration_mode } } add_or_update_record(db_session, CalibrationImage, {'filename': bpm_attributes['filename']}, bpm_attributes) db_session.commit()
def load_bpm(bpm_filename): bpm_hdu = fits_utils.open_fits_file(bpm_filename) bpm_extensions = fits_utils.get_extensions_by_name(bpm_hdu, 'BPM') # Filter out BPM extensions without data bpm_extensions = [ extension for extension in bpm_extensions if extension.data is not None ] if len(bpm_extensions) > 1: extension_shape = bpm_extensions[0].data.shape bpm_shape = (len(bpm_extensions), extension_shape[0], extension_shape[1]) bpm = np.zeros(bpm_shape, dtype=np.uint8) for i, extension in enumerate(bpm_extensions): bpm[i, :, :] = extension.data[:, :] elif len(bpm_extensions) == 1: bpm = np.array(bpm_extensions[0].data, dtype=np.uint8) else: bpm = np.array(bpm_hdu[0].data, dtype=np.uint8) return bpm