def do_stage(self, images):
for image in images:
image.header['RLEVEL'] = (self.pipeline_context.rlevel,
'Reduction level')
image.header['PIPEVER'] = (banzai.__version__, 'Pipeline version')
if instantly_public(image.header['PROPID']):
image.header['L1PUBDAT'] = (
image.header['DATE-OBS'],
'[UTC] Date the frame becomes public')
else:
# Wait a year
date_observed = date_utils.parse_date_obs(
image.header['DATE-OBS'])
next_year = date_observed + timedelta(days=365)
image.header['L1PUBDAT'] = (
date_utils.date_obs_to_string(next_year),
'[UTC] Date the frame becomes public')
logging_tags = logs.image_config_to_tags(image,
self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
logs.add_tag(logging_tags, 'rlevel', int(image.header['RLEVEL']))
logs.add_tag(logging_tags, 'pipeline_version',
image.header['PIPEVER'])
logs.add_tag(logging_tags, 'l1pubdat', image.header['L1PUBDAT'])
self.logger.info('Updating header', extra=logging_tags)
return images
def populate_calibration_table_with_bpms(directory, db_address=_DEFAULT_DB):
with get_session(db_address=db_address) as db_session:
bpm_filenames = glob(os.path.join(directory, 'bpm*.fits*'))
for bpm_filename in bpm_filenames:
if bpm_filename[-3:] == '.fz':
extension_number = 1
else:
extension_number = 0
header = fits.getheader(bpm_filename, extension_number)
ccdsum = fits.getval(bpm_filename, 'CCDSUM', extension_number)
dateobs = date_utils.parse_date_obs(fits.getval(bpm_filename, 'DATE-OBS', extension_number))
try:
instrument = get_instrument(header, db_address=db_address)
except ValueError:
logger.error('Instrument is missing from database', extra_tags={'site': header['SITEID'],
'camera': header['INSTRUME']})
continue
bpm_attributes = {'type': 'BPM',
'filename': os.path.basename(bpm_filename),
'filepath': os.path.abspath(directory),
'dateobs': dateobs,
'datecreated': dateobs,
'instrument_id': instrument.id,
'is_master': True,
'is_bad': False,
'attributes': {'ccdsum': ccdsum}}
add_or_update_record(db_session, CalibrationImage, {'filename': bpm_attributes['filename']}, bpm_attributes)
db_session.commit()
def _save_pipeline_metadata(self, runtime_context):
self.datecreated = datetime.datetime.utcnow()
self.header['DATE'] = (date_utils.date_obs_to_string(self.datecreated),
'[UTC] Date this FITS file was written')
self.header['RLEVEL'] = (runtime_context.rlevel, 'Reduction level')
self.header['PIPEVER'] = (banzai.__version__, 'Pipeline version')
if file_utils.instantly_public(self.header['PROPID']):
self.header['L1PUBDAT'] = (self.header['DATE-OBS'],
'[UTC] Date the frame becomes public')
else:
# Wait a year
date_observed = date_utils.parse_date_obs(self.header['DATE-OBS'])
next_year = date_observed + datetime.timedelta(days=365)
self.header['L1PUBDAT'] = (
date_utils.date_obs_to_string(next_year),
'[UTC] Date the frame becomes public')
logging_tags = {
'rlevel': int(self.header['RLEVEL']),
'pipeline_version': self.header['PIPEVER'],
'l1pubdat': self.header['L1PUBDAT'],
}
logger.info('Adding pipeline metadata to the header',
image=self,
extra_tags=logging_tags)
def __init__(self, pipeline_context, filename=None, data=None, header={}, bpm=None):
if filename is not None:
data, header, bpm = fits_utils.open_image(filename)
if '.fz' == filename[-3:]:
filename = filename[:-3]
self.filename = os.path.basename(filename)
self.data = data
self.header = header
self.bpm = bpm
self.site = header.get('SITEID')
self.instrument = header.get('INSTRUME')
self.epoch = header.get('DAY-OBS')
self.nx = header.get('NAXIS1')
self.ny = header.get('NAXIS2')
self.gain = header.get('GAIN')
self.ccdsum = header.get('CCDSUM')
self.filter = header.get('FILTER')
self.telescope_id = dbs.get_telescope_id(self.site, self.instrument,
db_address=pipeline_context.db_address)
self.obstype = header.get('OBSTYPE')
self.exptime = float(header.get('EXPTIME'))
self.dateobs = date_utils.parse_date_obs(header.get('DATE-OBS'))
self.readnoise = float(header.get('RDNOISE'))
self.ra, self.dec = fits_utils.parse_ra_dec(header)
self.pixel_scale = float(header.get('PIXSCALE'))
self.catalog = None
def __init__(self, filename=None, data=None, header={}, bpm=None):
if filename is not None:
data, header, bpm = fits_utils.open_image(filename)
if '.fz' == filename[-3:]:
filename = filename[:-3]
self.filename = os.path.basename(filename)
self.data = data
self.header = header
self.bpm = bpm
self.site = header.get('SITEID')
self.instrument = header.get('INSTRUME')
self.epoch = header.get('DAY-OBS')
self.nx = header.get('NAXIS1')
self.ny = header.get('NAXIS2')
self.gain = header.get('GAIN')
self.ccdsum = header.get('CCDSUM')
self.filter = header.get('FILTER')
self.telescope_id = dbs.get_telescope_id(self.site, self.instrument)
self.obstype = header.get('OBSTYPE')
self.exptime = float(header.get('EXPTIME'))
self.dateobs = date_utils.parse_date_obs(header.get('DATE-OBS'))
self.readnoise = float(header.get('RDNOISE'))
self.ra, self.dec = fits_utils.parse_ra_dec(header)
self.pixel_scale = float(header.get('PIXSCALE'))
self.catalog = None
def __init__(self,
pipeline_context,
filename=None,
data=None,
header={},
extension_headers=[],
bpm=None):
if filename is not None:
data, header, bpm, extension_headers = fits_utils.open_image(
filename)
if '.fz' == filename[-3:]:
filename = filename[:-3]
self.filename = os.path.basename(filename)
self.data = data
self.header = header
self.bpm = bpm
self.extension_headers = extension_headers
self.request_number = header.get('REQNUM')
self.site = header.get('SITEID')
self.instrument = header.get('INSTRUME')
self.epoch = str(header.get('DAY-OBS'))
self.nx = header.get('NAXIS1')
self.ny = header.get('NAXIS2')
if len(self.extension_headers
) > 0 and 'GAIN' in self.extension_headers[0]:
self.gain = [h['GAIN'] for h in extension_headers]
else:
self.gain = eval(str(header.get('GAIN')))
self.ccdsum = header.get('CCDSUM')
self.filter = header.get('FILTER')
self.telescope_id = dbs.get_telescope_id(
self.site, self.instrument, db_address=pipeline_context.db_address)
self.obstype = header.get('OBSTYPE')
self.exptime = float(header.get('EXPTIME'))
self.dateobs = date_utils.parse_date_obs(header.get('DATE-OBS'))
self.readnoise = float(header.get('RDNOISE'))
self.ra, self.dec = fits_utils.parse_ra_dec(header)
self.pixel_scale = float(header.get('PIXSCALE'))
self.catalog = None
def populate_calibration_table_with_bpms(directory, db_address=_DEFAULT_DB):
with get_session(db_address=db_address) as db_session:
bpm_filenames = glob(os.path.join(directory, '*bpm*.fits*'))
for bpm_filename in bpm_filenames:
hdu = fits_utils.open_fits_file(bpm_filename)
header = hdu[0].header
ccdsum = header.get('CCDSUM')
configuration_mode = fits_utils.get_configuration_mode(header)
dateobs = date_utils.parse_date_obs(header.get('DATE-OBS'))
try:
instrument = get_instrument(header, db_address=db_address)
except ValueError:
logger.error('Instrument is missing from database',
extra_tags={
'site': header['SITEID'],
'camera': header['INSTRUME']
})
continue
bpm_attributes = {
'type': 'BPM',
'filename': os.path.basename(bpm_filename),
'filepath': os.path.abspath(directory),
'dateobs': dateobs,
'datecreated': dateobs,
'instrument_id': instrument.id,
'is_master': True,
'is_bad': False,
'attributes': {
'ccdsum': ccdsum,
'configuration_mode': configuration_mode
}
}
add_or_update_record(db_session, CalibrationImage,
{'filename': bpm_attributes['filename']},
bpm_attributes)
db_session.commit()
def save_pipeline_metadata(image, pipeline_context):
image.header['RLEVEL'] = (pipeline_context.rlevel, 'Reduction level')
image.header['PIPEVER'] = (banzai.__version__, 'Pipeline version')
if file_utils.instantly_public(image.header['PROPID']):
image.header['L1PUBDAT'] = (image.header['DATE-OBS'],
'[UTC] Date the frame becomes public')
else:
# Wait a year
date_observed = date_utils.parse_date_obs(image.header['DATE-OBS'])
next_year = date_observed + timedelta(days=365)
image.header['L1PUBDAT'] = (date_utils.date_obs_to_string(next_year),
'[UTC] Date the frame becomes public')
logging_tags = logs.image_config_to_tags(image, None)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
logs.add_tag(logging_tags, 'rlevel', int(image.header['RLEVEL']))
logs.add_tag(logging_tags, 'pipeline_version', image.header['PIPEVER'])
logs.add_tag(logging_tags, 'l1pubdat', image.header['L1PUBDAT'])
logger.info('Updating header', extra=logging_tags)
def __init__(self,
runtime_context,
filename=None,
data=None,
data_tables=None,
header=None,
extension_headers=None,
bpm=None):
if header is None:
header = fits.Header()
if data_tables is None:
data_tables = {}
if extension_headers is None:
extension_headers = []
if filename is not None:
data, header, bpm, extension_headers = fits_utils.open_image(
filename)
if '.fz' == filename[-3:]:
filename = filename[:-3]
self.filename = os.path.basename(filename)
self.data = data
self.data_tables = data_tables
self.header = header
self.bpm = bpm
self.extension_headers = extension_headers
self.request_number = header.get('REQNUM')
self.instrument, self.site, self.camera = self._init_instrument_info(
runtime_context)
self.epoch = str(header.get('DAY-OBS'))
self.nx = header.get('NAXIS1')
self.ny = header.get('NAXIS2')
self.block_id = header.get('BLKUID')
self.block_start = date_utils.parse_date_obs(
header.get('BLKSDATE', '1900-01-01T00:00:00.00000'))
self.molecule_id = header.get('MOLUID')
if len(self.extension_headers
) > 0 and 'GAIN' in self.extension_headers[0]:
self.gain = [h['GAIN'] for h in extension_headers]
else:
self.gain = eval(str(header.get('GAIN')))
self.ccdsum = header.get('CCDSUM')
self.configuration_mode = fits_utils.get_configuration_mode(header)
self.filter = header.get('FILTER')
self.obstype = header.get('OBSTYPE')
self.exptime = float(header.get('EXPTIME', 0.0))
self.dateobs = date_utils.parse_date_obs(
header.get('DATE-OBS', '1900-01-01T00:00:00.00000'))
self.datecreated = date_utils.parse_date_obs(
header.get('DATE', date_utils.date_obs_to_string(self.dateobs)))
self.readnoise = float(header.get('RDNOISE', 0.0))
self.ra, self.dec = fits_utils.parse_ra_dec(header)
self.pixel_scale = float(header.get('PIXSCALE', 0.0))
self.is_bad = False
self.is_master = header.get('ISMASTER', False)
self.attributes = settings.CALIBRATION_SET_CRITERIA.get(
self.obstype, {})