def stack_calibrations(self, min_date: str, max_date: str, instrument_id: int, frame_type: str,
runtime_context: dict, observations: list):
runtime_context = Context(runtime_context)
instrument = dbs.get_instrument_by_id(instrument_id, db_address=runtime_context.db_address)
logger.info('Checking if we are ready to stack',
extra_tags={'site': instrument.site, 'min_date': min_date, 'max_date': max_date,
'instrument': instrument.camera, 'frame_type': frame_type})
completed_image_count = len(dbs.get_individual_calibration_images(instrument, frame_type,
min_date, max_date, include_bad_frames=True,
db_address=runtime_context.db_address))
expected_image_count = 0
for observation in observations:
for configuration in observation['request']['configurations']:
if frame_type.upper() == configuration['type']:
for instrument_config in configuration['instrument_configs']:
expected_image_count += instrument_config['exposure_count']
logger.info('expected image count: {0}, completed image count: {1}'.format(str(expected_image_count), str(completed_image_count)))
if completed_image_count < expected_image_count and self.request.retries < 3:
logger.info('Number of processed images less than expected. '
'Expected: {}, Completed: {}'.format(expected_image_count, completed_image_count),
extra_tags={'site': instrument.site, 'min_date': min_date, 'max_date': max_date,
'instrument': instrument.camera, 'frame_type': frame_type})
raise self.retry()
else:
logger.info('Starting to stack', extra_tags={'site': instrument.site, 'min_date': min_date,
'max_date': max_date, 'instrument': instrument.camera,
'frame_type': frame_type})
calibrations.process_master_maker(instrument, frame_type, min_date, max_date, runtime_context)
def parse_args(extra_console_arguments=None, parser_description='Process LCO data.'):
"""Parse arguments, including default command line argument, and set the overall log level"""
parser = argparse.ArgumentParser(description=parser_description)
parser.add_argument("--processed-path", default='/archive/engineering',
help='Top level directory where the processed data will be stored')
parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',
'critical', 'fatal', 'error'])
parser.add_argument('--post-to-archive', dest='post_to_archive', action='store_true',
default=False)
parser.add_argument('--post-to-elasticsearch', dest='post_to_elasticsearch', action='store_true',
default=False)
parser.add_argument('--fpack', dest='fpack', action='store_true', default=False,
help='Fpack the output files?')
parser.add_argument('--rlevel', dest='rlevel', default=91, type=int, help='Reduction level')
parser.add_argument('--db-address', dest='db_address',
default='mysql://*****:*****@localhost/test',
help='Database address: Should be in SQLAlchemy form')
parser.add_argument('--elasticsearch-url', dest='elasticsearch_url',
default='http://elasticsearch.lco.gtn:9200')
parser.add_argument('--es-index', dest='elasticsearch_qc_index', default='banzai_qc',
help='ElasticSearch index to use for QC results')
parser.add_argument('--es-doc-type', dest='elasticsearch_doc_type', default='qc',
help='Elasticsearch document type for QC records')
parser.add_argument('--no-bpm', dest='no_bpm', default=False, action='store_true',
help='Do not use a bad pixel mask to reduce data (BPM contains all zeros)')
parser.add_argument('--ignore-schedulability', dest='ignore_schedulability',
default=False, action='store_true',
help='Relax requirement that the instrument be schedulable')
parser.add_argument('--use-only-older-calibrations', dest='use_only_older_calibrations', default=False,
action='store_true', help='Only use calibrations that were created before the start of the block')
parser.add_argument('--preview-mode', dest='preview_mode', default=False, action='store_true',
help='Save the reductions to the preview directory')
parser.add_argument('--max-tries', dest='max_tries', default=5,
help='Maximum number of times to try to process a frame')
parser.add_argument('--broker-url', dest='broker_url',
help='URL for the FITS broker service.')
if extra_console_arguments is None:
extra_console_arguments = []
for argument in extra_console_arguments:
parser.add_argument(*argument['args'], **argument['kwargs'])
args = parser.parse_args()
logs.set_log_level(args.log_level)
# Get all of the settings that are not builtins and store them in the context object
for setting in dir(settings):
if '__' != setting[:2] and not isinstance(getattr(settings, setting), ModuleType):
setattr(args, setting, getattr(settings, setting))
return Context(args)
def process_image(path: str, runtime_context: dict):
runtime_context = Context(runtime_context)
logger.info('Running process image.')
try:
if realtime_utils.need_to_process_image(path, runtime_context):
logger.info('Reducing frame', extra_tags={'filename': os.path.basename(path)})
# Increment the number of tries for this file
realtime_utils.increment_try_number(path, db_address=runtime_context.db_address)
run(path, runtime_context)
realtime_utils.set_file_as_processed(path, db_address=runtime_context.db_address)
except Exception:
logger.error("Exception processing frame: {error}".format(error=logs.format_exception()),
extra_tags={'filename': os.path.basename(path)})
def setup(self):
self.site = 'coj'
self.min_date = '2019-02-19T20:27:49'
self.max_date = '2019-02-20T09:55:09'
self.context = Context({
'db_address': 'db_address',
'CALIBRATION_IMAGE_TYPES': ['BIAS'],
'CALIBRATION_STACK_DELAYS': {
'BIAS': 300
}
})
self.frame_type = 'BIAS'
self.fake_blocks_response_json = fake_blocks_response_json
self.fake_inst = FakeInstrument(site='coj',
camera='2m0-SciCam-Spectral',
enclosure='clma',
telescope='2m0a')
def schedule_calibration_stacking(site: str, runtime_context: dict, min_date=None, max_date=None, frame_types=None):
runtime_context = Context(runtime_context)
if min_date is None or max_date is None:
timezone_for_site = dbs.get_timezone(site, db_address=runtime_context.db_address)
min_date, max_date = date_utils.get_min_and_max_dates_for_calibration_scheduling(timezone_for_site)
calibration_blocks = observation_utils.get_calibration_blocks_for_time_range(site, max_date, min_date)
if frame_types is None:
frame_types = runtime_context.CALIBRATION_IMAGE_TYPES
for frame_type in frame_types:
logger.info('Scheduling stacking', extra_tags={'site': site, 'min_date': min_date, 'max_date': max_date,
'frame_type': frame_type})
instruments = dbs.get_instruments_at_site(site=site, db_address=runtime_context.db_address)
for instrument in instruments:
logger.info('Checking for scheduled calibration blocks', extra_tags={'site': site, 'min_date': min_date,
'max_date': max_date,
'instrument': instrument.camera,
'frame_type': frame_type})
blocks_for_calibration = observation_utils.filter_calibration_blocks_for_type(instrument, frame_type,
calibration_blocks)
if len(blocks_for_calibration) > 0:
# block_end should be the latest block end time
calibration_end_time = max([parse(block['end']) for block in blocks_for_calibration]).replace(tzinfo=None)
stack_delay = timedelta(seconds=runtime_context.CALIBRATION_STACK_DELAYS[frame_type.upper()])
now = datetime.utcnow().replace(microsecond=0)
message_delay = calibration_end_time - now + stack_delay
if message_delay.days < 0:
message_delay_in_seconds = 0 # Remove delay if block end is in the past
else:
message_delay_in_seconds = message_delay.seconds
schedule_time = now + timedelta(seconds=message_delay_in_seconds)
logger.info('Scheduling stacking at {}'.format(schedule_time.strftime(date_utils.TIMESTAMP_FORMAT)),
extra_tags={'site': site, 'min_date': min_date, 'max_date': max_date,
'instrument': instrument.camera, 'frame_type': frame_type})
stack_calibrations.apply_async(args=(min_date, max_date, instrument.id, frame_type,
vars(runtime_context), blocks_for_calibration),
countdown=message_delay_in_seconds)
else:
logger.warning('No scheduled calibration blocks found.',
extra_tags={'site': site, 'min_date': min_date, 'max_date': max_date,
'instrument': instrument.name, 'frame_type': frame_type})
def test_context_gets_arguments_from_argparse():
args = Namespace(a=1, b=2, c=5)
context = Context(args)
assert context.a == 1
assert context.b == 2
assert context.c == 5