def reduce_night():
parser = argparse.ArgumentParser(
description='Reduce all the data from a site at the end of a night.')
parser.add_argument('--site', dest='site', help='Site code (e.g. ogg)')
parser.add_argument('--dayobs', dest='dayobs',
default=None, help='Day-Obs to reduce (e.g. 20160201)')
parser.add_argument('--raw-path-root', dest='rawpath_root', default='/archive/engineering',
help='Top level directory with raw data.')
parser.add_argument("--processed-path", default='/archive/engineering',
help='Top level directory where the processed data will be stored')
parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',
'critical', 'fatal', 'error'])
parser.add_argument('--post-to-archive', dest='post_to_archive', action='store_true',
default=False)
parser.add_argument('--fpack', dest='fpack', action='store_true', default=False,
help='Fpack the output files?')
parser.add_argument('--rlevel', dest='rlevel', default=91, help='Reduction level')
parser.add_argument('--db-address', dest='db_address',
default='mysql://*****:*****@localhost/test',
help='Database address: Should be in SQLAlchemy form')
args = parser.parse_args()
args.preview_mode = False
args.raw_path = None
args.filename = None
pipeline_context = PipelineContext(args)
logs.start_logging(log_level=pipeline_context.log_level)
# Ping the configdb to get currently schedulable telescopes
try:
dbs.populate_telescope_tables(db_address=pipeline_context.db_address)
except Exception as e:
logger.error('Could not connect to the configdb.')
logger.error(e)
timezone = dbs.get_timezone(args.site, db_address=args.db_address)
telescopes = dbs.get_schedulable_telescopes(args.site, db_address=args.db_address)
if timezone is not None:
# If no dayobs is given, calculate it.
if args.dayobs is None:
args.dayobs = date_utils.get_dayobs(timezone=timezone)
# For each telescope at the given site
for telescope in telescopes:
pipeline_context.raw_path = os.path.join(args.rawpath_root, args.site,
telescope.instrument, args.dayobs, 'raw')
# Run the reductions on the given dayobs
make_master_bias(pipeline_context)
make_master_dark(pipeline_context)
make_master_flat(pipeline_context)
reduce_science_frames(pipeline_context)
logs.stop_logging()
def create_master_calibrations():
pipeline_context = parse_end_of_night_command_line_arguments()
logs.start_logging(log_level=pipeline_context.log_level)
make_master_bias()
make_master_dark()
make_master_flat()
logs.stop_logging()
def create_master_calibrations():
pipeline_context = parse_end_of_night_command_line_arguments()
logs.start_logging(log_level=pipeline_context.log_level)
make_master_bias()
make_master_dark()
make_master_flat()
logs.stop_logging()
def make_master_flat(cmd_args=None):
pipeline_context = parse_command_line_arguments(cmd_args=cmd_args)
logs.start_logging(log_level=pipeline_context.log_level)
stages_to_do = [munge.DataMunger, crosstalk.CrosstalkCorrector, bias.OverscanSubtractor,
gain.GainNormalizer, mosaic.MosaicCreator, trim.Trimmer, bias.BiasSubtractor,
dark.DarkSubtractor, flats.FlatMaker, headers.HeaderUpdater]
run(stages_to_do, pipeline_context, image_types=['SKYFLAT'], calibration_maker=True,
log_message='Making Master Flat')
logs.stop_logging()
def run_preview_pipeline():
parser = argparse.ArgumentParser(
description='Make master calibration frames from LCOGT imaging data.')
parser.add_argument("--processed-path", default='/archive/engineering',
help='Top level directory where the processed data will be stored')
parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',
'critical', 'fatal', 'error'])
parser.add_argument('--post-to-archive', dest='post_to_archive', action='store_true',
default=False)
parser.add_argument('--db-address', dest='db_address',
default='mysql://*****:*****@localhost/test',
help='Database address: Should be in SQLAlchemy form')
parser.add_argument('--fpack', dest='fpack', action='store_true', default=False,
help='Fpack the output files?')
parser.add_argument('--rlevel', dest='rlevel', default=11, help='Reduction level')
parser.add_argument('--n-processes', dest='n_processes', default=12,
help='Number of listener processes to spawn.')
parser.add_argument('--broker-url', dest='broker_url',
default='amqp://*****:*****@cerberus.lco.gtn',
help='URL for the broker service.')
parser.add_argument('--queue-name', dest='queue_name', default='preview_pipeline',
help='Name of the queue to listen to from the fits exchange.')
parser.add_argument('--max-preview-tries', dest='max_preview_tries', default=5,
help='Maximum number of tries to produce a preview image.')
args = parser.parse_args()
args.preview_mode = True
args.raw_path = None
args.filename = None
pipeline_context = PipelineContext(args)
logs.start_logging(log_level=pipeline_context.log_level)
try:
dbs.populate_telescope_tables(db_address=pipeline_context.db_address)
except Exception as e:
logger.error('Could not connect to the configdb.')
logger.error(e)
logger.info('Starting pipeline preview mode listener')
for i in range(args.n_processes):
p = multiprocessing.Process(target=run_indiviudal_listener, args=(args.broker_url,
args.queue_name,
PipelineContext(args)))
p.start()
logs.stop_logging()
def reduce_science_frames(cmd_args=None):
pipeline_context = parse_command_line_arguments(cmd_args=cmd_args)
logs.start_logging(log_level=pipeline_context.log_level)
stages_to_do = [munge.DataMunger, crosstalk.CrosstalkCorrector, bias.OverscanSubtractor,
gain.GainNormalizer, mosaic.MosaicCreator, trim.Trimmer, bias.BiasSubtractor,
dark.DarkSubtractor, flats.FlatDivider, photometry.SourceDetector,
astrometry.WCSSolver, headers.HeaderUpdater]
image_list = file_utils.make_image_list(pipeline_context)
for image in image_list:
pipeline_context.filename = os.path.basename(image)
run(stages_to_do, pipeline_context, image_types=['EXPOSE', 'STANDARD'],
log_message='Reducing Science Frames')
logs.stop_logging()
def run_preview_pipeline(cmd_args=None):
pipeline_context = parse_command_line_arguments(cmd_args=cmd_args)
logs.start_logging(log_level=pipeline_context.log_level)
logger.info('Starting pipeline preview mode listener')
crawl_exchange = Exchange('fits_files', type='fanout')
listener = PreviewModeListener('amqp://*****:*****@cerberus.lco.gtn', pipeline_context)
with Connection(listener.broker_url) as connection:
listener.connection = connection
listener.queue = Queue('preview_pipeline', crawl_exchange)
try:
listener.run()
except KeyboardInterrupt:
logger.info('Shutting down preview pipeline listener...')
logs.stop_logging()
sys.exit(0)
def make_master_flat_console():
pipeline_context = parse_end_of_night_command_line_arguments()
logs.start_logging(log_level=pipeline_context.log_level)
make_master_flat(pipeline_context)
logs.stop_logging()
def reduce_science_frames_console():
pipeline_context = parse_end_of_night_command_line_arguments()
logs.start_logging(log_level=pipeline_context.log_level)
reduce_science_frames(pipeline_context)
logs.stop_logging()
def run_end_of_night_from_console(scripts_to_run):
pipeline_context = parse_end_of_night_command_line_arguments()
logs.start_logging(log_level=pipeline_context.log_level)
for script in scripts_to_run:
script(pipeline_context)
logs.stop_logging()
def run_preview_pipeline():
parser = argparse.ArgumentParser(
description='Make master calibration frames from LCOGT imaging data.')
parser.add_argument(
"--processed-path",
default='/archive/engineering',
help='Top level directory where the processed data will be stored')
parser.add_argument(
"--log-level",
default='debug',
choices=['debug', 'info', 'warning', 'critical', 'fatal', 'error'])
parser.add_argument('--post-to-archive',
dest='post_to_archive',
action='store_true',
default=False)
parser.add_argument('--db-address',
dest='db_address',
default='mysql://*****:*****@localhost/test',
help='Database address: Should be in SQLAlchemy form')
parser.add_argument('--fpack',
dest='fpack',
action='store_true',
default=False,
help='Fpack the output files?')
parser.add_argument('--rlevel',
dest='rlevel',
default=11,
help='Reduction level')
parser.add_argument('--n-processes',
dest='n_processes',
default=12,
help='Number of listener processes to spawn.',
type=int)
parser.add_argument('--broker-url',
dest='broker_url',
default='amqp://*****:*****@rabbitmq.lco.gtn:5672/',
help='URL for the broker service.')
parser.add_argument(
'--queue-name',
dest='queue_name',
default='preview_pipeline',
help='Name of the queue to listen to from the fits exchange.')
parser.add_argument(
'--max-preview-tries',
dest='max_preview_tries',
default=5,
help='Maximum number of tries to produce a preview image.')
args = parser.parse_args()
args.preview_mode = True
args.raw_path = None
args.filename = None
pipeline_context = PipelineContext(args)
logs.start_logging(log_level=pipeline_context.log_level)
try:
dbs.populate_telescope_tables(db_address=pipeline_context.db_address)
except Exception as e:
logger.error('Could not connect to the configdb.')
logger.error(e)
logger.info('Starting pipeline preview mode listener')
for i in range(args.n_processes):
p = multiprocessing.Process(target=run_indiviudal_listener,
args=(args.broker_url, args.queue_name,
PipelineContext(args)))
p.start()
logs.stop_logging()
def reduce_night():
parser = argparse.ArgumentParser(
description='Reduce all the data from a site at the end of a night.')
parser.add_argument('--site', dest='site', help='Site code (e.g. ogg)')
parser.add_argument('--dayobs',
dest='dayobs',
default=None,
help='Day-Obs to reduce (e.g. 20160201)')
parser.add_argument('--raw-path-root',
dest='rawpath_root',
default='/archive/engineering',
help='Top level directory with raw data.')
parser.add_argument(
"--processed-path",
default='/archive/engineering',
help='Top level directory where the processed data will be stored')
parser.add_argument(
"--log-level",
default='debug',
choices=['debug', 'info', 'warning', 'critical', 'fatal', 'error'])
parser.add_argument('--post-to-archive',
dest='post_to_archive',
action='store_true',
default=False)
parser.add_argument('--fpack',
dest='fpack',
action='store_true',
default=False,
help='Fpack the output files?')
parser.add_argument('--rlevel',
dest='rlevel',
default=91,
help='Reduction level')
parser.add_argument('--db-address',
dest='db_address',
default='mysql://*****:*****@localhost/test',
help='Database address: Should be in SQLAlchemy form')
args = parser.parse_args()
args.preview_mode = False
args.raw_path = None
args.filename = None
args.max_preview_tries = 5
pipeline_context = PipelineContext(args)
logs.start_logging(log_level=pipeline_context.log_level)
# Ping the configdb to get currently schedulable telescopes
try:
dbs.populate_telescope_tables(db_address=pipeline_context.db_address)
except Exception as e:
logger.error('Could not connect to the configdb.')
logger.error(e)
timezone = dbs.get_timezone(args.site, db_address=args.db_address)
telescopes = dbs.get_schedulable_telescopes(args.site,
db_address=args.db_address)
if timezone is not None:
# If no dayobs is given, calculate it.
if args.dayobs is None:
args.dayobs = date_utils.get_dayobs(timezone=timezone)
# For each telescope at the given site
for telescope in telescopes:
pipeline_context.raw_path = os.path.join(args.rawpath_root,
args.site,
telescope.instrument,
args.dayobs, 'raw')
try:
# Run the reductions on the given dayobs
make_master_bias(pipeline_context)
make_master_dark(pipeline_context)
make_master_flat(pipeline_context)
reduce_science_frames(pipeline_context)
except Exception as e:
logger.error(e)
logs.stop_logging()
def make_master_flat_console():
pipeline_context = parse_end_of_night_command_line_arguments()
logs.start_logging(log_level=pipeline_context.log_level)
make_master_flat(pipeline_context)
logs.stop_logging()
def reduce_science_frames_console():
pipeline_context = parse_end_of_night_command_line_arguments()
logs.start_logging(log_level=pipeline_context.log_level)
reduce_science_frames(pipeline_context)
logs.stop_logging()
def run_end_of_night_from_console(scripts_to_run):
pipeline_context = parse_end_of_night_command_line_arguments()
logs.start_logging(log_level=pipeline_context.log_level)
for script in scripts_to_run:
script(pipeline_context)
logs.stop_logging()
