def reduce_data_set(rawDataSet, out_dir):
logger = logging.getLogger('main')
# generate reduced data set by reducing raw data set
reducedDataSet = reduce_frame.reduce_frame(rawDataSet, out_dir)
# produce data products from reduced data set
if config.params['products'] is True:
products.gen(reducedDataSet, out_dir)
else:
logger.info('data product generation inhibited by command line switch')
# if diagnostics mode is enabled, then produce diagnostic data products
if config.params['dgn'] is True:
logger.info('diagnostic mode enabled, generating diagnostic data products')
dgn.gen(reducedDataSet, out_dir)
def process_dir(in_dir, base_out_dir):
"""
NSDRP. Assembles raw data sets from FITS files in the input directory,
then generates reduced data sets from raw data sets. Level 1 data products
are generated from the reduced data sets and saved in the output directory.
"""
# use main logger for outer logging, reduce_frame makes the per frame logger
logger = logging.getLogger('main')
ssFptr = start_summary_ss(in_dir, base_out_dir)
# get list of raw data sets arranged in chronological order
rawDataSets = create_raw_data_sets.create(in_dir)
if rawDataSets is None:
return
# keep track of how many data sets have been successfully reduced
n_reduced = len(rawDataSets)
# instantiate a flat cacher so each flat or set of flats is only reduced once
flatCacher = FlatCacher.FlatCacher(logger, base_out_dir + '/flats')
reducedDataSets = []
nirspecConfig = None
# process each raw data set
for rawDataSet in rawDataSets:
# determine the output directory to use for this frame
#out_dir = get_out_dir(rawDataSet.objAFn, base_out_dir);
out_dir = get_out_dir(rawDataSet.baseNames['A'], base_out_dir)
if nirspecConfig is None:
nirspecConfig = NirspecConfig.NirspecConfig(rawDataSet.objHeader)
else:
if nirspecConfig.isTheSame(rawDataSet.objHeader) is not None:
if len(reducedDataSets) > 0:
if len(reducedDataSets) > 1:
mcal(reducedDataSets)
gen_data_products(reducedDataSets, nirspecConfig,
base_out_dir, ssFptr)
del reducedDataSets[:]
logger.info(
'starting new multi-frame set for nirpsec config: {}'.
format(nirspecConfig.toString()))
nirspecConfig = NirspecConfig.NirspecConfig(
rawDataSet.objHeader)
reducedDataSets.append(
reduce_frame.reduce_frame(rawDataSet, out_dir, flatCacher))
# except DrpException as e:
# n_reduced -= 1
# logger.error('failed to reduce {}: {}'.format(
# rawDataSet.objFileName, e.message))e
# except IOError as e:
# logger.critical('DRP failed due to I/O error: {}'.format(str(e)))
# sys.exit(1)
if len(reducedDataSets) > 1:
logger.info(
'doing multi-frame calibration on {} frames, nirspec config: {}'.
format(len(reducedDataSets), nirspecConfig.toString()))
mcal(reducedDataSets)
if len(rawDataSets) > 0:
gen_data_products(reducedDataSets, nirspecConfig, base_out_dir, ssFptr)
logger.info('n object frames reduced = {}/{}'.format(
n_reduced, len(rawDataSets)))
ssFptr.close()
logger.info('end nsdrp')
return
def process_frame(fn1, fn2, obj_B_fn, out_dir):
flat_fn = None
obj_fn = None
fn1_header = fits.PrimaryHDU.readfrom(fn1, ignore_missing_end=True).header
fn2_header = fits.PrimaryHDU.readfrom(fn2, ignore_missing_end=True).header
if fn1_header['flat'] == 1 and fn1_header['calmpos'] == 1:
flat_fn = fn1
obj_fn = fn2
obj_header = fn2_header
flat_header = fn1_header
if fn2_header['flat'] == 1 and fn2_header['calmpos'] == 1:
if flat_fn is not None:
raise DrpException.DrpException('two flats, no object frame')
else:
flat_fn = fn2
obj_fn = fn1
obj_header = fn1_header
flat_header = fn2_header
if flat_fn is None:
raise DrpException.DrpException('no flat')
if obj_header['ECHLPOS'] > 100:
print('ERROR: cannot reduce low-resolution image (ECHLPOS > 100')
exit(1)
if obj_header['NAXIS1'] != nirspec_constants.N_COLS:
raise DrpException.DrpException('NAXIS1 != {}'.format(
nirspec_constants.N_COLS))
if obj_header['NAXIS2'] != nirspec_constants.N_ROWS:
raise DrpException.DrpException('NAXIS2 != {}'.format(
nirspec_constants.N_COLS))
if obj_header['FILNAME'].upper() not in nirspec_constants.filter_names:
raise DrpException.DrpException('unsupported filter: {}'.format(
obj_header['FILNAME']))
if create_raw_data_sets.flat_criteria_met(
obj_header, flat_header, ignore_dispers=True) is False:
raise DrpException.DrpException(
'flat is not compatible with object frame')
if obj_B_fn is not None:
# confirm that A and B are not the same files
if obj_fn == obj_B_fn:
raise DrpException.DrpException('frames A and B are the same')
obj_B_header = fits.PrimaryHDU.readfrom(obj_B_fn,
ignore_missing_end=True).header
if create_raw_data_sets.is_valid_pair(obj_header, obj_B_header):
#print('Reducing AB pair, A=' + obj_fn + ', B=' + obj_B_fn)
rawDataSet = RawDataSet.RawDataSet(obj_fn, obj_B_fn, obj_header)
else:
raise DrpException.DrpException(
'frames A and B are not a valid pair')
else:
rawDataSet = RawDataSet.RawDataSet(obj_fn, None, obj_header)
rawDataSet.flatFns.append(flat_fn)
if not os.path.exists(out_dir):
try:
os.mkdir(out_dir)
except:
msg = 'output directory {} does not exist and cannot be created'.format(
out_dir)
raise IOError(msg)
logger = logging.getLogger('main')
# generate reduced data set by reducing raw data set
reducedDataSet = reduce_frame.reduce_frame(rawDataSet, out_dir)
# write reduction summary to log file
write_summary(reducedDataSet)
# produce data products from reduced data set
if config.params['no_products'] is True:
logger.info('data product generation inhibited by command line switch')
else:
products.gen(reducedDataSet, out_dir)
# if diagnostics mode is enabled, then produce diagnostic data products
if config.params['dgn'] is True:
logger.info(
'diagnostic mode enabled, generating diagnostic data products')
dgn.gen(reducedDataSet, out_dir)
return
def process_frame(fn1, fn2, obj_B_fn, out_dir, dark=None, eta=None, arc=None, override=False):
logger = logging.getLogger('main')
flat_fn = None
obj_fn = None
fn1_header = fits.PrimaryHDU.readfrom(fn1, ignore_missing_end=True).header
fn2_header = fits.PrimaryHDU.readfrom(fn2, ignore_missing_end=True).header
# Get the observation date to determine if this is upgraded NIRSPEC
if datetime.strptime(fn1_header['date-obs'], '%Y-%m-%d') > datetime.strptime('2018-10-10', '%Y-%m-%d'):
nirspec_constants.upgrade = True
# Do the case for the etalon lamp
if eta is not None:
eta_header = fits.PrimaryHDU.readfrom(eta, ignore_missing_end=True).header
if nirspec_constants.upgrade:
if eta_header['etalon'] == 'On' and eta_header['calmpos'] == 'In':
eta_fn = eta
else:
if eta_header['etalon'] == 1 and eta_header['calmpos'] == 1:
eta_fn = eta
# Do the case for the arc lamp
if arc is not None:
arc_header = fits.PrimaryHDU.readfrom(arc, ignore_missing_end=True).header
if nirspec_constants.upgrade:
if (arc_header['neon'] == 'On' or arc_header['argon'] == 'On' or arc_header['krypton'] == 'On' or \
arc_header['xenon'] == 'On') and arc_header['calmpos'] == 'In':
arc_fn = arc
else:
if (arc_header['neon'] == 1 or arc_header['argon'] == 1 or arc_header['krypton'] == 1 or \
arc_header['xenon'] == 1) and arc_header['calmpos'] == 1:
arc_fn = arc
# Get the master dark frame if given
if dark is not None:
dark_header = fits.PrimaryHDU.readfrom(dark, ignore_missing_end=True).header
if nirspec_constants.upgrade:
if dark_header['halogen'] == 'On' and dark_header['calmpos'] == 'In' and \
dark_header['neon'] != 'On' and dark_header['argon'] != 'On' and dark_header['krypton'] != 'On' and \
dark_header['xenon'] != 'On':
dark_fn = dark
else:
if dark_header['flat'] == 0 and dark_header['calmpos'] == 1 and \
dark_header['neon'] != 1 and dark_header['argon'] != 1 and dark_header['krypton'] != 1 and \
dark_header['xenon'] != 1:
dark_fn = dark
# Get the flat and object fits
if nirspec_constants.upgrade:
if fn1_header['halogen'] == 'On' and fn1_header['calmpos'] == 'In':
flat_fn = fn1
obj_fn = fn2
obj_header = fn2_header
flat_header = fn1_header
if fn1_header['halogen'] == 'On' and fn1_header['calmpos'] == 'In':
flat_fn = fn1
obj_fn = fn2
obj_header = fn2_header
flat_header = fn1_header
else:
if fn1_header['flat'] == 1 and fn1_header['calmpos'] == 1:
flat_fn = fn1
obj_fn = fn2
obj_header = fn2_header
flat_header = fn1_header
if fn2_header['flat'] == 1 and fn2_header['calmpos'] == 1:
if flat_fn is not None:
raise DrpException.DrpException('two flats, no object frame')
else:
flat_fn = fn2
obj_fn = fn1
obj_header = fn1_header
flat_header = fn2_header
if flat_fn is None:
raise DrpException.DrpException('no flat')
if obj_header['ECHLPOS'] > 100:
print('ERROR: cannot reduce low-resolution image (ECHLPOS > 100')
exit(1)
# Check the array size is correct
if nirspec_constants.upgrade:
if obj_header['NAXIS1'] != nirspec_constants.N_COLS_upgrade:
raise DrpException.DrpException('NAXIS1 != {}'.format(nirspec_constants.N_COLS_upgrade))
if obj_header['NAXIS2'] != nirspec_constants.N_ROWS_upgrade:
raise DrpException.DrpException('NAXIS2 != {}'.format(nirspec_constants.N_COLS_upgrade))
filtername1, filtername2 = obj_header['SCIFILT2'], ''
if obj_header['SCIFILT1'] == 'AO-stop': filtername2 = '-AO'
if filtername1.upper()+filtername2.upper() not in nirspec_constants.filter_names:
raise DrpException.DrpException('unsupported filter: {}'.format(obj_header['SCIFILT2']))
else:
if obj_header['NAXIS1'] != nirspec_constants.N_COLS:
raise DrpException.DrpException('NAXIS1 != {}'.format(nirspec_constants.N_COLS))
if obj_header['NAXIS2'] != nirspec_constants.N_ROWS:
raise DrpException.DrpException('NAXIS2 != {}'.format(nirspec_constants.N_COLS))
if obj_header['FILNAME'].upper() not in nirspec_constants.filter_names:
raise DrpException.DrpException('unsupported filter: {}'.format(obj_header['FILNAME']))
if create_raw_data_sets.flat_criteria_met(obj_header, flat_header, ignore_dispers=True) is False:
raise DrpException.DrpException('flat is not compatible with object frame')
if dark is not None:
if create_raw_data_sets.dark_criteria_met(obj_header, dark_header) is False:
raise DrpException.DrpException('dark is not compatible with object frame')
if obj_B_fn is not None:
# confirm that A and B are not the same files
if obj_fn == obj_B_fn:
raise DrpException.DrpException('frames A and B are the same')
obj_B_header = fits.PrimaryHDU.readfrom(obj_B_fn, ignore_missing_end=True).header
if create_raw_data_sets.is_valid_pair(obj_header, obj_B_header, override=override):
logger.debug('Reducing AB pair, A= ' + obj_fn + ' , B= ' + obj_B_fn)
rawDataSet = RawDataSet.RawDataSet(obj_fn, obj_B_fn, obj_header, eta=eta, arc=arc, dark=dark)
else:
raise DrpException.DrpException('frames A and B are not a valid pair')
else:
rawDataSet = RawDataSet.RawDataSet(obj_fn, None, obj_header, eta=eta, arc=arc, dark=dark)
rawDataSet.flatFns.append(flat_fn)
if not os.path.exists(out_dir):
try:
os.mkdir(out_dir)
except:
msg = 'output directory {} does not exist and cannot be created'.format(out_dir)
raise IOError(msg)
# generate reduced data set by reducing raw data set
reducedDataSet = reduce_frame.reduce_frame(rawDataSet, out_dir, eta=eta, arc=arc, dark=dark)
# write reduction summary to log file
write_summary(reducedDataSet)
# produce data products from reduced data set
if config.params['no_products'] is True:
logger.info('data product generation inhibited by command line switch')
else:
products.gen(reducedDataSet, out_dir, eta=eta, arc=arc)
# if diagnostics mode is enabled, then produce diagnostic data products
if config.params['dgn'] is True:
logger.info('diagnostic mode enabled, generating diagnostic data products')
dgn.gen(reducedDataSet, out_dir, eta=eta, arc=arc)
return