def __init__(self, objAFn, objBFn, objHeader):
if objAFn is None:
raise DrpException('objAFn cannot be None')
if objHeader is None:
raise DrpException('header cannot be None')
self.objAFn = objAFn
self.objBFn = objBFn
self.objHeader = objHeader
self.baseNames = {}
if self.objAFn is not None and self.objBFn is not None:
self.isPair = True
self.frames = ['A', 'B', 'AB']
self.baseNames['A'] = self.__deriveBaseName(self.objAFn)
self.baseNames['B'] = self.__deriveBaseName(self.objBFn)
self.baseNames['AB'] = self.__combineBaseNames(
self.baseNames['A'], self.baseNames['B'])
else:
self.isPair = False
self.frames = ['A']
self.baseNames['A'] = self.__deriveBaseName(self.objAFn)
self.flatFns = []
self.darkFns = []
def __find_spatial_profile_and_peak(order):
"""
"""
MARGIN = 5
for frame in order.frames:
# find spatial profile(s)
order.spatialProfile[frame] = order.ffObjImg[frame].mean(axis=1)
if len(order.spatialProfile[frame]) < (2 * MARGIN) + 2:
raise DrpException.DrpException(
'cannot find spatial profile for frame {} order {}'.format(
frame, order.flatOrder.orderNum))
# find peak locations
order.peakLocation[frame] = np.argmax(
order.spatialProfile[frame][MARGIN:-MARGIN]) + MARGIN
logger.info('frame {} spatial profile peak intensity row {:d}'.format(
frame, order.peakLocation[frame]))
# fit peak to Gaussian, save Gaussian parameters and real centroid
# location
p0 = order.peakLocation[frame] - (config.params['obj_window'] // 2)
p1 = order.peakLocation[frame] + (config.params['obj_window'] // 2)
order.centroid[frame] = (scipy.ndimage.measurements.center_of_mass(
order.spatialProfile[frame][p0:p1]))[0] + p0
logger.info('frame {} spatial profile peak centroid row {:.1f}'.format(
frame, float(order.centroid[frame])))
return
def reduce_frame(raw, out_dir, flatCacher=None):
"""
Arguments:
raw: RawDataSet object
out_dir: Data product root directory
"""
# initialize per-object logger and check output directory
if raw.isPair:
init(raw.baseNames['AB'], out_dir)
else:
init(raw.baseNames['A'], out_dir)
# if no flats in raw data set then fail
if (len(raw.flatFns) < 1):
logger.error("no flats for {}".format(raw.baseName))
raise DrpException.DrpException('no flats')
# create reduced data set
reduced = ReducedDataSet.ReducedDataSet(raw)
# read raw object image data into reduced data set object
reduced.objImg['A'] = fits.getdata(raw.objAFn, ignore_missing_end=True)
if raw.isPair:
reduced.objImg['B'] = fits.getdata(raw.objBFn, ignore_missing_end=True)
# put object summary info into per-object log
log_start_summary(reduced)
# Get fully processed flat in the form of a Flat object
reduced.Flat = getFlat(raw, flatCacher)
logger.info('using flat {}'.format(reduced.Flat.getBaseName()))
# clean cosmic ray hits on object frame(s)
if config.params['no_cosmic']:
logger.info(
"cosmic ray rejection on object frame inhibited by command line flag")
else:
logger.info('cosmic ray cleaning object frame A')
reduced.objImg['A'], cosmicMethod = image_lib.cosmic_clean(
reduced.objImg['A'])
logger.debug('cosmic ray cleaning object frame A complete')
if reduced.isPair:
logger.info('cosmic ray cleaning object frame B')
reduced.objImg['B'] = image_lib.cosmic_clean(reduced.objImg['B'])
logger.debug('cosmic ray cleaning object frame B complete')
reduced.cosmicCleaned = True
logger.info(cosmicMethod)
# if darks are available, combine them if there are more than one
# and subtract from object frame(s) and flat
process_darks(raw, reduced)
# if AB pair then subtract B from A
if reduced.isPair:
reduced.objImg['AB'] = np.subtract(
reduced.objImg['A'], reduced.objImg['B'])
# reduce orders
try:
reduce_orders(reduced)
except IOError as e:
# might want to do something else here
raise
# find and apply wavelength solution
imp.reload(wavelength_utils)
if find_global_wavelength_soln(reduced) is True:
apply_wavelength_soln(reduced)
else:
logger.info('not applying wavelength solution')
for order in reduced.orders:
order.waveScale = order.flatOrder.gratingEqWaveScale
order.calMethod = 'grating equation'
return(reduced)
def process_frame(fn1, fn2, obj_B_fn, out_dir):
flat_fn = None
obj_fn = None
fn1_header = fits.PrimaryHDU.readfrom(fn1, ignore_missing_end=True).header
fn2_header = fits.PrimaryHDU.readfrom(fn2, ignore_missing_end=True).header
if fn1_header['flat'] == 1 and fn1_header['calmpos'] == 1:
flat_fn = fn1
obj_fn = fn2
obj_header = fn2_header
flat_header = fn1_header
if fn2_header['flat'] == 1 and fn2_header['calmpos'] == 1:
if flat_fn is not None:
raise DrpException.DrpException('two flats, no object frame')
else:
flat_fn = fn2
obj_fn = fn1
obj_header = fn1_header
flat_header = fn2_header
if flat_fn is None:
raise DrpException.DrpException('no flat')
if obj_header['ECHLPOS'] > 100:
print('ERROR: cannot reduce low-resolution image (ECHLPOS > 100')
exit(1)
if obj_header['NAXIS1'] != nirspec_constants.N_COLS:
raise DrpException.DrpException('NAXIS1 != {}'.format(
nirspec_constants.N_COLS))
if obj_header['NAXIS2'] != nirspec_constants.N_ROWS:
raise DrpException.DrpException('NAXIS2 != {}'.format(
nirspec_constants.N_COLS))
if obj_header['FILNAME'].upper() not in nirspec_constants.filter_names:
raise DrpException.DrpException('unsupported filter: {}'.format(
obj_header['FILNAME']))
if create_raw_data_sets.flat_criteria_met(
obj_header, flat_header, ignore_dispers=True) is False:
raise DrpException.DrpException(
'flat is not compatible with object frame')
if obj_B_fn is not None:
# confirm that A and B are not the same files
if obj_fn == obj_B_fn:
raise DrpException.DrpException('frames A and B are the same')
obj_B_header = fits.PrimaryHDU.readfrom(obj_B_fn,
ignore_missing_end=True).header
if create_raw_data_sets.is_valid_pair(obj_header, obj_B_header):
#print('Reducing AB pair, A=' + obj_fn + ', B=' + obj_B_fn)
rawDataSet = RawDataSet.RawDataSet(obj_fn, obj_B_fn, obj_header)
else:
raise DrpException.DrpException(
'frames A and B are not a valid pair')
else:
rawDataSet = RawDataSet.RawDataSet(obj_fn, None, obj_header)
rawDataSet.flatFns.append(flat_fn)
if not os.path.exists(out_dir):
try:
os.mkdir(out_dir)
except:
msg = 'output directory {} does not exist and cannot be created'.format(
out_dir)
raise IOError(msg)
logger = logging.getLogger('main')
# generate reduced data set by reducing raw data set
reducedDataSet = reduce_frame.reduce_frame(rawDataSet, out_dir)
# write reduction summary to log file
write_summary(reducedDataSet)
# produce data products from reduced data set
if config.params['no_products'] is True:
logger.info('data product generation inhibited by command line switch')
else:
products.gen(reducedDataSet, out_dir)
# if diagnostics mode is enabled, then produce diagnostic data products
if config.params['dgn'] is True:
logger.info(
'diagnostic mode enabled, generating diagnostic data products')
dgn.gen(reducedDataSet, out_dir)
return
def reduce_frame(raw, out_dir, flatCacher=None, eta=None, arc=None, dark=None):
"""
Arguments:
raw: RawDataSet object
out_dir: Data product root directory
"""
# initialize per-object logger and check output directory
if raw.isPair:
init(raw.baseNames['AB'], out_dir)
else:
init(raw.baseNames['A'], out_dir)
# if no flats in raw data set then fail
if (len(raw.flatFns) < 1):
logger.error("no flats for {}".format(raw.baseName))
raise DrpException.DrpException('no flats')
# create reduced data set
reduced = ReducedDataSet.ReducedDataSet(raw)
# read raw object image data into reduced data set object
reduced.objImg['A'] = fits.getdata(raw.objAFn, ignore_missing_end=True)
#print('TEST', nirspec_constants.upgrade)
if nirspec_constants.upgrade:
reduced.objImg['A'] = np.rot90(reduced.objImg['A'], k=3)
### TEST PLOT XXX
'''
import matplotlib.pyplot as plt
from astropy.visualization import ImageNormalize, ZScaleInterval
norm = ImageNormalize(reduced.objImg['A'], interval=ZScaleInterval())
plt.imshow(reduced.objImg['A'], norm=norm, origin='lower', aspect='auto')
plt.show()
#sys.exit()
'''
### TEST PLOT XXX
if raw.isPair:
reduced.objImg['B'] = fits.getdata(raw.objBFn, ignore_missing_end=True)
if nirspec_constants.upgrade:
reduced.objImg['B'] = np.rot90(reduced.objImg['B'], k=3)
if eta is not None:
reduced.etaImg = fits.getdata(raw.etaFns, ignore_missing_end=True)
if nirspec_constants.upgrade:
reduced.etaImg = np.rot90(reduced.etaImg, k=3)
if arc is not None:
reduced.arcImg = fits.getdata(raw.arcFns, ignore_missing_end=True)
if nirspec_constants.upgrade:
reduced.arcImg = np.rot90(reduced.arcImg, k=3)
if dark is not None:
reduced.hasDark = True
# reduced.darkImg = fits.getdata(raw.darkFns, ignore_missing_end=True)
# put object summary info into per-object log
log_start_summary(reduced)
# Get fully processed flat in the form of a Flat object
reduced.Flat = getFlat(raw, flatCacher)
logger.info('using flat {}'.format(reduced.Flat.getBaseName()))
# clean cosmic ray hits on object frame(s)
if config.params['no_cosmic']:
logger.info(
"cosmic ray rejection on object frame inhibited by command line flag"
)
else:
logger.info('cosmic ray cleaning object frame A')
reduced.objImg['A'] = image_lib.cosmic_clean(reduced.objImg['A'])
logger.debug('cosmic ray cleaning object frame A complete')
if reduced.isPair:
logger.info('cosmic ray cleaning object frame B')
reduced.objImg['B'] = image_lib.cosmic_clean(reduced.objImg['B'])
logger.debug('cosmic ray cleaning object frame B complete')
if eta is not None:
logger.info('cosmic ray cleaning etalon frame')
reduced.etaImg = image_lib.cosmic_clean(reduced.etaImg)
logger.debug('cosmic ray cleaning etalon frame complete')
if arc is not None:
logger.info('cosmic ray cleaning arc lamp frame')
reduced.arcImg = image_lib.cosmic_clean(reduced.arcImg)
logger.debug('cosmic ray cleaning arc lamp frame complete')
reduced.cosmicCleaned = True
# if darks are available, combine them if there are more than one
# and subtract from object frame(s) and flat
process_darks(raw, reduced)
# if AB pair then subtract B from A
if reduced.isPair:
reduced.objImg['AB'] = np.subtract(reduced.objImg['A'],
reduced.objImg['B'])
# reduce orders
try:
reduce_orders(reduced, eta=eta, arc=arc)
except IOError as e:
# might want to do something else here
raise
# find and apply wavelength solution
imp.reload(wavelength_utils)
if find_global_wavelength_soln(reduced) is True:
apply_wavelength_soln(reduced)
else:
logger.info('not applying wavelength solution')
for order in reduced.orders:
order.waveScale = order.flatOrder.gratingEqWaveScale
order.calMethod = 'grating equation'
return (reduced)
def process_frame(fn1, fn2, obj_B_fn, out_dir, dark=None, eta=None, arc=None, override=False):
logger = logging.getLogger('main')
flat_fn = None
obj_fn = None
fn1_header = fits.PrimaryHDU.readfrom(fn1, ignore_missing_end=True).header
fn2_header = fits.PrimaryHDU.readfrom(fn2, ignore_missing_end=True).header
# Get the observation date to determine if this is upgraded NIRSPEC
if datetime.strptime(fn1_header['date-obs'], '%Y-%m-%d') > datetime.strptime('2018-10-10', '%Y-%m-%d'):
nirspec_constants.upgrade = True
# Do the case for the etalon lamp
if eta is not None:
eta_header = fits.PrimaryHDU.readfrom(eta, ignore_missing_end=True).header
if nirspec_constants.upgrade:
if eta_header['etalon'] == 'On' and eta_header['calmpos'] == 'In':
eta_fn = eta
else:
if eta_header['etalon'] == 1 and eta_header['calmpos'] == 1:
eta_fn = eta
# Do the case for the arc lamp
if arc is not None:
arc_header = fits.PrimaryHDU.readfrom(arc, ignore_missing_end=True).header
if nirspec_constants.upgrade:
if (arc_header['neon'] == 'On' or arc_header['argon'] == 'On' or arc_header['krypton'] == 'On' or \
arc_header['xenon'] == 'On') and arc_header['calmpos'] == 'In':
arc_fn = arc
else:
if (arc_header['neon'] == 1 or arc_header['argon'] == 1 or arc_header['krypton'] == 1 or \
arc_header['xenon'] == 1) and arc_header['calmpos'] == 1:
arc_fn = arc
# Get the master dark frame if given
if dark is not None:
dark_header = fits.PrimaryHDU.readfrom(dark, ignore_missing_end=True).header
if nirspec_constants.upgrade:
if dark_header['halogen'] == 'On' and dark_header['calmpos'] == 'In' and \
dark_header['neon'] != 'On' and dark_header['argon'] != 'On' and dark_header['krypton'] != 'On' and \
dark_header['xenon'] != 'On':
dark_fn = dark
else:
if dark_header['flat'] == 0 and dark_header['calmpos'] == 1 and \
dark_header['neon'] != 1 and dark_header['argon'] != 1 and dark_header['krypton'] != 1 and \
dark_header['xenon'] != 1:
dark_fn = dark
# Get the flat and object fits
if nirspec_constants.upgrade:
if fn1_header['halogen'] == 'On' and fn1_header['calmpos'] == 'In':
flat_fn = fn1
obj_fn = fn2
obj_header = fn2_header
flat_header = fn1_header
if fn1_header['halogen'] == 'On' and fn1_header['calmpos'] == 'In':
flat_fn = fn1
obj_fn = fn2
obj_header = fn2_header
flat_header = fn1_header
else:
if fn1_header['flat'] == 1 and fn1_header['calmpos'] == 1:
flat_fn = fn1
obj_fn = fn2
obj_header = fn2_header
flat_header = fn1_header
if fn2_header['flat'] == 1 and fn2_header['calmpos'] == 1:
if flat_fn is not None:
raise DrpException.DrpException('two flats, no object frame')
else:
flat_fn = fn2
obj_fn = fn1
obj_header = fn1_header
flat_header = fn2_header
if flat_fn is None:
raise DrpException.DrpException('no flat')
if obj_header['ECHLPOS'] > 100:
print('ERROR: cannot reduce low-resolution image (ECHLPOS > 100')
exit(1)
# Check the array size is correct
if nirspec_constants.upgrade:
if obj_header['NAXIS1'] != nirspec_constants.N_COLS_upgrade:
raise DrpException.DrpException('NAXIS1 != {}'.format(nirspec_constants.N_COLS_upgrade))
if obj_header['NAXIS2'] != nirspec_constants.N_ROWS_upgrade:
raise DrpException.DrpException('NAXIS2 != {}'.format(nirspec_constants.N_COLS_upgrade))
filtername1, filtername2 = obj_header['SCIFILT2'], ''
if obj_header['SCIFILT1'] == 'AO-stop': filtername2 = '-AO'
if filtername1.upper()+filtername2.upper() not in nirspec_constants.filter_names:
raise DrpException.DrpException('unsupported filter: {}'.format(obj_header['SCIFILT2']))
else:
if obj_header['NAXIS1'] != nirspec_constants.N_COLS:
raise DrpException.DrpException('NAXIS1 != {}'.format(nirspec_constants.N_COLS))
if obj_header['NAXIS2'] != nirspec_constants.N_ROWS:
raise DrpException.DrpException('NAXIS2 != {}'.format(nirspec_constants.N_COLS))
if obj_header['FILNAME'].upper() not in nirspec_constants.filter_names:
raise DrpException.DrpException('unsupported filter: {}'.format(obj_header['FILNAME']))
if create_raw_data_sets.flat_criteria_met(obj_header, flat_header, ignore_dispers=True) is False:
raise DrpException.DrpException('flat is not compatible with object frame')
if dark is not None:
if create_raw_data_sets.dark_criteria_met(obj_header, dark_header) is False:
raise DrpException.DrpException('dark is not compatible with object frame')
if obj_B_fn is not None:
# confirm that A and B are not the same files
if obj_fn == obj_B_fn:
raise DrpException.DrpException('frames A and B are the same')
obj_B_header = fits.PrimaryHDU.readfrom(obj_B_fn, ignore_missing_end=True).header
if create_raw_data_sets.is_valid_pair(obj_header, obj_B_header, override=override):
logger.debug('Reducing AB pair, A= ' + obj_fn + ' , B= ' + obj_B_fn)
rawDataSet = RawDataSet.RawDataSet(obj_fn, obj_B_fn, obj_header, eta=eta, arc=arc, dark=dark)
else:
raise DrpException.DrpException('frames A and B are not a valid pair')
else:
rawDataSet = RawDataSet.RawDataSet(obj_fn, None, obj_header, eta=eta, arc=arc, dark=dark)
rawDataSet.flatFns.append(flat_fn)
if not os.path.exists(out_dir):
try:
os.mkdir(out_dir)
except:
msg = 'output directory {} does not exist and cannot be created'.format(out_dir)
raise IOError(msg)
# generate reduced data set by reducing raw data set
reducedDataSet = reduce_frame.reduce_frame(rawDataSet, out_dir, eta=eta, arc=arc, dark=dark)
# write reduction summary to log file
write_summary(reducedDataSet)
# produce data products from reduced data set
if config.params['no_products'] is True:
logger.info('data product generation inhibited by command line switch')
else:
products.gen(reducedDataSet, out_dir, eta=eta, arc=arc)
# if diagnostics mode is enabled, then produce diagnostic data products
if config.params['dgn'] is True:
logger.info('diagnostic mode enabled, generating diagnostic data products')
dgn.gen(reducedDataSet, out_dir, eta=eta, arc=arc)
return