class PypeIt(object):
"""
This class runs the primary calibration and extraction in PypeIt
Args:
pypeit_file (:obj:`str`): PypeIt filename
verbosity (:obj:`int`, optional):
Verbosity level of system output. Can be::
- 0: No output
- 1: Minimal output (default)
- 2: All output
overwrite (:obj:`bool`, optional):
Flag to overwrite any existing files/directories.
reuse_masters (bool, optional): Reuse any pre-existing calibration files
logname (:obj:`str`, optional):
The name of an ascii log file with the details of the
reduction.
redux_path (:obj:`str`, optional):
Over-ride reduction path in PypeIt file (e.g. Notebook usage)
show: (:obj:`bool`, optional):
Show reduction steps via plots (which will block further
execution until clicked on) and outputs to ginga. Requires
remote control ginga session via "ginga --modules=RC &"
Attributes:
pypeit_file (:obj:`str`):
Name of the pypeit file to read. PypeIt files have a specific
set of valid formats. A description can be found `here`_
(include doc link).
fitstbl (:obj:`pypit.metadata.PypeItMetaData`): holds the meta info
"""
# __metaclass__ = ABCMeta
def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None,
show=False, redux_path=None):
# Load
cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(pypeit_file, runtime=True)
self.pypeit_file = pypeit_file
# Spectrograph
cfg = ConfigObj(cfg_lines)
spectrograph_name = cfg['rdx']['spectrograph']
self.spectrograph = load_spectrograph(spectrograph_name)
# Par
# Defaults
spectrograph_def_par = self.spectrograph.default_pypeit_par()
# Grab a science file for configuration specific parameters
sci_file = None
for idx, row in enumerate(usrdata):
if 'science' in row['frametype']:
sci_file = data_files[idx]
break
# Set
spectrograph_cfg_lines = self.spectrograph.config_specific_par(spectrograph_def_par, sci_file).to_config()
self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines)
# Fitstbl
self.fitstbl = PypeItMetaData(self.spectrograph, self.par, file_list=data_files,
usrdata=usrdata, strict=True)
# The following could be put in a prepare_to_run() method in PypeItMetaData
if 'setup' not in self.fitstbl.keys():
self.fitstbl['setup'] = setups[0]
self.fitstbl.get_frame_types(user=frametype) # This sets them using the user inputs
self.fitstbl.set_defaults() # Only does something if values not set in PypeIt file
self.fitstbl._set_calib_group_bits()
self.fitstbl._check_calib_groups()
# Write .calib file (For QA naming amongst other things)
calib_file = pypeit_file.replace('.pypeit', '.calib')
self.fitstbl.write_calib(calib_file)
# Other Internals
self.logname = logname
self.overwrite = overwrite
# Currently the runtime argument determines the behavior for reuse_masters. There is also a reuse_masters
# parameter in the parset but it is currently ignored.
self.reuse_masters=reuse_masters
self.show = show
# Make the output directories
self.par['rdx']['redux_path'] = os.getcwd() if redux_path is None else redux_path
msgs.info("Setting reduction path to {:s}".format(self.par['rdx']['redux_path']))
paths.make_dirs(self.spectrograph.spectrograph, self.par['calibrations']['caldir'],
self.par['rdx']['scidir'], self.par['rdx']['qadir'],
overwrite=self.overwrite, redux_path=self.par['rdx']['redux_path'])
# Instantiate Calibrations class
self.caliBrate \
= calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], self.spectrograph,
redux_path=self.par['rdx']['redux_path'],
reuse_masters=self.reuse_masters,
save_masters=True, write_qa=True,
show=self.show)
# Init
self.verbosity = verbosity
# TODO: I don't think this ever used
self.frame = None
self.det = None
self.tstart = None
self.basename = None
self.sciI = None
self.obstime = None
def build_qa(self):
"""
Generate QA wrappers
"""
qa.gen_mf_html(self.pypeit_file)
qa.gen_exp_html()
def outfile_exists(self, frame):
"""
Check whether the 2D outfile of a given frame already exists
Args:
frame (int): Frame index from fitstbl
Returns:
bool: True if the 2d file exists
False if it does not exist
"""
# Check if the 2d output file exists
scidir = os.path.join(self.par['rdx']['redux_path'], self.par['rdx']['scidir'])
basename = self.fitstbl.construct_basename(frame)
outfile = scidir + '/spec2d_{:s}.fits'.format(basename)
return os.path.isfile(outfile)
def get_std_outfile(self, standard_frames):
"""
Grab the output filename from an input list of standard_frame indices
If more than one index is provided, the first is taken
Args:
standard_frames (list): List of indices corresponding to standard stars
Returns:
str: Full path to the standard spec1d output file
"""
# TODO: Need to decide how to associate standards with
# science frames in the case where there is more than one
# standard associated with a given science frame. Below, I
# just use the first standard
std_outfile = None
std_frame = None if len(standard_frames) == 0 else standard_frames[0]
# Prepare to load up standard?
if std_frame is not None:
std_outfile = os.path.join(self.par['rdx']['redux_path'], self.par['rdx']['scidir'],
'spec1d_{:s}.fits'.format(self.fitstbl.construct_basename(std_frame))) \
if isinstance(std_frame, (int,np.integer)) else None
if std_outfile is not None and not os.path.isfile(std_outfile):
msgs.error('Could not find standard file: {0}'.format(std_outfile))
return std_outfile
def reduce_all(self):
"""
Main driver of the entire reduction
Calibration and extraction via a series of calls to reduce_exposure()
"""
# Validate the parameter set
required = ['rdx', 'calibrations', 'scienceframe', 'scienceimage', 'flexure', 'fluxcalib']
can_be_None = ['flexure', 'fluxcalib']
self.par.validate_keys(required=required, can_be_None=can_be_None)
self.tstart = time.time()
# Find the standard frames
is_standard = self.fitstbl.find_frames('standard')
# Find the science frames
is_science = self.fitstbl.find_frames('science')
# Frame indices
frame_indx = np.arange(len(self.fitstbl))
# Iterate over each calibration group and reduce the standards
for i in range(self.fitstbl.n_calib_groups):
# Find all the frames in this calibration group
in_grp = self.fitstbl.find_calib_group(i)
# Find the indices of the standard frames in this calibration group:
grp_standards = frame_indx[is_standard & in_grp]
# Reduce all the standard frames, loop on unique comb_id
u_combid_std= np.unique(self.fitstbl['comb_id'][grp_standards])
for j, comb_id in enumerate(u_combid_std):
frames = np.where(self.fitstbl['comb_id'] == comb_id)[0]
bg_frames = np.where(self.fitstbl['bkg_id'] == comb_id)[0]
if not self.outfile_exists(frames[0]) or self.overwrite:
std_dict = self.reduce_exposure(frames, bg_frames=bg_frames)
# TODO come up with sensible naming convention for save_exposure for combined files
self.save_exposure(frames[0], std_dict, self.basename)
else:
msgs.info('Output file: {:s} already exists'.format(self.fitstbl.construct_basename(frames[0])) +
'. Set overwrite=True to recreate and overwrite.')
# Iterate over each calibration group again and reduce the science frames
for i in range(self.fitstbl.n_calib_groups):
# Find all the frames in this calibration group
in_grp = self.fitstbl.find_calib_group(i)
# Find the indices of the science frames in this calibration group:
grp_science = frame_indx[is_science & in_grp]
# Associate standards (previously reduced above) for this setup
std_outfile = self.get_std_outfile(frame_indx[is_standard])
# Reduce all the science frames; keep the basenames of the science frames for use in flux calibration
science_basename = [None]*len(grp_science)
# Loop on unique comb_id
u_combid = np.unique(self.fitstbl['comb_id'][grp_science])
for j, comb_id in enumerate(u_combid):
frames = np.where(self.fitstbl['comb_id'] == comb_id)[0]
bg_frames = np.where(self.fitstbl['bkg_id'] == comb_id)[0]
if not self.outfile_exists(frames[0]) or self.overwrite:
sci_dict = self.reduce_exposure(frames, bg_frames=bg_frames, std_outfile=std_outfile)
science_basename[j] = self.basename
# TODO come up with sensible naming convention for save_exposure for combined files
self.save_exposure(frames[0], sci_dict, self.basename)
else:
msgs.warn('Output file: {:s} already exists'.format(self.fitstbl.construct_basename(frames[0])) +
'. Set overwrite=True to recreate and overwrite.')
msgs.info('Finished calibration group {0}'.format(i))
# Finish
self.print_end_time()
def select_detectors(self):
"""
Return the 1-indexed list of detectors to reduce.
Returns:
list: List of detectors to be reduced
"""
if self.par['rdx']['detnum'] is None:
return np.arange(self.spectrograph.ndet)+1
return [self.par['rdx']['detnum']] if isinstance(self.par['rdx']['detnum'], int) \
else self.par['rdx']['detnum']
def reduce_exposure(self, frames, bg_frames=[], std_outfile=None):
"""
Reduce a single exposure
Args:
frame (:obj:`int`):
0-indexed row in :attr:`fitstbl` with the frame to
reduce
bgframes (:obj:`list`, optional):
List of frame indices for the background
std_outfile (:obj:`str`, optional):
the name of a file with a previously PypeIt-reduced standard spectrum.
Returns:
dict: The dictionary containing the primary outputs of extraction
"""
# if show is set, clear the ginga channels at the start of each new sci_ID
if self.show:
ginga.clear_all()
# Save the frame
self.frames = frames
self.bg_frames = bg_frames
# Is this an IR reduction?
self.ir_redux = True if len(bg_frames) > 0 else False
# JFH Why does this need to be ordered?
sci_dict = OrderedDict() # This needs to be ordered
sci_dict['meta'] = {}
sci_dict['meta']['vel_corr'] = 0.
sci_dict['meta']['ir_redux'] = self.ir_redux
# Print status message
msgs_string = 'Reducing target {:s}'.format(self.fitstbl['target'][self.frames[0]]) + msgs.newline()
msgs_string += 'Combining frames:' + msgs.newline()
for iframe in self.frames:
msgs_string += '{0:s}'.format(self.fitstbl['filename'][iframe]) + msgs.newline()
msgs.info(msgs_string)
if len(bg_frames) > 0:
bg_msgs_string = ''
for iframe in self.bg_frames:
bg_msgs_string += '{0:s}'.format(self.fitstbl['filename'][iframe]) + msgs.newline()
bg_msgs_string = msgs.newline() + 'Using background from frames:' + msgs.newline() + bg_msgs_string
msgs.info(bg_msgs_string)
# Find the detectors to reduce
detectors = self.select_detectors()
if len(detectors) != self.spectrograph.ndet:
msgs.warn('Not reducing detectors: {0}'.format(' '.join([ str(d) for d in
set(np.arange(self.spectrograph.ndet))-set(detectors)])))
# Loop on Detectors
for self.det in detectors:
msgs.info("Working on detector {0}".format(self.det))
sci_dict[self.det] = {}
# Calibrate
#TODO Is the right behavior to just use the first frame?
self.caliBrate.set_config(self.frames[0], self.det, self.par['calibrations'])
self.caliBrate.run_the_steps()
# Extract
# TODO: pass back the background frame, pass in background
# files as an argument. extract one takes a file list as an
# argument and instantiates science within
sci_dict[self.det]['sciimg'], sci_dict[self.det]['sciivar'], sci_dict[self.det]['skymodel'], \
sci_dict[self.det]['objmodel'], sci_dict[self.det]['ivarmodel'], sci_dict[self.det]['outmask'], \
sci_dict[self.det]['specobjs'], vel_corr \
= self.extract_one(self.frames, self.det, bg_frames = self.bg_frames, std_outfile = std_outfile)
if vel_corr is not None:
sci_dict['meta']['vel_corr'] = vel_corr
# JFH TODO write out the background frame?
# Return
return sci_dict
def flexure_correct(self, sobjs, maskslits):
"""
Correct for flexure
Spectra are modified in place (wavelengths are shifted)
Args:
sobjs (SpecObjs):
maskslits (ndarray): Mask of SpecObjs
"""
if self.par['flexure']['method'] != 'skip':
flex_list = wave.flexure_obj(sobjs, maskslits, self.par['flexure']['method'],
self.par['flexure']['spectrum'],
mxshft=self.par['flexure']['maxshift'])
# QA
wave.flexure_qa(sobjs, maskslits, self.basename, self.det, flex_list,
out_dir=self.par['rdx']['redux_path'])
else:
msgs.info('Skipping flexure correction.')
def helio_correct(self, sobjs, maskslits, frame, obstime):
"""
Perform a heliocentric correction on a set of spectra
Args:
sobjs (pypeit.specobjs.SpecObjs): Spectra
maskslits (ndarray): Slits that are masked
frame (int): Frame to use for meta info
obstime (astropy.time.Time):
Returns:
astropy.units.Quantity: Velocity correction in km/s
"""
# Helio, correct Earth's motion
if (self.caliBrate.par['wavelengths']['frame'] in ['heliocentric', 'barycentric']) \
and (self.caliBrate.par['wavelengths']['reference'] != 'pixel'):
# TODO change this keyword to refframe instead of frame
msgs.info("Performing a {0} correction".format(self.caliBrate.par['wavelengths']['frame']))
vel, vel_corr = wave.geomotion_correct(sobjs, maskslits, self.fitstbl, frame, obstime,
self.spectrograph.telescope['longitude'],
self.spectrograph.telescope['latitude'],
self.spectrograph.telescope['elevation'],
self.caliBrate.par['wavelengths']['frame'])
else:
msgs.info('A wavelength reference-frame correction will not be performed.')
vel_corr = None
return vel_corr
def get_sci_metadata(self, frame, det):
"""
Grab the meta data for a given science frame and specific detector
Args:
frame (int): Frame index
det (int): Detector index
Returns:
5 objects are returned::
- str: Object type; science or standard
- str: Setup string from master_key()
- astropy.time.Time: Time of observation
- str: Basename of the frame
- str: Binning of the detector
"""
# Set binning, obstime, basename, and objtype
binning = self.fitstbl['binning'][frame]
obstime = self.fitstbl.construct_obstime(frame)
basename = self.fitstbl.construct_basename(frame, obstime=obstime)
objtype = self.fitstbl['frametype'][frame]
if 'science' in objtype:
objtype_out = 'science'
elif 'standard' in objtype:
objtype_out = 'standard'
else:
msgs.error('Unrecognized objtype')
setup = self.fitstbl.master_key(frame, det=det)
return objtype_out, setup, obstime, basename, binning
def get_std_trace(self, std_redux, det, std_outfile):
"""
Returns the trace of the standard if it is applicable to the current reduction
Args:
std_redux (bool): If False, proceed
det (int): Detector index
std_outfile (str): Filename for the standard star spec1d file
Returns:
ndarray: Trace of the standard star on input detector
"""
if std_redux is False and std_outfile is not None:
sobjs, hdr_std = load.load_specobjs(std_outfile)
# Does the detector match?
# TODO Instrument specific logic here could be implemented with the parset. For example LRIS-B or LRIS-R we
# we would use the standard from another detector
this_det = sobjs.det == det
if np.any(this_det):
sobjs_det = sobjs[this_det]
sobjs_std = sobjs_det.get_std()
std_trace = sobjs_std.trace_spat
# flatten the array if this multislit
if 'MultiSlit' in self.spectrograph.pypeline:
std_trace = std_trace.flatten()
elif 'Echelle' in self.spectrograph.pypeline:
std_trace = std_trace.T
else:
msgs.error('Unrecognized pypeline')
else:
std_trace = None
else:
std_trace = None
return std_trace
def extract_one(self, frames, det, bg_frames=[], std_outfile=None):
"""
Extract a single exposure/detector pair
sci_ID and det need to have been set internally prior to calling this method
Args:
frames (list): List of frames to extract; stacked if more than one is provided
det (int):
bg_frames (list, optional): List of frames to use as the background
std_outfile (str, optional):
Returns:
eight objects are returned::
- ndarray: Science image
- ndarray: Science inverse variance image
- ndarray: Model of the sky
- ndarray: Model of the object
- ndarray: Model of inverse variance
- ndarray: Mask
- :obj:`pypeit.specobjs.SpecObjs`: spectra
- astropy.units.Quantity: velocity correction
"""
# Grab some meta-data needed for the reduction from the fitstbl
self.objtype, self.setup, self.obstime, self.basename, self.binning = self.get_sci_metadata(frames[0], det)
# Is this a standard star?
self.std_redux = 'standard' in self.objtype
# Get the standard trace if need be
std_trace = self.get_std_trace(self.std_redux, det, std_outfile)
# Instantiate ScienceImage for the files we will reduce
sci_files = self.fitstbl.frame_paths(frames)
self.sciI = scienceimage.ScienceImage(self.spectrograph, sci_files,
bg_file_list=self.fitstbl.frame_paths(bg_frames),
ir_redux = self.ir_redux,
par=self.par['scienceframe'],
det=det,
binning=self.binning)
# For QA on crash.
msgs.sciexp = self.sciI
# Process images (includes inverse variance image, rn2 image, and CR mask)
self.sciimg, self.sciivar, self.rn2img, self.mask, self.crmask = \
self.sciI.proc(self.caliBrate.msbias, self.caliBrate.mspixflatnrm.copy(),
self.caliBrate.msbpm, illum_flat=self.caliBrate.msillumflat,
show=self.show)
# Object finding, first pass on frame without sky subtraction
self.maskslits = self.caliBrate.maskslits.copy()
self.redux = reduce.instantiate_me(self.spectrograph, self.caliBrate.tslits_dict,
self.mask, self.par,
ir_redux = self.ir_redux,
objtype=self.objtype, setup=self.setup,
det=det, binning=self.binning)
# Prep for manual extraction (if requested)
manual_extract_dict = self.fitstbl.get_manual_extract(frames, det)
# Do one iteration of object finding, and sky subtract to get initial sky model
self.sobjs_obj, self.nobj, skymask_init = \
self.redux.find_objects(self.sciimg, self.sciivar, std=self.std_redux, ir_redux=self.ir_redux,
std_trace=std_trace,maskslits=self.maskslits,
show=self.show & (not self.std_redux),
manual_extract_dict=manual_extract_dict)
# Global sky subtraction, first pass. Uses skymask from object finding step above
self.initial_sky = \
self.redux.global_skysub(self.sciimg, self.sciivar, self.caliBrate.tilts_dict['tilts'], skymask=skymask_init,
std=self.std_redux, maskslits=self.maskslits, show=self.show)
if not self.std_redux:
# Object finding, second pass on frame *with* sky subtraction. Show here if requested
self.sobjs_obj, self.nobj, self.skymask = \
self.redux.find_objects(self.sciimg - self.initial_sky, self.sciivar, std=self.std_redux, ir_redux=self.ir_redux,
std_trace=std_trace,maskslits=self.maskslits,show=self.show,
manual_extract_dict=manual_extract_dict)
# If there are objects, do 2nd round of global_skysub, local_skysub_extract, flexure, geo_motion
if self.nobj > 0:
# Global sky subtraction second pass. Uses skymask from object finding
self.global_sky = self.initial_sky if self.std_redux else \
self.redux.global_skysub(self.sciimg, self.sciivar, self.caliBrate.tilts_dict['tilts'],
skymask=self.skymask, maskslits=self.maskslits, show=self.show)
self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs = \
self.redux.local_skysub_extract(self.sciimg, self.sciivar, self.caliBrate.tilts_dict['tilts'], self.caliBrate.mswave,
self.global_sky, self.rn2img, self.sobjs_obj,
model_noise=(not self.ir_redux),std = self.std_redux,
maskslits=self.maskslits, show_profile=self.show,show=self.show)
# Purge out the negative objects if this was a near-IR reduction.
# TODO should we move this purge call to local_skysub_extract??
if self.ir_redux:
self.sobjs.purge_neg()
# Flexure correction if this is not a standard star
if not self.std_redux:
self.redux.flexure_correct(self.sobjs, self.basename)
# Grab coord
radec = ltu.radec_to_coord((self.fitstbl["ra"][frames[0]], self.fitstbl["dec"][frames[0]]))
self.vel_corr = self.redux.helio_correct(self.sobjs, radec, self.obstime)
else:
# Print status message
msgs_string = 'No objects to extract for target {:s}'.format(self.fitstbl['target'][frames[0]]) + msgs.newline()
msgs_string += 'On frames:' + msgs.newline()
for iframe in frames:
msgs_string += '{0:s}'.format(self.fitstbl['filename'][iframe]) + msgs.newline()
msgs.warn(msgs_string)
# set to first pass global sky
self.skymodel = self.initial_sky
self.objmodel = np.zeros_like(self.sciimg)
# Set to sciivar. Could create a model but what is the point?
self.ivarmodel = np.copy(self.sciivar)
# Set to the initial mask in case no objects were found
self.outmask = self.redux.mask
# empty specobjs object from object finding
if self.ir_redux:
self.sobjs_obj.purge_neg()
self.sobjs = self.sobjs_obj
self.vel_corr = None
return self.sciimg, self.sciivar, self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs, self.vel_corr
# TODO: Why not use self.frame?
def save_exposure(self, frame, sci_dict, basename):
"""
Save the outputs from extraction for a given exposure
Args:
frame (:obj:`int`):
0-indexed row in the metadata table with the frame that
has been reduced.
sci_dict (:obj:`dict`):
Dictionary containing the primary outputs of extraction
basename (:obj:`str`):
The root name for the output file.
Returns:
None or SpecObjs: All of the objects saved to disk
"""
# TODO: Need some checks here that the exposure has been reduced
# Determine the headers
head1d = self.fitstbl[frame]
# Need raw file header information
rawfile = self.fitstbl.frame_paths(frame)
head2d = fits.getheader(rawfile, ext=self.spectrograph.primary_hdrext,)
refframe = 'pixel' if self.caliBrate.par['wavelengths']['reference'] == 'pixel' else \
self.caliBrate.par['wavelengths']['frame']
# Determine the paths/filenames
scipath = os.path.join(self.par['rdx']['redux_path'], self.par['rdx']['scidir'])
save.save_all(sci_dict, self.caliBrate.master_key_dict, self.caliBrate.master_dir, self.spectrograph,
head1d, head2d, scipath, basename, refframe=refframe,
update_det=self.par['rdx']['detnum'], binning=self.fitstbl['binning'][frame])
return
def msgs_reset(self):
"""
Reset the msgs object
"""
# Reset the global logger
msgs.reset(log=self.logname, verbosity=self.verbosity)
msgs.pypeit_file = self.pypeit_file
def print_end_time(self):
"""
Print the elapsed time
"""
# Capture the end time and print it to user
tend = time.time()
codetime = tend-self.tstart
if codetime < 60.0:
msgs.info('Execution time: {0:.2f}s'.format(codetime))
elif codetime/60.0 < 60.0:
mns = int(codetime/60.0)
scs = codetime - 60.0*mns
msgs.info('Execution time: {0:d}m {1:.2f}s'.format(mns, scs))
else:
hrs = int(codetime/3600.0)
mns = int(60.0*(codetime/3600.0 - hrs))
scs = codetime - 60.0*mns - 3600.0*hrs
msgs.info('Execution time: {0:d}h {1:d}m {2:.2f}s'.format(hrs, mns, scs))
# TODO: Move this to fitstbl?
def show_science(self):
"""
Simple print of science frames
"""
indx = self.fitstbl.find_frames('science')
print(self.fitstbl[['target','ra','dec','exptime','dispname']][indx])
def __repr__(self):
# Generate sets string
return '<{:s}: pypeit_file={}>'.format(self.__class__.__name__, self.pypeit_file)
class PypeIt(object):
"""
This class runs the primary calibration and extraction in PypeIt
.. todo::
Fill in list of attributes!
Args:
pypeit_file (:obj:`str`):
PypeIt filename.
verbosity (:obj:`int`, optional):
Verbosity level of system output. Can be:
- 0: No output
- 1: Minimal output (default)
- 2: All output
overwrite (:obj:`bool`, optional):
Flag to overwrite any existing files/directories.
reuse_masters (:obj:`bool`, optional):
Reuse any pre-existing calibration files
logname (:obj:`str`, optional):
The name of an ascii log file with the details of the
reduction.
show: (:obj:`bool`, optional):
Show reduction steps via plots (which will block further
execution until clicked on) and outputs to ginga. Requires
remote control ginga session via ``ginga --modules=RC &``
redux_path (:obj:`str`, optional):
Over-ride reduction path in PypeIt file (e.g. Notebook usage)
Attributes:
pypeit_file (:obj:`str`):
Name of the pypeit file to read. PypeIt files have a
specific set of valid formats. A description can be found
:ref:`pypeit_file`.
fitstbl (:obj:`pypit.metadata.PypeItMetaData`): holds the meta info
"""
# __metaclass__ = ABCMeta
def __init__(self,
pypeit_file,
verbosity=2,
overwrite=True,
reuse_masters=False,
logname=None,
show=False,
redux_path=None):
# Load
cfg_lines, data_files, frametype, usrdata, setups \
= parse_pypeit_file(pypeit_file, runtime=True)
self.pypeit_file = pypeit_file
# Spectrograph
cfg = ConfigObj(cfg_lines)
spectrograph_name = cfg['rdx']['spectrograph']
self.spectrograph = load_spectrograph(spectrograph_name,
ifile=data_files[0])
msgs.info('Loaded spectrograph {0}'.format(
self.spectrograph.spectrograph))
# --------------------------------------------------------------
# Get the full set of PypeIt parameters
# - Grab a science or standard file for configuration specific parameters
scistd_file = None
for idx, row in enumerate(usrdata):
if ('science' in row['frametype']) or ('standard'
in row['frametype']):
scistd_file = data_files[idx]
break
# - Configuration specific parameters for the spectrograph
if scistd_file is not None:
msgs.info(
'Setting configuration-specific parameters using {0}'.format(
os.path.split(scistd_file)[1]))
spectrograph_cfg_lines = self.spectrograph.config_specific_par(
scistd_file).to_config()
# - Build the full set, merging with any user-provided
# parameters
self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines,
merge_with=cfg_lines)
msgs.info('Built full PypeIt parameter set.')
# Check the output paths are ready
if redux_path is not None:
self.par['rdx']['redux_path'] = redux_path
# TODO: Write the full parameter set here?
# --------------------------------------------------------------
# --------------------------------------------------------------
# Build the meta data
# - Re-initilize based on the file data
msgs.info('Compiling metadata')
self.fitstbl = PypeItMetaData(self.spectrograph,
self.par,
files=data_files,
usrdata=usrdata,
strict=True)
# - Interpret automated or user-provided data from the PypeIt
# file
self.fitstbl.finalize_usr_build(frametype, setups[0])
# --------------------------------------------------------------
# - Write .calib file (For QA naming amongst other things)
calib_file = pypeit_file.replace('.pypeit', '.calib')
self.fitstbl.write_calib(calib_file)
# Other Internals
self.logname = logname
self.overwrite = overwrite
# Currently the runtime argument determines the behavior for
# reuse_masters.
self.reuse_masters = reuse_masters
self.show = show
# Set paths
if self.par['calibrations']['caldir'] == 'default':
self.calibrations_path = os.path.join(
self.par['rdx']['redux_path'], 'Masters')
else:
self.calibrations_path = self.par['calibrations']['caldir']
# Report paths
msgs.info('Setting reduction path to {0}'.format(
self.par['rdx']['redux_path']))
msgs.info('Master calibration data output to: {0}'.format(
self.calibrations_path))
msgs.info('Science data output to: {0}'.format(self.science_path))
msgs.info('Quality assessment plots output to: {0}'.format(
self.qa_path))
# TODO: Is anything written to the qa dir or only to qa/PNGs?
# Should we have separate calibration and science QA
# directories?
# Instantiate Calibrations class
self.caliBrate \
= calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'],
self.spectrograph,
caldir=self.calibrations_path,
qadir=self.qa_path,
reuse_masters=self.reuse_masters,
show=self.show)
# Init
self.verbosity = verbosity
# TODO: I don't think this ever used
self.frame = None
self.det = None
self.tstart = None
self.basename = None
self.sciI = None
self.obstime = None
@property
def science_path(self):
"""Return the path to the science directory."""
return os.path.join(self.par['rdx']['redux_path'],
self.par['rdx']['scidir'])
@property
def qa_path(self):
"""Return the path to the top-level QA directory."""
return os.path.join(self.par['rdx']['redux_path'],
self.par['rdx']['qadir'])
def build_qa(self):
"""
Generate QA wrappers
"""
qa.gen_mf_html(self.pypeit_file, self.qa_path)
qa.gen_exp_html()
# TODO: This should go in a more relevant place
def spec_output_file(self, frame, twod=False):
"""
Return the path to the spectral output data file.
Args:
frame (:obj:`int`):
Frame index from :attr:`fitstbl`.
twod (:obj:`bool`):
Name for the 2D output file; 1D file otherwise.
Returns:
:obj:`str`: The path for the output file
"""
return os.path.join(
self.science_path,
'spec{0}d_{1}.fits'.format('2' if twod else '1',
self.fitstbl.construct_basename(frame)))
def outfile_exists(self, frame):
"""
Check whether the 2D outfile of a given frame already exists
Args:
frame (int): Frame index from fitstbl
Returns:
bool: True if the 2d file exists, False if it does not exist
"""
return os.path.isfile(self.spec_output_file(frame, twod=True))
def get_std_outfile(self, standard_frames):
"""
Grab the output filename from an input list of standard_frame indices
If more than one index is provided, the first is taken
Args:
standard_frames (list): List of indices corresponding to standard stars
Returns:
str: Full path to the standard spec1d output file
"""
# TODO: Need to decide how to associate standards with
# science frames in the case where there is more than one
# standard associated with a given science frame. Below, I
# just use the first standard
std_outfile = None
std_frame = None if len(standard_frames) == 0 else standard_frames[0]
# Prepare to load up standard?
if std_frame is not None:
std_outfile = self.spec_output_file(std_frame) \
if isinstance(std_frame, (int,np.integer)) else None
if std_outfile is not None and not os.path.isfile(std_outfile):
msgs.error('Could not find standard file: {0}'.format(std_outfile))
return std_outfile
def reduce_all(self):
"""
Main driver of the entire reduction
Calibration and extraction via a series of calls to reduce_exposure()
"""
# Validate the parameter set
required = [
'rdx', 'calibrations', 'scienceframe', 'reduce', 'flexure',
'fluxcalib'
]
can_be_None = ['flexure', 'fluxcalib']
self.par.validate_keys(required=required, can_be_None=can_be_None)
self.tstart = time.time()
# Find the standard frames
is_standard = self.fitstbl.find_frames('standard')
# Find the science frames
is_science = self.fitstbl.find_frames('science')
# Frame indices
frame_indx = np.arange(len(self.fitstbl))
# Iterate over each calibration group and reduce the standards
for i in range(self.fitstbl.n_calib_groups):
# Find all the frames in this calibration group
in_grp = self.fitstbl.find_calib_group(i)
# Find the indices of the standard frames in this calibration group:
grp_standards = frame_indx[is_standard & in_grp]
# Reduce all the standard frames, loop on unique comb_id
u_combid_std = np.unique(self.fitstbl['comb_id'][grp_standards])
for j, comb_id in enumerate(u_combid_std):
frames = np.where(self.fitstbl['comb_id'] == comb_id)[0]
bg_frames = np.where(self.fitstbl['bkg_id'] == comb_id)[0]
if not self.outfile_exists(frames[0]) or self.overwrite:
std_dict = self.reduce_exposure(frames,
bg_frames=bg_frames)
# TODO come up with sensible naming convention for save_exposure for combined files
self.save_exposure(frames[0], std_dict, self.basename)
else:
msgs.info(
'Output file: {:s} already exists'.format(
self.fitstbl.construct_basename(frames[0])) +
'. Set overwrite=True to recreate and overwrite.')
# Iterate over each calibration group again and reduce the science frames
for i in range(self.fitstbl.n_calib_groups):
# Find all the frames in this calibration group
in_grp = self.fitstbl.find_calib_group(i)
# Find the indices of the science frames in this calibration group:
grp_science = frame_indx[is_science & in_grp]
# Associate standards (previously reduced above) for this setup
std_outfile = self.get_std_outfile(frame_indx[is_standard])
# Reduce all the science frames; keep the basenames of the science frames for use in flux calibration
science_basename = [None] * len(grp_science)
# Loop on unique comb_id
u_combid = np.unique(self.fitstbl['comb_id'][grp_science])
for j, comb_id in enumerate(u_combid):
frames = np.where(self.fitstbl['comb_id'] == comb_id)[0]
# Find all frames whose comb_id matches the current frames bkg_id.
bg_frames = np.where((self.fitstbl['comb_id'] ==
self.fitstbl['bkg_id'][frames][0])
& (self.fitstbl['comb_id'] >= 0))[0]
# JFH changed the syntax below to that above, which allows frames to be used more than once
# as a background image. The syntax below would require that we could somehow list multiple
# numbers for the bkg_id which is impossible without a comma separated list
# bg_frames = np.where(self.fitstbl['bkg_id'] == comb_id)[0]
if not self.outfile_exists(frames[0]) or self.overwrite:
sci_dict = self.reduce_exposure(frames,
bg_frames=bg_frames,
std_outfile=std_outfile)
science_basename[j] = self.basename
# TODO come up with sensible naming convention for save_exposure for combined files
self.save_exposure(frames[0], sci_dict, self.basename)
else:
msgs.warn(
'Output file: {:s} already exists'.format(
self.fitstbl.construct_basename(frames[0])) +
'. Set overwrite=True to recreate and overwrite.')
msgs.info('Finished calibration group {0}'.format(i))
# Finish
self.print_end_time()
# This is a static method to allow for use in coadding script
@staticmethod
def select_detectors(detnum=None, ndet=1):
"""
Return the 1-indexed list of detectors to reduce.
Args:
detnum (:obj:`int`, :obj:`list`, optional):
One or more detectors to reduce. If None, return the
full list for the provided number of detectors (`ndet`).
ndet (:obj:`int`, optional):
The number of detectors for this instrument. Only used
if `detnum is None`.
Returns:
list: List of detectors to be reduced
"""
if detnum is None:
return np.arange(1, ndet + 1).tolist()
return [detnum] if isinstance(detnum, int) else detnum
def reduce_exposure(self, frames, bg_frames=None, std_outfile=None):
"""
Reduce a single exposure
Args:
frame (:obj:`int`):
0-indexed row in :attr:`fitstbl` with the frame to
reduce.
bg_frames (:obj:`list`, optional):
List of frame indices for the background.
std_outfile (:obj:`str`, optional):
File with a previously reduced standard spectrum from
PypeIt.
Returns:
dict: The dictionary containing the primary outputs of
extraction.
"""
# TODO:
# - bg_frames should be None by default
# - change doc string to reflect that more than one frame can be
# provided
# if show is set, clear the ginga channels at the start of each new sci_ID
if self.show:
# TODO: Put this in a try/except block?
ginga.clear_all()
has_bg = True if bg_frames is not None and len(
bg_frames) > 0 else False
# Is this an IR reduction?
# TODO: Why specific to IR?
self.ir_redux = True if has_bg else False
# TODO: JFH Why does this need to be ordered?
sci_dict = OrderedDict() # This needs to be ordered
sci_dict['meta'] = {}
sci_dict['meta']['ir_redux'] = self.ir_redux
# Print status message
msgs_string = 'Reducing target {:s}'.format(
self.fitstbl['target'][frames[0]]) + msgs.newline()
# TODO: Print these when the frames are actually combined,
# backgrounds are used, etc?
msgs_string += 'Combining frames:' + msgs.newline()
for iframe in frames:
msgs_string += '{0:s}'.format(
self.fitstbl['filename'][iframe]) + msgs.newline()
msgs.info(msgs_string)
if has_bg:
bg_msgs_string = ''
for iframe in bg_frames:
bg_msgs_string += '{0:s}'.format(
self.fitstbl['filename'][iframe]) + msgs.newline()
bg_msgs_string = msgs.newline(
) + 'Using background from frames:' + msgs.newline(
) + bg_msgs_string
msgs.info(bg_msgs_string)
# Find the detectors to reduce
detectors = PypeIt.select_detectors(detnum=self.par['rdx']['detnum'],
ndet=self.spectrograph.ndet)
if len(detectors) != self.spectrograph.ndet:
msgs.warn('Not reducing detectors: {0}'.format(' '.join([
str(d) for d in set(np.arange(self.spectrograph.ndet)) -
set(detectors)
])))
# Loop on Detectors
for self.det in detectors:
msgs.info("Working on detector {0}".format(self.det))
sci_dict[self.det] = {}
# Calibrate
#TODO Is the right behavior to just use the first frame?
self.caliBrate.set_config(frames[0], self.det,
self.par['calibrations'])
self.caliBrate.run_the_steps()
# Extract
# TODO: pass back the background frame, pass in background
# files as an argument. extract one takes a file list as an
# argument and instantiates science within
sci_dict[self.det]['sciimg'], sci_dict[self.det]['sciivar'], \
sci_dict[self.det]['skymodel'], sci_dict[self.det]['objmodel'], \
sci_dict[self.det]['ivarmodel'], sci_dict[self.det]['outmask'], \
sci_dict[self.det]['specobjs'], \
= self.extract_one(frames, self.det, bg_frames,
std_outfile=std_outfile)
# JFH TODO write out the background frame?
# Return
return sci_dict
def get_sci_metadata(self, frame, det):
"""
Grab the meta data for a given science frame and specific detector
Args:
frame (int): Frame index
det (int): Detector index
Returns:
5 objects are returned::
- str: Object type; science or standard
- str: Setup string from master_key()
- astropy.time.Time: Time of observation
- str: Basename of the frame
- str: Binning of the detector
"""
# Set binning, obstime, basename, and objtype
binning = self.fitstbl['binning'][frame]
obstime = self.fitstbl.construct_obstime(frame)
basename = self.fitstbl.construct_basename(frame, obstime=obstime)
objtype = self.fitstbl['frametype'][frame]
if 'science' in objtype:
objtype_out = 'science'
elif 'standard' in objtype:
objtype_out = 'standard'
else:
msgs.error('Unrecognized objtype')
setup = self.fitstbl.master_key(frame, det=det)
return objtype_out, setup, obstime, basename, binning
def get_std_trace(self, std_redux, det, std_outfile):
"""
Returns the trace of the standard if it is applicable to the current reduction
Args:
std_redux (bool): If False, proceed
det (int): Detector index
std_outfile (str): Filename for the standard star spec1d file
Returns:
ndarray or None: Trace of the standard star on input detector
"""
if std_redux is False and std_outfile is not None:
sobjs = specobjs.SpecObjs.from_fitsfile(std_outfile)
# Does the detector match?
# TODO Instrument specific logic here could be implemented with the parset. For example LRIS-B or LRIS-R we
# we would use the standard from another detector
this_det = sobjs.DET == det
if np.any(this_det):
sobjs_det = sobjs[this_det]
sobjs_std = sobjs_det.get_std()
# No standard extracted on this detector??
if sobjs_std is None:
return None
std_trace = sobjs_std.TRACE_SPAT
# flatten the array if this multislit
if 'MultiSlit' in self.spectrograph.pypeline:
std_trace = std_trace.flatten()
elif 'Echelle' in self.spectrograph.pypeline:
std_trace = std_trace.T
else:
msgs.error('Unrecognized pypeline')
else:
std_trace = None
else:
std_trace = None
return std_trace
def extract_one(self, frames, det, bg_frames, std_outfile=None):
"""
Extract a single exposure/detector pair
sci_ID and det need to have been set internally prior to calling this method
Args:
frames (list):
List of frames to extract; stacked if more than one is provided
det (int):
bg_frames (list):
List of frames to use as the background
Can be empty
std_outfile (str, optional):
Returns:
seven objects are returned::
- ndarray: Science image
- ndarray: Science inverse variance image
- ndarray: Model of the sky
- ndarray: Model of the object
- ndarray: Model of inverse variance
- ndarray: Mask
- :obj:`pypeit.specobjs.SpecObjs`: spectra
"""
# Grab some meta-data needed for the reduction from the fitstbl
self.objtype, self.setup, self.obstime, self.basename, self.binning = self.get_sci_metadata(
frames[0], det)
# Is this a standard star?
self.std_redux = 'standard' in self.objtype
# Get the standard trace if need be
std_trace = self.get_std_trace(self.std_redux, det, std_outfile)
# Build Science image
sci_files = self.fitstbl.frame_paths(frames)
self.sciImg = scienceimage.build_from_file_list(
self.spectrograph,
det,
self.par['scienceframe']['process'],
self.caliBrate.msbpm,
sci_files,
self.caliBrate.msbias,
self.caliBrate.mspixelflat,
illum_flat=self.caliBrate.msillumflat)
# Background Image?
if len(bg_frames) > 0:
bg_file_list = self.fitstbl.frame_paths(bg_frames)
self.sciImg = self.sciImg - scienceimage.build_from_file_list(
self.spectrograph,
det,
self.par['scienceframe']['process'],
self.caliBrate.msbpm,
bg_file_list,
self.caliBrate.msbias,
self.caliBrate.mspixelflat,
illum_flat=self.caliBrate.msillumflat)
# Update mask for slitmask
slitmask = pixels.tslits2mask(self.caliBrate.tslits_dict)
self.sciImg.update_mask_slitmask(slitmask)
# For QA on crash
msgs.sciexp = self.sciImg
# Instantiate Reduce object
self.maskslits = self.caliBrate.tslits_dict['maskslits'].copy()
# Required for pypeline specific object
# TODO -- caliBrate should be replaced by the ~3 primary Objects needed
# once we have the data models in place.
self.redux = reduce.instantiate_me(self.sciImg,
self.spectrograph,
self.par,
self.caliBrate,
maskslits=self.maskslits,
ir_redux=self.ir_redux,
std_redux=self.std_redux,
objtype=self.objtype,
setup=self.setup,
show=self.show,
det=det,
binning=self.binning)
# Show?
if self.show:
self.redux.show('image',
image=self.sciImg.image,
chname='processed',
slits=True,
clear=True)
# Prep for manual extraction (if requested)
manual_extract_dict = self.fitstbl.get_manual_extract(frames, det)
self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs = self.redux.run(
std_trace=std_trace,
manual_extract_dict=manual_extract_dict,
show_peaks=self.show,
basename=self.basename,
ra=self.fitstbl["ra"][frames[0]],
dec=self.fitstbl["dec"][frames[0]],
obstime=self.obstime)
# Return
return self.sciImg.image, self.sciImg.ivar, self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs
# TODO: Why not use self.frame?
def save_exposure(self, frame, sci_dict, basename):
"""
Save the outputs from extraction for a given exposure
Args:
frame (:obj:`int`):
0-indexed row in the metadata table with the frame that
has been reduced.
sci_dict (:obj:`dict`):
Dictionary containing the primary outputs of extraction
basename (:obj:`str`):
The root name for the output file.
Returns:
None or SpecObjs: All of the objects saved to disk
"""
# TODO: Need some checks here that the exposure has been reduced
# Determine the headers
head1d = self.fitstbl[frame]
# Need raw file header information
rawfile = self.fitstbl.frame_paths(frame)
head2d = fits.getheader(rawfile, ext=self.spectrograph.primary_hdrext)
refframe = 'pixel' if self.caliBrate.par['wavelengths']['reference'] == 'pixel' else \
self.caliBrate.par['wavelengths']['frame']
# Determine the paths/filenames
save.save_all(sci_dict,
self.caliBrate.master_key_dict,
self.caliBrate.master_dir,
self.spectrograph,
head1d,
head2d,
self.science_path,
basename,
update_det=self.par['rdx']['detnum'],
binning=self.fitstbl['binning'][frame])
def msgs_reset(self):
"""
Reset the msgs object
"""
# Reset the global logger
msgs.reset(log=self.logname, verbosity=self.verbosity)
msgs.pypeit_file = self.pypeit_file
def print_end_time(self):
"""
Print the elapsed time
"""
# Capture the end time and print it to user
tend = time.time()
codetime = tend - self.tstart
if codetime < 60.0:
msgs.info('Execution time: {0:.2f}s'.format(codetime))
elif codetime / 60.0 < 60.0:
mns = int(codetime / 60.0)
scs = codetime - 60.0 * mns
msgs.info('Execution time: {0:d}m {1:.2f}s'.format(mns, scs))
else:
hrs = int(codetime / 3600.0)
mns = int(60.0 * (codetime / 3600.0 - hrs))
scs = codetime - 60.0 * mns - 3600.0 * hrs
msgs.info('Execution time: {0:d}h {1:d}m {2:.2f}s'.format(
hrs, mns, scs))
# TODO: Move this to fitstbl?
def show_science(self):
"""
Simple print of science frames
"""
indx = self.fitstbl.find_frames('science')
print(self.fitstbl[['target', 'ra', 'dec', 'exptime',
'dispname']][indx])
def __repr__(self):
# Generate sets string
return '<{:s}: pypeit_file={}>'.format(self.__class__.__name__,
self.pypeit_file)