def MasterStandard(self, scidx, fitsdict):
"""
Generate Master Standard frame for a given detector
and generates a sensitivity function
Currently only uses first standard star exposure
Currently takes brightest source on the mosaic
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
Returns
-------
boolean : bool
"""
if len(self._msstd[0]) != 0:
msgs.info("Using existing standard frame")
return False
#
msgs.info("Preparing the standard")
# Get all of the pixel flat frames for this science frame
ind = self._idx_std
msgs.warn("Taking only the first standard frame for now")
ind = [ind[0]]
# Extract
all_specobj = []
for kk in xrange(self._spect['mosaic']['ndet']):
det = kk+1
# Load the frame(s)
# set_trace()
frame = arload.load_frames(self, fitsdict, ind, det, frametype='standard',
msbias=self._msbias[det-1],
transpose=self._transpose)
# msgs.warn("Taking only the first standard frame for now")
# ind = ind[0]
sciframe = frame[:, :, 0]
# Save RA/DEC
if kk == 0:
self._msstd[det-1]['RA'] = fitsdict['ra'][ind[0]]
self._msstd[det-1]['DEC'] = fitsdict['dec'][ind[0]]
#debugger.set_trace()
arproc.reduce_frame(self, sciframe, ind[0], fitsdict, det, standard=True)
#
all_specobj += self._msstd[det-1]['spobjs']
# debugger.set_trace()
# If standard, generate a sensitivity function
sensfunc = arflux.generate_sensfunc(self, scidx, all_specobj, fitsdict)
# Set the sensitivity function
self.SetMasterFrame(sensfunc, "sensfunc", None, copy=False)
return True
def ARMLSD(argflag, spect, fitsdict, reuseMaster=False):
"""
Automatic Reduction and Modeling of Long Slit Data
Parameters
----------
argflag : dict
Arguments and flags used for reduction
spect : dict
Properties of the spectrograph.
fitsdict : dict
Contains relevant information from fits header files
reuseMaster : bool
If True, a master frame that will be used for another science frame
will not be regenerated after it is first made.
This setting comes with a price, and if a large number of science frames are
being generated, it may be more efficient to simply regenerate the master
calibrations on the fly.
Returns
-------
status : int
Status of the reduction procedure
0 = Successful execution
1 = ...
"""
status = 0
# Create a list of science exposure classes
sciexp = armbase.SetupScience(argflag, spect, fitsdict)
numsci = len(sciexp)
# Create a list of master calibration frames
masters = armasters.MasterFrames(spect['mosaic']['ndet'])
# Use Masters? Requires setup file
setup_file = argflag['out']['sorted'] + '.setup'
try:
calib_dict = ltu.loadjson(setup_file)
except:
msgs.info("No setup file {:s} for MasterFrames".format(setup_file))
calib_dict = {}
else:
argflag['masters']['setup_file'] = setup_file
# Start reducing the data
for sc in range(numsci):
slf = sciexp[sc]
scidx = slf._idx_sci[0]
msgs.info("Reducing file {0:s}, target {1:s}".format(
fitsdict['filename'][scidx], slf._target_name))
msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too
# Loop on Detectors
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
slf.det = det
###############
# Get amplifier sections
arproc.get_ampsec_trimmed(slf, fitsdict, det, scidx)
# Setup
setup = arsort.calib_setup(slf,
sc,
det,
fitsdict,
calib_dict,
write=False)
slf._argflag['masters']['setup'] = setup
###############
# Generate master bias frame
update = slf.MasterBias(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
###############
# Generate a bad pixel mask (should not repeat)
update = slf.BadPixelMask(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Generate a master arc frame
update = slf.MasterArc(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Determine the dispersion direction (and transpose if necessary)
slf.GetDispersionDirection(fitsdict, det, scidx)
if slf._bpix[
det -
1] is None: # Needs to be done here after nspec is set
slf.SetFrame(
slf._bpix,
np.zeros((slf._nspec[det - 1], slf._nspat[det - 1])), det)
'''
###############
# Estimate gain and readout noise for the amplifiers
msgs.work("Estimate Gain and Readout noise from the raw frames...")
update = slf.MasterRN(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise")
'''
###############
# Generate a master trace frame
update = slf.MasterTrace(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="flat",
chktype="trace")
###############
# Generate an array that provides the physical pixel locations on the detector
slf.GetPixelLocations(det)
# Determine the edges of the spectrum (spatial)
if 'trace' + slf._argflag['masters']['setup'] not in slf._argflag[
'masters']['loaded']:
###############
# Determine the edges of the spectrum (spatial)
lordloc, rordloc, extord = artrace.trace_orders(
slf,
slf._mstrace[det - 1],
det,
singleSlit=True,
pcadesc="PCA trace of the slit edges")
slf.SetFrame(slf._lordloc, lordloc, det)
slf.SetFrame(slf._rordloc, rordloc, det)
# Convert physical trace into a pixel trace
msgs.info(
"Converting physical trace locations to nearest pixel")
pixcen = artrace.phys_to_pix(
0.5 * (slf._lordloc[det - 1] + slf._rordloc[det - 1]),
slf._pixlocn[det - 1], 1)
pixwid = (slf._rordloc[det - 1] -
slf._lordloc[det - 1]).mean(0).astype(np.int)
lordpix = artrace.phys_to_pix(slf._lordloc[det - 1],
slf._pixlocn[det - 1], 1)
rordpix = artrace.phys_to_pix(slf._rordloc[det - 1],
slf._pixlocn[det - 1], 1)
slf.SetFrame(slf._pixcen, pixcen, det)
slf.SetFrame(slf._pixwid, pixwid, det)
slf.SetFrame(slf._lordpix, lordpix, det)
slf.SetFrame(slf._rordpix, rordpix, det)
# Save QA for slit traces
arqa.slit_trace_qa(slf,
slf._mstrace[det - 1],
slf._lordpix[det - 1],
slf._rordpix[det - 1],
extord,
desc="Trace of the slit edges")
###############
# Prepare the pixel flat field frame
update = slf.MasterFlatField(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="flat",
chktype="pixflat")
###############
# Generate the 1D wavelength solution
update = slf.MasterWaveCalib(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="arc",
chktype="trace")
###############
# Derive the spectral tilt
if slf._tilts[det - 1] is None:
if slf._argflag['masters']['use']:
mstilt_name = armasters.master_name(
slf._argflag['run']['masterdir'], 'tilts',
slf._argflag['masters']['setup'])
try:
tilts, head = arload.load_master(mstilt_name,
frametype="tilts")
except IOError:
pass
else:
slf.SetFrame(slf._tilts, tilts, det)
slf._argflag['masters']['loaded'].append(
'tilts' + slf._argflag['masters']['setup'])
if 'tilts' + slf._argflag['masters'][
'setup'] not in slf._argflag['masters']['loaded']:
# First time tilts are derived for this arc frame --> derive the order tilts
tilts, satmask, outpar = artrace.model_tilt(
slf, det, slf._msarc[det - 1])
slf.SetFrame(slf._tilts, tilts, det)
slf.SetFrame(slf._satmask, satmask, det)
slf.SetFrame(slf._tiltpar, outpar, det)
###############
# Generate/load a master wave frame
update = slf.MasterWave(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="arc",
chktype="wave")
# Check if the user only wants to prepare the calibrations only
msgs.info("All calibration frames have been prepared")
if slf._argflag['run']['preponly']:
msgs.info("If you would like to continue with the reduction," +
msgs.newline() + "disable the run+preponly command")
continue
# Write setup
setup = arsort.calib_setup(slf,
sc,
det,
fitsdict,
calib_dict,
write=True)
# Write MasterFrames (currently per detector)
armasters.save_masters(slf, det, setup)
###############
# Load the science frame and from this generate a Poisson error frame
msgs.info("Loading science frame")
sciframe = arload.load_frames(slf,
fitsdict, [scidx],
det,
frametype='science',
msbias=slf._msbias[det - 1],
transpose=slf._transpose)
sciframe = sciframe[:, :, 0]
# Extract
msgs.info("Processing science frame")
arproc.reduce_frame(slf, sciframe, scidx, fitsdict, det)
#continue
#msgs.error("UP TO HERE")
###############
# Perform a velocity correction
if (slf._argflag['reduce']['heliocorr'] == True) & False:
if slf._argflag['science']['load']['extracted'] == True:
msgs.warn(
"Heliocentric correction will not be applied if an extracted science frame exists, and is used"
)
msgs.work("Perform a full barycentric correction")
msgs.work(
"Include the facility to correct for gravitational redshifts and time delays (see Pulsar timing work)"
)
msgs.info("Performing a heliocentric correction")
# Load the header for the science frame
slf._waveids = arvcorr.helio_corr(slf, scidx[0])
else:
msgs.info("A heliocentric correction will not be performed")
###############
# Using model sky, calculate a flexure correction
# Close the QA for this object
slf._qa.close()
###############
# Flux
###############
# Standard star (is this a calibration, e.g. goes above?)
msgs.info("Processing standard star")
msgs.info("Assuming one star per detector mosaic")
msgs.info("Waited until last detector to process")
msgs.work("Need to check for existing sensfunc")
update = slf.MasterStandard(scidx, fitsdict)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, 0, ftype="standard")
#
msgs.work("Consider using archived sensitivity if not found")
msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name']))
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
arflux.apply_sensfunc(slf, det, scidx, fitsdict)
# Write 1D spectra
arsave.save_1d_spectra(slf)
# Write 2D images for the Science Frame
arsave.save_2d_images(slf)
# Free up some memory by replacing the reduced ScienceExposure class
sciexp[sc] = None
return status
def ARMLSD(argflag, spect, fitsdict, reuseMaster=False):
"""
Automatic Reduction and Modeling of Long Slit Data
Parameters
----------
argflag : dict
Arguments and flags used for reduction
spect : dict
Properties of the spectrograph.
fitsdict : dict
Contains relevant information from fits header files
reuseMaster : bool
If True, a master frame that will be used for another science frame
will not be regenerated after it is first made.
This setting comes with a price, and if a large number of science frames are
being generated, it may be more efficient to simply regenerate the master
calibrations on the fly.
Returns
-------
status : int
Status of the reduction procedure
0 = Successful execution
1 = ...
"""
status = 0
# Create a list of science exposure classes
sciexp = armbase.SetupScience(argflag, spect, fitsdict)
numsci = len(sciexp)
# Create a list of master calibration frames
masters = armasters.MasterFrames(spect['mosaic']['ndet'])
# Use Masters? Requires setup file
setup_file = argflag['out']['sorted']+'.setup'
try:
calib_dict = ltu.loadjson(setup_file)
except:
msgs.info("No setup file {:s} for MasterFrames".format(setup_file))
calib_dict = {}
else:
argflag['masters']['setup_file'] = setup_file
# Start reducing the data
for sc in range(numsci):
slf = sciexp[sc]
scidx = slf._idx_sci[0]
msgs.info("Reducing file {0:s}, target {1:s}".format(fitsdict['filename'][scidx], slf._target_name))
msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too
# Loop on Detectors
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
slf.det = det
###############
# Get amplifier sections
arproc.get_ampsec_trimmed(slf, fitsdict, det, scidx)
# Setup
setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=False)
slf._argflag['masters']['setup'] = setup
###############
# Generate master bias frame
update = slf.MasterBias(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
###############
# Generate a bad pixel mask (should not repeat)
update = slf.BadPixelMask(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Generate a master arc frame
update = slf.MasterArc(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Determine the dispersion direction (and transpose if necessary)
slf.GetDispersionDirection(fitsdict, det, scidx)
if slf._bpix[det-1] is None: # Needs to be done here after nspec is set
slf.SetFrame(slf._bpix, np.zeros((slf._nspec[det-1], slf._nspat[det-1])), det)
'''
###############
# Estimate gain and readout noise for the amplifiers
msgs.work("Estimate Gain and Readout noise from the raw frames...")
update = slf.MasterRN(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise")
'''
###############
# Generate a master trace frame
update = slf.MasterTrace(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
###############
# Generate an array that provides the physical pixel locations on the detector
slf.GetPixelLocations(det)
# Determine the edges of the spectrum (spatial)
if 'trace'+slf._argflag['masters']['setup'] not in slf._argflag['masters']['loaded']:
###############
# Determine the edges of the spectrum (spatial)
lordloc, rordloc, extord = artrace.trace_orders(slf, slf._mstrace[det-1], det, singleSlit=True, pcadesc="PCA trace of the slit edges")
slf.SetFrame(slf._lordloc, lordloc, det)
slf.SetFrame(slf._rordloc, rordloc, det)
# Convert physical trace into a pixel trace
msgs.info("Converting physical trace locations to nearest pixel")
pixcen = artrace.phys_to_pix(0.5*(slf._lordloc[det-1]+slf._rordloc[det-1]), slf._pixlocn[det-1], 1)
pixwid = (slf._rordloc[det-1]-slf._lordloc[det-1]).mean(0).astype(np.int)
lordpix = artrace.phys_to_pix(slf._lordloc[det-1], slf._pixlocn[det-1], 1)
rordpix = artrace.phys_to_pix(slf._rordloc[det-1], slf._pixlocn[det-1], 1)
slf.SetFrame(slf._pixcen, pixcen, det)
slf.SetFrame(slf._pixwid, pixwid, det)
slf.SetFrame(slf._lordpix, lordpix, det)
slf.SetFrame(slf._rordpix, rordpix, det)
# Save QA for slit traces
arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det-1], extord, desc="Trace of the slit edges")
###############
# Prepare the pixel flat field frame
update = slf.MasterFlatField(fitsdict, det)
if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixflat")
###############
# Generate the 1D wavelength solution
update = slf.MasterWaveCalib(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace")
###############
# Derive the spectral tilt
if slf._tilts[det-1] is None:
if slf._argflag['masters']['use']:
mstilt_name = armasters.master_name(slf._argflag['run']['masterdir'],
'tilts', slf._argflag['masters']['setup'])
try:
tilts, head = arload.load_master(mstilt_name, frametype="tilts")
except IOError:
pass
else:
slf.SetFrame(slf._tilts, tilts, det)
slf._argflag['masters']['loaded'].append('tilts'+slf._argflag['masters']['setup'])
if 'tilts'+slf._argflag['masters']['setup'] not in slf._argflag['masters']['loaded']:
# First time tilts are derived for this arc frame --> derive the order tilts
tilts, satmask, outpar = artrace.model_tilt(slf, det, slf._msarc[det-1])
slf.SetFrame(slf._tilts, tilts, det)
slf.SetFrame(slf._satmask, satmask, det)
slf.SetFrame(slf._tiltpar, outpar, det)
###############
# Generate/load a master wave frame
update = slf.MasterWave(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave")
# Check if the user only wants to prepare the calibrations only
msgs.info("All calibration frames have been prepared")
if slf._argflag['run']['preponly']:
msgs.info("If you would like to continue with the reduction,"
+msgs.newline()+"disable the run+preponly command")
continue
# Write setup
setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=True)
# Write MasterFrames (currently per detector)
armasters.save_masters(slf, det, setup)
###############
# Load the science frame and from this generate a Poisson error frame
msgs.info("Loading science frame")
sciframe = arload.load_frames(slf, fitsdict, [scidx], det,
frametype='science',
msbias=slf._msbias[det-1],
transpose=slf._transpose)
sciframe = sciframe[:, :, 0]
# Extract
msgs.info("Processing science frame")
arproc.reduce_frame(slf, sciframe, scidx, fitsdict, det)
#continue
#msgs.error("UP TO HERE")
###############
# Perform a velocity correction
if (slf._argflag['reduce']['heliocorr'] == True) & False:
if slf._argflag['science']['load']['extracted'] == True:
msgs.warn("Heliocentric correction will not be applied if an extracted science frame exists, and is used")
msgs.work("Perform a full barycentric correction")
msgs.work("Include the facility to correct for gravitational redshifts and time delays (see Pulsar timing work)")
msgs.info("Performing a heliocentric correction")
# Load the header for the science frame
slf._waveids = arvcorr.helio_corr(slf, scidx[0])
else:
msgs.info("A heliocentric correction will not be performed")
###############
# Using model sky, calculate a flexure correction
# Close the QA for this object
slf._qa.close()
###############
# Flux
###############
# Standard star (is this a calibration, e.g. goes above?)
msgs.info("Processing standard star")
msgs.info("Assuming one star per detector mosaic")
msgs.info("Waited until last detector to process")
msgs.work("Need to check for existing sensfunc")
update = slf.MasterStandard(scidx, fitsdict)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, 0, ftype="standard")
#
msgs.work("Consider using archived sensitivity if not found")
msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name']))
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
arflux.apply_sensfunc(slf, det, scidx, fitsdict)
# Write 1D spectra
arsave.save_1d_spectra(slf)
# Write 2D images for the Science Frame
arsave.save_2d_images(slf)
# Free up some memory by replacing the reduced ScienceExposure class
sciexp[sc] = None
return status
