def MasterBias(self, fitsdict, det):
"""
Generate Master Bias frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
# If the master bias is already made, use it
if self._msbias[det - 1] is not None:
msgs.info("An identical master {0:s} frame already exists".format(
settings.argflag['bias']['useframe']))
return False
elif settings.argflag['bias']['useframe'] in ['bias', 'dark']:
try:
msbias = armasters.get_master_frame(self, "bias")
except IOError:
msgs.info("Preparing a master {0:s} frame".format(
settings.argflag['bias']['useframe']))
# Get all of the bias frames for this science frame
ind = self._idx_bias
# Load the Bias/Dark frames
frames = arload.load_frames(
fitsdict,
ind,
det,
frametype=settings.argflag['bias']['useframe'])
msbias = arcomb.comb_frames(
frames,
det,
'bias',
printtype=settings.argflag['bias']['useframe'])
del frames
elif settings.argflag['bias']['useframe'] == 'overscan':
self.SetMasterFrame('overscan', "bias", det, mkcopy=False)
return False
elif settings.argflag['bias']['useframe'] == 'none':
msgs.info("Not performing a bias/dark subtraction")
self.SetMasterFrame(None, "bias", det, mkcopy=False)
return False
else: # It must be the name of a file the user wishes to load
msbias_name = settings.argflag['run']['directory'][
'master'] + u'/' + settings.argflag['bias']['useframe']
msbias, head = arload.load_master(msbias_name, frametype="bias")
settings.argflag['reduce']['masters']['loaded'].append('bias')
# Set and then delete the Master Bias frame
self.SetMasterFrame(msbias, "bias", det)
armasters.save_masters(self, det, mftype='bias')
del msbias
return True
def MasterPinhole(self, fitsdict, det, msbias):
"""
Generate Master pinhole frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
dnum = settings.get_dnum(det)
# If the master pinhole is already made, use it
if self._mspinhole[det - 1] is not None:
msgs.info("An identical master pinhole frame already exists")
return False
if settings.argflag['reduce']['slitcen']['useframe'] in ['trace', 'pinhole']:
try:
mspinhole = armasters.get_master_frame(self, "pinhole")
except IOError:
msgs.info("Preparing a master pinhole frame with {0:s}".format(
settings.argflag['reduce']['slitcen']['useframe']))
ind = self._idx_cent
# Load the pinhole frames
frames = arload.load_frames(fitsdict, ind, det, frametype='pinhole', msbias=msbias, # self._msbias[det - 1],
trim=settings.argflag['reduce']['trim'])
if settings.argflag['pinhole']['combine']['match'] > 0.0:
sframes = arsort.match_frames(frames, settings.argflag['pinhole']['combine']['match'],
frametype='pinhole', satlevel=settings.spect[dnum]['saturation'] *
settings.spect['det'][det - 1]['nonlinear'])
subframes = np.zeros((frames.shape[0], frames.shape[1], len(sframes)))
numarr = np.array([])
for i in range(len(sframes)):
numarr = np.append(numarr, sframes[i].shape[2])
mspinhole = arcomb.comb_frames(sframes[i], det, 'pinhole')
subframes[:, :, i] = mspinhole.copy()
del sframes
# Combine all sub-frames
mspinhole = arcomb.comb_frames(subframes, det, 'pinhole', weights=numarr)
del subframes
else:
mspinhole = arcomb.comb_frames(frames, det, 'pinhole')
del frames
else: # It must be the name of a file the user wishes to load
mspinhole_name = settings.argflag['run']['directory']['master'] + '/' + \
settings.argflag['reduce']['slitcen']['useframe']
mspinhole, head = armasters.load_master(mspinhole_name, frametype=None)
debugger.set_trace() # NEED TO LOAD EXTRAS AS ABOVE
# Set and then delete the Master Trace frame
self.SetMasterFrame(mspinhole, "pinhole", det)
#armasters.save_masters(self, det, mftype='pinhole')
del mspinhole
return True
def MasterWaveCalib(self, fitsdict, sc, det):
"""
Generate Master 1D Wave Solution (down slit center)
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
from pypit import ararc
if self._wvcalib[det - 1] is not None:
msgs.info("An identical master wave calib frame already exists")
return False
else:
wv_calib = None
# Attempt to load the Master Frame
try:
wv_calib = armasters.get_master_frame(self, "wv_calib")
except IOError:
if settings.argflag["reduce"]["calibrate"][
"wavelength"] == "pixel":
msgs.info("A wavelength calibration will not be performed")
else:
# Setup arc parameters (e.g. linelist)
arcparam = ararc.setup_param(self, sc, det, fitsdict)
self.SetFrame(self._arcparam, arcparam, det)
###############
# Extract arc and identify lines
if settings.argflag['arc']['calibrate']['method'] == 'simple':
wv_calib = ararc.simple_calib(self, det)
elif settings.argflag['arc']['calibrate'][
'method'] == 'arclines':
wv_calib = ararc.calib_with_arclines(self, det)
# Set
if wv_calib is not None:
self.SetFrame(self._wvcalib, wv_calib, det)
armasters.save_masters(self, det, mftype='wv_calib')
del wv_calib
return True
def BadPixelMask(self, fitsdict, det):
"""
Generate Bad Pixel Mask for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
bpix = None
if settings.argflag['reduce']['badpix'] == 'bias':
try:
bpix = armasters.get_master_frame(self, "badpix")
except IOError:
msgs.info("Preparing a bad pixel mask")
# Get all of the bias frames for this science frame
if len(self._idx_bias) == 0:
msgs.warn(
"No bias frames available to determine bad pixel mask")
msgs.info("Not preparing a bad pixel mask")
return False
# Load the Bias frames
bpix = arproc.badpix(self, det,
self.GetMasterFrame('bias', det))
else:
# Instrument dependent
if settings.argflag['run']['spectrograph'] in ['keck_lris_red']:
bpix = arlris.bpm(self, 'red', fitsdict, det)
else:
msgs.info("Not preparing a bad pixel mask")
return False
# Save
self.SetFrame(self._bpix, bpix, det)
armasters.save_masters(self, det, mftype='badpix')
del bpix
return True
def MasterWave(self, fitsdict, sc, det):
"""
Generate Master Wave frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
if self._mswave[det - 1] is not None:
msgs.info("An identical master arc frame already exists")
return False
try:
mswave = armasters.get_master_frame(self, "wave")
except IOError:
msgs.info("Preparing a master wave frame")
if settings.argflag["reduce"]["calibrate"][
"wavelength"] == "pixel":
mswave = self._tilts[det -
1] * (self._tilts[det - 1].shape[0] - 1.0)
else:
wv_calib = self._wvcalib[det - 1]
mswave = arutils.func_val(wv_calib['fitc'],
self._tilts[det - 1],
wv_calib['function'],
minv=wv_calib['fmin'],
maxv=wv_calib['fmax'])
# Set and then delete the Master Arc frame
self.SetMasterFrame(mswave, "wave", det)
armasters.save_masters(self, det, mftype='wave')
del mswave
return True
def ARMED(fitsdict, reuseMaster=False, reloadMaster=True):
"""
Automatic Reduction and Modeling of Echelle Data
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
reuseMaster : bool
If True, a master frame that will be used for another science frame
will not be regenerated after it is first made.
This setting comes with a price, and if a large number of science frames are
being generated, it may be more efficient to simply regenerate the master
calibrations on the fly.
Returns
-------
status : int
Status of the reduction procedure
0 = Successful execution
1 = ...
"""
status = 0
# Create a list of science exposure classes
sciexp, setup_dict = armbase.SetupScience(fitsdict)
if sciexp == 'setup':
status = 1
return status
elif sciexp == 'calcheck':
status = 2
return status
else:
numsci = len(sciexp)
# Create a list of master calibration frames
#masters = armasters.MasterFrames(settings.spect['mosaic']['ndet'])
# Masters
#settings.argflag['reduce']['masters']['file'] = setup_file
# Start reducing the data
for sc in range(numsci):
slf = sciexp[sc]
scidx = slf._idx_sci[0]
msgs.info("Reducing file {0:s}, target {1:s}".format(fitsdict['filename'][scidx], slf._target_name))
msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too
if reloadMaster and (sc > 0):
settings.argflag['reduce']['masters']['reuse'] = True
# Loop on Detectors
for kk in range(settings.spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
if settings.argflag['reduce']['detnum'] is not None:
if det != settings.argflag['reduce']['detnum']:
continue
else:
msgs.warn("Restricting the reduction to detector {:d}".format(det))
slf.det = det
###############
# Get data sections
arproc.get_datasec_trimmed(slf, fitsdict, det, scidx)
# Setup
setup = arsort.instr_setup(slf, det, fitsdict, setup_dict, must_exist=True)
settings.argflag['reduce']['masters']['setup'] = setup
slf.setup = setup
###############
# Generate master bias frame
update = slf.MasterBias(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
###############
# Generate a bad pixel mask (should not repeat)
update = slf.BadPixelMask(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Estimate gain and readout noise for the amplifiers
msgs.work("Estimate Gain and Readout noise from the raw frames...")
###############
# Generate a master arc frame
update = slf.MasterArc(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Set the number of spectral and spatial pixels, and the bad pixel mask is it does not exist
slf._nspec[det-1], slf._nspat[det-1] = slf._msarc[det-1].shape
if slf._bpix[det-1] is None:
slf.SetFrame(slf._bpix, np.zeros((slf._nspec[det-1], slf._nspat[det-1])), det)
###############
# Generate a master trace frame
update = slf.MasterTrace(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
###############
# Generate a master pinhole frame
update = slf.MasterPinhole(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pinhole")
###############
# Generate an array that provides the physical pixel locations on the detector
slf.GetPixelLocations(det)
###############
# Determine the edges of the spectrum (spatial)
if ('trace'+settings.argflag['reduce']['masters']['setup'] not in settings.argflag['reduce']['masters']['loaded']):
if True:#not msgs._debug['develop']:
msgs.info("Tracing slit edges with a {0:s} frame".format(settings.argflag['trace']['useframe']))
if settings.argflag['trace']['useframe'] == 'pinhole':
###############
# Determine the centroid of the spectrum (spatial)
lordloc, rordloc, extord = artrace.trace_slits(slf, slf._mspinhole[det-1], det,
pcadesc="PCA trace of the slit edges")
# Using the order centroid, expand the order edges until the edge of the science slit is found
if settings.argflag['trace']['slits']['expand']:
lordloc, rordloc = artrace.expand_slits(slf, slf._mstrace[det-1], det,
0.5*(lordloc+rordloc), extord)
elif settings.argflag['trace']['useframe'] == 'trace':
###############
# Determine the edges of the slit using a trace frame
lordloc, rordloc, extord = artrace.trace_slits(slf, slf._mstrace[det-1], det,
pcadesc="PCA trace of the slit edges")
else:
msgs.error("Cannot trace slit edges using {0:s}".format(settings.argflag['trace']['useframe']))
else:
lordloc, rordloc, extord = np.load("lordloc.npy"), np.load("rordloc.npy"), np.load("extord.npy")
# Save the locations of the order edges
slf.SetFrame(slf._lordloc, lordloc, det)
slf.SetFrame(slf._rordloc, rordloc, det)
# Convert physical trace into a pixel trace
msgs.info("Converting physical trace locations to nearest pixel")
pixcen = artrace.phys_to_pix(0.5 * (slf._lordloc[det - 1] + slf._rordloc[det - 1]), slf._pixlocn[det - 1], 1)
pixwid = (slf._rordloc[det - 1] - slf._lordloc[det - 1]).mean(0).astype(np.int)
lordpix = artrace.phys_to_pix(slf._lordloc[det - 1], slf._pixlocn[det - 1], 1)
rordpix = artrace.phys_to_pix(slf._rordloc[det - 1], slf._pixlocn[det - 1], 1)
slf.SetFrame(slf._pixcen, pixcen, det)
slf.SetFrame(slf._pixwid, pixwid, det)
slf.SetFrame(slf._lordpix, lordpix, det)
slf.SetFrame(slf._rordpix, rordpix, det)
msgs.info("Identifying the pixels belonging to each slit")
slitpix = arproc.slit_pixels(slf, slf._mstrace[det-1].shape, det)
slf.SetFrame(slf._slitpix, slitpix, det)
# Save to disk
armasters.save_masters(slf, det, mftype='trace')
# Save QA for slit traces
arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det - 1], extord,
desc="Trace of the slit edges", normalize=False)
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
###############
# Generate the 1D wavelength solution
update = slf.MasterWaveCalib(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace")
###############
# Derive the spectral tilt
if slf._tilts[det-1] is None:
try:
tilts = armasters.get_master_frame(slf, "tilts")
except IOError:
# First time tilts are derived for this arc frame --> derive the order tilts
tilts, satmask, outpar = artrace.echelle_tilt(slf, slf._msarc[det - 1], det)
slf.SetFrame(slf._tilts, tilts, det)
slf.SetFrame(slf._satmask, satmask, det)
slf.SetFrame(slf._tiltpar, outpar, det)
armasters.save_masters(slf, det, mftype='tilts')
else:
slf.SetFrame(slf._tilts, tilts, det)
###############
# Prepare the pixel flat field frame
update = slf.MasterFlatField(fitsdict, det)
if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixelflat")
###############
# Derive the spatial profile and blaze function
if slf._slitprof[det-1] is None:
if settings.argflag['reduce']['masters']['reuse']:
msslitprof_name = armasters.master_name('slitprof', settings.argflag['reduce']['masters']['setup'])
try:
slit_profiles, head = arload.load_master(msslitprof_name, frametype="slit profile")
except IOError:
pass
else:
slf.SetFrame(slf._slitprof, slit_profiles, det)
settings.argflag['reduce']['masters']['loaded'].append('slitprof'+settings.argflag['reduce']['masters']['setup'])
if 'slitprof'+settings.argflag['reduce']['masters']['setup'] not in settings.argflag['reduce']['masters']['loaded']:
# First time slit profile is derived
msgs.info("Calculating slit profile from master trace frame")
slit_profiles, mstracenrm, msblaze, flat_ext1d, extrap_slit = arproc.slit_profile(slf, slf._mstrace[det - 1], det)
# If some slit profiles/blaze functions need to be extrapolated, do that now
if np.sum(extrap_slit) != 0.0:
slit_profiles, mstracenrm, msblaze = arproc.slit_profile_pca(slf, slf._mstrace[det - 1], det, msblaze, extrap_slit)
slf.SetFrame(slf._slitprof, slit_profiles, det)
slf.SetFrame(slf._msblaze, msblaze, det)
# Prepare some QA for the average slit profile along the slit
msgs.info("Preparing QA of each slit profile")
arqa.slit_profile(slf, mstracenrm, slit_profiles, slf._lordloc[det - 1], slf._rordloc[det - 1],
slf._slitpix[det - 1], desc="Slit profile")
msgs.info("Saving blaze function QA")
arqa.plot_orderfits(slf, msblaze, flat_ext1d, desc="Blaze function", textplt="Order")
###############
# Generate/load a master wave frame
update = slf.MasterWave(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave")
###############
# Check if the user only wants to prepare the calibrations only
msgs.info("All calibration frames have been prepared")
if settings.argflag['run']['preponly']:
msgs.info("If you would like to continue with the reduction, disable the command:" + msgs.newline() +
"run preponly False")
continue
###############
# Write setup
#setup = arsort.calib_setup(sc, det, fitsdict, setup_dict, write=True)
# Write MasterFrames (currently per detector)
#armasters.save_masters(slf, det, setup)
###############
# Load the science frame and from this generate a Poisson error frame
msgs.info("Loading science frame")
sciframe = arload.load_frames(fitsdict, [scidx], det,
frametype='science',
msbias=slf._msbias[det - 1])
sciframe = sciframe[:, :, 0]
# Extract
msgs.info("Processing science frame")
arproc.reduce_echelle(slf, sciframe, scidx, fitsdict, det)
# Write 1D spectra
save_format = 'fits'
if save_format == 'fits':
arsave.save_1d_spectra_fits(slf, fitsdict)
elif save_format == 'hdf5':
arsave.save_1d_spectra_hdf5(slf)
else:
msgs.error(save_format + ' is not a recognized output format!')
# Write 2D images for the Science Frame
arsave.save_2d_images(slf, fitsdict)
# Free up some memory by replacing the reduced ScienceExposure class
sciexp[sc] = None
return status
def ARMLSD(fitsdict, reuseMaster=False, reloadMaster=True):
"""
Automatic Reduction and Modeling of Long Slit Data
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
reuseMaster : bool
If True, a master frame that will be used for another science frame
will not be regenerated after it is first made.
This setting comes with a price, and if a large number of science frames are
being generated, it may be more efficient to simply regenerate the master
calibrations on the fly.
Returns
-------
status : int
Status of the reduction procedure
0 = Successful full execution
1 = Successful processing of setup or calcheck
"""
status = 0
# Create a list of science exposure classes
sciexp, setup_dict = armbase.SetupScience(fitsdict)
if sciexp == 'setup':
status = 1
return status
elif sciexp == 'calcheck':
status = 2
return status
else:
numsci = len(sciexp)
# Create a list of master calibration frames
#masters = armasters.MasterFrames(settings.spect['mosaic']['ndet'])
# Masters
#settings.argflag['reduce']['masters']['file'] = setup_file
# Start reducing the data
for sc in range(numsci):
slf = sciexp[sc]
scidx = slf._idx_sci[0]
msgs.info("Reducing file {0:s}, target {1:s}".format(fitsdict['filename'][scidx], slf._target_name))
msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too
if reloadMaster and (sc > 0):
settings.argflag['reduce']['masters']['reuse'] = True
# Loop on Detectors
for kk in range(settings.spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
if settings.argflag['reduce']['detnum'] is not None:
if det != settings.argflag['reduce']['detnum']:
continue
else:
msgs.warn("Restricting the reduction to detector {:d}".format(det))
slf.det = det
###############
# Get data sections
arproc.get_datasec_trimmed(slf, fitsdict, det, scidx)
# Setup
setup = arsort.instr_setup(slf, det, fitsdict, setup_dict, must_exist=True)
settings.argflag['reduce']['masters']['setup'] = setup
slf.setup = setup
###############
# Generate master bias frame
update = slf.MasterBias(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
###############
# Generate a bad pixel mask (should not repeat)
update = slf.BadPixelMask(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Generate a master arc frame
update = slf.MasterArc(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Set the number of spectral and spatial pixels, and the bad pixel mask is it does not exist
slf._nspec[det-1], slf._nspat[det-1] = slf._msarc[det-1].shape
if slf._bpix[det-1] is None:
slf.SetFrame(slf._bpix, np.zeros((slf._nspec[det-1], slf._nspat[det-1])), det)
'''
###############
# Estimate gain and readout noise for the amplifiers
msgs.work("Estimate Gain and Readout noise from the raw frames...")
update = slf.MasterRN(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise")
'''
###############
# Generate a master trace frame
update = slf.MasterTrace(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
###############
# Generate an array that provides the physical pixel locations on the detector
slf.GetPixelLocations(det)
# Determine the edges of the spectrum (spatial)
if ('trace'+settings.argflag['reduce']['masters']['setup'] not in settings.argflag['reduce']['masters']['loaded']):
###############
# Determine the edges of the spectrum (spatial)
lordloc, rordloc, extord = artrace.trace_slits(slf, slf._mstrace[det-1], det, pcadesc="PCA trace of the slit edges")
slf.SetFrame(slf._lordloc, lordloc, det)
slf.SetFrame(slf._rordloc, rordloc, det)
# Convert physical trace into a pixel trace
msgs.info("Converting physical trace locations to nearest pixel")
pixcen = artrace.phys_to_pix(0.5*(slf._lordloc[det-1]+slf._rordloc[det-1]), slf._pixlocn[det-1], 1)
pixwid = (slf._rordloc[det-1]-slf._lordloc[det-1]).mean(0).astype(np.int)
lordpix = artrace.phys_to_pix(slf._lordloc[det-1], slf._pixlocn[det-1], 1)
rordpix = artrace.phys_to_pix(slf._rordloc[det-1], slf._pixlocn[det-1], 1)
slf.SetFrame(slf._pixcen, pixcen, det)
slf.SetFrame(slf._pixwid, pixwid, det)
slf.SetFrame(slf._lordpix, lordpix, det)
slf.SetFrame(slf._rordpix, rordpix, det)
msgs.info("Identifying the pixels belonging to each slit")
slitpix = arproc.slit_pixels(slf, slf._mstrace[det-1].shape, det)
slf.SetFrame(slf._slitpix, slitpix, det)
# Save to disk
armasters.save_masters(slf, det, mftype='trace')
# Save QA for slit traces
arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1],
slf._rordpix[det-1], extord,
desc="Trace of the slit edges D{:02d}".format(det), use_slitid=det)
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
###############
# Generate the 1D wavelength solution
update = slf.MasterWaveCalib(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace")
###############
# Derive the spectral tilt
if slf._tilts[det-1] is None:
try:
tilts = armasters.get_master_frame(slf, "tilts")
except IOError:
# First time tilts are derived for this arc frame --> derive the order tilts
tilts, satmask, outpar = artrace.multislit_tilt(slf, slf._msarc[det-1], det)
slf.SetFrame(slf._tilts, tilts, det)
slf.SetFrame(slf._satmask, satmask, det)
slf.SetFrame(slf._tiltpar, outpar, det)
armasters.save_masters(slf, det, mftype='tilts')
else:
slf.SetFrame(slf._tilts, tilts, det)
###############
# Prepare the pixel flat field frame
update = slf.MasterFlatField(fitsdict, det)
if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixelflat")
###############
# Generate/load a master wave frame
update = slf.MasterWave(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave")
###############
# Check if the user only wants to prepare the calibrations only
msgs.info("All calibration frames have been prepared")
if settings.argflag['run']['preponly']:
msgs.info("If you would like to continue with the reduction, disable the command:" + msgs.newline() +
"run preponly False")
continue
###############
# Write setup
#setup = arsort.calib_setup(sc, det, fitsdict, setup_dict, write=True)
# Write MasterFrames (currently per detector)
#armasters.save_masters(slf, det, setup)
###############
# Load the science frame and from this generate a Poisson error frame
msgs.info("Loading science frame")
sciframe = arload.load_frames(fitsdict, [scidx], det,
frametype='science',
msbias=slf._msbias[det-1])
sciframe = sciframe[:, :, 0]
# Extract
msgs.info("Processing science frame")
arproc.reduce_multislit(slf, sciframe, scidx, fitsdict, det)
###############
# Using model sky, calculate a flexure correction
###############
# Flux
###############
# Standard star (is this a calibration, e.g. goes above?)
msgs.info("Processing standard star")
msgs.info("Assuming one star per detector mosaic")
msgs.info("Waited until last detector to process")
update = slf.MasterStandard(fitsdict)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, 0, ftype="standard")
#
msgs.work("Consider using archived sensitivity if not found")
msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name']))
for kk in range(settings.spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
if slf._specobjs[det-1] is not None:
arflux.apply_sensfunc(slf, det, scidx, fitsdict)
else:
msgs.info("There are no objects on detector {0:d} to apply a flux calibration".format(det))
# Write 1D spectra
save_format = 'fits'
if save_format == 'fits':
arsave.save_1d_spectra_fits(slf, fitsdict)
elif save_format == 'hdf5':
arsave.save_1d_spectra_hdf5(slf)
else:
msgs.error(save_format + ' is not a recognized output format!')
arsave.save_obj_info(slf, fitsdict)
# Write 2D images for the Science Frame
arsave.save_2d_images(slf, fitsdict)
# Free up some memory by replacing the reduced ScienceExposure class
sciexp[sc] = None
return status
def MasterStandard(self, fitsdict):
"""
Generate Master Standard frame for a given detector
and generates a sensitivity function
Currently only uses first standard star exposure
Currently takes brightest source on the mosaic
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
Returns
-------
boolean : bool
"""
if self._sensfunc is not None:
msgs.info("Using existing sensitivity function.")
return False
# Attempt to load the Master Frame
try:
sensfunc = armasters.get_master_frame(self, "sensfunc")
except IOError:
# Grab the standard star frames
msgs.info("Preparing the standard")
ind = self._idx_std
msgs.warn("Taking only the first standard frame for now")
ind = [ind[0]]
# Extract
all_specobj = []
for kk in range(settings.spect['mosaic']['ndet']):
det = kk + 1
# Load the frame(s)
frame = arload.load_frames(fitsdict,
ind,
det,
frametype='standard',
msbias=self._msbias[det - 1])
sciframe = frame[:, :, 0] # First exposure
# Save RA/DEC
self._msstd[det - 1]['RA'] = fitsdict['ra'][ind[0]]
self._msstd[det - 1]['DEC'] = fitsdict['dec'][ind[0]]
self._msstd[det - 1]['spobjs'] = None
# Use this detector? Need to check this after setting RA/DEC above
if settings.argflag['reduce']['detnum'] is not None:
msgs.warn(
"If your standard wasnt on this detector, you will have trouble.."
)
if det != settings.argflag['reduce']['detnum']:
continue
if settings.spect["mosaic"]["reduction"] == "ARMLSD":
arproc.reduce_multislit(self,
sciframe,
ind[0],
fitsdict,
det,
standard=True)
elif settings.spect["mosaic"]["reduction"] == "ARMED":
arproc.reduce_echelle(self,
sciframe,
ind[0],
fitsdict,
det,
standard=True)
else:
msgs.error("Not ready for reduction type {0:s}".format(
settings.spect["mosaic"]["reduction"]))
if self._msstd[det - 1]['spobjs'] is not None:
all_specobj += self._msstd[det - 1]['spobjs']
# If standard, generate a sensitivity function
sensfunc = arflux.generate_sensfunc(self, ind[0], all_specobj,
fitsdict)
# Set the sensitivity function
self.SetMasterFrame(sensfunc, "sensfunc", None, mkcopy=False)
# Apply to Standard
for kk in range(settings.spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
arflux.apply_sensfunc(self,
det,
ind[0],
fitsdict,
standard=True)
# Save
armasters.save_sensfunc(
self, settings.argflag['reduce']['masters']['setup'])
# Save standard star spectrum to disk
outfile = settings.argflag['run']['directory'][
'science'] + '/spec1d_{:s}.fits'.format(
fitsdict['filename'][ind[0]].split('.')[0])
arsave.save_1d_spectra_fits(self,
fitsdict,
standard=True,
outfile=outfile)
return True
else:
self._sensfunc = sensfunc.copy()
return True
def MasterFlatField(self, fitsdict, det):
"""
Generate Master Flat-field frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
from pypit import arqa
if settings.argflag['reduce']['flatfield'][
'perform']: # Only do it if the user wants to flat field
# If the master pixelflat is already made, use it
if self._mspixelflat[det - 1] is not None:
msgs.info("An identical master pixelflat frame already exists")
if self._mspixelflatnrm[det - 1] is None:
# Normalize the flat field
msgs.info("Normalizing the pixel flat")
slit_profiles, mstracenrm, msblaze, flat_ext1d, extrap_slit = \
arproc.slit_profile(self, self.GetMasterFrame("pixelflat", det),
det, ntcky=settings.argflag['reduce']['flatfield']['params'][0])
# If some slit profiles/blaze functions need to be extrapolated, do that now
if np.sum(extrap_slit) != 0.0:
slit_profiles, mstracenrm, msblaze = arproc.slit_profile_pca(
self, self.GetMasterFrame("pixelflat", det), det,
msblaze, extrap_slit, slit_profiles)
mspixelflatnrm = mstracenrm.copy()
winpp = np.where(slit_profiles != 0.0)
mspixelflatnrm[winpp] /= slit_profiles[winpp]
self.SetMasterFrame(mspixelflatnrm, "normpixelflat", det)
armasters.save_masters(self, det, mftype='normpixelflat')
if np.array_equal(self._idx_flat, self._idx_trace):
# The flat field frame is also being used to trace the slit edges and determine the slit
# profile. Avoid recalculating the slit profile and blaze function and save them here.
self.SetFrame(self._msblaze, msblaze, det)
self.SetFrame(self._slitprof, slit_profiles, det)
armasters.save_masters(self, det, mftype='slitprof')
if settings.argflag["reduce"]["slitprofile"][
"perform"]:
msgs.info("Preparing QA of each slit profile")
arqa.slit_profile(self,
mstracenrm,
slit_profiles,
self._lordloc[det - 1],
self._rordloc[det - 1],
self._slitpix[det - 1],
desc="Slit profile")
msgs.info("Saving blaze function QA")
arqa.plot_orderfits(self,
msblaze,
flat_ext1d,
desc="Blaze function")
return False
###############
# Generate/load a master pixel flat frame
if settings.argflag['reduce']['flatfield']['useframe'] in [
'pixelflat', 'trace'
]:
try:
mspixelflatnrm = armasters.get_master_frame(
self, "normpixelflat")
except IOError:
msgs.info("Preparing a master pixel flat frame with {0:s}".
format(settings.argflag['reduce']['flatfield']
['useframe']))
# Get all of the pixel flat frames for this science frame
ind = self._idx_flat
# Load the frames for tracing
frames = arload.load_frames(fitsdict,
ind,
det,
frametype='pixel flat',
msbias=self._msbias[det - 1])
if settings.argflag['pixelflat']['combine']['match'] > 0.0:
sframes = arsort.match_frames(
frames,
settings.argflag['pixelflat']['combine']['match'],
frametype='pixel flat',
satlevel=self._nonlinear)
subframes = np.zeros(
(frames.shape[0], frames.shape[1], len(sframes)))
numarr = np.array([])
for i in range(len(sframes)):
numarr = np.append(numarr, sframes[i].shape[2])
mspixelflat = arcomb.comb_frames(
sframes[i],
det,
'pixelflat',
printtype='pixel flat')
subframes[:, :, i] = mspixelflat.copy()
del sframes
# Combine all sub-frames
mspixelflat = arcomb.comb_frames(
subframes,
det,
'pixelflat',
weights=numarr,
printtype='pixel flat')
del subframes
else:
mspixelflat = arcomb.comb_frames(
frames, det, 'pixelflat', printtype='pixel flat')
del frames
# Apply gain (instead of ampsec scale)
mspixelflat *= arproc.gain_frame(self, det)
# Normalize the flat field
msgs.info("Normalizing the pixel flat")
slit_profiles, mstracenrm, msblaze, flat_ext1d, extrap_slit = \
arproc.slit_profile(self, mspixelflat, det, ntcky=settings.argflag['reduce']['flatfield']['params'][0])
# If some slit profiles/blaze functions need to be extrapolated, do that now
if np.sum(extrap_slit) != 0.0:
slit_profiles, mstracenrm, msblaze = arproc.slit_profile_pca(
self, mspixelflat, det, msblaze, extrap_slit,
slit_profiles)
mspixelflatnrm = mstracenrm.copy()
winpp = np.where(slit_profiles != 0.0)
mspixelflatnrm[winpp] /= slit_profiles[winpp]
if np.array_equal(self._idx_flat, self._idx_trace):
# The flat field frame is also being used to trace the slit edges and determine the slit
# profile. Avoid recalculating the slit profile and blaze function and save them here.
self.SetFrame(self._msblaze, msblaze, det)
self.SetFrame(self._slitprof, slit_profiles, det)
armasters.save_masters(self, det, mftype='slitprof')
if settings.argflag["reduce"]["slitprofile"][
"perform"]:
msgs.info("Preparing QA of each slit profile")
arqa.slit_profile(self,
mstracenrm,
slit_profiles,
self._lordloc[det - 1],
self._rordloc[det - 1],
self._slitpix[det - 1],
desc="Slit profile")
msgs.info("Saving blaze function QA")
arqa.plot_orderfits(self,
msblaze,
flat_ext1d,
desc="Blaze function")
else:
mspixelflat = mspixelflatnrm
else: # It must be the name of a file the user wishes to load
mspixelflat_name = armasters.user_master_name(
settings.argflag['run']['directory']['master'],
settings.argflag['reduce']['flatfield']['useframe'])
mspixelflatnrm, head = arload.load_master(mspixelflat_name,
exten=det,
frametype=None)
mspixelflat = mspixelflatnrm
# Now that the combined, master flat field frame is loaded...
else:
msgs.work(
"Pixel Flat arrays need to be generated when not flat fielding"
)
msgs.bug("Blaze is currently undefined")
mspixelflat = np.ones_like(self._msarc)
mspixelflatnrm = np.ones_like(self._msarc)
# Set Master Frames
self.SetMasterFrame(mspixelflat, "pixelflat", det)
self.SetMasterFrame(mspixelflatnrm, "normpixelflat", det)
armasters.save_masters(self, det, mftype='normpixelflat')
return True
def MasterArc(self, fitsdict, det):
"""
Generate Master Arc frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
dnum = settings.get_dnum(det)
if self._msarc[det - 1] is not None:
msgs.info("A master arc frame already exists for this frame")
return False
if settings.argflag['arc']['useframe'] in ['arc']:
# Master Frame
try:
msarc = armasters.get_master_frame(self, "arc")
except IOError:
msgs.info("Preparing a master arc frame")
ind = self._idx_arcs
# Load the arc frames
frames = arload.load_frames(fitsdict,
ind,
det,
frametype='arc',
msbias=self._msbias[det - 1])
if settings.argflag['arc']['combine']['match'] > 0.0:
sframes = arsort.match_frames(
frames,
settings.argflag['arc']['combine']['match'],
frametype='arc',
satlevel=settings.spect[dnum]['saturation'] *
settings.spect[dnum]['nonlinear'])
subframes = np.zeros(
(frames.shape[0], frames.shape[1], len(sframes)))
numarr = np.array([])
for i in range(len(sframes)):
numarr = np.append(numarr, sframes[i].shape[2])
msarc = arcomb.comb_frames(sframes[i], det, 'arc')
# Send the data away to be saved
subframes[:, :, i] = msarc.copy()
del sframes
# Combine all sub-frames
msarc = arcomb.comb_frames(subframes,
det,
'arc',
weights=numarr)
del subframes
else:
msarc = arcomb.comb_frames(frames, det, 'arc')
del frames
else: # Use input frame name located in MasterFrame directory
msarc_name = settings.argflag['run']['directory'][
'master'] + '/' + settings.argflag['arc']['useframe']
msarc, _ = arload.load_master(msarc_name, frametype=None)
# Set and then delete the Master Arc frame
self.SetMasterFrame(msarc, "arc", det)
armasters.save_masters(self, det, mftype='arc')
del msarc
return True