def MasterBias(self, fitsdict, det):
"""
Generate Master Bias frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
# If the master bias is already made, use it
if self._msbias[det - 1] is not None:
msgs.info("An identical master {0:s} frame already exists".format(
settings.argflag['bias']['useframe']))
return False
elif settings.argflag['bias']['useframe'] in ['bias', 'dark']:
try:
msbias = armasters.get_master_frame(self, "bias")
except IOError:
msgs.info("Preparing a master {0:s} frame".format(
settings.argflag['bias']['useframe']))
# Get all of the bias frames for this science frame
ind = self._idx_bias
# Load the Bias/Dark frames
frames = arload.load_frames(
fitsdict,
ind,
det,
frametype=settings.argflag['bias']['useframe'])
msbias = arcomb.comb_frames(
frames,
det,
'bias',
printtype=settings.argflag['bias']['useframe'])
del frames
elif settings.argflag['bias']['useframe'] == 'overscan':
self.SetMasterFrame('overscan', "bias", det, mkcopy=False)
return False
elif settings.argflag['bias']['useframe'] == 'none':
msgs.info("Not performing a bias/dark subtraction")
self.SetMasterFrame(None, "bias", det, mkcopy=False)
return False
else: # It must be the name of a file the user wishes to load
msbias_name = settings.argflag['run']['directory'][
'master'] + u'/' + settings.argflag['bias']['useframe']
msbias, head = arload.load_master(msbias_name, frametype="bias")
settings.argflag['reduce']['masters']['loaded'].append('bias')
# Set and then delete the Master Bias frame
self.SetMasterFrame(msbias, "bias", det)
armasters.save_masters(self, det, mftype='bias')
del msbias
return True
def MasterPinhole(self, fitsdict, det, msbias):
"""
Generate Master pinhole frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
dnum = settings.get_dnum(det)
# If the master pinhole is already made, use it
if self._mspinhole[det - 1] is not None:
msgs.info("An identical master pinhole frame already exists")
return False
if settings.argflag['reduce']['slitcen']['useframe'] in ['trace', 'pinhole']:
try:
mspinhole = armasters.get_master_frame(self, "pinhole")
except IOError:
msgs.info("Preparing a master pinhole frame with {0:s}".format(
settings.argflag['reduce']['slitcen']['useframe']))
ind = self._idx_cent
# Load the pinhole frames
frames = arload.load_frames(fitsdict, ind, det, frametype='pinhole', msbias=msbias, # self._msbias[det - 1],
trim=settings.argflag['reduce']['trim'])
if settings.argflag['pinhole']['combine']['match'] > 0.0:
sframes = arsort.match_frames(frames, settings.argflag['pinhole']['combine']['match'],
frametype='pinhole', satlevel=settings.spect[dnum]['saturation'] *
settings.spect['det'][det - 1]['nonlinear'])
subframes = np.zeros((frames.shape[0], frames.shape[1], len(sframes)))
numarr = np.array([])
for i in range(len(sframes)):
numarr = np.append(numarr, sframes[i].shape[2])
mspinhole = arcomb.comb_frames(sframes[i], det, 'pinhole')
subframes[:, :, i] = mspinhole.copy()
del sframes
# Combine all sub-frames
mspinhole = arcomb.comb_frames(subframes, det, 'pinhole', weights=numarr)
del subframes
else:
mspinhole = arcomb.comb_frames(frames, det, 'pinhole')
del frames
else: # It must be the name of a file the user wishes to load
mspinhole_name = settings.argflag['run']['directory']['master'] + '/' + \
settings.argflag['reduce']['slitcen']['useframe']
mspinhole, head = armasters.load_master(mspinhole_name, frametype=None)
debugger.set_trace() # NEED TO LOAD EXTRAS AS ABOVE
# Set and then delete the Master Trace frame
self.SetMasterFrame(mspinhole, "pinhole", det)
#armasters.save_masters(self, det, mftype='pinhole')
del mspinhole
return True
def MasterBias(self, fitsdict, det):
"""
Generate Master Bias frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
# If the master bias is already made, use it
if self._msbias[det-1] is not None:
msgs.info("An identical master {0:s} frame already exists".format(self._argflag['reduce']['usebias']))
return False
elif self._argflag['reduce']['usebias'] in ['bias', 'dark']:
# Load from hard-drive?
if self._argflag['masters']['use']:
# Attempt to load the Master Frame
msbias_name = armasters.master_name(self._argflag['run']['masterdir'],
'bias', self._argflag['masters']['setup'])
try:
msbias, head = arload.load_master(msbias_name, frametype="bias")
except IOError:
msgs.warn("No MasterBias frame found {:s}".format(msbias_name))
else:
self._argflag['masters']['loaded'].append('bias'+self._argflag['masters']['setup'])
if 'bias'+self._argflag['masters']['setup'] not in self._argflag['masters']['loaded']:
msgs.info("Preparing a master {0:s} frame".format(self._argflag['reduce']['usebias']))
# Get all of the bias frames for this science frame
ind = self._idx_bias
# Load the Bias/Dark frames
frames = arload.load_frames(self, fitsdict, ind, det, frametype=self._argflag['reduce']['usebias'], transpose=self._transpose)
msbias = arcomb.comb_frames(frames, det, spect=self._spect, frametype=self._argflag['reduce']['usebias'], **self._argflag['bias']['comb'])
del frames
elif self._argflag['reduce']['usebias'] == 'overscan':
self.SetMasterFrame('overscan', "bias", det, copy=False)
return False
elif self._argflag['reduce']['usebias'] == 'none':
msgs.info("Not performing a bias/dark subtraction")
self.SetMasterFrame(None, "bias", det, copy=False)
return False
else: # It must be the name of a file the user wishes to load
msbias_name = self._argflag['run']['masterdir']+'/'+self._argflag['reduce']['usebias']
msbias, head = arload.load_master(msbias_name, frametype="bias")
self._argflag['masters']['loaded'].append('bias')
# Set and then delete the Master Bias frame
self.SetMasterFrame(msbias, "bias", det)
del msbias
return True
def MasterStandard(self, scidx, fitsdict):
"""
Generate Master Standard frame for a given detector
and generates a sensitivity function
Currently only uses first standard star exposure
Currently takes brightest source on the mosaic
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
Returns
-------
boolean : bool
"""
if len(self._msstd[0]) != 0:
msgs.info("Using existing standard frame")
return False
#
msgs.info("Preparing the standard")
# Get all of the pixel flat frames for this science frame
ind = self._idx_std
msgs.warn("Taking only the first standard frame for now")
ind = [ind[0]]
# Extract
all_specobj = []
for kk in xrange(self._spect['mosaic']['ndet']):
det = kk+1
# Load the frame(s)
# set_trace()
frame = arload.load_frames(self, fitsdict, ind, det, frametype='standard',
msbias=self._msbias[det-1],
transpose=self._transpose)
# msgs.warn("Taking only the first standard frame for now")
# ind = ind[0]
sciframe = frame[:, :, 0]
# Save RA/DEC
if kk == 0:
self._msstd[det-1]['RA'] = fitsdict['ra'][ind[0]]
self._msstd[det-1]['DEC'] = fitsdict['dec'][ind[0]]
#debugger.set_trace()
arproc.reduce_frame(self, sciframe, ind[0], fitsdict, det, standard=True)
#
all_specobj += self._msstd[det-1]['spobjs']
# debugger.set_trace()
# If standard, generate a sensitivity function
sensfunc = arflux.generate_sensfunc(self, scidx, all_specobj, fitsdict)
# Set the sensitivity function
self.SetMasterFrame(sensfunc, "sensfunc", None, copy=False)
return True
def MasterTrace(self, fitsdict, det):
"""
Generate Master Trace frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
# If the master trace is already made, use it
if self._mstrace[det-1] is not None:
msgs.info("An identical master trace frame already exists")
return False
if self._argflag['reduce']['usetrace'] in ['trace', 'blzflat']:
if self._argflag['masters']['use']:
# Attempt to load the Master Frame
mstrace_name = armasters.master_name(self._argflag['run']['masterdir'],
'trace', self._argflag['masters']['setup'])
try:
mstrace, head = arload.load_master(mstrace_name, frametype="trace")
except IOError:
msgs.warn("No MasterTrace frame found {:s}".format(mstrace_name))
else:
# Extras
lordloc, _ = arload.load_master(mstrace_name, frametype="trace", exten=1)
rordloc, _ = arload.load_master(mstrace_name, frametype="trace", exten=2)
pixcen, _ = arload.load_master(mstrace_name, frametype="trace", exten=3)
pixwid, _ = arload.load_master(mstrace_name, frametype="trace", exten=4)
lordpix, _ = arload.load_master(mstrace_name, frametype="trace", exten=5)
rordpix, _ = arload.load_master(mstrace_name, frametype="trace", exten=6)
self.SetFrame(self._lordloc, lordloc, det)
self.SetFrame(self._rordloc, rordloc, det)
self.SetFrame(self._pixcen, pixcen.astype(np.int), det)
self.SetFrame(self._pixwid, pixwid.astype(np.int), det)
self.SetFrame(self._lordpix, lordpix.astype(np.int), det)
self.SetFrame(self._rordpix, rordpix.astype(np.int), det)
#
self._argflag['masters']['loaded'].append('trace'+self._argflag['masters']['setup'])
if 'trace'+self._argflag['masters']['setup'] not in self._argflag['masters']['loaded']:
msgs.info("Preparing a master trace frame with {0:s}".format(self._argflag['reduce']['usetrace']))
ind = self._idx_trace
# Load the frames for tracing
frames = arload.load_frames(self, fitsdict, ind, det, frametype='trace', msbias=self._msbias[det-1],
trim=self._argflag['reduce']['trim'], transpose=self._transpose)
if self._argflag['reduce']['flatmatch'] > 0.0:
sframes = arsort.match_frames(frames, self._argflag['reduce']['flatmatch'], msgs, frametype='trace', satlevel=self._spect['det'][det-1]['saturation']*self._spect['det'][det-1]['nonlinear'])
subframes = np.zeros((frames.shape[0], frames.shape[1], len(sframes)))
numarr = np.array([])
for i in xrange(len(sframes)):
numarr = np.append(numarr, sframes[i].shape[2])
mstrace = arcomb.comb_frames(sframes[i], det, spect=self._spect, frametype='trace', **self._argflag['trace']['comb'])
subframes[:,:,i] = mstrace.copy()
del sframes
# Combine all sub-frames
mstrace = arcomb.comb_frames(subframes, det, spect=self._spect, frametype='trace', weights=numarr, **self._argflag['trace']['comb'])
del subframes
else:
mstrace = arcomb.comb_frames(frames, det, spect=self._spect, frametype='trace', **self._argflag['trace']['comb'])
del frames
elif self._argflag['reduce']['usetrace'] == 'science':
msgs.error("Tracing with a science frame is not yet implemented")
else: # It must be the name of a file the user wishes to load
mstrace_name = self._argflag['run']['masterdir']+'/'+self._argflag['reduce']['usetrace']
mstrace, head = arload.load_master(mstrace_name, frametype=None)
debugger.set_trace() # NEED TO LOAD EXTRAS AS ABOVE
# Set and then delete the Master Trace frame
self.SetMasterFrame(mstrace, "trace", det)
del mstrace
return True
def MasterFlatField(self, fitsdict, det):
"""
Generate Master Flat-field frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
if self._argflag['reduce']['flatfield']: # Only do it if the user wants to flat field
# If the master pixflat is already made, use it
if self._mspixflat[det-1] is not None:
msgs.info("An identical master pixflat frame already exists")
if self._mspixflatnrm[det-1] is None:
# Normalize the flat field
msgs.info("Normalizing the pixel flat")
mspixflatnrm, msblaze = arproc.flatnorm(self, det, self.GetMasterFrame("pixflat", det),
overpix=0, plotdesc="Blaze function")
self.SetFrame(self._msblaze, msblaze, det)
self.SetMasterFrame(mspixflatnrm, "normpixflat", det)
return False
###############
# Generate a master pixel flat frame
if self._argflag['reduce']['useflat'] in ['pixflat', 'blzflat']:
if self._argflag['masters']['use']:
# Attempt to load the Master Frame
msflat_name = armasters.master_name(self._argflag['run']['masterdir'],
'normpixflat', self._argflag['masters']['setup'])
try:
mspixflatnrm, head = arload.load_master(msflat_name, frametype="normpixflat")
except IOError:
msgs.warn("No MasterFlatField frame found {:s}".format(msflat_name))
else:
self._argflag['masters']['loaded'].append('normpixflat'+self._argflag['masters']['setup'])
mspixflat = mspixflatnrm
if 'normpixflat'+self._argflag['masters']['setup'] not in self._argflag['masters']['loaded']:
msgs.info("Preparing a master pixel flat frame with {0:s}".format(self._argflag['reduce']['useflat']))
# Get all of the pixel flat frames for this science frame
ind = self._idx_flat
# Load the frames for tracing
frames = arload.load_frames(self, fitsdict, ind, det, frametype='pixel flat',
msbias=self._msbias[det-1], transpose=self._transpose)
if self._argflag['reduce']['flatmatch'] > 0.0:
sframes = arsort.match_frames(frames, self._argflag['reduce']['flatmatch'],
frametype='pixel flat', satlevel=self._nonlinear)
subframes = np.zeros((frames.shape[0], frames.shape[1], len(sframes)))
numarr = np.array([])
for i in xrange(len(sframes)):
numarr = np.append(numarr, sframes[i].shape[2])
mspixflat = arcomb.comb_frames(sframes[i], det, spect=self._spect, frametype='pixel flat',
**self._argflag['pixflat']['comb'])
subframes[:,:,i] = mspixflat.copy()
del sframes
# Combine all sub-frames
mspixflat = arcomb.comb_frames(subframes, det, spect=self._spect, frametype='pixel flat',
weights=numarr, **self._argflag['pixflat']['comb'])
del subframes
else:
mspixflat = arcomb.comb_frames(frames, det, spect=self._spect, frametype='pixel flat',
**self._argflag['pixflat']['comb'])
del frames
# Apply gain (instead of ampsec scale)
mspixflat *= arproc.gain_frame(self, det)
# Normalize the flat field
mspixflatnrm, msblaze = arproc.flatnorm(self, det, mspixflat, overpix=0, plotdesc="Blaze function")
self.SetFrame(self._msblaze, msblaze, det)
else: # It must be the name of a file the user wishes to load
mspixflat_name = armasters.user_master_name(self._argflag['run']['masterdir'],
self._argflag['reduce']['useflat'])
mspixflatnrm, head = arload.load_master(mspixflat_name, exten=det, frametype=None)
mspixflat = mspixflatnrm
# Now that the combined, master flat field frame is loaded...
else:
msgs.work("Pixel Flat arrays need to be generated when not flat fielding")
msgs.bug("Blaze is currently undefined")
mspixflat = np.ones_like(self._msarc)
mspixflatnrm = np.ones_like(self._msarc)
# Set Master Frames
self.SetMasterFrame(mspixflat, "pixflat", det)
self.SetMasterFrame(mspixflatnrm, "normpixflat", det)
return True
def MasterArc(self, fitsdict, det):
"""
Generate Master Arc frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
if self._msarc[det-1] is not None:
msgs.info("An identical master arc frame already exists")
return False
if self._argflag['reduce']['usearc'] in ['arc']:
# Attempt to load the Master Frame
if self._argflag['masters']['use']:
msarc_name = armasters.master_name(self._argflag['run']['masterdir'],
'arc', self._argflag['masters']['setup'])
try:
msarc, head = arload.load_master(msarc_name, frametype="arc")
except IOError:
msgs.warn("No MasterArc frame found {:s}".format(msarc_name))
else:
self._transpose = head['transp']
if self._transpose: # Need to setup for flipping
self._argflag['trace']['disp']['direction'] = 1
else:
self._argflag['trace']['disp']['direction'] = 0
# Append as loaded
self._argflag['masters']['loaded'].append('arc'+self._argflag['masters']['setup'])
if 'arc'+self._argflag['masters']['setup'] not in self._argflag['masters']['loaded']:
msgs.info("Preparing a master arc frame")
ind = self._idx_arcs
# Load the arc frames
frames = arload.load_frames(self, fitsdict, ind, det, frametype='arc',
msbias=self._msbias[det-1])
if self._argflag['reduce']['arcmatch'] > 0.0:
sframes = arsort.match_frames(frames, self._argflag['reduce']['arcmatch'], msgs, frametype='arc',
satlevel=self._spect['det']['saturation']*self._spect['det']['nonlinear'])
subframes = np.zeros((frames.shape[0], frames.shape[1], len(sframes)))
numarr = np.array([])
for i in xrange(len(sframes)):
numarr = np.append(numarr, sframes[i].shape[2])
msarc = arcomb.comb_frames(sframes[i], det, spect=self._spect,
frametype='arc', **self._argflag['arc']['comb'])
# Send the data away to be saved
subframes[:,:,i] = msarc.copy()
del sframes
# Combine all sub-frames
msarc = arcomb.comb_frames(subframes, det, spect=self._spect,
frametype='arc', weights=numarr, **self._argflag['arc']['comb'])
del subframes
else:
msarc = arcomb.comb_frames(frames, det, spect=self._spect,
frametype='arc', **self._argflag['arc']['comb'])
del frames
# # Derive a suitable name for the master arc frame
# msarc_name = "{0:s}/{1:s}/msarc{2:s}_{3:03d}.fits".format(os.getcwd(),self._argflag['run']['masterdir'],self._spect["det"][det-1]["suffix"],len(self._done_arcs))
# self._tltprefix = os.path.splitext(os.path.basename(msarc_name))[0]
# # Send the data away to be saved
# arsave.save_master(self, msarc, filename=msarc_name, frametype='arc', ind=ind)
# # Store the files used and the master bias name in case it can be used during the later reduction processes
# self._done_arcs.append(ind)
# self._name_arcs.append(msarc_name)
else:
msarc_name = self._argflag['run']['masterdir']+'/'+self._argflag['reduce']['usearc']
msarc, head = arload.load_master(msarc_name, frametype=None)
# Set and then delete the Master Arc frame
self.SetMasterFrame(msarc, "arc", det)
del msarc
return True
def ARMED(fitsdict, reuseMaster=False, reloadMaster=True):
"""
Automatic Reduction and Modeling of Echelle Data
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
reuseMaster : bool
If True, a master frame that will be used for another science frame
will not be regenerated after it is first made.
This setting comes with a price, and if a large number of science frames are
being generated, it may be more efficient to simply regenerate the master
calibrations on the fly.
Returns
-------
status : int
Status of the reduction procedure
0 = Successful execution
1 = ...
"""
status = 0
# Create a list of science exposure classes
sciexp, setup_dict = armbase.SetupScience(fitsdict)
if sciexp == 'setup':
status = 1
return status
elif sciexp == 'calcheck':
status = 2
return status
else:
numsci = len(sciexp)
# Create a list of master calibration frames
#masters = armasters.MasterFrames(settings.spect['mosaic']['ndet'])
# Masters
#settings.argflag['reduce']['masters']['file'] = setup_file
# Start reducing the data
for sc in range(numsci):
slf = sciexp[sc]
scidx = slf._idx_sci[0]
msgs.info("Reducing file {0:s}, target {1:s}".format(fitsdict['filename'][scidx], slf._target_name))
msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too
if reloadMaster and (sc > 0):
settings.argflag['reduce']['masters']['reuse'] = True
# Loop on Detectors
for kk in range(settings.spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
if settings.argflag['reduce']['detnum'] is not None:
if det != settings.argflag['reduce']['detnum']:
continue
else:
msgs.warn("Restricting the reduction to detector {:d}".format(det))
slf.det = det
###############
# Get data sections
arproc.get_datasec_trimmed(slf, fitsdict, det, scidx)
# Setup
setup = arsort.instr_setup(slf, det, fitsdict, setup_dict, must_exist=True)
settings.argflag['reduce']['masters']['setup'] = setup
slf.setup = setup
###############
# Generate master bias frame
update = slf.MasterBias(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
###############
# Generate a bad pixel mask (should not repeat)
update = slf.BadPixelMask(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Estimate gain and readout noise for the amplifiers
msgs.work("Estimate Gain and Readout noise from the raw frames...")
###############
# Generate a master arc frame
update = slf.MasterArc(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Set the number of spectral and spatial pixels, and the bad pixel mask is it does not exist
slf._nspec[det-1], slf._nspat[det-1] = slf._msarc[det-1].shape
if slf._bpix[det-1] is None:
slf.SetFrame(slf._bpix, np.zeros((slf._nspec[det-1], slf._nspat[det-1])), det)
###############
# Generate a master trace frame
update = slf.MasterTrace(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
###############
# Generate a master pinhole frame
update = slf.MasterPinhole(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pinhole")
###############
# Generate an array that provides the physical pixel locations on the detector
slf.GetPixelLocations(det)
###############
# Determine the edges of the spectrum (spatial)
if ('trace'+settings.argflag['reduce']['masters']['setup'] not in settings.argflag['reduce']['masters']['loaded']):
if True:#not msgs._debug['develop']:
msgs.info("Tracing slit edges with a {0:s} frame".format(settings.argflag['trace']['useframe']))
if settings.argflag['trace']['useframe'] == 'pinhole':
###############
# Determine the centroid of the spectrum (spatial)
lordloc, rordloc, extord = artrace.trace_slits(slf, slf._mspinhole[det-1], det,
pcadesc="PCA trace of the slit edges")
# Using the order centroid, expand the order edges until the edge of the science slit is found
if settings.argflag['trace']['slits']['expand']:
lordloc, rordloc = artrace.expand_slits(slf, slf._mstrace[det-1], det,
0.5*(lordloc+rordloc), extord)
elif settings.argflag['trace']['useframe'] == 'trace':
###############
# Determine the edges of the slit using a trace frame
lordloc, rordloc, extord = artrace.trace_slits(slf, slf._mstrace[det-1], det,
pcadesc="PCA trace of the slit edges")
else:
msgs.error("Cannot trace slit edges using {0:s}".format(settings.argflag['trace']['useframe']))
else:
lordloc, rordloc, extord = np.load("lordloc.npy"), np.load("rordloc.npy"), np.load("extord.npy")
# Save the locations of the order edges
slf.SetFrame(slf._lordloc, lordloc, det)
slf.SetFrame(slf._rordloc, rordloc, det)
# Convert physical trace into a pixel trace
msgs.info("Converting physical trace locations to nearest pixel")
pixcen = artrace.phys_to_pix(0.5 * (slf._lordloc[det - 1] + slf._rordloc[det - 1]), slf._pixlocn[det - 1], 1)
pixwid = (slf._rordloc[det - 1] - slf._lordloc[det - 1]).mean(0).astype(np.int)
lordpix = artrace.phys_to_pix(slf._lordloc[det - 1], slf._pixlocn[det - 1], 1)
rordpix = artrace.phys_to_pix(slf._rordloc[det - 1], slf._pixlocn[det - 1], 1)
slf.SetFrame(slf._pixcen, pixcen, det)
slf.SetFrame(slf._pixwid, pixwid, det)
slf.SetFrame(slf._lordpix, lordpix, det)
slf.SetFrame(slf._rordpix, rordpix, det)
msgs.info("Identifying the pixels belonging to each slit")
slitpix = arproc.slit_pixels(slf, slf._mstrace[det-1].shape, det)
slf.SetFrame(slf._slitpix, slitpix, det)
# Save to disk
armasters.save_masters(slf, det, mftype='trace')
# Save QA for slit traces
arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det - 1], extord,
desc="Trace of the slit edges", normalize=False)
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
###############
# Generate the 1D wavelength solution
update = slf.MasterWaveCalib(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace")
###############
# Derive the spectral tilt
if slf._tilts[det-1] is None:
try:
tilts = armasters.get_master_frame(slf, "tilts")
except IOError:
# First time tilts are derived for this arc frame --> derive the order tilts
tilts, satmask, outpar = artrace.echelle_tilt(slf, slf._msarc[det - 1], det)
slf.SetFrame(slf._tilts, tilts, det)
slf.SetFrame(slf._satmask, satmask, det)
slf.SetFrame(slf._tiltpar, outpar, det)
armasters.save_masters(slf, det, mftype='tilts')
else:
slf.SetFrame(slf._tilts, tilts, det)
###############
# Prepare the pixel flat field frame
update = slf.MasterFlatField(fitsdict, det)
if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixelflat")
###############
# Derive the spatial profile and blaze function
if slf._slitprof[det-1] is None:
if settings.argflag['reduce']['masters']['reuse']:
msslitprof_name = armasters.master_name('slitprof', settings.argflag['reduce']['masters']['setup'])
try:
slit_profiles, head = arload.load_master(msslitprof_name, frametype="slit profile")
except IOError:
pass
else:
slf.SetFrame(slf._slitprof, slit_profiles, det)
settings.argflag['reduce']['masters']['loaded'].append('slitprof'+settings.argflag['reduce']['masters']['setup'])
if 'slitprof'+settings.argflag['reduce']['masters']['setup'] not in settings.argflag['reduce']['masters']['loaded']:
# First time slit profile is derived
msgs.info("Calculating slit profile from master trace frame")
slit_profiles, mstracenrm, msblaze, flat_ext1d, extrap_slit = arproc.slit_profile(slf, slf._mstrace[det - 1], det)
# If some slit profiles/blaze functions need to be extrapolated, do that now
if np.sum(extrap_slit) != 0.0:
slit_profiles, mstracenrm, msblaze = arproc.slit_profile_pca(slf, slf._mstrace[det - 1], det, msblaze, extrap_slit)
slf.SetFrame(slf._slitprof, slit_profiles, det)
slf.SetFrame(slf._msblaze, msblaze, det)
# Prepare some QA for the average slit profile along the slit
msgs.info("Preparing QA of each slit profile")
arqa.slit_profile(slf, mstracenrm, slit_profiles, slf._lordloc[det - 1], slf._rordloc[det - 1],
slf._slitpix[det - 1], desc="Slit profile")
msgs.info("Saving blaze function QA")
arqa.plot_orderfits(slf, msblaze, flat_ext1d, desc="Blaze function", textplt="Order")
###############
# Generate/load a master wave frame
update = slf.MasterWave(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave")
###############
# Check if the user only wants to prepare the calibrations only
msgs.info("All calibration frames have been prepared")
if settings.argflag['run']['preponly']:
msgs.info("If you would like to continue with the reduction, disable the command:" + msgs.newline() +
"run preponly False")
continue
###############
# Write setup
#setup = arsort.calib_setup(sc, det, fitsdict, setup_dict, write=True)
# Write MasterFrames (currently per detector)
#armasters.save_masters(slf, det, setup)
###############
# Load the science frame and from this generate a Poisson error frame
msgs.info("Loading science frame")
sciframe = arload.load_frames(fitsdict, [scidx], det,
frametype='science',
msbias=slf._msbias[det - 1])
sciframe = sciframe[:, :, 0]
# Extract
msgs.info("Processing science frame")
arproc.reduce_echelle(slf, sciframe, scidx, fitsdict, det)
# Write 1D spectra
save_format = 'fits'
if save_format == 'fits':
arsave.save_1d_spectra_fits(slf, fitsdict)
elif save_format == 'hdf5':
arsave.save_1d_spectra_hdf5(slf)
else:
msgs.error(save_format + ' is not a recognized output format!')
# Write 2D images for the Science Frame
arsave.save_2d_images(slf, fitsdict)
# Free up some memory by replacing the reduced ScienceExposure class
sciexp[sc] = None
return status
def ARMLSD(argflag, spect, fitsdict, reuseMaster=False):
"""
Automatic Reduction and Modeling of Long Slit Data
Parameters
----------
argflag : dict
Arguments and flags used for reduction
spect : dict
Properties of the spectrograph.
fitsdict : dict
Contains relevant information from fits header files
reuseMaster : bool
If True, a master frame that will be used for another science frame
will not be regenerated after it is first made.
This setting comes with a price, and if a large number of science frames are
being generated, it may be more efficient to simply regenerate the master
calibrations on the fly.
Returns
-------
status : int
Status of the reduction procedure
0 = Successful execution
1 = ...
"""
status = 0
# Create a list of science exposure classes
sciexp = armbase.SetupScience(argflag, spect, fitsdict)
numsci = len(sciexp)
# Create a list of master calibration frames
masters = armasters.MasterFrames(spect['mosaic']['ndet'])
# Use Masters? Requires setup file
setup_file = argflag['out']['sorted'] + '.setup'
try:
calib_dict = ltu.loadjson(setup_file)
except:
msgs.info("No setup file {:s} for MasterFrames".format(setup_file))
calib_dict = {}
else:
argflag['masters']['setup_file'] = setup_file
# Start reducing the data
for sc in range(numsci):
slf = sciexp[sc]
scidx = slf._idx_sci[0]
msgs.info("Reducing file {0:s}, target {1:s}".format(
fitsdict['filename'][scidx], slf._target_name))
msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too
# Loop on Detectors
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
slf.det = det
###############
# Get amplifier sections
arproc.get_ampsec_trimmed(slf, fitsdict, det, scidx)
# Setup
setup = arsort.calib_setup(slf,
sc,
det,
fitsdict,
calib_dict,
write=False)
slf._argflag['masters']['setup'] = setup
###############
# Generate master bias frame
update = slf.MasterBias(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
###############
# Generate a bad pixel mask (should not repeat)
update = slf.BadPixelMask(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Generate a master arc frame
update = slf.MasterArc(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Determine the dispersion direction (and transpose if necessary)
slf.GetDispersionDirection(fitsdict, det, scidx)
if slf._bpix[
det -
1] is None: # Needs to be done here after nspec is set
slf.SetFrame(
slf._bpix,
np.zeros((slf._nspec[det - 1], slf._nspat[det - 1])), det)
'''
###############
# Estimate gain and readout noise for the amplifiers
msgs.work("Estimate Gain and Readout noise from the raw frames...")
update = slf.MasterRN(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise")
'''
###############
# Generate a master trace frame
update = slf.MasterTrace(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="flat",
chktype="trace")
###############
# Generate an array that provides the physical pixel locations on the detector
slf.GetPixelLocations(det)
# Determine the edges of the spectrum (spatial)
if 'trace' + slf._argflag['masters']['setup'] not in slf._argflag[
'masters']['loaded']:
###############
# Determine the edges of the spectrum (spatial)
lordloc, rordloc, extord = artrace.trace_orders(
slf,
slf._mstrace[det - 1],
det,
singleSlit=True,
pcadesc="PCA trace of the slit edges")
slf.SetFrame(slf._lordloc, lordloc, det)
slf.SetFrame(slf._rordloc, rordloc, det)
# Convert physical trace into a pixel trace
msgs.info(
"Converting physical trace locations to nearest pixel")
pixcen = artrace.phys_to_pix(
0.5 * (slf._lordloc[det - 1] + slf._rordloc[det - 1]),
slf._pixlocn[det - 1], 1)
pixwid = (slf._rordloc[det - 1] -
slf._lordloc[det - 1]).mean(0).astype(np.int)
lordpix = artrace.phys_to_pix(slf._lordloc[det - 1],
slf._pixlocn[det - 1], 1)
rordpix = artrace.phys_to_pix(slf._rordloc[det - 1],
slf._pixlocn[det - 1], 1)
slf.SetFrame(slf._pixcen, pixcen, det)
slf.SetFrame(slf._pixwid, pixwid, det)
slf.SetFrame(slf._lordpix, lordpix, det)
slf.SetFrame(slf._rordpix, rordpix, det)
# Save QA for slit traces
arqa.slit_trace_qa(slf,
slf._mstrace[det - 1],
slf._lordpix[det - 1],
slf._rordpix[det - 1],
extord,
desc="Trace of the slit edges")
###############
# Prepare the pixel flat field frame
update = slf.MasterFlatField(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="flat",
chktype="pixflat")
###############
# Generate the 1D wavelength solution
update = slf.MasterWaveCalib(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="arc",
chktype="trace")
###############
# Derive the spectral tilt
if slf._tilts[det - 1] is None:
if slf._argflag['masters']['use']:
mstilt_name = armasters.master_name(
slf._argflag['run']['masterdir'], 'tilts',
slf._argflag['masters']['setup'])
try:
tilts, head = arload.load_master(mstilt_name,
frametype="tilts")
except IOError:
pass
else:
slf.SetFrame(slf._tilts, tilts, det)
slf._argflag['masters']['loaded'].append(
'tilts' + slf._argflag['masters']['setup'])
if 'tilts' + slf._argflag['masters'][
'setup'] not in slf._argflag['masters']['loaded']:
# First time tilts are derived for this arc frame --> derive the order tilts
tilts, satmask, outpar = artrace.model_tilt(
slf, det, slf._msarc[det - 1])
slf.SetFrame(slf._tilts, tilts, det)
slf.SetFrame(slf._satmask, satmask, det)
slf.SetFrame(slf._tiltpar, outpar, det)
###############
# Generate/load a master wave frame
update = slf.MasterWave(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp,
sc,
det,
ftype="arc",
chktype="wave")
# Check if the user only wants to prepare the calibrations only
msgs.info("All calibration frames have been prepared")
if slf._argflag['run']['preponly']:
msgs.info("If you would like to continue with the reduction," +
msgs.newline() + "disable the run+preponly command")
continue
# Write setup
setup = arsort.calib_setup(slf,
sc,
det,
fitsdict,
calib_dict,
write=True)
# Write MasterFrames (currently per detector)
armasters.save_masters(slf, det, setup)
###############
# Load the science frame and from this generate a Poisson error frame
msgs.info("Loading science frame")
sciframe = arload.load_frames(slf,
fitsdict, [scidx],
det,
frametype='science',
msbias=slf._msbias[det - 1],
transpose=slf._transpose)
sciframe = sciframe[:, :, 0]
# Extract
msgs.info("Processing science frame")
arproc.reduce_frame(slf, sciframe, scidx, fitsdict, det)
#continue
#msgs.error("UP TO HERE")
###############
# Perform a velocity correction
if (slf._argflag['reduce']['heliocorr'] == True) & False:
if slf._argflag['science']['load']['extracted'] == True:
msgs.warn(
"Heliocentric correction will not be applied if an extracted science frame exists, and is used"
)
msgs.work("Perform a full barycentric correction")
msgs.work(
"Include the facility to correct for gravitational redshifts and time delays (see Pulsar timing work)"
)
msgs.info("Performing a heliocentric correction")
# Load the header for the science frame
slf._waveids = arvcorr.helio_corr(slf, scidx[0])
else:
msgs.info("A heliocentric correction will not be performed")
###############
# Using model sky, calculate a flexure correction
# Close the QA for this object
slf._qa.close()
###############
# Flux
###############
# Standard star (is this a calibration, e.g. goes above?)
msgs.info("Processing standard star")
msgs.info("Assuming one star per detector mosaic")
msgs.info("Waited until last detector to process")
msgs.work("Need to check for existing sensfunc")
update = slf.MasterStandard(scidx, fitsdict)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, 0, ftype="standard")
#
msgs.work("Consider using archived sensitivity if not found")
msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name']))
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
arflux.apply_sensfunc(slf, det, scidx, fitsdict)
# Write 1D spectra
arsave.save_1d_spectra(slf)
# Write 2D images for the Science Frame
arsave.save_2d_images(slf)
# Free up some memory by replacing the reduced ScienceExposure class
sciexp[sc] = None
return status
def ARMLSD(argflag, spect, fitsdict, reuseMaster=False):
"""
Automatic Reduction and Modeling of Long Slit Data
Parameters
----------
argflag : dict
Arguments and flags used for reduction
spect : dict
Properties of the spectrograph.
fitsdict : dict
Contains relevant information from fits header files
reuseMaster : bool
If True, a master frame that will be used for another science frame
will not be regenerated after it is first made.
This setting comes with a price, and if a large number of science frames are
being generated, it may be more efficient to simply regenerate the master
calibrations on the fly.
Returns
-------
status : int
Status of the reduction procedure
0 = Successful execution
1 = ...
"""
status = 0
# Create a list of science exposure classes
sciexp = armbase.SetupScience(argflag, spect, fitsdict)
numsci = len(sciexp)
# Create a list of master calibration frames
masters = armasters.MasterFrames(spect['mosaic']['ndet'])
# Use Masters? Requires setup file
setup_file = argflag['out']['sorted']+'.setup'
try:
calib_dict = ltu.loadjson(setup_file)
except:
msgs.info("No setup file {:s} for MasterFrames".format(setup_file))
calib_dict = {}
else:
argflag['masters']['setup_file'] = setup_file
# Start reducing the data
for sc in range(numsci):
slf = sciexp[sc]
scidx = slf._idx_sci[0]
msgs.info("Reducing file {0:s}, target {1:s}".format(fitsdict['filename'][scidx], slf._target_name))
msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too
# Loop on Detectors
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
slf.det = det
###############
# Get amplifier sections
arproc.get_ampsec_trimmed(slf, fitsdict, det, scidx)
# Setup
setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=False)
slf._argflag['masters']['setup'] = setup
###############
# Generate master bias frame
update = slf.MasterBias(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
###############
# Generate a bad pixel mask (should not repeat)
update = slf.BadPixelMask(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Generate a master arc frame
update = slf.MasterArc(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Determine the dispersion direction (and transpose if necessary)
slf.GetDispersionDirection(fitsdict, det, scidx)
if slf._bpix[det-1] is None: # Needs to be done here after nspec is set
slf.SetFrame(slf._bpix, np.zeros((slf._nspec[det-1], slf._nspat[det-1])), det)
'''
###############
# Estimate gain and readout noise for the amplifiers
msgs.work("Estimate Gain and Readout noise from the raw frames...")
update = slf.MasterRN(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise")
'''
###############
# Generate a master trace frame
update = slf.MasterTrace(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
###############
# Generate an array that provides the physical pixel locations on the detector
slf.GetPixelLocations(det)
# Determine the edges of the spectrum (spatial)
if 'trace'+slf._argflag['masters']['setup'] not in slf._argflag['masters']['loaded']:
###############
# Determine the edges of the spectrum (spatial)
lordloc, rordloc, extord = artrace.trace_orders(slf, slf._mstrace[det-1], det, singleSlit=True, pcadesc="PCA trace of the slit edges")
slf.SetFrame(slf._lordloc, lordloc, det)
slf.SetFrame(slf._rordloc, rordloc, det)
# Convert physical trace into a pixel trace
msgs.info("Converting physical trace locations to nearest pixel")
pixcen = artrace.phys_to_pix(0.5*(slf._lordloc[det-1]+slf._rordloc[det-1]), slf._pixlocn[det-1], 1)
pixwid = (slf._rordloc[det-1]-slf._lordloc[det-1]).mean(0).astype(np.int)
lordpix = artrace.phys_to_pix(slf._lordloc[det-1], slf._pixlocn[det-1], 1)
rordpix = artrace.phys_to_pix(slf._rordloc[det-1], slf._pixlocn[det-1], 1)
slf.SetFrame(slf._pixcen, pixcen, det)
slf.SetFrame(slf._pixwid, pixwid, det)
slf.SetFrame(slf._lordpix, lordpix, det)
slf.SetFrame(slf._rordpix, rordpix, det)
# Save QA for slit traces
arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det-1], extord, desc="Trace of the slit edges")
###############
# Prepare the pixel flat field frame
update = slf.MasterFlatField(fitsdict, det)
if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixflat")
###############
# Generate the 1D wavelength solution
update = slf.MasterWaveCalib(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace")
###############
# Derive the spectral tilt
if slf._tilts[det-1] is None:
if slf._argflag['masters']['use']:
mstilt_name = armasters.master_name(slf._argflag['run']['masterdir'],
'tilts', slf._argflag['masters']['setup'])
try:
tilts, head = arload.load_master(mstilt_name, frametype="tilts")
except IOError:
pass
else:
slf.SetFrame(slf._tilts, tilts, det)
slf._argflag['masters']['loaded'].append('tilts'+slf._argflag['masters']['setup'])
if 'tilts'+slf._argflag['masters']['setup'] not in slf._argflag['masters']['loaded']:
# First time tilts are derived for this arc frame --> derive the order tilts
tilts, satmask, outpar = artrace.model_tilt(slf, det, slf._msarc[det-1])
slf.SetFrame(slf._tilts, tilts, det)
slf.SetFrame(slf._satmask, satmask, det)
slf.SetFrame(slf._tiltpar, outpar, det)
###############
# Generate/load a master wave frame
update = slf.MasterWave(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave")
# Check if the user only wants to prepare the calibrations only
msgs.info("All calibration frames have been prepared")
if slf._argflag['run']['preponly']:
msgs.info("If you would like to continue with the reduction,"
+msgs.newline()+"disable the run+preponly command")
continue
# Write setup
setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=True)
# Write MasterFrames (currently per detector)
armasters.save_masters(slf, det, setup)
###############
# Load the science frame and from this generate a Poisson error frame
msgs.info("Loading science frame")
sciframe = arload.load_frames(slf, fitsdict, [scidx], det,
frametype='science',
msbias=slf._msbias[det-1],
transpose=slf._transpose)
sciframe = sciframe[:, :, 0]
# Extract
msgs.info("Processing science frame")
arproc.reduce_frame(slf, sciframe, scidx, fitsdict, det)
#continue
#msgs.error("UP TO HERE")
###############
# Perform a velocity correction
if (slf._argflag['reduce']['heliocorr'] == True) & False:
if slf._argflag['science']['load']['extracted'] == True:
msgs.warn("Heliocentric correction will not be applied if an extracted science frame exists, and is used")
msgs.work("Perform a full barycentric correction")
msgs.work("Include the facility to correct for gravitational redshifts and time delays (see Pulsar timing work)")
msgs.info("Performing a heliocentric correction")
# Load the header for the science frame
slf._waveids = arvcorr.helio_corr(slf, scidx[0])
else:
msgs.info("A heliocentric correction will not be performed")
###############
# Using model sky, calculate a flexure correction
# Close the QA for this object
slf._qa.close()
###############
# Flux
###############
# Standard star (is this a calibration, e.g. goes above?)
msgs.info("Processing standard star")
msgs.info("Assuming one star per detector mosaic")
msgs.info("Waited until last detector to process")
msgs.work("Need to check for existing sensfunc")
update = slf.MasterStandard(scidx, fitsdict)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, 0, ftype="standard")
#
msgs.work("Consider using archived sensitivity if not found")
msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name']))
for kk in xrange(slf._spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
arflux.apply_sensfunc(slf, det, scidx, fitsdict)
# Write 1D spectra
arsave.save_1d_spectra(slf)
# Write 2D images for the Science Frame
arsave.save_2d_images(slf)
# Free up some memory by replacing the reduced ScienceExposure class
sciexp[sc] = None
return status
def ARMLSD(fitsdict, reuseMaster=False, reloadMaster=True):
"""
Automatic Reduction and Modeling of Long Slit Data
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
reuseMaster : bool
If True, a master frame that will be used for another science frame
will not be regenerated after it is first made.
This setting comes with a price, and if a large number of science frames are
being generated, it may be more efficient to simply regenerate the master
calibrations on the fly.
Returns
-------
status : int
Status of the reduction procedure
0 = Successful full execution
1 = Successful processing of setup or calcheck
"""
status = 0
# Create a list of science exposure classes
sciexp, setup_dict = armbase.SetupScience(fitsdict)
if sciexp == 'setup':
status = 1
return status
elif sciexp == 'calcheck':
status = 2
return status
else:
numsci = len(sciexp)
# Create a list of master calibration frames
#masters = armasters.MasterFrames(settings.spect['mosaic']['ndet'])
# Masters
#settings.argflag['reduce']['masters']['file'] = setup_file
# Start reducing the data
for sc in range(numsci):
slf = sciexp[sc]
scidx = slf._idx_sci[0]
msgs.info("Reducing file {0:s}, target {1:s}".format(fitsdict['filename'][scidx], slf._target_name))
msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too
if reloadMaster and (sc > 0):
settings.argflag['reduce']['masters']['reuse'] = True
# Loop on Detectors
for kk in range(settings.spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
if settings.argflag['reduce']['detnum'] is not None:
if det != settings.argflag['reduce']['detnum']:
continue
else:
msgs.warn("Restricting the reduction to detector {:d}".format(det))
slf.det = det
###############
# Get data sections
arproc.get_datasec_trimmed(slf, fitsdict, det, scidx)
# Setup
setup = arsort.instr_setup(slf, det, fitsdict, setup_dict, must_exist=True)
settings.argflag['reduce']['masters']['setup'] = setup
slf.setup = setup
###############
# Generate master bias frame
update = slf.MasterBias(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
###############
# Generate a bad pixel mask (should not repeat)
update = slf.BadPixelMask(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Generate a master arc frame
update = slf.MasterArc(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
###############
# Set the number of spectral and spatial pixels, and the bad pixel mask is it does not exist
slf._nspec[det-1], slf._nspat[det-1] = slf._msarc[det-1].shape
if slf._bpix[det-1] is None:
slf.SetFrame(slf._bpix, np.zeros((slf._nspec[det-1], slf._nspat[det-1])), det)
'''
###############
# Estimate gain and readout noise for the amplifiers
msgs.work("Estimate Gain and Readout noise from the raw frames...")
update = slf.MasterRN(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise")
'''
###############
# Generate a master trace frame
update = slf.MasterTrace(fitsdict, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
###############
# Generate an array that provides the physical pixel locations on the detector
slf.GetPixelLocations(det)
# Determine the edges of the spectrum (spatial)
if ('trace'+settings.argflag['reduce']['masters']['setup'] not in settings.argflag['reduce']['masters']['loaded']):
###############
# Determine the edges of the spectrum (spatial)
lordloc, rordloc, extord = artrace.trace_slits(slf, slf._mstrace[det-1], det, pcadesc="PCA trace of the slit edges")
slf.SetFrame(slf._lordloc, lordloc, det)
slf.SetFrame(slf._rordloc, rordloc, det)
# Convert physical trace into a pixel trace
msgs.info("Converting physical trace locations to nearest pixel")
pixcen = artrace.phys_to_pix(0.5*(slf._lordloc[det-1]+slf._rordloc[det-1]), slf._pixlocn[det-1], 1)
pixwid = (slf._rordloc[det-1]-slf._lordloc[det-1]).mean(0).astype(np.int)
lordpix = artrace.phys_to_pix(slf._lordloc[det-1], slf._pixlocn[det-1], 1)
rordpix = artrace.phys_to_pix(slf._rordloc[det-1], slf._pixlocn[det-1], 1)
slf.SetFrame(slf._pixcen, pixcen, det)
slf.SetFrame(slf._pixwid, pixwid, det)
slf.SetFrame(slf._lordpix, lordpix, det)
slf.SetFrame(slf._rordpix, rordpix, det)
msgs.info("Identifying the pixels belonging to each slit")
slitpix = arproc.slit_pixels(slf, slf._mstrace[det-1].shape, det)
slf.SetFrame(slf._slitpix, slitpix, det)
# Save to disk
armasters.save_masters(slf, det, mftype='trace')
# Save QA for slit traces
arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1],
slf._rordpix[det-1], extord,
desc="Trace of the slit edges D{:02d}".format(det), use_slitid=det)
armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
###############
# Generate the 1D wavelength solution
update = slf.MasterWaveCalib(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace")
###############
# Derive the spectral tilt
if slf._tilts[det-1] is None:
try:
tilts = armasters.get_master_frame(slf, "tilts")
except IOError:
# First time tilts are derived for this arc frame --> derive the order tilts
tilts, satmask, outpar = artrace.multislit_tilt(slf, slf._msarc[det-1], det)
slf.SetFrame(slf._tilts, tilts, det)
slf.SetFrame(slf._satmask, satmask, det)
slf.SetFrame(slf._tiltpar, outpar, det)
armasters.save_masters(slf, det, mftype='tilts')
else:
slf.SetFrame(slf._tilts, tilts, det)
###############
# Prepare the pixel flat field frame
update = slf.MasterFlatField(fitsdict, det)
if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixelflat")
###############
# Generate/load a master wave frame
update = slf.MasterWave(fitsdict, sc, det)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave")
###############
# Check if the user only wants to prepare the calibrations only
msgs.info("All calibration frames have been prepared")
if settings.argflag['run']['preponly']:
msgs.info("If you would like to continue with the reduction, disable the command:" + msgs.newline() +
"run preponly False")
continue
###############
# Write setup
#setup = arsort.calib_setup(sc, det, fitsdict, setup_dict, write=True)
# Write MasterFrames (currently per detector)
#armasters.save_masters(slf, det, setup)
###############
# Load the science frame and from this generate a Poisson error frame
msgs.info("Loading science frame")
sciframe = arload.load_frames(fitsdict, [scidx], det,
frametype='science',
msbias=slf._msbias[det-1])
sciframe = sciframe[:, :, 0]
# Extract
msgs.info("Processing science frame")
arproc.reduce_multislit(slf, sciframe, scidx, fitsdict, det)
###############
# Using model sky, calculate a flexure correction
###############
# Flux
###############
# Standard star (is this a calibration, e.g. goes above?)
msgs.info("Processing standard star")
msgs.info("Assuming one star per detector mosaic")
msgs.info("Waited until last detector to process")
update = slf.MasterStandard(fitsdict)
if update and reuseMaster:
armbase.UpdateMasters(sciexp, sc, 0, ftype="standard")
#
msgs.work("Consider using archived sensitivity if not found")
msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name']))
for kk in range(settings.spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
if slf._specobjs[det-1] is not None:
arflux.apply_sensfunc(slf, det, scidx, fitsdict)
else:
msgs.info("There are no objects on detector {0:d} to apply a flux calibration".format(det))
# Write 1D spectra
save_format = 'fits'
if save_format == 'fits':
arsave.save_1d_spectra_fits(slf, fitsdict)
elif save_format == 'hdf5':
arsave.save_1d_spectra_hdf5(slf)
else:
msgs.error(save_format + ' is not a recognized output format!')
arsave.save_obj_info(slf, fitsdict)
# Write 2D images for the Science Frame
arsave.save_2d_images(slf, fitsdict)
# Free up some memory by replacing the reduced ScienceExposure class
sciexp[sc] = None
return status
def MasterStandard(self, fitsdict):
"""
Generate Master Standard frame for a given detector
and generates a sensitivity function
Currently only uses first standard star exposure
Currently takes brightest source on the mosaic
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
Returns
-------
boolean : bool
"""
if self._sensfunc is not None:
msgs.info("Using existing sensitivity function.")
return False
# Attempt to load the Master Frame
try:
sensfunc = armasters.get_master_frame(self, "sensfunc")
except IOError:
# Grab the standard star frames
msgs.info("Preparing the standard")
ind = self._idx_std
msgs.warn("Taking only the first standard frame for now")
ind = [ind[0]]
# Extract
all_specobj = []
for kk in range(settings.spect['mosaic']['ndet']):
det = kk + 1
# Load the frame(s)
frame = arload.load_frames(fitsdict,
ind,
det,
frametype='standard',
msbias=self._msbias[det - 1])
sciframe = frame[:, :, 0] # First exposure
# Save RA/DEC
self._msstd[det - 1]['RA'] = fitsdict['ra'][ind[0]]
self._msstd[det - 1]['DEC'] = fitsdict['dec'][ind[0]]
self._msstd[det - 1]['spobjs'] = None
# Use this detector? Need to check this after setting RA/DEC above
if settings.argflag['reduce']['detnum'] is not None:
msgs.warn(
"If your standard wasnt on this detector, you will have trouble.."
)
if det != settings.argflag['reduce']['detnum']:
continue
if settings.spect["mosaic"]["reduction"] == "ARMLSD":
arproc.reduce_multislit(self,
sciframe,
ind[0],
fitsdict,
det,
standard=True)
elif settings.spect["mosaic"]["reduction"] == "ARMED":
arproc.reduce_echelle(self,
sciframe,
ind[0],
fitsdict,
det,
standard=True)
else:
msgs.error("Not ready for reduction type {0:s}".format(
settings.spect["mosaic"]["reduction"]))
if self._msstd[det - 1]['spobjs'] is not None:
all_specobj += self._msstd[det - 1]['spobjs']
# If standard, generate a sensitivity function
sensfunc = arflux.generate_sensfunc(self, ind[0], all_specobj,
fitsdict)
# Set the sensitivity function
self.SetMasterFrame(sensfunc, "sensfunc", None, mkcopy=False)
# Apply to Standard
for kk in range(settings.spect['mosaic']['ndet']):
det = kk + 1 # Detectors indexed from 1
arflux.apply_sensfunc(self,
det,
ind[0],
fitsdict,
standard=True)
# Save
armasters.save_sensfunc(
self, settings.argflag['reduce']['masters']['setup'])
# Save standard star spectrum to disk
outfile = settings.argflag['run']['directory'][
'science'] + '/spec1d_{:s}.fits'.format(
fitsdict['filename'][ind[0]].split('.')[0])
arsave.save_1d_spectra_fits(self,
fitsdict,
standard=True,
outfile=outfile)
return True
else:
self._sensfunc = sensfunc.copy()
return True
def MasterFlatField(self, fitsdict, det):
"""
Generate Master Flat-field frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
from pypit import arqa
if settings.argflag['reduce']['flatfield'][
'perform']: # Only do it if the user wants to flat field
# If the master pixelflat is already made, use it
if self._mspixelflat[det - 1] is not None:
msgs.info("An identical master pixelflat frame already exists")
if self._mspixelflatnrm[det - 1] is None:
# Normalize the flat field
msgs.info("Normalizing the pixel flat")
slit_profiles, mstracenrm, msblaze, flat_ext1d, extrap_slit = \
arproc.slit_profile(self, self.GetMasterFrame("pixelflat", det),
det, ntcky=settings.argflag['reduce']['flatfield']['params'][0])
# If some slit profiles/blaze functions need to be extrapolated, do that now
if np.sum(extrap_slit) != 0.0:
slit_profiles, mstracenrm, msblaze = arproc.slit_profile_pca(
self, self.GetMasterFrame("pixelflat", det), det,
msblaze, extrap_slit, slit_profiles)
mspixelflatnrm = mstracenrm.copy()
winpp = np.where(slit_profiles != 0.0)
mspixelflatnrm[winpp] /= slit_profiles[winpp]
self.SetMasterFrame(mspixelflatnrm, "normpixelflat", det)
armasters.save_masters(self, det, mftype='normpixelflat')
if np.array_equal(self._idx_flat, self._idx_trace):
# The flat field frame is also being used to trace the slit edges and determine the slit
# profile. Avoid recalculating the slit profile and blaze function and save them here.
self.SetFrame(self._msblaze, msblaze, det)
self.SetFrame(self._slitprof, slit_profiles, det)
armasters.save_masters(self, det, mftype='slitprof')
if settings.argflag["reduce"]["slitprofile"][
"perform"]:
msgs.info("Preparing QA of each slit profile")
arqa.slit_profile(self,
mstracenrm,
slit_profiles,
self._lordloc[det - 1],
self._rordloc[det - 1],
self._slitpix[det - 1],
desc="Slit profile")
msgs.info("Saving blaze function QA")
arqa.plot_orderfits(self,
msblaze,
flat_ext1d,
desc="Blaze function")
return False
###############
# Generate/load a master pixel flat frame
if settings.argflag['reduce']['flatfield']['useframe'] in [
'pixelflat', 'trace'
]:
try:
mspixelflatnrm = armasters.get_master_frame(
self, "normpixelflat")
except IOError:
msgs.info("Preparing a master pixel flat frame with {0:s}".
format(settings.argflag['reduce']['flatfield']
['useframe']))
# Get all of the pixel flat frames for this science frame
ind = self._idx_flat
# Load the frames for tracing
frames = arload.load_frames(fitsdict,
ind,
det,
frametype='pixel flat',
msbias=self._msbias[det - 1])
if settings.argflag['pixelflat']['combine']['match'] > 0.0:
sframes = arsort.match_frames(
frames,
settings.argflag['pixelflat']['combine']['match'],
frametype='pixel flat',
satlevel=self._nonlinear)
subframes = np.zeros(
(frames.shape[0], frames.shape[1], len(sframes)))
numarr = np.array([])
for i in range(len(sframes)):
numarr = np.append(numarr, sframes[i].shape[2])
mspixelflat = arcomb.comb_frames(
sframes[i],
det,
'pixelflat',
printtype='pixel flat')
subframes[:, :, i] = mspixelflat.copy()
del sframes
# Combine all sub-frames
mspixelflat = arcomb.comb_frames(
subframes,
det,
'pixelflat',
weights=numarr,
printtype='pixel flat')
del subframes
else:
mspixelflat = arcomb.comb_frames(
frames, det, 'pixelflat', printtype='pixel flat')
del frames
# Apply gain (instead of ampsec scale)
mspixelflat *= arproc.gain_frame(self, det)
# Normalize the flat field
msgs.info("Normalizing the pixel flat")
slit_profiles, mstracenrm, msblaze, flat_ext1d, extrap_slit = \
arproc.slit_profile(self, mspixelflat, det, ntcky=settings.argflag['reduce']['flatfield']['params'][0])
# If some slit profiles/blaze functions need to be extrapolated, do that now
if np.sum(extrap_slit) != 0.0:
slit_profiles, mstracenrm, msblaze = arproc.slit_profile_pca(
self, mspixelflat, det, msblaze, extrap_slit,
slit_profiles)
mspixelflatnrm = mstracenrm.copy()
winpp = np.where(slit_profiles != 0.0)
mspixelflatnrm[winpp] /= slit_profiles[winpp]
if np.array_equal(self._idx_flat, self._idx_trace):
# The flat field frame is also being used to trace the slit edges and determine the slit
# profile. Avoid recalculating the slit profile and blaze function and save them here.
self.SetFrame(self._msblaze, msblaze, det)
self.SetFrame(self._slitprof, slit_profiles, det)
armasters.save_masters(self, det, mftype='slitprof')
if settings.argflag["reduce"]["slitprofile"][
"perform"]:
msgs.info("Preparing QA of each slit profile")
arqa.slit_profile(self,
mstracenrm,
slit_profiles,
self._lordloc[det - 1],
self._rordloc[det - 1],
self._slitpix[det - 1],
desc="Slit profile")
msgs.info("Saving blaze function QA")
arqa.plot_orderfits(self,
msblaze,
flat_ext1d,
desc="Blaze function")
else:
mspixelflat = mspixelflatnrm
else: # It must be the name of a file the user wishes to load
mspixelflat_name = armasters.user_master_name(
settings.argflag['run']['directory']['master'],
settings.argflag['reduce']['flatfield']['useframe'])
mspixelflatnrm, head = arload.load_master(mspixelflat_name,
exten=det,
frametype=None)
mspixelflat = mspixelflatnrm
# Now that the combined, master flat field frame is loaded...
else:
msgs.work(
"Pixel Flat arrays need to be generated when not flat fielding"
)
msgs.bug("Blaze is currently undefined")
mspixelflat = np.ones_like(self._msarc)
mspixelflatnrm = np.ones_like(self._msarc)
# Set Master Frames
self.SetMasterFrame(mspixelflat, "pixelflat", det)
self.SetMasterFrame(mspixelflatnrm, "normpixelflat", det)
armasters.save_masters(self, det, mftype='normpixelflat')
return True
def MasterArc(self, fitsdict, det):
"""
Generate Master Arc frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
dnum = settings.get_dnum(det)
if self._msarc[det - 1] is not None:
msgs.info("A master arc frame already exists for this frame")
return False
if settings.argflag['arc']['useframe'] in ['arc']:
# Master Frame
try:
msarc = armasters.get_master_frame(self, "arc")
except IOError:
msgs.info("Preparing a master arc frame")
ind = self._idx_arcs
# Load the arc frames
frames = arload.load_frames(fitsdict,
ind,
det,
frametype='arc',
msbias=self._msbias[det - 1])
if settings.argflag['arc']['combine']['match'] > 0.0:
sframes = arsort.match_frames(
frames,
settings.argflag['arc']['combine']['match'],
frametype='arc',
satlevel=settings.spect[dnum]['saturation'] *
settings.spect[dnum]['nonlinear'])
subframes = np.zeros(
(frames.shape[0], frames.shape[1], len(sframes)))
numarr = np.array([])
for i in range(len(sframes)):
numarr = np.append(numarr, sframes[i].shape[2])
msarc = arcomb.comb_frames(sframes[i], det, 'arc')
# Send the data away to be saved
subframes[:, :, i] = msarc.copy()
del sframes
# Combine all sub-frames
msarc = arcomb.comb_frames(subframes,
det,
'arc',
weights=numarr)
del subframes
else:
msarc = arcomb.comb_frames(frames, det, 'arc')
del frames
else: # Use input frame name located in MasterFrame directory
msarc_name = settings.argflag['run']['directory'][
'master'] + '/' + settings.argflag['arc']['useframe']
msarc, _ = arload.load_master(msarc_name, frametype=None)
# Set and then delete the Master Arc frame
self.SetMasterFrame(msarc, "arc", det)
armasters.save_masters(self, det, mftype='arc')
del msarc
return True
