def MasterStandard(self, scidx, fitsdict): """ Generate Master Standard frame for a given detector and generates a sensitivity function Currently only uses first standard star exposure Currently takes brightest source on the mosaic Parameters ---------- fitsdict : dict Contains relevant information from fits header files Returns ------- boolean : bool """ if len(self._msstd[0]) != 0: msgs.info("Using existing standard frame") return False # msgs.info("Preparing the standard") # Get all of the pixel flat frames for this science frame ind = self._idx_std msgs.warn("Taking only the first standard frame for now") ind = [ind[0]] # Extract all_specobj = [] for kk in xrange(self._spect['mosaic']['ndet']): det = kk+1 # Load the frame(s) # set_trace() frame = arload.load_frames(self, fitsdict, ind, det, frametype='standard', msbias=self._msbias[det-1], transpose=self._transpose) # msgs.warn("Taking only the first standard frame for now") # ind = ind[0] sciframe = frame[:, :, 0] # Save RA/DEC if kk == 0: self._msstd[det-1]['RA'] = fitsdict['ra'][ind[0]] self._msstd[det-1]['DEC'] = fitsdict['dec'][ind[0]] #debugger.set_trace() arproc.reduce_frame(self, sciframe, ind[0], fitsdict, det, standard=True) # all_specobj += self._msstd[det-1]['spobjs'] # debugger.set_trace() # If standard, generate a sensitivity function sensfunc = arflux.generate_sensfunc(self, scidx, all_specobj, fitsdict) # Set the sensitivity function self.SetMasterFrame(sensfunc, "sensfunc", None, copy=False) return True
def ARMLSD(argflag, spect, fitsdict, reuseMaster=False): """ Automatic Reduction and Modeling of Long Slit Data Parameters ---------- argflag : dict Arguments and flags used for reduction spect : dict Properties of the spectrograph. fitsdict : dict Contains relevant information from fits header files reuseMaster : bool If True, a master frame that will be used for another science frame will not be regenerated after it is first made. This setting comes with a price, and if a large number of science frames are being generated, it may be more efficient to simply regenerate the master calibrations on the fly. Returns ------- status : int Status of the reduction procedure 0 = Successful execution 1 = ... """ status = 0 # Create a list of science exposure classes sciexp = armbase.SetupScience(argflag, spect, fitsdict) numsci = len(sciexp) # Create a list of master calibration frames masters = armasters.MasterFrames(spect['mosaic']['ndet']) # Use Masters? Requires setup file setup_file = argflag['out']['sorted'] + '.setup' try: calib_dict = ltu.loadjson(setup_file) except: msgs.info("No setup file {:s} for MasterFrames".format(setup_file)) calib_dict = {} else: argflag['masters']['setup_file'] = setup_file # Start reducing the data for sc in range(numsci): slf = sciexp[sc] scidx = slf._idx_sci[0] msgs.info("Reducing file {0:s}, target {1:s}".format( fitsdict['filename'][scidx], slf._target_name)) msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too # Loop on Detectors for kk in xrange(slf._spect['mosaic']['ndet']): det = kk + 1 # Detectors indexed from 1 slf.det = det ############### # Get amplifier sections arproc.get_ampsec_trimmed(slf, fitsdict, det, scidx) # Setup setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=False) slf._argflag['masters']['setup'] = setup ############### # Generate master bias frame update = slf.MasterBias(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="bias") ############### # Generate a bad pixel mask (should not repeat) update = slf.BadPixelMask(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="arc") ############### # Generate a master arc frame update = slf.MasterArc(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="arc") ############### # Determine the dispersion direction (and transpose if necessary) slf.GetDispersionDirection(fitsdict, det, scidx) if slf._bpix[ det - 1] is None: # Needs to be done here after nspec is set slf.SetFrame( slf._bpix, np.zeros((slf._nspec[det - 1], slf._nspat[det - 1])), det) ''' ############### # Estimate gain and readout noise for the amplifiers msgs.work("Estimate Gain and Readout noise from the raw frames...") update = slf.MasterRN(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise") ''' ############### # Generate a master trace frame update = slf.MasterTrace(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace") ############### # Generate an array that provides the physical pixel locations on the detector slf.GetPixelLocations(det) # Determine the edges of the spectrum (spatial) if 'trace' + slf._argflag['masters']['setup'] not in slf._argflag[ 'masters']['loaded']: ############### # Determine the edges of the spectrum (spatial) lordloc, rordloc, extord = artrace.trace_orders( slf, slf._mstrace[det - 1], det, singleSlit=True, pcadesc="PCA trace of the slit edges") slf.SetFrame(slf._lordloc, lordloc, det) slf.SetFrame(slf._rordloc, rordloc, det) # Convert physical trace into a pixel trace msgs.info( "Converting physical trace locations to nearest pixel") pixcen = artrace.phys_to_pix( 0.5 * (slf._lordloc[det - 1] + slf._rordloc[det - 1]), slf._pixlocn[det - 1], 1) pixwid = (slf._rordloc[det - 1] - slf._lordloc[det - 1]).mean(0).astype(np.int) lordpix = artrace.phys_to_pix(slf._lordloc[det - 1], slf._pixlocn[det - 1], 1) rordpix = artrace.phys_to_pix(slf._rordloc[det - 1], slf._pixlocn[det - 1], 1) slf.SetFrame(slf._pixcen, pixcen, det) slf.SetFrame(slf._pixwid, pixwid, det) slf.SetFrame(slf._lordpix, lordpix, det) slf.SetFrame(slf._rordpix, rordpix, det) # Save QA for slit traces arqa.slit_trace_qa(slf, slf._mstrace[det - 1], slf._lordpix[det - 1], slf._rordpix[det - 1], extord, desc="Trace of the slit edges") ############### # Prepare the pixel flat field frame update = slf.MasterFlatField(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixflat") ############### # Generate the 1D wavelength solution update = slf.MasterWaveCalib(fitsdict, sc, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace") ############### # Derive the spectral tilt if slf._tilts[det - 1] is None: if slf._argflag['masters']['use']: mstilt_name = armasters.master_name( slf._argflag['run']['masterdir'], 'tilts', slf._argflag['masters']['setup']) try: tilts, head = arload.load_master(mstilt_name, frametype="tilts") except IOError: pass else: slf.SetFrame(slf._tilts, tilts, det) slf._argflag['masters']['loaded'].append( 'tilts' + slf._argflag['masters']['setup']) if 'tilts' + slf._argflag['masters'][ 'setup'] not in slf._argflag['masters']['loaded']: # First time tilts are derived for this arc frame --> derive the order tilts tilts, satmask, outpar = artrace.model_tilt( slf, det, slf._msarc[det - 1]) slf.SetFrame(slf._tilts, tilts, det) slf.SetFrame(slf._satmask, satmask, det) slf.SetFrame(slf._tiltpar, outpar, det) ############### # Generate/load a master wave frame update = slf.MasterWave(fitsdict, sc, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave") # Check if the user only wants to prepare the calibrations only msgs.info("All calibration frames have been prepared") if slf._argflag['run']['preponly']: msgs.info("If you would like to continue with the reduction," + msgs.newline() + "disable the run+preponly command") continue # Write setup setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=True) # Write MasterFrames (currently per detector) armasters.save_masters(slf, det, setup) ############### # Load the science frame and from this generate a Poisson error frame msgs.info("Loading science frame") sciframe = arload.load_frames(slf, fitsdict, [scidx], det, frametype='science', msbias=slf._msbias[det - 1], transpose=slf._transpose) sciframe = sciframe[:, :, 0] # Extract msgs.info("Processing science frame") arproc.reduce_frame(slf, sciframe, scidx, fitsdict, det) #continue #msgs.error("UP TO HERE") ############### # Perform a velocity correction if (slf._argflag['reduce']['heliocorr'] == True) & False: if slf._argflag['science']['load']['extracted'] == True: msgs.warn( "Heliocentric correction will not be applied if an extracted science frame exists, and is used" ) msgs.work("Perform a full barycentric correction") msgs.work( "Include the facility to correct for gravitational redshifts and time delays (see Pulsar timing work)" ) msgs.info("Performing a heliocentric correction") # Load the header for the science frame slf._waveids = arvcorr.helio_corr(slf, scidx[0]) else: msgs.info("A heliocentric correction will not be performed") ############### # Using model sky, calculate a flexure correction # Close the QA for this object slf._qa.close() ############### # Flux ############### # Standard star (is this a calibration, e.g. goes above?) msgs.info("Processing standard star") msgs.info("Assuming one star per detector mosaic") msgs.info("Waited until last detector to process") msgs.work("Need to check for existing sensfunc") update = slf.MasterStandard(scidx, fitsdict) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, 0, ftype="standard") # msgs.work("Consider using archived sensitivity if not found") msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name'])) for kk in xrange(slf._spect['mosaic']['ndet']): det = kk + 1 # Detectors indexed from 1 arflux.apply_sensfunc(slf, det, scidx, fitsdict) # Write 1D spectra arsave.save_1d_spectra(slf) # Write 2D images for the Science Frame arsave.save_2d_images(slf) # Free up some memory by replacing the reduced ScienceExposure class sciexp[sc] = None return status
def ARMLSD(argflag, spect, fitsdict, reuseMaster=False): """ Automatic Reduction and Modeling of Long Slit Data Parameters ---------- argflag : dict Arguments and flags used for reduction spect : dict Properties of the spectrograph. fitsdict : dict Contains relevant information from fits header files reuseMaster : bool If True, a master frame that will be used for another science frame will not be regenerated after it is first made. This setting comes with a price, and if a large number of science frames are being generated, it may be more efficient to simply regenerate the master calibrations on the fly. Returns ------- status : int Status of the reduction procedure 0 = Successful execution 1 = ... """ status = 0 # Create a list of science exposure classes sciexp = armbase.SetupScience(argflag, spect, fitsdict) numsci = len(sciexp) # Create a list of master calibration frames masters = armasters.MasterFrames(spect['mosaic']['ndet']) # Use Masters? Requires setup file setup_file = argflag['out']['sorted']+'.setup' try: calib_dict = ltu.loadjson(setup_file) except: msgs.info("No setup file {:s} for MasterFrames".format(setup_file)) calib_dict = {} else: argflag['masters']['setup_file'] = setup_file # Start reducing the data for sc in range(numsci): slf = sciexp[sc] scidx = slf._idx_sci[0] msgs.info("Reducing file {0:s}, target {1:s}".format(fitsdict['filename'][scidx], slf._target_name)) msgs.sciexp = slf # For QA writing on exit, if nothing else. Could write Masters too # Loop on Detectors for kk in xrange(slf._spect['mosaic']['ndet']): det = kk + 1 # Detectors indexed from 1 slf.det = det ############### # Get amplifier sections arproc.get_ampsec_trimmed(slf, fitsdict, det, scidx) # Setup setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=False) slf._argflag['masters']['setup'] = setup ############### # Generate master bias frame update = slf.MasterBias(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="bias") ############### # Generate a bad pixel mask (should not repeat) update = slf.BadPixelMask(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="arc") ############### # Generate a master arc frame update = slf.MasterArc(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="arc") ############### # Determine the dispersion direction (and transpose if necessary) slf.GetDispersionDirection(fitsdict, det, scidx) if slf._bpix[det-1] is None: # Needs to be done here after nspec is set slf.SetFrame(slf._bpix, np.zeros((slf._nspec[det-1], slf._nspat[det-1])), det) ''' ############### # Estimate gain and readout noise for the amplifiers msgs.work("Estimate Gain and Readout noise from the raw frames...") update = slf.MasterRN(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise") ''' ############### # Generate a master trace frame update = slf.MasterTrace(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace") ############### # Generate an array that provides the physical pixel locations on the detector slf.GetPixelLocations(det) # Determine the edges of the spectrum (spatial) if 'trace'+slf._argflag['masters']['setup'] not in slf._argflag['masters']['loaded']: ############### # Determine the edges of the spectrum (spatial) lordloc, rordloc, extord = artrace.trace_orders(slf, slf._mstrace[det-1], det, singleSlit=True, pcadesc="PCA trace of the slit edges") slf.SetFrame(slf._lordloc, lordloc, det) slf.SetFrame(slf._rordloc, rordloc, det) # Convert physical trace into a pixel trace msgs.info("Converting physical trace locations to nearest pixel") pixcen = artrace.phys_to_pix(0.5*(slf._lordloc[det-1]+slf._rordloc[det-1]), slf._pixlocn[det-1], 1) pixwid = (slf._rordloc[det-1]-slf._lordloc[det-1]).mean(0).astype(np.int) lordpix = artrace.phys_to_pix(slf._lordloc[det-1], slf._pixlocn[det-1], 1) rordpix = artrace.phys_to_pix(slf._rordloc[det-1], slf._pixlocn[det-1], 1) slf.SetFrame(slf._pixcen, pixcen, det) slf.SetFrame(slf._pixwid, pixwid, det) slf.SetFrame(slf._lordpix, lordpix, det) slf.SetFrame(slf._rordpix, rordpix, det) # Save QA for slit traces arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det-1], extord, desc="Trace of the slit edges") ############### # Prepare the pixel flat field frame update = slf.MasterFlatField(fitsdict, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixflat") ############### # Generate the 1D wavelength solution update = slf.MasterWaveCalib(fitsdict, sc, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace") ############### # Derive the spectral tilt if slf._tilts[det-1] is None: if slf._argflag['masters']['use']: mstilt_name = armasters.master_name(slf._argflag['run']['masterdir'], 'tilts', slf._argflag['masters']['setup']) try: tilts, head = arload.load_master(mstilt_name, frametype="tilts") except IOError: pass else: slf.SetFrame(slf._tilts, tilts, det) slf._argflag['masters']['loaded'].append('tilts'+slf._argflag['masters']['setup']) if 'tilts'+slf._argflag['masters']['setup'] not in slf._argflag['masters']['loaded']: # First time tilts are derived for this arc frame --> derive the order tilts tilts, satmask, outpar = artrace.model_tilt(slf, det, slf._msarc[det-1]) slf.SetFrame(slf._tilts, tilts, det) slf.SetFrame(slf._satmask, satmask, det) slf.SetFrame(slf._tiltpar, outpar, det) ############### # Generate/load a master wave frame update = slf.MasterWave(fitsdict, sc, det) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave") # Check if the user only wants to prepare the calibrations only msgs.info("All calibration frames have been prepared") if slf._argflag['run']['preponly']: msgs.info("If you would like to continue with the reduction," +msgs.newline()+"disable the run+preponly command") continue # Write setup setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=True) # Write MasterFrames (currently per detector) armasters.save_masters(slf, det, setup) ############### # Load the science frame and from this generate a Poisson error frame msgs.info("Loading science frame") sciframe = arload.load_frames(slf, fitsdict, [scidx], det, frametype='science', msbias=slf._msbias[det-1], transpose=slf._transpose) sciframe = sciframe[:, :, 0] # Extract msgs.info("Processing science frame") arproc.reduce_frame(slf, sciframe, scidx, fitsdict, det) #continue #msgs.error("UP TO HERE") ############### # Perform a velocity correction if (slf._argflag['reduce']['heliocorr'] == True) & False: if slf._argflag['science']['load']['extracted'] == True: msgs.warn("Heliocentric correction will not be applied if an extracted science frame exists, and is used") msgs.work("Perform a full barycentric correction") msgs.work("Include the facility to correct for gravitational redshifts and time delays (see Pulsar timing work)") msgs.info("Performing a heliocentric correction") # Load the header for the science frame slf._waveids = arvcorr.helio_corr(slf, scidx[0]) else: msgs.info("A heliocentric correction will not be performed") ############### # Using model sky, calculate a flexure correction # Close the QA for this object slf._qa.close() ############### # Flux ############### # Standard star (is this a calibration, e.g. goes above?) msgs.info("Processing standard star") msgs.info("Assuming one star per detector mosaic") msgs.info("Waited until last detector to process") msgs.work("Need to check for existing sensfunc") update = slf.MasterStandard(scidx, fitsdict) if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, 0, ftype="standard") # msgs.work("Consider using archived sensitivity if not found") msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name'])) for kk in xrange(slf._spect['mosaic']['ndet']): det = kk + 1 # Detectors indexed from 1 arflux.apply_sensfunc(slf, det, scidx, fitsdict) # Write 1D spectra arsave.save_1d_spectra(slf) # Write 2D images for the Science Frame arsave.save_2d_images(slf) # Free up some memory by replacing the reduced ScienceExposure class sciexp[sc] = None return status