def GetMasterFrame(self, ftype, det, mkcopy=True): det -= 1 # Get the frame if mkcopy: #if ftype == "arc": return self._msarc[det].copy() if ftype == "wave": return self._mswave[det].copy() #elif ftype == "bias": return self._msbias[det].copy() elif ftype == "normpixelflat": return self._mspixelflatnrm[det].copy() elif ftype == "pixelflat": return self._mspixelflat[det].copy() elif ftype == "trace": return self._mstrace[det].copy() elif ftype == "pinhole": return self._mspinhole[det].copy() elif ftype == "standard": return mkcopy.copy(self._msstd[det]) elif ftype == "sensfunc": return mkcopy.copy(self._sensfunc) else: msgs.bug("I could not get master frame of type: {0:s}".format(ftype)) msgs.error("Please contact the authors") else: #if ftype == "arc": return self._msarc[det] if ftype == "wave": return self._mswave[det] #elif ftype == "bias": return self._msbias[det] elif ftype == "normpixelflat": return self._mspixelflatnrm[det] elif ftype == "pixelflat": return self._mspixelflat[det] elif ftype == "trace": return self._mstrace[det] elif ftype == "pinhole": return self._mspinhole[det] elif ftype == "standard": return self._msstd[det] elif ftype == "sensfunc": return self._sensfunc else: msgs.bug("I could not get master frame of type: {0:s}".format(ftype)) msgs.error("Please contact the authors") return None
def update_sci_pixmask(self, det, mask_pix, mask_type): """ Update the binary pixel mask for a given science frame Parameters ---------- det : int mask_pix : ndarray Image of pixels to mask (anything >0) mask_type : str Type of masked pixel BadPix = 1 CR = 2 """ mask_dict = dict(BadPix=1, CR=2) if mask_type not in mask_dict.keys(): msgs.error("Bad pixel mask type") # Find pixels to mask mask = np.where(mask_pix > 0) if len(mask[0]) == 0: return # Update those that need it prev_val = self._scimask[det-1][mask] upd = np.where((prev_val % 2**(mask_dict[mask_type]+1)) < 2**(mask_dict[mask_type]))[0] if len(upd) > 0: self._scimask[det-1][mask[0][upd], mask[1][upd]] += 2**mask_dict[mask_type] # Return return
def MasterStandard(self, fitsdict, msbias): """ Generate Master Standard frame for a given detector and generates a sensitivity function Currently only uses first standard star exposure Currently takes brightest source on the mosaic Parameters ---------- fitsdict : dict Contains relevant information from fits header files Returns ------- boolean : bool """ msgs.error("THIS IS DEPRECATED") '''
def MasterWave(self, det, all_wvcalib, tilts): """ Generate Master Wave frame for a given detector Parameters ---------- fitsdict : dict Contains relevant information from fits header files det : int Index of the detector Returns ------- boolean : bool Should other ScienceExposure classes be updated? """ msgs.error("SHOULD NOT BE HERE ANYMORE") '''
def MasterFlatField(self, fitsdict, det, msbias, datasec_img, tilts): """ Generate Master Flat-field frame for a given detector Parameters ---------- fitsdict : dict Contains relevant information from fits header files det : int Index of the detector Returns ------- boolean : bool Should other ScienceExposure classes be updated? """ msgs.error("SHOULD NOT GET HERE") '''
def SetMasterFrame(self, frame, ftype, det, mkcopy=True): """ Set the Master Frame Parameters ---------- frame : object ftype : str frame type det : int Detector index mkcopy Returns ------- """ if det is not None: det -= 1 if mkcopy: cpf = frame.copy() else: cpf = frame # Set the frame #if ftype == "arc": self._msarc[det] = cpf if ftype == "wave": self._mswave[det] = cpf #elif ftype == "bias": self._msbias[det] = cpf elif ftype == "readnoise": self._msrn[det] = cpf elif ftype == "normpixelflat": self._mspixelflatnrm[det] = cpf elif ftype == "pixelflat": self._mspixelflat[det] = cpf elif ftype == "trace": self._mstrace[det] = cpf elif ftype == "pinhole": self._mspinhole[det] = cpf elif ftype == "standard": self._msstd[det] = cpf elif ftype == "sensfunc": self._sensfunc = cpf else: msgs.bug("I could not set master frame of type: {0:s}".format(ftype)) msgs.error("Please contact the authors") return
def reduce_echelle(slf, sciframe, scidx, fitsdict, det, standard=False, triml=1, trimr=1, mspixelflatnrm=None, doqa=True): """ Run standard extraction steps on an echelle frame Parameters ---------- sciframe : image Bias subtracted image (using arload.load_frame) scidx : int Index of the frame fitsdict : dict Contains relevant information from fits header files det : int Detector index standard : bool, optional Standard star frame? triml : int (optional) Number of pixels to trim from the left slit edge trimr : int (optional) Number of pixels to trim from the right slit edge """ msgs.work("Multiprocess this algorithm") nspec = sciframe.shape[0] nord = slf._lordloc[det - 1].shape[1] # Prepare the frames for tracing and extraction sciframe, rawvarframe, crmask = reduce_prepare( slf, sciframe, scidx, fitsdict, det, mspixelflatnrm=mspixelflatnrm, standard=standard, slitprof=slitprof) bgframe = np.zeros_like(sciframe) bgnl, bgnr = np.zeros(nord, dtype=np.int), np.zeros(nord, dtype=np.int) skysub = True if settings.argflag['reduce']['skysub']['perform']: # Identify background pixels, and generate an image of the sky spectrum in each slit for o in range(nord): word = np.where((slf._slitpix[det - 1] == o + 1) & (slf._scimask[det - 1] == 0)) if word[0].size == 0: msgs.warn("There are no pixels in slit {0:d}".format(o + 1)) continue tbgframe, nl, nr = background_subtraction(slf, sciframe, rawvarframe, o, det) bgnl[o], bgnr[o] = nl, nr bgframe += tbgframe if nl == 0 and nr == 0: pass # If just one slit cannot do sky subtraction, don't do sky subtraction # msgs.warn("A sky subtraction will not be performed") # skysub = False # bgframe = np.zeros_like(sciframe) # modelvarframe = rawvarframe.copy() # break if skysub: # Provided the for loop above didn't break early, model the variance frame dnum = settings.get_dnum(det) modelvarframe = arprocimg.variance_frame(datasec_img, det, sciframe, scidx, settings.spect[dnum], fitsdict=fitsdict, skyframe=bgframe) else: modelvarframe = rawvarframe.copy() bgframe = np.zeros_like(sciframe) if not standard: # Need to save slf._modelvarframe[det - 1] = modelvarframe slf._bgframe[det - 1] = bgframe # Obtain a first estimate of the object trace then # fit the traces and perform a PCA for the refinements trccoeff = np.zeros( (settings.argflag['trace']['object']['order'] + 1, nord)) trcxfit = np.arange(nspec) extrap_slit = np.zeros(nord) for o in range(nord): trace, error = artrace.trace_weighted(sciframe - bgframe, slf._lordloc[det - 1][:, o], slf._rordloc[det - 1][:, o], mask=slf._scimask[det - 1], wght="flux") if trace is None: extrap_slit[o] = 1 continue # Find only the good pixels w = np.where((error != 0.0) & (~np.isnan(error))) if w[0].size <= 2 * settings.argflag['trace']['object']['order']: extrap_slit[o] = 1 continue # Convert the trace locations to be a fraction of the slit length, # measured from the left slit edge. trace -= slf._lordloc[det - 1][:, o] trace /= (slf._rordloc[det - 1][:, o] - slf._lordloc[det - 1][:, o]) try: msk, trccoeff[:, o] = arutils.robust_polyfit( trcxfit[w], trace[w], settings.argflag['trace']['object']['order'], function=settings.argflag['trace']['object']['function'], weights=1.0 / error[w]**2, minv=0.0, maxv=nspec - 1.0) except: msgs.info("arproc.reduce_echelle") debugger.set_trace() refine = 0.0 if settings.argflag['trace']['object']['method'] == "pca": # Identify the orders to be extrapolated during reconstruction orders = 1.0 + np.arange(nord) msgs.info("Performing a PCA on the object trace") ofit = settings.argflag['trace']['object']['params'] lnpc = len(ofit) - 1 maskord = np.where(extrap_slit == 1)[0] xcen = trcxfit[:, np.newaxis].repeat(nord, axis=1) trccen = arutils.func_val( trccoeff, trcxfit, settings.argflag['trace']['object']['function'], minv=0.0, maxv=nspec - 1.0).T if np.sum(1.0 - extrap_slit) > ofit[0] + 1: fitted, outpar = arpca.basis( xcen, trccen, trccoeff, lnpc, ofit, skipx0=False, mask=maskord, function=settings.argflag['trace']['object']['function']) if doqa: # arqa.pca_plot(slf, outpar, ofit, "Object_Trace", pcadesc="PCA of object trace") arpca.pca_plot(slf.setup, outpar, ofit, "Object_Trace", pcadesc="PCA of object trace") # Extrapolate the remaining orders requested trccen, outpar = arpca.extrapolate( outpar, orders, function=settings.argflag['trace']['object']['function']) #refine = trccen-trccen[nspec//2, :].reshape((1, nord)) else: msgs.warn("Could not perform a PCA on the object trace" + msgs.newline() + "Not enough well-traced orders") msgs.info("Using direct determination of the object trace instead") pass else: msgs.error("Not ready for object trace method:" + msgs.newline() + settings.argflag['trace']['object']['method']) # Construct the left and right traces of the object profile # The following code ensures that the fraction of the slit # containing the object remains constant along the spectral # direction trcmean = np.mean(trccen, axis=0) trobjl = (trcmean - (1 + bgnl) / slf._pixwid[det - 1].astype(np.float)).reshape( (1, nord)).repeat(nspec, axis=0) trobjl = trccen - trobjl trobjr = (-trcmean + (slf._pixwid[det - 1] - bgnr - 1) / slf._pixwid[det - 1].astype(np.float)).reshape( (1, nord)).repeat(nspec, axis=0) trobjr = trccen + trobjr # Convert trccen to the actual trace locations trccen *= (slf._rordloc[det - 1] - slf._lordloc[det - 1]) trccen += slf._lordloc[det - 1] trobjl *= (slf._rordloc[det - 1] - slf._lordloc[det - 1]) trobjl += slf._lordloc[det - 1] trobjr *= (slf._rordloc[det - 1] - slf._lordloc[det - 1]) trobjr += slf._lordloc[det - 1] # Generate an image of pixel weights for each object. Each weight can # take any floating point value from 0 to 1 (inclusive). For the rec_obj_img, # a weight of 1 means that the pixel is fully contained within the object # region, and 0 means that the pixel is fully contained within the background # region. The opposite is true for the rec_bg_img array. A pixel that is on # the border of object/background is assigned a value between 0 and 1. msgs.work( "Eventually allow ARMED to find multiple objects in the one slit") nobj = 1 rec_obj_img = np.zeros(sciframe.shape + (nobj, )) rec_bg_img = np.zeros(sciframe.shape + (nobj, )) for o in range(nord): # Prepare object/background regions objl = np.array([bgnl[o]]) objr = np.array([slf._pixwid[det - 1][o] - bgnr[o] - triml - trimr]) bckl = np.zeros((slf._pixwid[det - 1][o] - triml - trimr, 1)) bckr = np.zeros((slf._pixwid[det - 1][o] - triml - trimr, 1)) bckl[:bgnl[o]] = 1 if bgnr[o] != 0: bckr[-bgnr[o]:] = 1 tobj_img, tbg_img = artrace.trace_objbg_image(slf, det, sciframe - bgframe, o, [objl, objr], [bckl, bckr], triml=triml, trimr=trimr) rec_obj_img += tobj_img rec_bg_img += tbg_img # Create trace dict scitrace = artrace.trace_object_dict(nobj, trccen[:, 0].reshape(trccen.shape[0], 1), object=rec_obj_img, background=rec_bg_img) for o in range(1, nord): scitrace = artrace.trace_object_dict(nobj, trccen[:, o].reshape( trccen.shape[0], 1), tracelist=scitrace) # Save the quality control if doqa: artrace.obj_trace_qa(slf, sciframe, trobjl, trobjr, None, det, root="object_trace", normalize=False) # Finalize the Sky Background image if settings.argflag['reduce']['skysub']['perform'] and (nobj > 0) and skysub: msgs.info("Finalizing the sky background image") # Identify background pixels, and generate an image of the sky spectrum in each slit bgframe = np.zeros_like(sciframe) for o in range(nord): tbgframe, nl, nr = background_subtraction(slf, sciframe, rawvarframe, o, det, refine=refine) bgnl[o], bgnr[o] = nl, nr bgframe += tbgframe modelvarframe = arprocimg.variance_frame(datasec_img, det, sciframe, scidx, settings.spect[dnum], fitsdict=fitsdict, skyframe=bgframe) # Perform an optimal extraction return reduce_frame(slf, sciframe, rawvarframe, modelvarframe, bgframe, scidx, fitsdict, det, crmask, scitrace=scitrace, standard=standard)
def background_subtraction(slf, sciframe, varframe, slitn, det, refine=0.0, doqa=True): """ Generate a frame containing the background sky spectrum Parameters ---------- slf : Class Science Exposure Class sciframe : ndarray science frame varframe : ndarray variance frame slitn : int Slit number det : int Detector index refine : float or ndarray refine the object traces. This should be a small value around 0.0. If a float, a constant offset will be applied. Otherwise, an array needs to be specified of the same length as sciframe.shape[0] that contains the refinement of each pixel along the spectral direction. Returns ------- bgframe : ndarray An image, the same size as sciframe, that contains the background spectrum within the specified slit. nl : int number of pixels from the left slit edge to use as background pixels nr : int number of pixels from the right slit edge to use as background pixels """ # Obtain all pixels that are within the slit edges, and are not masked word = np.where((slf._slitpix[det - 1] == slitn + 1) & (slf._scimask[det - 1] == 0)) if word[0].size == 0: msgs.warn("There are no pixels in slit {0:d}".format(slitn)) debugger.set_trace() nl, nr = 0, 0 return np.zeros_like(sciframe), nl, nr # Calculate the oversampled object profiles oversampling_factor = 3 # should be an integer according to the description in object_profile() xedges, modvals = object_profile(slf, sciframe, slitn, det, refine=refine, factor=oversampling_factor) bincent = 0.5 * (xedges[1:] + xedges[:-1]) npix = slf._pixwid[det - 1][slitn] tilts = slf._tilts[det - 1].copy() lordloc = slf._lordloc[det - 1][:, slitn] rordloc = slf._rordloc[det - 1][:, slitn] # For each pixel, calculate the fraction along the slit's spatial direction spatval = (word[1] - lordloc[word[0]] + refine) / (rordloc[word[0]] - lordloc[word[0]]) # Cumulative sum and normalize csum = np.cumsum(modvals) csum -= csum[0] csum /= csum[-1] # Find a first guess of the edges of the object profile - assume this is the innermost 90 percent of the flux argl = np.argmin(np.abs(csum - 0.05)) argr = np.argmin(np.abs(csum - 0.95)) # Considering the possible background pixels that are left of the object, # find the first time where the object profile no longer decreases as you # move toward the edge of the slit. This is the beginning of the noisy # object profile, which is where the object can no longer be distinguished # from the background. wl = np.where((modvals[1:] < modvals[:-1]) & (bincent[1:] < bincent[argl])) wr = np.where((modvals[1:] > modvals[:-1]) & (bincent[1:] > bincent[argr])) nl, nr = 0, 0 if wl[0].size != 0: # This is the index of the first time where the object profile # no longer decreases as you move towards the slit edge nl_index = np.max(wl[0]) # Calculate nl, defined as: # "number of pixels from the left slit edge to use as background pixels", # which is just nl_index with the sampling factor taken out nl_index_origscale = int(nl_index / oversampling_factor + 0.5) nl = nl_index_origscale if wr[0].size != 0: # This is the index of the first time where the object profile # no longer decreases as you move towards the slit edge nr_index = np.min(wr[0]) # Calculate nr, defined as: # "number of pixels from the right slit edge to use as background pixels", # which is npix minus nr_index with the sampling factor taken out nr_index_origscale = int(nr_index / oversampling_factor + 0.5) nr = npix - nr_index_origscale if nl + nr < 5: msgs.warn( "The object profile appears to extrapolate to the edge of the slit" ) msgs.info( "A background subtraction will not be performed for slit {0:d}". format(slitn + 1)) nl, nr = 0, 0 return np.zeros_like(sciframe), nl, nr # Find background pixels and fit wbgpix_spatval = np.where( (spatval <= float(nl) / npix) | (spatval >= float(npix - nr) / npix)) # this cannot be used to index the 2D array tilts wbgpix = (word[0][wbgpix_spatval], word[1][wbgpix_spatval] ) # this may be approproate for indexing the 2D array tilts if settings.argflag['reduce']['skysub']['method'].lower() == 'bspline': msgs.info("Using bspline sky subtraction") srt = np.argsort(tilts[wbgpix]) ivar = arutils.calc_ivar(varframe) # Perform a weighted b-spline fit to the sky background pixels mask, bspl = arutils.robust_polyfit( tilts[wbgpix][srt], sciframe[wbgpix][srt], 3, function='bspline', weights=np.sqrt(ivar)[wbgpix][srt], sigma=5., maxone=False, **settings.argflag['reduce']['skysub']['bspline']) bgf_flat = arutils.func_val(bspl, tilts.flatten(), 'bspline') bgframe = bgf_flat.reshape(tilts.shape) if doqa: plt_bspline_sky(tilts, sciframe, bgf_flat, gdp) debugger.set_trace() else: msgs.error('Not ready for this method for skysub {:s}'.format( settings.argflag['reduce']['skysub']['method'].lower())) if np.any(np.isnan(bgframe)): msgs.warn("NAN in bgframe. Replacing with 0") bad = np.isnan(bgframe) bgframe[bad] = 0. return bgframe, nl, nr
def __init__(self, sci_ID, fitstbl, settings_argflag, settings_spect, do_qa=True, idx_sci=None): # Set indices used for frame combination msgs.error("DEPRECATED") self.sci_ID = sci_ID # Binary 1,2,4,8,.. self._idx_sci = np.where((fitstbl['sci_ID'] == sci_ID) & fitstbl['science'])[0] if idx_sci is not None: self._idx_sci = np.array([idx_sci]) if settings_argflag['reduce']['masters']['force']: #self._idx_bias = [] #self._idx_flat = [] self._idx_cent = [] #self._idx_trace = [] #self._idx_arcs = [] #self._idx_std = [] else: #self._idx_arcs = arsort.ftype_indices(fitstbl, 'arc', self.sci_ID) #self._idx_std = arsort.ftype_indices(fitstbl, 'standard', self.sci_ID) # Bias #if settings_argflag['bias']['useframe'] == 'bias': # self._idx_bias = arsort.ftype_indices(fitstbl, 'bias', self.sci_ID) #elif settings_argflag['bias']['useframe'] == 'dark': # self._idx_bias = arsort.ftype_indices(fitstbl, 'dark', self.sci_ID) #else: self._idx_bias = [] # Trace #self._idx_trace = arsort.ftype_indices(fitstbl, 'trace', self.sci_ID) # Flat #if settings_argflag['reduce']['flatfield']['useframe'] == 'pixelflat': # self._idx_flat = arsort.ftype_indices(fitstbl, 'pixelflat', self.sci_ID) #elif settings_argflag['reduce']['flatfield']['useframe'] == 'trace': # self._idx_flat = arsort.ftype_indices(fitstbl, 'trace', self.sci_ID) #else: self._idx_flat = [] # Cent if settings_argflag['reduce']['slitcen']['useframe'] == 'trace': self._idx_cent = arsort.ftype_indices(fitstbl, 'trace', self.sci_ID) elif settings_argflag['reduce']['slitcen']['useframe'] == 'pinhole': # Not sure this will work self._idx_cent = arsort.ftype_indices(fitstbl, 'pinhole', self.sci_ID) else: self._idx_cent = [] # Set the base name and extract other names that will be used for output files # Also parses the time input self.SetBaseName(fitstbl) # Setup self.setup = '' # Velocity correction (e.g. heliocentric) self.vel_correction = 0. # Initialize the QA for this science exposure qafn = "{0:s}/QA_{1:s}.pdf".format(settings_argflag['run']['directory']['qa'], self._basename) self.qaroot = "{0:s}/PNGs/QA_{1:s}".format(settings_argflag['run']['directory']['qa'], self._basename) # Initialize Variables ndet = settings_spect['mosaic']['ndet'] self.extracted = [False for all in range(ndet)] # Mainly for standard stars self._nonlinear = [settings_spect[settings.get_dnum(det+1)]['saturation'] * settings_spect[settings.get_dnum(det+1)]['nonlinear'] for det in range(ndet)] #self._nspec = [None for all in range(ndet)] # Number of spectral pixels #self._nspat = [None for all in range(ndet)] # Number of spatial pixels #self._datasec = [None for all in range(ndet)] # Locations of the data on each detector self._pixlocn = [None for all in range(ndet)] # Physical locations of each pixel on the detector self._lordloc = [None for all in range(ndet)] # Array of slit traces (left side) in physical pixel coordinates self._rordloc = [None for all in range(ndet)] # Array of slit traces (left side) in physical pixel coordinates self._pixcen = [None for all in range(ndet)] # Central slit traces in apparent pixel coordinates self._pixwid = [None for all in range(ndet)] # Width of slit (at each row) in apparent pixel coordinates self._lordpix = [None for all in range(ndet)] # Array of slit traces (left side) in apparent pixel coordinates self._rordpix = [None for all in range(ndet)] # Array of slit traces (right side) in apparent pixel coordinates self._slitpix = [None for all in range(ndet)] # Array identifying if a given pixel belongs to a given slit #self._tilts = [None for all in range(ndet)] # Array of spectral tilts at each position on the detector #self._tiltpar = [None for all in range(ndet)] # Dict parameters for tilt fitting self._satmask = [None for all in range(ndet)] # Array of Arc saturation streaks #self._arcparam = [None for all in range(ndet)] # Dict guiding wavelength calibration #self._wvcalib = [None for all in range(ndet)] # List of dict's self._resnarr = [None for all in range(ndet)] # Resolution array self._maskslits = [None for all in range(ndet)] # Mask for whether to analyze a given slit (True=masked) # Initialize the Master Calibration frames #self._bpix = [None for all in range(ndet)] # Bad Pixel Mask #self._msarc = [None for all in range(ndet)] # Master Arc self._mswave = [None for all in range(ndet)] # Master Wavelength image #self._msbias = [None for all in range(ndet)] # Master Bias self._msrn = [None for all in range(ndet)] # Master ReadNoise image #self._mstrace = [None for all in range(ndet)] # Master Trace self._mspinhole = [None for all in range(ndet)] # Master Pinhole #self._mspixelflat = [None for all in range(ndet)] # Master Pixel Flat #self._mspixelflatnrm = [None for all in range(ndet)] # Normalized Master pixel flat #self._msblaze = [None for all in range(ndet)] # Blaze function #self._msstd = [{} for all in range(ndet)] # Master Standard dict #self._sensfunc = None # Sensitivity function # Initialize the Master Calibration frame names #self._msarc_name = [None for all in range(ndet)] # Master Arc Name #self._msbias_name = [None for all in range(ndet)] # Master Bias Name #self._mstrace_name = [None for all in range(ndet)] # Master Trace Name self._mspinhole_name = [None for all in range(ndet)] # Master Pinhole Name #self._mspixelflat_name = [None for all in range(ndet)] # Master Pixel Flat Name # Initialize the science, variance, and background frames self._sciframe = [None for all in range(ndet)] self._rawvarframe = [None for all in range(ndet)] # Variance based on detected counts + RN self._modelvarframe = [None for all in range(ndet)] # Variance from sky and object models self._bgframe = [None for all in range(ndet)] self._scimask = [None for all in range(ndet)] # Mask (1=Bad pix; 2=CR) self._scitrace = [None for all in range(ndet)] #self._slitprof = [None for all in range(ndet)] # Slit profiles at each position on the detector self._specobjs = [None for all in range(ndet)] # Initialize some extraction products self._ext_boxcar = [None for all in range(ndet)] self._ext_optimal = [None for all in range(ndet)] return