def optimal_extract(slf, det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, pickle_file=None, profiles=None): """ Preform optimal extraction Standard Horne approach Parameters ---------- slf det specobjs sciframe varframe crmask scitrace COUNT_LIM pickle_file Returns ------- newvar : ndarray Updated variance array that includes object model """ from pypit import arproc # Setup #rnimg = arproc.rn_frame(slf,det) #model_var = np.abs(skyframe + sciframe - np.sqrt(2)*rnimg + rnimg**2) # sqrt 2 term deals with negative flux/sky #model_ivar = 1./model_var # Inverse variance model_ivar = np.zeros_like(varframe) gdvar = varframe > 0. model_ivar[gdvar] = arutils.calc_ivar(varframe[gdvar]) cr_mask = 1.0 - crmask # Object model image obj_model = np.zeros_like(varframe) # Loop on slits for sl in range(len(specobjs)): # Loop on objects nobj = scitrace[sl]['traces'].shape[1] for o in range(nobj): msgs.info( "Performing optimal extraction of object {0:d}/{1:d} in slit {2:d}/{3:d}" .format(o + 1, nobj, sl + 1, len(specobjs))) # Get object pixels if scitrace[sl]['background'] is None: # The object for all slits is provided in the first extension objreg = np.copy(scitrace[0]['object'][:, :, o]) wzro = np.where(slf._slitpix[det - 1] != sl + 1) objreg[wzro] = 0.0 else: objreg = scitrace[sl]['object'][:, :, o] # Fit dict fit_dict = scitrace[sl]['opt_profile'][o] if 'param' not in fit_dict.keys(): continue # Slit image slit_img = artrace.slit_image(slf, det, scitrace[sl], o) #, tilts=tilts) #msgs.warn("Turn off tilts") # Object pixels weight = objreg.copy() gdo = (weight > 0) & (model_ivar > 0) # Profile image prof_img = np.zeros_like(weight) prof_img[gdo] = arutils.func_val(fit_dict['param'], slit_img[gdo], fit_dict['func']) # Normalize norm_prof = np.sum(prof_img, axis=1) prof_img /= np.outer(norm_prof + (norm_prof == 0.), np.ones(prof_img.shape[1])) # Mask (1=good) mask = np.zeros_like(prof_img) mask[gdo] = 1. mask *= cr_mask # Optimal flux opt_num = np.sum(mask * sciframe * model_ivar * prof_img, axis=1) opt_den = np.sum(mask * model_ivar * prof_img**2, axis=1) opt_flux = opt_num / (opt_den + (opt_den == 0.)) # Optimal wave opt_num = np.sum(slf._mswave[det - 1] * model_ivar * prof_img**2, axis=1) opt_den = np.sum(model_ivar * prof_img**2, axis=1) opt_wave = opt_num / (opt_den + (opt_den == 0.)) if (np.sum(opt_wave < 1.) > 0) and settings.argflag["reduce"][ "calibrate"]["wavelength"] != "pixel": debugger.set_trace() msgs.error("Zero value in wavelength array. Uh-oh") # Optimal ivar opt_num = np.sum(mask * model_ivar * prof_img**2, axis=1) ivar_den = np.sum(mask * prof_img, axis=1) opt_ivar = opt_num * arutils.calc_ivar(ivar_den) # Save specobjs[sl][o].optimal['wave'] = opt_wave.copy( ) * u.AA # Yes, units enter here specobjs[sl][o].optimal['counts'] = opt_flux.copy() gdiv = (opt_ivar > 0.) & (ivar_den > 0.) opt_var = np.zeros_like(opt_ivar) opt_var[gdiv] = arutils.calc_ivar(opt_ivar[gdiv]) specobjs[sl][o].optimal['var'] = opt_var.copy() #specobjs[o].boxcar['sky'] = skysum # per pixel # Update object model counts_image = np.outer(opt_flux, np.ones(prof_img.shape[1])) obj_model += prof_img * counts_image ''' if 'OPTIMAL' in msgs._debug: debugger.set_trace() debugger.xplot(opt_wave, opt_flux, np.sqrt(opt_var)) ''' # Generate new variance image newvar = arproc.variance_frame(slf, det, sciframe, -1, skyframe=skyframe, objframe=obj_model) # Return return newvar
def optimal_extract(slf, det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, pickle_file=None, profiles=None): """ Preform optimal extraction Standard Horne approach Parameters ---------- slf det specobjs sciframe varframe crmask scitrace COUNT_LIM pickle_file Returns ------- newvar : ndarray Updated variance array that includes object model """ from pypit import arproc # Setup #rnimg = arproc.rn_frame(slf,det) #model_var = np.abs(skyframe + sciframe - np.sqrt(2)*rnimg + rnimg**2) # sqrt 2 term deals with negative flux/sky #model_ivar = 1./model_var # Inverse variance model_ivar = np.zeros_like(varframe) gdvar = varframe > 0. model_ivar[gdvar] = 1./varframe[gdvar] cr_mask = 1.0-crmask # Object model image obj_model = np.zeros_like(varframe) # Loop nobj = scitrace['traces'].shape[1] for o in range(nobj): # Fit dict fit_dict = scitrace['opt_profile'][o] if 'param' not in fit_dict.keys(): continue # Slit image slit_img = artrace.slit_image(slf, det, scitrace, o)#, tilts=tilts) #msgs.warn("Turn off tilts") # Object pixels weight = scitrace['object'][:,:,o] gdo = (weight > 0) & (model_ivar > 0) # Profile image prof_img = np.zeros_like(weight) prof_img[gdo] = arutils.func_val(fit_dict['param'], slit_img[gdo], fit_dict['func']) # Normalize norm_prof = np.sum(prof_img, axis=1) prof_img /= np.outer(norm_prof + (norm_prof == 0.), np.ones(prof_img.shape[1])) # Mask (1=good) mask = np.zeros_like(prof_img) mask[gdo] = 1. mask *= cr_mask # Optimal flux opt_num = np.sum(mask * sciframe * model_ivar * prof_img, axis=1) opt_den = np.sum(mask * model_ivar * prof_img**2, axis=1) opt_flux = opt_num / (opt_den + (opt_den == 0.)) # Optimal wave opt_num = np.sum(slf._mswave[det-1] * model_ivar * prof_img**2, axis=1) opt_den = np.sum(model_ivar * prof_img**2, axis=1) opt_wave = opt_num / (opt_den + (opt_den == 0.)) if np.sum(opt_wave < 1.) > 0: debugger.set_trace() msgs.error("Zero value in wavelength array. Uh-oh") # Optimal ivar opt_num = np.sum(mask * model_ivar * prof_img**2, axis=1) ivar_den = np.sum(mask * prof_img, axis=1) opt_ivar = opt_num / (ivar_den + (ivar_den==0.)) # Save specobjs[o].optimal['wave'] = opt_wave.copy()*u.AA # Yes, units enter here specobjs[o].optimal['counts'] = opt_flux.copy() gdiv = (opt_ivar > 0.) & (ivar_den > 0.) opt_var = np.zeros_like(opt_ivar) opt_var[gdiv] = 1./opt_ivar[gdiv] specobjs[o].optimal['var'] = opt_var.copy() #specobjs[o].boxcar['sky'] = skysum # per pixel # Update object model counts_image = np.outer(opt_flux, np.ones(prof_img.shape[1])) obj_model += prof_img * counts_image ''' if 'OPTIMAL' in msgs._debug: debugger.set_trace() debugger.xplot(opt_wave, opt_flux, np.sqrt(opt_var)) ''' # Generate new variance image newvar = arproc.variance_frame(slf, det, sciframe, -1, skyframe=skyframe, objframe=obj_model) # Return return newvar
def obj_profiles(slf, det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, COUNT_LIM=25., doqa=True, pickle_file=None): """ Derive spatial profiles for each object Parameters ---------- slf det specobjs sciframe varframe skyframe crmask scitrace Returns ------- """ ''' FOR DEVELOPING import pickle if False: tilts = slf._tilts[det-1] args = [det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts] msgs.warn("Pickling in the profile code") with open("trc_pickle.p",'wb') as f: pickle.dump(args,f) debugger.set_trace() if pickle_file is not None: f = open(pickle_file,'r') args = pickle.load(f) f.close() det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts = args slf = None else: tilts = slf._tilts[det-1] ''' # Init QA # sigframe = np.sqrt(varframe) # Loop on slits for sl in range(len(specobjs)): # Loop on objects nobj = scitrace[sl]['traces'].shape[1] scitrace[sl]['opt_profile'] = [] msgs.work("Should probably loop on S/N") for o in range(nobj): msgs.info( "Deriving spatial profile of object {0:d}/{1:d} in slit {2:d}/{3:d}" .format(o + 1, nobj, sl + 1, len(specobjs))) # Get object pixels if scitrace[sl]['background'] is None: # The object for all slits is provided in the first extension objreg = np.copy(scitrace[0]['object'][:, :, o]) wzro = np.where(slf._slitpix[det - 1] != sl + 1) objreg[wzro] = 0.0 else: objreg = scitrace[sl]['object'][:, :, o] # Calculate slit image slit_img = artrace.slit_image(slf, det, scitrace[sl], o) #, tilts=tilts) # Object pixels weight = objreg.copy() # Identify good rows gdrow = np.where(specobjs[sl][o].boxcar['counts'] > COUNT_LIM)[0] # Normalized image norm_img = sciframe / np.outer(specobjs[sl][o].boxcar['counts'], np.ones(sciframe.shape[1])) # Eliminate rows with CRs (wipes out boxcar) crspec = np.sum(crmask * weight, axis=1) cr_rows = np.where(crspec > 0)[0] weight[cr_rows, :] = 0. # if len(gdrow) > 100: # Good S/N regime msgs.info("Good S/N for profile") # Eliminate low count regions badrow = np.where( specobjs[sl][o].boxcar['counts'] < COUNT_LIM)[0] weight[badrow, :] = 0. # Extract profile gdprof = (weight > 0) & (sigframe > 0.) & ( ~np.isnan(slit_img) ) # slit_img=nan if the slit is partially on the chip slit_val = slit_img[gdprof] flux_val = norm_img[gdprof] #weight_val = sciframe[gdprof]/sigframe[gdprof] # S/N weight_val = 1. / sigframe[gdprof] # 1/N msgs.work( "Weight by S/N in boxcar extraction? [avoid CRs; smooth?]") # Fit fdict = dict( func=settings.argflag['science']['extraction']['profile'], deg=3, extrap=False) if fdict['func'] == 'gaussian': fdict['deg'] = 2 elif fdict['func'] == 'moffat': fdict['deg'] = 3 else: msgs.error("Not ready for this type of object profile") msgs.work( "Might give our own guess here instead of using default") guess = None # Check if there are enough pixels in the slit to perform fit if slit_val.size <= fdict['deg'] + 1: msgs.warn( "Not enough pixels to determine profile of object={0:s} in slit {1:d}" .format(specobjs[sl][o].idx, sl + 1) + msgs.newline() + "Skipping Optimal") fdict['extrap'] = True scitrace[sl]['opt_profile'].append(copy.deepcopy(fdict)) continue # Fit the profile try: mask, gfit = arutils.robust_polyfit(slit_val, flux_val, fdict['deg'], function=fdict['func'], weights=weight_val, maxone=False, guesses=guess) except RuntimeError: msgs.warn("Bad Profile fit for object={:s}." + msgs.newline() + "Skipping Optimal".format(specobjs[sl][o].idx)) fdict['extrap'] = True scitrace[sl]['opt_profile'].append(copy.deepcopy(fdict)) continue except ValueError: debugger.set_trace() # NaNs in the values? Check msgs.work("Consider flagging/removing CRs here") # Record fdict['param'] = gfit.copy() fdict['mask'] = mask fdict['slit_val'] = slit_val fdict['flux_val'] = flux_val scitrace[sl]['opt_profile'].append(copy.deepcopy(fdict)) specobjs[sl][o].optimal['fwhm'] = fdict['param'][1] # Pixels if msgs._debug['obj_profile']: gdp = mask == 0 mn = np.min(slit_val[gdp]) mx = np.max(slit_val[gdp]) xval = np.linspace(mn, mx, 1000) model = arutils.func_val(gfit, xval, fdict['func']) import matplotlib.pyplot as plt plt.clf() ax = plt.gca() ax.scatter(slit_val[gdp], flux_val[gdp], marker='.', s=0.7, edgecolor='none', facecolor='black') ax.plot(xval, model, 'b') # Gaussian too? if False: fdictg = dict(func='gaussian', deg=2) maskg, gfitg = arutils.robust_polyfit( slit_val, flux_val, fdict['deg'], function=fdictg['func'], weights=weight_val, maxone=False) modelg = arutils.func_val(gfitg, xval, fdictg['func']) ax.plot(xval, modelg, 'r') plt.show() debugger.set_trace() elif len(gdrow) > 10: # msgs.warn( "Low extracted flux for obj={:s} in slit {:d}. Not ready for Optimal" .format(specobjs[sl][o].idx, sl + 1)) scitrace[sl]['opt_profile'].append({}) continue elif len(gdrow) >= 0: # limit is ">= 0" to avoid crash for gdrow=0 msgs.warn( "Low extracted flux for obj={:s} in slit {:d}. Not ready for Optimal" .format(specobjs[sl][o].idx, sl + 1)) scitrace[sl]['opt_profile'].append({}) continue # QA if doqa: #not msgs._debug['no_qa'] and doqa: msgs.info("Preparing QA for spatial object profiles") arqa.obj_profile_qa(slf, specobjs, scitrace, det) return
def obj_profiles(slf, det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, COUNT_LIM=25., pickle_file=None): """ Derive spatial profiles for each object Parameters ---------- slf det specobjs sciframe varframe skyframe crmask scitrace Returns ------- """ ''' FOR DEVELOPING import pickle if False: tilts = slf._tilts[det-1] args = [det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts] msgs.warn("Pickling in the profile code") with open("trc_pickle.p",'wb') as f: pickle.dump(args,f) debugger.set_trace() if pickle_file is not None: f = open(pickle_file,'r') args = pickle.load(f) f.close() det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts = args slf = None else: tilts = slf._tilts[det-1] ''' # Init QA # sigframe = np.sqrt(varframe) # Loop nobj = scitrace['traces'].shape[1] scitrace['opt_profile'] = [] msgs.work("Should probably loop on S/N") for o in range(nobj): # Calculate slit image slit_img = artrace.slit_image(slf, det, scitrace, o)#, tilts=tilts) # Object pixels weight = scitrace['object'][:,:,o].copy() # Identify good rows gdrow = np.where(specobjs[o].boxcar['counts'] > COUNT_LIM)[0] # Normalized image norm_img = sciframe / np.outer(specobjs[o].boxcar['counts'], np.ones(sciframe.shape[1])) # Eliminate rows with CRs (wipes out boxcar) crspec = np.sum(crmask*weight,axis=1) cr_rows = np.where(crspec > 0)[0] weight[cr_rows,:] = 0. # if len(gdrow) > 100: # Good S/N regime msgs.info("Good S/N for profile") # Eliminate low count regions badrow = np.where(specobjs[o].boxcar['counts'] < COUNT_LIM)[0] weight[badrow,:] = 0. # Extract profile gdprof = (weight > 0) & (sigframe > 0.) slit_val = slit_img[gdprof] flux_val = norm_img[gdprof] #weight_val = sciframe[gdprof]/sigframe[gdprof] # S/N weight_val = 1./sigframe[gdprof] # 1/N msgs.work("Weight by S/N in boxcar extraction? [avoid CRs; smooth?]") # Fit fdict = dict(func=slf._argflag['science']['extraction']['profile'], deg=3) if fdict['func'] == 'gaussian': fdict['deg'] = 2 elif fdict['func'] == 'moffat': fdict['deg'] = 3 else: msgs.error("Not ready for this type of object profile") msgs.work("Might give our own guess here instead of using default") guess = None try: mask, gfit = arutils.robust_polyfit(slit_val, flux_val, fdict['deg'], function=fdict['func'], weights=weight_val, maxone=False, guesses=guess) except RuntimeError: msgs.warn("Bad Profile fit for object={:s}. Skipping Optimal".format(specobjs[o].idx)) scitrace['opt_profile'].append(fdict) continue except ValueError: debugger.set_trace() # NaNs in the values? Check msgs.work("Consider flagging/removing CRs here") # Record fdict['param'] = gfit fdict['mask'] = mask fdict['slit_val'] = slit_val fdict['flux_val'] = flux_val scitrace['opt_profile'].append(fdict) if msgs._debug['obj_profile']: # gdp = mask == 0 mn = np.min(slit_val[gdp]) mx = np.max(slit_val[gdp]) xval = np.linspace(mn,mx,1000) model = arutils.func_val(gfit, xval, fdict['func']) import matplotlib.pyplot as plt plt.clf() ax = plt.gca() ax.scatter(slit_val[gdp], flux_val[gdp], marker='.', s=0.7, edgecolor='none', facecolor='black') ax.plot(xval, model, 'b') # Gaussian too? if False: fdictg = dict(func='gaussian', deg=2) maskg, gfitg = arutils.robust_polyfit(slit_val, flux_val, fdict['deg'], function=fdictg['func'], weights=weight_val, maxone=False) modelg = arutils.func_val(gfitg, xval, fdictg['func']) ax.plot(xval, modelg, 'r') plt.show() debugger.set_trace() elif len(gdrow) > 10: # msgs.warn("Low extracted flux for obj={:s}. Not ready for Optimal".format(specobjs[o].idx)) scitrace['opt_profile'].append({}) continue # QA arqa.obj_profile_qa(slf, specobjs, scitrace) return scitrace['opt_profile']