def proc_diff(self, file_list, bg_file_list, reject_cr=True, sigma_clip=False, sigrej=None, maxiters=5): """ Process a list of science images and their background frames Primarily for near-IR reductions Wrapper to proc_sci for Needed in part to set self.sciframe, although I could kludge it another way.. Returns ------- self.sciframe self.rawvarframe self.crmask """ sciimg_sci, sciivar_sci, rn2img_sci, mask_sci, crmask_sci = self.proc_sci( file_list, reject_cr=reject_cr, sigma_clip=sigma_clip, sigrej=sigrej, maxiters=maxiters) sciimg_bg, sciivar_bg, rn2img_bg, mask_bg, crmask_bg = self.proc_sci( bg_file_list, reject_cr=reject_cr, sigma_clip=sigma_clip, sigrej=sigrej, maxiters=maxiters) # Combine the images outmask_comb = (mask_sci == 0) & (mask_bg == 0) sciimg = sciimg_sci - sciimg_bg varcomb = utils.calc_ivar(sciivar_sci) + utils.calc_ivar(sciivar_bg) sciivar = utils.calc_ivar(varcomb) * outmask_comb rn2img = rn2img_sci + rn2img_bg # Now reject CRs again on the differenced image crmask_diff = self.build_crmask(sciimg, self.par['process'], self.det, self.spectrograph, ivar=sciivar, binning=self.binning) # crmask_eff assumes evertything masked in the outmask_comb is a CR in the individual images crmask = crmask_diff | np.invert(outmask_comb) # Create a mask for this image now mask = self.build_mask(sciimg, sciivar, crmask, self.bpm, saturation=self.saturation) return sciimg, sciivar, rn2img, mask, crmask
def test_calc_ivar(): """ Run the parameter setup script """ x = np.array([-1.0, -0.1, 0.0, 0.1, 1.0]) res = utils.calc_ivar(x) assert np.array_equal(res, np.array([0.0, 0.0, 0.0, 10.0, 1.0])) assert np.array_equal(utils.calc_ivar(res), np.array([0.0, 0.0, 0.0, 0.1, 1.0]))
def proc_sci(self, file_list, reject_cr=True, sigma_clip=False, sigrej=None, maxiters=5): """ Process a list of science images This includes stacking the images if there is more than 1 Args: file_list (list): List of filenames for science frames reject_cr (bool, optional): sigrej (int or float, optional): Rejection threshold for sigma clipping. Code defaults to determining this automatically based on the numberr of images provided. maxiters (int, optional): show (bool, optional): Returns: ndarray, ndarray, ndarray, ndarray, ndarray: sciimg sciivar rn2img mask crmask """ nsci = len(file_list) weights = np.ones(nsci)/float(nsci) sciimg_stack, sciivar_stack, rn2img_stack, crmask_stack, mask_stack = \ self.read_stack(file_list, self.bias, self.pixel_flat, self.bpm, self.det, self.par['process'], self.spectrograph, illum_flat=self.illum_flat, reject_cr=reject_cr, binning=self.binning) # ToDO The bitmask is not being properly propagated here! if nsci > 1: sci_list = [sciimg_stack] var_stack = utils.calc_ivar(sciivar_stack) var_list = [var_stack, rn2img_stack] sci_list_out, var_list_out, outmask, nused = coadd2d.weighted_combine( weights, sci_list, var_list, (mask_stack == 0), sigma_clip=sigma_clip, sigma_clip_stack = sciimg_stack, sigrej=sigrej, maxiters=maxiters) sciimg = sci_list_out[0] sciivar = utils.calc_ivar(var_list_out[0]) rn2img = var_list_out[1] # assumes everything masked in the outmask is a CR in the individual images crmask = np.invert(outmask) # Create a mask for this image now mask = self.build_mask(sciimg, sciivar, crmask, self.bpm, saturation=self.saturation, mincounts=self.mincounts) else: mask = mask_stack[0, :, :] crmask = crmask_stack[0, :, :] sciimg = sciimg_stack[0, :, :] sciivar = sciivar_stack[0, :, :] rn2img = rn2img_stack[0, :, :] return sciimg, sciivar, rn2img, mask, crmask
def build_crmask(stack, proc_par, det, spectrograph, ivar=None, binning=None): """ Generate the CR mask frame Wrapper to procimg.lacosmic Parameters ---------- varframe : ndarray, optional Returns ------- self.crmask : ndarray 1. = Masked CR """ # Run LA Cosmic to get the cosmic ray mask varframe = utils.calc_ivar(ivar) saturation = spectrograph.detector[det-1]['saturation'] nonlinear = spectrograph.detector[det-1]['nonlinear'] #sigclip, objlim = spectrograph.get_lacosmics_par(proc_par,binning=binning) crmask = procimg.lacosmic(det, stack, saturation, nonlinear, varframe=varframe, maxiter=proc_par['lamaxiter'], grow=proc_par['grow'], remove_compact_obj=proc_par['rmcompact'], sigclip=proc_par['sigclip'], sigfrac=proc_par['sigfrac'], objlim=proc_par['objlim']) # Return return crmask
def read_lris_stack(): stackfile = '/Users/joe/REDUX/lris_redux/Nov_2004/Final/SDSSJ073522.43+295710.1_N.fits' hdu = fits.open(stackfile) flux_ref = hdu[0].data sig_ref = hdu[1].data ivar_ref = utils.calc_ivar(sig_ref) mask_ref = (ivar_ref > 0.0) wave_ref = hdu[2].data infiles = [ '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0063.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0064.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0065.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0066.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0219.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0220.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0221.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0222.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0063.fits.gz' ] nfiles = len(infiles) objid = 0 for idx, file in enumerate(infiles): obj = Table.read(file, hdu=5) flux = obj[objid]['FLUX_OPT'] ivar = obj[objid]['IVAR_OPT'] wave = obj[objid]['WAVE_OPT'] if idx == 0: nspec = flux.size flux_arr = np.zeros((nfiles, nspec)) wave_arr = np.zeros((nfiles, nspec)) ivar_arr = np.zeros((nfiles, nspec)) flux_arr[idx, :] = flux ivar_arr[idx, :] = ivar wave_arr[idx, :] = wave mask_arr = (ivar_arr > 0.0) return wave_arr, flux_arr, ivar_arr, mask_arr #plt.plot(wave_ref, flux_ref, drawstyle='steps-mid') #for ii in range(nfiles): # plt.plot(wave_arr[:,ii], flux_arr[:,ii],drawstyle='steps-mid') #flux_inter, ivar_inter, mask_inter = interp_spec(wave_ref, wave_arr, flux_arr, ivar_arr, mask_arr) #idx = 5 #wave = wave_ref #flux = flux_inter[idx, :] #ivar = ivar_inter[idx, :] #mask = ivar > 0 #norder = 3 #ymult,flux_rescale, ivar_rescale, outmask = solve_poly_ratio(wave, flux, ivar, flux_ref, ivar_ref, norder, # mask=mask, mask_ref=mask_ref, debug=True)
def apply_sens_tell_spec(wave, counts, ivar, sensfunc, airmass, exptime, mask=None, extinct_correct=True, telluric=None, longitude=None, latitude=None, debug=False): if mask is None: mask = ivar > 0.0 # Did the user request a telluric correction from the same file? if telluric is not None: # This assumes there is a separate telluric key in this dict. msgs.info('Applying telluric correction') sensfunc = sensfunc*(telluric > 1e-10)/(telluric + (telluric < 1e-10)) if extinct_correct: if longitude is None or latitude is None: msgs.error('You must specify longitude and latitude if we are extinction correcting') # Apply Extinction if optical bands msgs.info("Applying extinction correction") msgs.warn("Extinction correction applyed only if the spectra covers <10000Ang.") extinct = load_extinction_data(longitude,latitude) ext_corr = extinction_correction(wave* units.AA, airmass, extinct) senstot = sensfunc * ext_corr else: senstot = sensfunc.copy() flam = counts * senstot/ exptime flam_ivar = ivar / (senstot / exptime) **2 # Mask bad pixels msgs.info(" Masking bad pixels") outmask = mask & (senstot>0.) # debug if debug: wave_mask = wave > 1.0 fig = plt.figure(figsize=(12, 8)) ymin, ymax = coadd1d.get_ylim(flam, flam_ivar, outmask) plt.plot(wave[wave_mask], flam[wave_mask], color='black', drawstyle='steps-mid', zorder=1, alpha=0.8) plt.plot(wave[wave_mask], np.sqrt(utils.calc_ivar(flam_ivar[wave_mask])), zorder=2, color='red', alpha=0.7, drawstyle='steps-mid', linestyle=':') plt.ylim([ymin,ymax]) plt.xlim([wave[wave_mask].min(),wave[wave_mask].max()]) plt.xlabel('Wavelength (Angstrom)') plt.ylabel('Flux') plt.show() return flam, flam_ivar, outmask
def read_stack(cls, files, bias, pixel_flat, bpm, det, proc_par, spectrograph, illum_flat=None, reject_cr=False, binning=None): """ Utility function for reading in image stacks using ProcessImages Parameters file_list: bias: pixel_flat: bpm: illum_flat: Returns: """ nfiles = len(files) for ifile in range(nfiles): this_proc = ProcessImages(spectrograph, proc_par, [files[ifile]], det=det) # TODO I think trim should be hard wired, and am not letting it be a free parameter sciimg = this_proc.process(bias_subtract=bias,pixel_flat=pixel_flat, illum_flat=illum_flat, bpm=bpm, apply_gain=True, trim=True) # Allocate the images if ifile == 0: # numpy is row major so stacking will be fastest with nfiles as the first dimensions shape = (nfiles, sciimg.shape[0],sciimg.shape[1]) sciimg_stack = np.zeros(shape) sciivar_stack = np.zeros(shape) rn2img_stack = np.zeros(shape) crmask_stack = np.zeros(shape,dtype=bool) mask_stack = np.zeros(shape,this_proc.bitmask.minimum_dtype(asuint=True)) # Construct raw variance image rawvarframe = this_proc.build_rawvarframe(trim=True) # Mask cosmic rays sciivar_stack[ifile,:,:] = utils.calc_ivar(rawvarframe) if reject_cr: crmask_stack[ifile,:,:] = this_proc.build_crmask(sciimg, proc_par, det, spectrograph, ivar=sciivar_stack[ifile,:,:], binning=binning) sciimg_stack[ifile,:,:] = sciimg # Build read noise squared image rn2img_stack[ifile,:,:] = this_proc.build_rn2img() # Final mask for this image mask_stack[ifile,:,:] = this_proc.build_mask(sciimg, sciivar_stack[ifile,:,:], crmask_stack[ifile,:,:], bpm, saturation = spectrograph.detector[det - 1]['saturation'], mincounts = spectrograph.detector[det - 1]['mincounts']) return sciimg_stack, sciivar_stack, rn2img_stack, crmask_stack, mask_stack
def read_lris_stack(): stackfile = '/Users/joe/REDUX/lris_redux/Nov_2004/Final/SDSSJ073522.43+295710.1_N.fits' hdu = fits.open(stackfile) flux_ref = hdu[0].data sig_ref = hdu[1].data ivar_ref = utils.calc_ivar(sig_ref) mask_ref = (ivar_ref > 0.0) wave_ref = hdu[2].data infiles = [ '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0063.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0064.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0065.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0066.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0219.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0220.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0221.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0222.fits.gz', '/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0063.fits.gz' ] nfiles = len(infiles) objid = 0 for idx, file in enumerate(infiles): obj = Table.read(file, hdu=5) flux = obj[objid]['FLUX_OPT'] ivar = obj[objid]['IVAR_OPT'] wave = obj[objid]['WAVE_OPT'] if idx == 0: nspec = flux.size flux_arr = np.zeros((nspec, nfiles)) wave_arr = np.zeros((nspec, nfiles)) ivar_arr = np.zeros((nspec, nfiles)) flux_arr[:, idx] = np.flip(flux) ivar_arr[:, idx] = np.flip(ivar) wave_arr[:, idx] = np.flip(wave) mask_arr = (ivar_arr > 0.0) return wave_arr, flux_arr, ivar_arr, mask_arr
def coadd2d(trace_stack, sciimg_stack, sciivar_stack, skymodel_stack, inmask_stack, tilts_stack, waveimg_stack, thismask_stack, weights=None, loglam_grid=None, wave_grid=None): """ Construct a 2d co-add of a stack of PypeIt spec2d reduction outputs. Slits are 'rectified' onto a spatial and spectral grid, which encompasses the spectral and spatial coverage of the image stacks. The rectification uses nearest grid point interpolation to avoid covariant errors. Dithering is supported as all images are centered relative to a set of reference traces in trace_stack. Args: trace_stack (`numpy.ndarray`_): Stack of reference traces about which the images are rectified and coadded. If the images were not dithered then this reference trace can simply be the center of the slit:: slitcen = (slit_left + slit_righ)/2 If the images were dithered, then this object can either be the slitcen appropriately shifted with the dither pattern, or it could the trace of the object of interest in each exposure determined by running PypeIt on the individual images. Shape is (nimgs, nspec). sciimg_stack (`numpy.ndarray`_): Stack of science images. Shape is (nimgs, nspec, nspat). sciivar_stack (`numpy.ndarray`_): Stack of inverse variance images. Shape is (nimgs, nspec, nspat). skymodel_stack (`numpy.ndarray`_): Stack of the model sky. Shape is (nimgs, nspec, nspat). inmask_stack (`numpy.ndarray`_): Boolean array with the input masks for each image; `True` values are *good*, `False` values are *bad*. Shape is (nimgs, nspec, nspat). tilts_stack (`numpy.ndarray`_): Stack of the wavelength tilts traces. Shape is (nimgs, nspec, nspat). waveimg_stack (`numpy.ndarray`_): Stack of the wavelength images. Shape is (nimgs, nspec, nspat). thismask_stack (`numpy.ndarray`_): Boolean array with the masks indicating which pixels are on the slit in question. `True` values are on the slit; `False` values are off the slit. Shape is (nimgs, nspec, nspat). weights (`numpy.ndarray`_, optional): The weights used when combining the rectified images (see :func:`weighted_combine`). If no weights are provided, uniform weighting is used. Weights are broadast to the correct size of the image stacks (see :func:`broadcast_weights`), as necessary. Shape must be (nimgs,), (nimgs, nspec), or (nimgs, nspec, nspat). loglam_grid (`numpy.ndarray`_, optional): Wavelength grid in log10(wave) onto which the image stacks will be rectified. The code will automatically choose the subset of this grid encompassing the wavelength coverage of the image stacks provided (see :func:`waveimg_stack`). Either `loglam_grid` or `wave_grid` must be provided. wave_grid (`numpy.ndarray`_, optional): Same as `loglam_grid` but in angstroms instead of log(angstroms). (TODO: Check units...) Returns: TODO: This needs to be updated. (sciimg, sciivar, imgminsky, outmask, nused, tilts, waveimg, dspat, thismask, tslits_dict) sciimg: float ndarray shape = (nspec_coadd, nspat_coadd) Rectified and coadded science image sciivar: float ndarray shape = (nspec_coadd, nspat_coadd) Rectified and coadded inverse variance image with correct error propagation imgminsky: float ndarray shape = (nspec_coadd, nspat_coadd) Rectified and coadded sky subtracted image outmask: bool ndarray shape = (nspec_coadd, nspat_coadd) Output mask for rectified and coadded images. True = Good, False=Bad. nused: int ndarray shape = (nspec_coadd, nspat_coadd) Image of integers indicating the number of images from the image stack that contributed to each pixel tilts: float ndarray shape = (nspec_coadd, nspat_coadd) The averaged tilts image corresponding to the rectified and coadded data. waveimg: float ndarray shape = (nspec_coadd, nspat_coadd) The averaged wavelength image corresponding to the rectified and coadded data. dspat: float ndarray shape = (nspec_coadd, nspat_coadd) The average spatial offsets in pixels from the reference trace trace_stack corresponding to the rectified and coadded data. thismask: bool ndarray shape = (nspec_coadd, nspat_coadd) Output mask for rectified and coadded images. True = Good, False=Bad. This image is trivial, and is simply an image of True values the same shape as the rectified and coadded data. tslits_dict: dict tslits_dict dictionary containing the information about the slits boundaries. The slit boundaries are trivial and are simply vertical traces at 0 and nspat_coadd-1. """ nimgs, nspec, nspat = sciimg_stack.shape # Determine the wavelength grid that we will use for the current slit/order # TODO This cut on waveimg_stack should not be necessary wavemask = thismask_stack & (waveimg_stack > 1.0) wave_lower = waveimg_stack[wavemask].min() wave_upper = waveimg_stack[wavemask].max() if loglam_grid is not None: ind_lower, ind_upper = get_wave_ind(loglam_grid, np.log10(wave_lower), np.log10(wave_upper)) loglam_bins = loglam_grid[ind_lower:ind_upper + 1] wave_bins = np.power(10.0, loglam_bins) elif wave_grid is not None: ind_lower, ind_upper = get_wave_ind(wave_grid, wave_lower, wave_upper) wave_bins = wave_grid[ind_lower:ind_upper + 1] loglam_bins = np.log10(wave_bins) else: msgs.error('You must input either a uniformly space loglam grid or wave grid') if weights is None: msgs.info('No weights were provided. Using uniform weights.') weights = np.ones(nimgs)/float(nimgs) weights_stack = broadcast_weights(weights, sciimg_stack.shape) # Create the slit_cen_stack and determine the minimum and maximum # spatial offsets that we need to cover to determine the spatial # bins spat_img = np.outer(np.ones(nspec), np.arange(nspat)) dspat_stack = np.zeros_like(sciimg_stack) spat_min = np.inf spat_max = -np.inf for img in range(nimgs): # center of the slit replicated spatially slit_cen_img = np.outer(trace_stack[img, :], np.ones(nspat)) dspat_iexp = (spat_img - slit_cen_img) dspat_stack[img, :, :] = dspat_iexp thismask_now = thismask_stack[img, :, :] spat_min = np.fmin(spat_min, dspat_iexp[thismask_now].min()) spat_max = np.fmax(spat_max, dspat_iexp[thismask_now].max()) spat_min_int = int(np.floor(spat_min)) spat_max_int = int(np.ceil(spat_max)) dspat_bins = np.arange(spat_min_int, spat_max_int + 1, 1) sci_list = [weights_stack, sciimg_stack, sciimg_stack - skymodel_stack, tilts_stack, waveimg_stack, dspat_stack] var_list = [utils.calc_ivar(sciivar_stack)] sci_list_rebin, var_list_rebin, norm_rebin_stack, nsmp_rebin_stack \ = rebin2d(wave_bins, dspat_bins, waveimg_stack, dspat_stack, thismask_stack, inmask_stack, sci_list, var_list) # Now compute the final stack with sigma clipping sigrej = 3.0 maxiters = 10 # sci_list_rebin[0] = rebinned weights image stack # sci_list_rebin[1:] = stacks of images that we want to weighted combine # sci_list_rebin[2] = rebinned sciimg-sky_model images that we used for the sigma clipping sci_list_out, var_list_out, outmask, nused \ = weighted_combine(sci_list_rebin[0], sci_list_rebin[1:], var_list_rebin, norm_rebin_stack != 0, sigma_clip=True, sigma_clip_stack=sci_list_rebin[2], sigrej=sigrej, maxiters=maxiters) sciimg, imgminsky, tilts, waveimg, dspat = sci_list_out sciivar = utils.calc_ivar(var_list_out[0]) # Compute the midpoints vectors, and lower/upper bins of the rectified image wave_mid = ((wave_bins + np.roll(wave_bins,1))/2.0)[1:] wave_min = wave_bins[:-1] wave_max = wave_bins[1:] dspat_mid = ((dspat_bins + np.roll(dspat_bins,1))/2.0)[1:] # Interpolate the dspat images wherever the coadds are masked # because a given pixel was not sampled. This is done because the # dspat image is not allowed to have holes if it is going to work # with local_skysub_extract nspec_coadd, nspat_coadd = imgminsky.shape spat_img_coadd, spec_img_coadd = np.meshgrid(np.arange(nspat_coadd), np.arange(nspec_coadd)) if np.any(np.invert(outmask)): points_good = np.stack((spec_img_coadd[outmask], spat_img_coadd[outmask]), axis=1) points_bad = np.stack((spec_img_coadd[np.invert(outmask)], spat_img_coadd[np.invert(outmask)]), axis=1) values_dspat = dspat[outmask] dspat_bad = scipy.interpolate.griddata(points_good, values_dspat, points_bad, method='cubic') dspat[np.invert(outmask)] = dspat_bad # Points outside the convex hull of the data are set to nan. We # identify those and simply assume them values from the # dspat_img_fake, which is what dspat would be on a regular # perfectly rectified image grid. nanpix = np.isnan(dspat) if np.any(nanpix): dspat_img_fake = spat_img_coadd + dspat_mid[0] dspat[nanpix] = dspat_img_fake[nanpix] return dict(wave_bins=wave_bins, dspat_bins=dspat_bins, wave_mid=wave_mid, wave_min=wave_min, wave_max=wave_max, dspat_mid=dspat_mid, sciimg=sciimg, sciivar=sciivar, imgminsky=imgminsky, outmask=outmask, nused=nused, tilts=tilts, waveimg=waveimg, dspat=dspat, nspec=imgminsky.shape[0], nspat=imgminsky.shape[1])
objminsky = hdu[0].data - hdu[2].data ivar = hdu[4].data #ivar = utils.calc_ivar(var) mask = (ivar > 0.0) slit_left = readsav(user + 'Dropbox/hires_fndobj/MAGE/left_edge.sav',python_dict=False)['left_edge'].T slit_righ = readsav(user + 'Dropbox/hires_fndobj/MAGE/right_edge.sav',python_dict=False)['right_edge'].T plate_scale = 0.3 hdu_std = fits.open(user + 'Dropbox/hires_fndobj/MAGE/ObjStr1046.fits') std_trace = hdu_std[1].data['trace'].T # Standard is the ObjStr1046.fits exten = 1 elif spectro == 'ESI': # ESI hdu = fits.open(user + '/Dropbox/Cowie_2002-02-17/Final/fringe_ES.20020217.35453.fits.gz') sciimg = hdu[0].data var = hdu[1].data ivar = utils.calc_ivar(var) mask = (var > 0.0) skyimg = hdu[2].data objminsky = sciimg - skyimg hdu_sedg = fits.open(user + '/Dropbox/Cowie_2002-02-17/Flats/SEdgECH75_1x1.fits') data = hdu_sedg[0].data slit_left = data[0,:,:].T slit_righ = data[1,:,:].T plate_scale = 0.149 hdu_std =fits.open(user + '/Dropbox/Cowie_2002-02-17/Extract/Obj_ES.20020217.20037.fits.gz') std_trace = hdu_std[1].data['trace'][:,0:slit_left.shape[0]].T std_trace[:,-1] = std_trace[:,-2]+ (np.median(std_trace[:200,-1])-np.median(std_trace[:200,-2])) #The structure is exten = 1, and the xpos,ypos are the standard trace elif spectro == 'NIRES': from linetools import utils as ltu jdict = ltu.loadjson(user + '/Dropbox/hires_fndobj/tilt_nires.json')
def ech_objfind(image, ivar, ordermask, slit_left, slit_righ, inmask=None, plate_scale=0.2, npca=2, ncoeff=5, min_snr=0.0, nabove_min_snr=0, pca_percentile=20.0, snr_pca=3.0, box_radius=2.0, show_peaks=False, show_fits=False, show_trace=False): if inmask is None: inmask = (ordermask > 0) frameshape = image.shape nspec = frameshape[0] norders = slit_left.shape[1] if isinstance(plate_scale, (float, int)): plate_scale_ord = np.full( norders, plate_scale) # 0.12 binned by 3 spatially for HIRES elif isinstance(plate_scale, (np.ndarray, list, tuple)): if len(plate_scale) == norders: plate_scale_ord = plate_scale elif len(plate_scale) == 1: plate_scale_ord = np.full(norders, plate_scale[0]) else: msgs.error( 'Invalid size for plate_scale. It must either have one element or norders elements' ) else: msgs.error('Invalid type for plate scale') specmid = nspec // 2 slit_width = slit_righ - slit_left spec_vec = np.arange(nspec) slit_spec_pos = nspec / 2.0 slit_spat_pos = np.zeros((norders, 2)) for iord in range(norders): slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec, slit_left[:, iord]), np.interp(slit_spec_pos, spec_vec, slit_righ[:, iord])) # Loop over orders and find objects sobjs = specobjs.SpecObjs() show_peaks = True show_fits = True # ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx! for iord in range(norders): msgs.info('Finding objects on slit # {:d}'.format(iord + 1)) thismask = ordermask == (iord + 1) inmask_iord = inmask & thismask specobj_dict = { 'setup': 'HIRES', 'slitid': iord + 1, 'scidx': 0, 'det': 1, 'objtype': 'science' } sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \ extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks, show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0) # ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord for spec in sobjs_slit: spec.ech_orderindx = iord sobjs.add_sobj(sobjs_slit) nfound = len(sobjs) # Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0" FOFSEP = 1.0 # separation of FOF algorithm in arcseconds FOF_frac = FOFSEP / (np.median(slit_width) * np.median(plate_scale_ord)) # Feige: made the code also works for only one object found in one order # Run the FOF. We use fake coordinaes fracpos = sobjs.spat_fracpos ra_fake = fracpos / 1000.0 # Divide all angles by 1000 to make geometry euclidian dec_fake = 0.0 * fracpos if nfound > 1: (ingroup, multgroup, firstgroup, nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac / 1000.0) group = ingroup.copy() uni_group, uni_ind = np.unique(group, return_index=True) nobj = len(uni_group) msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects') elif nfound == 1: group = np.zeros(1, dtype='int') uni_group, uni_ind = np.unique(group, return_index=True) nobj = len(group) msgs.warn('Only find one object no FOF matching is needed') gfrac = np.zeros(nfound) for jj in range(nobj): this_group = group == uni_group[jj] gfrac[this_group] = np.median(fracpos[this_group]) uni_frac = gfrac[uni_ind] sobjs_align = sobjs.copy() # Now fill in the missing objects and their traces for iobj in range(nobj): for iord in range(norders): # Is there an object on this order that grouped into the current group in question? on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx == iord) if not np.any(on_slit): # Add this to the sobjs_align, and assign required tags thisobj = specobjs.SpecObj(frameshape, slit_spat_pos[iord, :], slit_spec_pos, det=sobjs_align[0].det, setup=sobjs_align[0].setup, slitid=(iord + 1), scidx=sobjs_align[0].scidx, objtype=sobjs_align[0].objtype) thisobj.ech_orderindx = iord thisobj.spat_fracpos = uni_frac[iobj] thisobj.trace_spat = slit_left[:, iord] + slit_width[:, iord] * uni_frac[ iobj] # new trace thisobj.trace_spec = spec_vec thisobj.spat_pixpos = thisobj.trace_spat[specmid] thisobj.set_idx() # Use the real detections of this objects for the FWHM this_group = group == uni_group[iobj] # Assign to the fwhm of the nearest detected order imin = np.argmin( np.abs(sobjs_align[this_group].ech_orderindx - iord)) thisobj.fwhm = sobjs_align[imin].fwhm thisobj.maskwidth = sobjs_align[imin].maskwidth thisobj.ech_fracpos = uni_frac[iobj] thisobj.ech_group = uni_group[iobj] thisobj.ech_usepca = True sobjs_align.add_sobj(thisobj) group = np.append(group, uni_group[iobj]) gfrac = np.append(gfrac, uni_frac[iobj]) else: # ToDo fix specobjs to get rid of these crappy loops! for spec in sobjs_align[on_slit]: spec.ech_fracpos = uni_frac[iobj] spec.ech_group = uni_group[iobj] spec.ech_usepca = False # Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order # respectively sobjs_sort = specobjs.SpecObjs() for iobj in range(nobj): this_group = group == uni_group[iobj] this_sobj = sobjs_align[this_group] sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)]) # Loop over the objects and perform a quick and dirty extraction to assess S/N. varimg = utils.calc_ivar(ivar) flux_box = np.zeros((nspec, norders, nobj)) ivar_box = np.zeros((nspec, norders, nobj)) mask_box = np.zeros((nspec, norders, nobj)) SNR_arr = np.zeros((norders, nobj)) for iobj in range(nobj): for iord in range(norders): indx = (sobjs_sort.ech_group == uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord) spec = sobjs_sort[indx] thismask = ordermask == (iord + 1) inmask_iord = inmask & thismask box_rad_pix = box_radius / plate_scale_ord[iord] flux_tmp = extract.extract_boxcar(image * inmask_iord, spec.trace_spat, box_rad_pix, ycen=spec.trace_spec) var_tmp = extract.extract_boxcar(varimg * inmask_iord, spec.trace_spat, box_rad_pix, ycen=spec.trace_spec) ivar_tmp = utils.calc_ivar(var_tmp) pixtot = extract.extract_boxcar(ivar * 0 + 1.0, spec.trace_spat, box_rad_pix, ycen=spec.trace_spec) mask_tmp = (extract.extract_boxcar(ivar * inmask_iord == 0.0, spec.trace_spat, box_rad_pix, ycen=spec.trace_spec) != pixtot) flux_box[:, iord, iobj] = flux_tmp * mask_tmp ivar_box[:, iord, iobj] = np.fmax(ivar_tmp * mask_tmp, 0.0) mask_box[:, iord, iobj] = mask_tmp (mean, med_sn, stddev) = sigma_clipped_stats( flux_box[mask_tmp, iord, iobj] * np.sqrt(ivar_box[mask_tmp, iord, iobj]), sigma_lower=5.0, sigma_upper=5.0) SNR_arr[iord, iobj] = med_sn # Purge objects with low SNR and that don't show up in enough orders keep_obj = np.zeros(nobj, dtype=bool) sobjs_trim = specobjs.SpecObjs() uni_group_trim = np.array([], dtype=int) uni_frac_trim = np.array([], dtype=float) for iobj in range(nobj): if (np.sum(SNR_arr[:, iobj] > min_snr) >= nabove_min_snr): keep_obj[iobj] = True ikeep = sobjs_sort.ech_group == uni_group[iobj] sobjs_trim.add_sobj(sobjs_sort[ikeep]) uni_group_trim = np.append(uni_group_trim, uni_group[iobj]) uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj]) else: msgs.info( 'Purging object #{:d}'.format(iobj) + ' which does not satisfy min_snr > {:5.2f}'.format(min_snr) + ' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) + ' orders') nobj_trim = np.sum(keep_obj) if nobj_trim == 0: return specobjs.SpecObjs() SNR_arr_trim = SNR_arr[:, keep_obj] # Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA for iobj in range(nobj_trim): SNR_now = SNR_arr_trim[:, iobj] indx = (sobjs_trim.ech_group == uni_group_trim[iobj]) # PCA interp/extrap if: # (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca) # OR # (if this order was not originally traced by the object finding, see above) usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) & (SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca # ToDo fix specobjs to get rid of these crappy loops! for iord, spec in enumerate(sobjs_trim[indx]): spec.ech_usepca = usepca[iord] if usepca[iord]: msgs.info('Using PCA to predict trace for object #{:d}'.format( iobj) + ' on order #{:d}'.format(iord)) sobjs_final = sobjs_trim.copy() # Loop over the objects one by one and adjust/predict the traces npoly_cen = 3 pca_fits = np.zeros((nspec, norders, nobj_trim)) for iobj in range(nobj_trim): igroup = sobjs_final.ech_group == uni_group_trim[iobj] # PCA predict the masked orders which were not traced pca_fits[:, :, iobj] = pca_trace((sobjs_final[igroup].trace_spat).T, usepca=None, npca=npca, npoly_cen=npoly_cen) # usepca = sobjs_final[igroup].ech_usepca, # Perform iterative flux weighted centroiding using new PCA predictions xinit_fweight = pca_fits[:, :, iobj].copy() inmask_now = inmask & (ordermask > 0) xfit_fweight = extract.iter_tracefit(image, xinit_fweight, ncoeff, inmask=inmask_now, show_fits=show_fits) # Perform iterative Gaussian weighted centroiding xinit_gweight = xfit_fweight.copy() xfit_gweight = extract.iter_tracefit(image, xinit_gweight, ncoeff, inmask=inmask_now, gweight=True, show_fits=show_fits) # Assign the new traces for iord, spec in enumerate(sobjs_final[igroup]): spec.trace_spat = xfit_gweight[:, iord] spec.spat_pixpos = spec.trace_spat[specmid] # Set the IDs sobjs_final.set_idx() if show_trace: viewer, ch = ginga.show_image(objminsky * (ordermask > 0)) for iobj in range(nobj_trim): for iord in range(norders): ginga.show_trace(viewer, ch, pca_fits[:, iord, iobj], str(uni_frac[iobj]), color='yellow') for spec in sobjs_trim: color = 'green' if spec.ech_usepca else 'magenta' ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color) #for spec in sobjs_final: # color = 'red' if spec.ech_usepca else 'green' # ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color) return sobjs_final
hdu = fits.open('/Users/feige/Dropbox/hires_fndobj/f_hires0184G.fits.gz') objminsky = hdu[2].data ivar = hdu[1].data mask = (ivar > 0.0) order_str = Table.read('/Users/feige/Dropbox/hires_fndobj/OStr_G_04.fits') slit_left = (order_str['LHEDG']).T slit_righ = (order_str['RHEDG']).T plate_scale = 0.36 elif spectro == 'ESI': # ESI hdu = fits.open( '/Users/feige/Dropbox/Cowie_2002-02-17/Final/fringe_ES.20020217.35453.fits.gz' ) sciimg = hdu[0].data var = hdu[1].data ivar = utils.calc_ivar(var) mask = (var > 0.0) skyimg = hdu[2].data objminsky = sciimg - skyimg hdu_sedg = fits.open( '/Users/feige/Dropbox/Cowie_2002-02-17/Flats/SEdgECH75_1x1.fits') data = hdu_sedg[0].data slit_left = data[0, :, :].T slit_righ = data[1, :, :].T plate_scale = 0.149 elif spectro == 'NIRES': from linetools import utils as ltu jdict = ltu.loadjson('/Users/feige/Dropbox/hires_fndobj/tilt_nires.json') slit_left = np.array(jdict['lcen']) slit_righ = np.array(jdict['rcen']) hdu = fits.open(
def ech_objfind(image, ivar, ordermask, slit_left, slit_righ,inmask=None,plate_scale=0.2,npca=2,ncoeff = 5,min_snr=0.0,nabove_min_snr=0, pca_percentile=20.0,snr_pca=3.0,box_radius=2.0,show_peaks=False,show_fits=False,show_trace=False): if inmask is None: inmask = (ordermask > 0) frameshape = image.shape nspec = frameshape[0] norders = slit_left.shape[1] if isinstance(plate_scale,(float, int)): plate_scale_ord = np.full(norders, plate_scale) # 0.12 binned by 3 spatially for HIRES elif isinstance(plate_scale,(np.ndarray, list, tuple)): if len(plate_scale) == norders: plate_scale_ord = plate_scale elif len(plate_scale) == 1: plate_scale_ord = np.full(norders, plate_scale[0]) else: msgs.error('Invalid size for plate_scale. It must either have one element or norders elements') else: msgs.error('Invalid type for plate scale') specmid = nspec // 2 slit_width = slit_righ - slit_left spec_vec = np.arange(nspec) slit_spec_pos = nspec/2.0 slit_spat_pos = np.zeros((norders, 2)) for iord in range(norders): slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec, slit_left[:,iord]), np.interp(slit_spec_pos, spec_vec, slit_righ[:,iord])) # Loop over orders and find objects sobjs = specobjs.SpecObjs() show_peaks=True show_fits=True # ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx! for iord in range(norders): msgs.info('Finding objects on slit # {:d}'.format(iord + 1)) thismask = ordermask == (iord + 1) inmask_iord = inmask & thismask specobj_dict = {'setup': 'HIRES', 'slitid': iord + 1, 'scidx': 0,'det': 1, 'objtype': 'science'} sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \ extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks, show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0) # ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord for spec in sobjs_slit: spec.ech_orderindx = iord sobjs.add_sobj(sobjs_slit) nfound = len(sobjs) # Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0" FOFSEP = 1.0 # separation of FOF algorithm in arcseconds FOF_frac = FOFSEP/(np.median(slit_width)*np.median(plate_scale_ord)) # Feige: made the code also works for only one object found in one order # Run the FOF. We use fake coordinaes fracpos = sobjs.spat_fracpos ra_fake = fracpos/1000.0 # Divide all angles by 1000 to make geometry euclidian dec_fake = 0.0*fracpos if nfound>1: (ingroup, multgroup, firstgroup, nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac/1000.0) group = ingroup.copy() uni_group, uni_ind = np.unique(group, return_index=True) nobj = len(uni_group) msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects') elif nfound==1: group = np.zeros(1,dtype='int') uni_group, uni_ind = np.unique(group, return_index=True) nobj = len(group) msgs.warn('Only find one object no FOF matching is needed') gfrac = np.zeros(nfound) for jj in range(nobj): this_group = group == uni_group[jj] gfrac[this_group] = np.median(fracpos[this_group]) uni_frac = gfrac[uni_ind] sobjs_align = sobjs.copy() # Now fill in the missing objects and their traces for iobj in range(nobj): for iord in range(norders): # Is there an object on this order that grouped into the current group in question? on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx == iord) if not np.any(on_slit): # Add this to the sobjs_align, and assign required tags thisobj = specobjs.SpecObj(frameshape, slit_spat_pos[iord,:], slit_spec_pos, det = sobjs_align[0].det, setup = sobjs_align[0].setup, slitid = (iord + 1), scidx = sobjs_align[0].scidx, objtype=sobjs_align[0].objtype) thisobj.ech_orderindx = iord thisobj.spat_fracpos = uni_frac[iobj] thisobj.trace_spat = slit_left[:,iord] + slit_width[:,iord]*uni_frac[iobj] # new trace thisobj.trace_spec = spec_vec thisobj.spat_pixpos = thisobj.trace_spat[specmid] thisobj.set_idx() # Use the real detections of this objects for the FWHM this_group = group == uni_group[iobj] # Assign to the fwhm of the nearest detected order imin = np.argmin(np.abs(sobjs_align[this_group].ech_orderindx - iord)) thisobj.fwhm = sobjs_align[imin].fwhm thisobj.maskwidth = sobjs_align[imin].maskwidth thisobj.ech_fracpos = uni_frac[iobj] thisobj.ech_group = uni_group[iobj] thisobj.ech_usepca = True sobjs_align.add_sobj(thisobj) group = np.append(group, uni_group[iobj]) gfrac = np.append(gfrac, uni_frac[iobj]) else: # ToDo fix specobjs to get rid of these crappy loops! for spec in sobjs_align[on_slit]: spec.ech_fracpos = uni_frac[iobj] spec.ech_group = uni_group[iobj] spec.ech_usepca = False # Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order # respectively sobjs_sort = specobjs.SpecObjs() for iobj in range(nobj): this_group = group == uni_group[iobj] this_sobj = sobjs_align[this_group] sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)]) # Loop over the objects and perform a quick and dirty extraction to assess S/N. varimg = utils.calc_ivar(ivar) flux_box = np.zeros((nspec, norders, nobj)) ivar_box = np.zeros((nspec, norders, nobj)) mask_box = np.zeros((nspec, norders, nobj)) SNR_arr = np.zeros((norders, nobj)) for iobj in range(nobj): for iord in range(norders): indx = (sobjs_sort.ech_group == uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord) spec = sobjs_sort[indx] thismask = ordermask == (iord + 1) inmask_iord = inmask & thismask box_rad_pix = box_radius/plate_scale_ord[iord] flux_tmp = extract.extract_boxcar(image*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) var_tmp = extract.extract_boxcar(varimg*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) ivar_tmp = utils.calc_ivar(var_tmp) pixtot = extract.extract_boxcar(ivar*0 + 1.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) mask_tmp = (extract.extract_boxcar(ivar*inmask_iord == 0.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) != pixtot) flux_box[:,iord,iobj] = flux_tmp*mask_tmp ivar_box[:,iord,iobj] = np.fmax(ivar_tmp*mask_tmp,0.0) mask_box[:,iord,iobj] = mask_tmp (mean, med_sn, stddev) = sigma_clipped_stats(flux_box[mask_tmp,iord,iobj]*np.sqrt(ivar_box[mask_tmp,iord,iobj]), sigma_lower=5.0,sigma_upper=5.0) SNR_arr[iord,iobj] = med_sn # Purge objects with low SNR and that don't show up in enough orders keep_obj = np.zeros(nobj,dtype=bool) sobjs_trim = specobjs.SpecObjs() uni_group_trim = np.array([],dtype=int) uni_frac_trim = np.array([],dtype=float) for iobj in range(nobj): if (np.sum(SNR_arr[:,iobj] > min_snr) >= nabove_min_snr): keep_obj[iobj] = True ikeep = sobjs_sort.ech_group == uni_group[iobj] sobjs_trim.add_sobj(sobjs_sort[ikeep]) uni_group_trim = np.append(uni_group_trim, uni_group[iobj]) uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj]) else: msgs.info('Purging object #{:d}'.format(iobj) + ' which does not satisfy min_snr > {:5.2f}'.format(min_snr) + ' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) + ' orders') nobj_trim = np.sum(keep_obj) if nobj_trim == 0: return specobjs.SpecObjs() SNR_arr_trim = SNR_arr[:,keep_obj] # Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA for iobj in range(nobj_trim): SNR_now = SNR_arr_trim[:,iobj] indx = (sobjs_trim.ech_group == uni_group_trim[iobj]) # PCA interp/extrap if: # (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca) # OR # (if this order was not originally traced by the object finding, see above) usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) & (SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca # ToDo fix specobjs to get rid of these crappy loops! for iord, spec in enumerate(sobjs_trim[indx]): spec.ech_usepca = usepca[iord] if usepca[iord]: msgs.info('Using PCA to predict trace for object #{:d}'.format(iobj) + ' on order #{:d}'.format(iord)) sobjs_final = sobjs_trim.copy() # Loop over the objects one by one and adjust/predict the traces npoly_cen = 3 pca_fits = np.zeros((nspec, norders, nobj_trim)) for iobj in range(nobj_trim): igroup = sobjs_final.ech_group == uni_group_trim[iobj] # PCA predict the masked orders which were not traced pca_fits[:,:,iobj] = pca_trace((sobjs_final[igroup].trace_spat).T, usepca = None, npca = npca, npoly_cen = npoly_cen) # usepca = sobjs_final[igroup].ech_usepca, # Perform iterative flux weighted centroiding using new PCA predictions xinit_fweight = pca_fits[:,:,iobj].copy() inmask_now = inmask & (ordermask > 0) xfit_fweight = extract.iter_tracefit(image, xinit_fweight, ncoeff, inmask = inmask_now, show_fits=show_fits) # Perform iterative Gaussian weighted centroiding xinit_gweight = xfit_fweight.copy() xfit_gweight = extract.iter_tracefit(image, xinit_gweight, ncoeff, inmask = inmask_now, gweight=True,show_fits=show_fits) # Assign the new traces for iord, spec in enumerate(sobjs_final[igroup]): spec.trace_spat = xfit_gweight[:,iord] spec.spat_pixpos = spec.trace_spat[specmid] # Set the IDs sobjs_final.set_idx() if show_trace: viewer, ch = ginga.show_image(objminsky*(ordermask > 0)) for iobj in range(nobj_trim): for iord in range(norders): ginga.show_trace(viewer, ch, pca_fits[:,iord, iobj], str(uni_frac[iobj]), color='yellow') for spec in sobjs_trim: color = 'green' if spec.ech_usepca else 'magenta' ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color) #for spec in sobjs_final: # color = 'red' if spec.ech_usepca else 'green' # ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color) return sobjs_final
#tslits_dict=tslits_dict, #tilts=mstilts, det=1, #datasec_img=datasec_img, #bpm=bpm, #pixlocn=pixlocn, #fitstbl=fitstbl) ) sciframe_B, sciivar_B, rawvarframe_B, crmask_B = sciIMG_B.process(bias_subtract=None, pixel_flat=None, bpm=bpm, apply_gain=True, trim=False) # This addition of a tiny bit of effor imposes a maximum S/N ratio which stabilizes the fits adderr = 0.01 # Additional error to add to the formal errors, as a fraction of the flux; default to 0.01 (1 per cent). ivar_A = utils.calc_ivar(rawvarframe_A) * (crmask_A == False) gmask_A = (ivar_A > 0.0).astype(int) # = 1 for good points, 0 for bad points ivar_A = gmask_A / (1.0 / (ivar_A + (1.0 - gmask_A)) + (adderr ** 2) * (np.abs(sciframe_A)) ** 2) ivar_B = utils.calc_ivar(rawvarframe_B) * (crmask_B == False) gmask_B = (ivar_B > 0.0).astype(int) # = 1 for good points, 0 for bad points ivar_B = gmask_B / (1.0 / (ivar_B + (1.0 - gmask_B)) + (adderr ** 2) * (np.abs(sciframe_B)) ** 2) # Diff the images diff_AB = sciframe_A - sciframe_B var_A = utils.calc_ivar(ivar_A) var_B = utils.calc_ivar(ivar_B) var_AB = var_A + var_B # I think processimages already masks saturation but am not sure mask_AB = (sciframe_A > 0.0) & (sciframe_B > 0.0) & \ (sciframe_A < spectrograph.detector[0]['saturation']) & \ (sciframe_B < spectrograph.detector[0]['saturation']) & \ (crmask_A == False) & (crmask_B == False)