def dummy_setup_dict(file_list, setup): """ Generates a dummy setup_dict. .. todo:: - Describe how this is used. - Improve the description of setup and/or point to the function used to generate it. Args: file_list (:obj:`list`): List of files to set as science exposures. setup (:obj:`str`): String representation of the instrument setup. Returns: dict: Dictionary with the instrument setup. """ # setup_dict setup_dict = {} setup_dict[setup[0]] = {} # Fill with dummy dicts for ii in range(1,20): # Dummy detectors setup_dict[setup[0]][parse.get_dnum(ii)] = dict(binning='1x1') setup_dict[setup[0]][setup[-2:]] = {} # Fill up filenames setup_dict[setup[0]][setup[-2:]]['sci'] = file_list # Write # TODO: Why is this called here? _ = write_calib(setup_dict) return setup_dict
def init_hdus(update_det, outfile): hdus, prihdu = None, None if (update_det is not None) and os.path.isfile(outfile): hdus = fits.open(outfile) msgs.info("Using existing spec1d file, including the Header") msgs.info( "Will only update the data extension for {} detector(s)".format( update_det)) prihdu = hdus[0] # Names hdu_names = [hdu.name for hdu in hdus] # Remove the detector(s) being updated if not isinstance(update_det, list): update_det = [update_det] popme = [] # Find em for ss, hdu_name in enumerate(hdu_names): for det in update_det: sdet = parse.get_dnum(det, prefix=False) idx = '{:s}{:s}'.format(specobjs.naming_model['det'], sdet) if idx in hdu_name: popme.append(ss) # Remove em (and the bit in the Header too) for popthis in reversed(popme): hdus.pop(popthis) keywd = 'EXT{:04d}'.format(popthis) prihdu.header.remove(keywd) # Return return hdus, prihdu
def det_setup(isetup_dict, ddict): """ Return detector setup on config dict or add to it if new Parameters ---------- isetup_dict : dict Selected setup_dict ddict : dict detector dict Returns ------- dkey : str Name of the detector string associated to the input ddict May be new or previously used """ det_str = [parse.get_dnum(i+1, prefix=False) for i in range(99)] # Init for dkey in det_str: mtch = True if dkey not in isetup_dict.keys(): isetup_dict[dkey] = ddict break for key in isetup_dict[dkey].keys(): mtch &= isetup_dict[dkey][key] == ddict[key] if mtch: break # Return return dkey
def load_std_trace(spec1dfile, det): sdet = parse.get_dnum(det, prefix=False) hdulist_1d = fits.open(spec1dfile) det_nm = 'DET{:s}'.format(sdet) for hdu in hdulist_1d: if det_nm in hdu.name: tbl = Table(hdu.data) # TODO what is the data model for echelle standards? This routine needs to distinguish between echelle and longslit trace = tbl['TRACE'] return trace
def main(args): raise NotImplementedError('This script is currently out of date.') # List only? hdu = fits.open(args.file) head0 = hdu[0].header if args.list: hdu.info() return # Init sdet = get_dnum(args.det, prefix=False) # One detector, sky sub for now names = [hdu[i].name for i in range(len(hdu))] try: exten = names.index('DET{:s}-PROCESSED'.format(sdet)) except: # Backwards compatability msgs.error('Requested detector {:s} was not processed.\n' 'Maybe you chose the wrong one to view?\n' 'Set with --det= or check file contents with --list'.format(sdet)) sciimg = hdu[exten].data try: exten = names.index('DET{:s}-SKY'.format(sdet)) except: # Backwards compatability msgs.error('Requested detector {:s} has no sky model.\n' 'Maybe you chose the wrong one to view?\n' 'Set with --det= or check file contents with --list'.format(sdet)) skymodel = hdu[exten].data try: exten = names.index('DET{:s}-MASK'.format(sdet)) except ValueError: # Backwards compatability msgs.error('Requested detector {:s} has no bit mask.\n' 'Maybe you chose the wrong one to view?\n' 'Set with --det= or check file contents with --list'.format(sdet)) mask = hdu[exten].data frame = (sciimg - skymodel) * (mask == 0) mdir = head0['PYPMFDIR'] mkey = head0['FRAMMKEY'] mast_key = '{0}_{1:02d}'.format(mkey, args.det) if not os.path.exists(mdir): mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir)) msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base)) mdir = mdir_base # Assumes a MasterSlit file has been written #slits = slittrace.SlitTraceSet.from_master('{0}_{1:02d}'.format(head0['TRACMKEY'], args.det), mdir)
def set_idx(self): """ Generate a unique index for this spectrum Sets self.idx internally Returns: str: :attr:`self.idx` """ # Detector string sdet = parse.get_dnum(self.det, prefix=False) if 'Echelle' in self.pypeline: # ObjID self.idx = naming_model['obj'] if self.ech_objid is None: self.idx += '----' else: self.idx += '{:04d}'.format(self.ech_objid) self.idx += '-' + naming_model['order'] # Order if self.ech_orderindx is None: self.idx += '----' else: self.idx += '{:04d}'.format(self.ech_order) else: # Spat self.idx = naming_model['spat'] if self.spat_pixpos is None: self.idx += '----' else: self.idx += '{:04d}'.format(int(np.rint(self.spat_pixpos))) # Slit self.idx += '-' + naming_model['slit'] if self.slitid is None: self.idx += '----' else: self.idx += '{:04d}'.format(self.slitid) self.idx += '-{:s}{:s}'.format(naming_model['det'], sdet) # Return return self.idx
def init_hdus(update_det, outfile): """ Load up existing header and HDUList ..todo:: Confirm this works when you are modifying an inner HDU Args: update_det (int or list): outfile (str): Returns: fits.HDUList, fits.PrimaryHDU """ # hdus = fits.open(outfile) msgs.info("Using existing output file, including the Header") msgs.info("Will only update the data extension for {} detector(s)".format( update_det)) prihdu = hdus[0] # Names hdu_names = [hdu.name for hdu in hdus] # Remove the detector(s) being updated if not isinstance(update_det, list): update_det = [update_det] popme = [] # Find em for ss, hdu_name in enumerate(hdu_names): for det in update_det: sdet = parse.get_dnum(det, prefix=False) idx = '{:s}{:s}'.format(specobj.naming_model['det'], sdet) if idx in hdu_name: popme.append(ss) # Remove em (and the bit in the Header too) for popthis in reversed(popme): hdus.pop(popthis) keywd = 'EXT{:04d}'.format(popthis) prihdu.header.remove(keywd) # Return return hdus, prihdu
def lacosmic(det, sciframe, saturation, nonlinear, varframe=None, maxiter=1, grow=1.5, remove_compact_obj=True, sigclip=5.0, sigfrac=0.3, objlim=5.0): """ Identify cosmic rays using the L.A.Cosmic algorithm U{http://www.astro.yale.edu/dokkum/lacosmic/} (article : U{http://arxiv.org/abs/astro-ph/0108003}) This routine is mostly courtesy of Malte Tewes Args: det: sciframe: saturation: nonlinear: varframe: maxiter: grow: remove_compact_obj: sigclip: sigfrac: objlim: Returns: ndarray: mask of cosmic rays (0=no CR, 1=CR) """ dnum = parse.get_dnum(det) msgs.info("Detecting cosmic rays with the L.A.Cosmic algorithm") # msgs.work("Include these parameters in the settings files to be adjusted by the user") # Set the settings scicopy = sciframe.copy() crmask = np.cast['bool'](np.zeros(sciframe.shape)) sigcliplow = sigclip * sigfrac # Determine if there are saturated pixels satpix = np.zeros_like(sciframe) # satlev = settings_det['saturation']*settings_det['nonlinear'] satlev = saturation * nonlinear wsat = np.where(sciframe >= satlev) if wsat[0].size == 0: satpix = None else: satpix[wsat] = 1.0 satpix = np.cast['bool'](satpix) # Define the kernels laplkernel = np.array([[0.0, -1.0, 0.0], [-1.0, 4.0, -1.0], [0.0, -1.0, 0.0]]) # Laplacian kernal growkernel = np.ones((3, 3)) for i in range(1, maxiter + 1): msgs.info("Convolving image with Laplacian kernel") # Subsample, convolve, clip negative values, and rebin to original size subsam = utils.subsample(scicopy) conved = signal.convolve2d(subsam, laplkernel, mode="same", boundary="symm") cliped = conved.clip(min=0.0) lplus = utils.rebin_evlist(cliped, np.array(cliped.shape) / 2.0) msgs.info("Creating noise model") # Build a custom noise map, and compare this to the laplacian m5 = ndimage.filters.median_filter(scicopy, size=5, mode='mirror') if varframe is None: noise = np.sqrt(np.abs(m5)) else: noise = np.sqrt(varframe) msgs.info("Calculating Laplacian signal to noise ratio") # Laplacian S/N s = lplus / (2.0 * noise ) # Note that the 2.0 is from the 2x2 subsampling # Remove the large structures sp = s - ndimage.filters.median_filter(s, size=5, mode='mirror') msgs.info("Selecting candidate cosmic rays") # Candidate cosmic rays (this will include HII regions) candidates = sp > sigclip nbcandidates = np.sum(candidates) msgs.info("{0:5d} candidate pixels".format(nbcandidates)) # At this stage we use the saturated stars to mask the candidates, if available : if satpix is not None: msgs.info("Masking saturated pixels") candidates = np.logical_and(np.logical_not(satpix), candidates) nbcandidates = np.sum(candidates) msgs.info( "{0:5d} candidate pixels not part of saturated stars".format( nbcandidates)) msgs.info("Building fine structure image") # We build the fine structure image : m3 = ndimage.filters.median_filter(scicopy, size=3, mode='mirror') m37 = ndimage.filters.median_filter(m3, size=7, mode='mirror') f = m3 - m37 f /= noise f = f.clip(min=0.01) msgs.info("Removing suspected compact bright objects") # Now we have our better selection of cosmics : if remove_compact_obj: cosmics = np.logical_and(candidates, sp / f > objlim) else: cosmics = candidates nbcosmics = np.sum(cosmics) msgs.info("{0:5d} remaining candidate pixels".format(nbcosmics)) # What follows is a special treatment for neighbors, with more relaxed constains. msgs.info("Finding neighboring pixels affected by cosmic rays") # We grow these cosmics a first time to determine the immediate neighborhod : growcosmics = np.cast['bool'](signal.convolve2d( np.cast['float32'](cosmics), growkernel, mode="same", boundary="symm")) # From this grown set, we keep those that have sp > sigmalim # so obviously not requiring sp/f > objlim, otherwise it would be pointless growcosmics = np.logical_and(sp > sigclip, growcosmics) # Now we repeat this procedure, but lower the detection limit to sigmalimlow : finalsel = np.cast['bool'](signal.convolve2d( np.cast['float32'](growcosmics), growkernel, mode="same", boundary="symm")) finalsel = np.logical_and(sp > sigcliplow, finalsel) # Unmask saturated pixels: if satpix is not None: msgs.info("Masking saturated stars") finalsel = np.logical_and(np.logical_not(satpix), finalsel) ncrp = np.sum(finalsel) msgs.info("{0:5d} pixels detected as cosmics".format(ncrp)) # We find how many cosmics are not yet known : newmask = np.logical_and(np.logical_not(crmask), finalsel) nnew = np.sum(newmask) # We update the mask with the cosmics we have found : crmask = np.logical_or(crmask, finalsel) msgs.info( "Iteration {0:d} -- {1:d} pixels identified as cosmic rays ({2:d} new)" .format(i, ncrp, nnew)) if ncrp == 0: break # Additional algorithms (not traditionally implemented by LA cosmic) to remove some false positives. msgs.work( "The following algorithm would be better on the rectified, tilts-corrected image" ) filt = ndimage.sobel(sciframe, axis=1, mode='constant') filty = ndimage.sobel(filt / np.sqrt(np.abs(sciframe)), axis=0, mode='constant') filty[np.where(np.isnan(filty))] = 0.0 sigimg = cr_screen(filty) sigsmth = ndimage.filters.gaussian_filter(sigimg, 1.5) sigsmth[np.where(np.isnan(sigsmth))] = 0.0 sigmask = np.cast['bool'](np.zeros(sciframe.shape)) sigmask[np.where(sigsmth > sigclip)] = True crmask = np.logical_and(crmask, sigmask) msgs.info("Growing cosmic ray mask by 1 pixel") crmask = grow_masked(crmask.astype(np.float), grow, 1.0) return crmask.astype(bool)
def main(args): # Build the fitstable since we currently need it for output. This should not be the case! A_files = [os.path.join(args.full_rawpath, file) for file in args.Afiles] B_files = [os.path.join(args.full_rawpath, file) for file in args.Bfiles] data_files = A_files + B_files ps = pypeitsetup.PypeItSetup(A_files, path='./', spectrograph_name='keck_mosfire') ps.build_fitstbl() fitstbl = ps.fitstbl # Read in the spectrograph, config the parset spectrograph = load_spectrograph('keck_mosfire') spectrograph_def_par = spectrograph.default_pypeit_par() parset = par.PypeItPar.from_cfg_lines( cfg_lines=spectrograph_def_par.to_config(), merge_with=config_lines(args)) science_path = os.path.join(parset['rdx']['redux_path'], parset['rdx']['scidir']) # Calibration Master directory if args.master_dir is None: msgs.error( "You need to set an Environmental variable MOSFIRE_MASTERS that points at the Master Calibs" ) # Define some hard wired master files here to be later parsed out of the directory slit_masterframe_name = os.path.join(args.master_dir, 'MasterSlits_E_15_01.fits.gz') tilts_masterframe_name = os.path.join(args.master_dir, 'MasterTilts_E_1_01.fits') wvcalib_masterframe_name = os.path.join(args.master_dir, 'MasterWaveCalib_E_1_01.fits') # For now don't require a standard std_outfile = None #std_outfile = os.path.join('/Users/joe/Dropbox/PypeIt_Redux/MOSFIRE/Nov19/quicklook/Science/', # 'spec1d_m191118_0064-GD71_MOSFIRE_2019Nov18T104704.507.fits') # make the get_std from pypeit a utility function or class method det = 1 # MOSFIRE has a single detector if std_outfile is not None: # Get the standard trace if need be sobjs = specobjs.SpecObjs.from_fitsfile(std_outfile) this_det = sobjs.DET == det if np.any(this_det): sobjs_det = sobjs[this_det] sobjs_std = sobjs_det.get_std() std_trace = None if sobjs_std is None else sobjs_std.TRACE_SPAT.flatten( ) else: std_trace = None else: std_trace = None # Read in the msbpm sdet = get_dnum(det, prefix=False) msbpm = spectrograph.bpm(A_files[0], det) # Read in the slits slits = slittrace.SlitTraceSet.from_file(slit_masterframe_name) # Reset the bitmask slits.mask = slits.mask_init.copy() # Read in the wv_calib wv_calib = wavecalib.WaveCalib.from_file(wvcalib_masterframe_name) wv_calib.is_synced(slits) slits.mask_wvcalib(wv_calib) # Read in the tilts tilts_obj = wavetilts.WaveTilts.from_file(tilts_masterframe_name) tilts_obj.is_synced(slits) slits.mask_wavetilts(tilts_obj) # Build Science image sciImg = buildimage.buildimage_fromlist(spectrograph, det, parset['scienceframe'], A_files, bpm=msbpm, slits=slits, ignore_saturation=False) # Background Image? sciImg = sciImg.sub( buildimage.buildimage_fromlist(spectrograph, det, parset['scienceframe'], B_files, bpm=msbpm, slits=slits, ignore_saturation=False), parset['scienceframe']['process']) # Build the Calibrate object caliBrate = calibrations.Calibrations(None, parset['calibrations'], spectrograph, None) caliBrate.slits = slits caliBrate.wavetilts = tilts_obj caliBrate.wv_calib = wv_calib # Instantiate Reduce object # Required for pypeline specific object # At instantiaton, the fullmask in self.sciImg is modified redux = reduce.Reduce.get_instance(sciImg, spectrograph, parset, caliBrate, 'science', ir_redux=True, show=args.show, det=det, std_outfile=std_outfile) manual_extract_dict = None skymodel, objmodel, ivarmodel, outmask, sobjs, waveImg, tilts = redux.run( std_trace=std_trace, return_negative=True, manual_extract_dict=manual_extract_dict, show_peaks=args.show) # TODO -- Do this upstream # Tack on detector for sobj in sobjs: sobj.DETECTOR = sciImg.detector # Construct the Spec2DObj with the positive image spec2DObj_A = spec2dobj.Spec2DObj(det=det, sciimg=sciImg.image, ivarraw=sciImg.ivar, skymodel=skymodel, objmodel=objmodel, ivarmodel=ivarmodel, waveimg=waveImg, bpmmask=outmask, detector=sciImg.detector, sci_spat_flexure=sciImg.spat_flexure, tilts=tilts, slits=copy.deepcopy(caliBrate.slits)) spec2DObj_A.process_steps = sciImg.process_steps all_spec2d = spec2dobj.AllSpec2DObj() all_spec2d['meta']['ir_redux'] = True all_spec2d[det] = spec2DObj_A # Save image A but with all the objects extracted, i.e. positive and negative #outfile2d, outfile1d = save_exposure(fitstbl, 0, spectrograph, science_path, parset, caliBrate, all_spec2d, sobjs) # Construct the Spec2DObj with the negative image spec2DObj_B = spec2dobj.Spec2DObj(det=det, sciimg=-sciImg.image, ivarraw=sciImg.ivar, skymodel=-skymodel, objmodel=-objmodel, ivarmodel=ivarmodel, waveimg=waveImg, bpmmask=outmask, detector=sciImg.detector, sci_spat_flexure=sciImg.spat_flexure, tilts=tilts, slits=copy.deepcopy(caliBrate.slits)) # Parse the offset information out of the headers. TODO in the future get this out of fitstable dither_pattern_A, dither_id_A, offset_arcsec_A = parse_dither_pattern( A_files, spectrograph.primary_hdrext) dither_pattern_B, dither_id_B, offset_arcsec_B = parse_dither_pattern( B_files, spectrograph.primary_hdrext) # Print out a report on the offsets msg_string = msgs.newline( ) + '****************************************************' msg_string += msgs.newline( ) + ' Summary of offsets for dither pattern: {:s}'.format( dither_pattern_A[0]) msg_string += msgs.newline( ) + '****************************************************' msg_string += msgs.newline( ) + 'Position filename arcsec pixels ' msg_string += msgs.newline( ) + '----------------------------------------------------' for iexp, file in enumerate(A_files): msg_string += msgs.newline( ) + ' A {:s} {:6.2f} {:6.2f}'.format( os.path.basename(file), offset_arcsec_A[iexp], offset_arcsec_A[iexp] / sciImg.detector.platescale) for iexp, file in enumerate(B_files): msg_string += msgs.newline( ) + ' B {:s} {:6.2f} {:6.2f}'.format( os.path.basename(file), offset_arcsec_B[iexp], offset_arcsec_B[iexp] / sciImg.detector.platescale) msg_string += msgs.newline( ) + '****************************************************' msgs.info(msg_string) #offset_dith_pix = offset_dith_pix = offset_arcsec_A[0]/sciImg.detector.platescale offsets_dith_pix = (np.array([ 0.0, np.mean(offset_arcsec_B) - np.mean(offset_arcsec_A) ])) / sciImg.detector.platescale if args.offset is not None: offsets_pixels = np.array([0.0, args.offset]) msgs.info('Using user specified offsets instead: {:5.2f}'.format( args.offset)) else: offsets_pixels = offsets_dith_pix spec2d_list = [spec2DObj_A, spec2DObj_B] # Instantiate Coadd2d coadd = coadd2d.CoAdd2D.get_instance(spec2d_list, spectrograph, parset, det=det, offsets=offsets_pixels, weights='uniform', ir_redux=True, debug=args.show, samp_fact=args.samp_fact) # Coadd the slits coadd_dict_list = coadd.coadd( only_slits=None, interp_dspat=False) # TODO implement only_slits later # Create the pseudo images pseudo_dict = coadd.create_pseudo_image(coadd_dict_list) ########################## # Now display the images # ########################## display.display.connect_to_ginga(raise_err=True, allow_new=True) # Bug in ginga prevents me from using cuts here for some reason #mean, med, sigma = sigma_clipped_stats(pseudo_dict['imgminsky'][pseudo_dict['inmask']], sigma_lower=5.0,sigma_upper=5.0) #cut_min = mean - 4.0 * sigma #cut_max = mean + 4.0 * sigma chname_skysub = 'skysub-det{:s}'.format(sdet) # Clear all channels at the beginning # TODO: JFH For some reason Ginga crashes when I try to put cuts in here. viewer, ch = ginga.show_image(pseudo_dict['imgminsky'], chname=chname_skysub, waveimg=pseudo_dict['waveimg'], clear=True) # cuts=(cut_min, cut_max), slit_left, slit_righ, _ = pseudo_dict['slits'].select_edges() slit_id = slits.slitord_id[0] ginga.show_slits(viewer, ch, slit_left, slit_righ, slit_ids=slit_id) # SKRESIDS chname_skyresids = 'sky_resid-det{:s}'.format(sdet) image = pseudo_dict['imgminsky'] * np.sqrt( pseudo_dict['sciivar']) * pseudo_dict['inmask'] # sky residual map viewer, ch = ginga.show_image( image, chname_skyresids, waveimg=pseudo_dict['waveimg'], cuts=(-5.0, 5.0), ) ginga.show_slits(viewer, ch, slit_left, slit_righ, slit_ids=slits.slitord_id[0]) shell = viewer.shell() out = shell.start_global_plugin('WCSMatch') out = shell.call_global_plugin_method('WCSMatch', 'set_reference_channel', [chname_skyresids], {}) if args.embed: embed() return 0
def main(args): raise NotImplementedError('This script is currently out of date.') # List only? hdu = fits.open(args.file) head0 = hdu[0].header if args.list: hdu.info() return # Init sdet = get_dnum(args.det, prefix=False) # One detector, sky sub for now names = [hdu[i].name for i in range(len(hdu))] try: exten = names.index('DET{:s}-PROCESSED'.format(sdet)) except: # Backwards compatability msgs.error( 'Requested detector {:s} was not processed.\n' 'Maybe you chose the wrong one to view?\n' 'Set with --det= or check file contents with --list'.format(sdet)) sciimg = hdu[exten].data try: exten = names.index('DET{:s}-SKY'.format(sdet)) except: # Backwards compatability msgs.error( 'Requested detector {:s} has no sky model.\n' 'Maybe you chose the wrong one to view?\n' 'Set with --det= or check file contents with --list'.format(sdet)) skymodel = hdu[exten].data try: exten = names.index('DET{:s}-MASK'.format(sdet)) except ValueError: # Backwards compatability msgs.error( 'Requested detector {:s} has no bit mask.\n' 'Maybe you chose the wrong one to view?\n' 'Set with --det= or check file contents with --list'.format(sdet)) mask = hdu[exten].data frame = (sciimg - skymodel) * (mask == 0) mdir = head0['PYPMFDIR'] mkey = head0['FRAMMKEY'] mast_key = '{0}_{1:02d}'.format(mkey, args.det) if not os.path.exists(mdir): mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir)) msgs.warn('Master file dir: {0} does not exist. Using {1}'.format( mdir, mdir_base)) mdir = mdir_base # Assumes a MasterSlit file has been written #slits = slittrace.SlitTraceSet.from_master('{0}_{1:02d}'.format(head0['TRACMKEY'], args.det), # mdir) # Load the slits information slits = slittrace.SlitTraceSet.from_master(mast_key, mdir) # Object traces left, right, mask = slits.select_edges() msgs.error("You need to choose which slits you care about here") # Get object traces spec1d_file = args.file.replace('spec2d', 'spec1d') if os.path.isfile(spec1d_file): hdulist_1d = fits.open(spec1d_file) else: hdulist_1d = [] msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) + msgs.newline() + ' No objects were extracted.') msgs.error( "This code needs to be refactored since tslits_dict was removed...") import pdb pdb.set_trace() tslits_dict['objtrc'] = parse_traces(hdulist_1d, det_nm) obj_trace = parse_traces(hdulist_1d, 'DET{:s}'.format(sdet)) # TODO :: Need to include standard star trace in the spec2d files std_trace = None # Extract some trace models fwhm = 2 # Start with some default value # TODO: Dictionaries like this are a pet peeve of mine. I'd prefer # either individual objects or a class with a well-formed data model. # TODO: Why do all of these dictionary elements need fwhm? Can they # be different? trace_models = dict() # Brightest object on slit trace_models['object'] = dict(trace_model=None, fwhm=fwhm) if len(obj_trace['pkflux']) > 0: smash_peakflux = obj_trace['pkflux'] ibri = smash_peakflux.argmax() trace_models['object']['trace_model'] = obj_trace['traces'][ibri] trace_models['object']['fwhm'] = obj_trace['fwhm'][ibri] # Standard star trace trace_models['std'] = dict(trace_model=std_trace, fwhm=trace_models['object']['fwhm']) # Trace of the slit edge # TODO: Any particular reason to use the lefts? trace_models['slit'] = dict(trace_model=left.copy(), fwhm=trace_models['object']['fwhm']) # Finally, initialise the GUI gui.object_find.initialise(args.det, frame, left, right, obj_trace, trace_models, None, printout=True, slit_ids=slits.id) ofgui = gui_object_find.initialise(args.det, frame, tslits_dict, None, printout=True, slit_ids=slits.id)
def set_name(self): """ Generate a unique index for this spectrum based on the slit/order, its position and for multi-slit the detector. Multi-slit Each object is named by its: - spatial position (pixel number) on the reduced image [SPAT] - the slit number based on SPAT center of the slit or SlitMask ID [SLIT] - the detector number [DET] For example:: SPAT0176-SLIT0185-DET01 Echelle Returns: str: """ if 'Echelle' in self.PYPELINE: # ObjID name = naming_model['obj'] ech_name = naming_model['obj'] if self['ECH_FRACPOS'] is None: name += '----' else: # JFH TODO Why not just write it out with the decimal place. That is clearer than this?? name += '{:04d}'.format(int(np.rint(1000 * self.ECH_FRACPOS))) ech_name += '{:04d}'.format( int(np.rint(1000 * self.ECH_FRACPOS))) sdet = parse.get_dnum(self.DET, prefix=False) name += '-{:s}{:s}'.format(naming_model['det'], sdet) ech_name += '-{:s}{:s}'.format(naming_model['det'], sdet) # Order number name += '-' + naming_model['order'] name += '{:04d}'.format(self.ECH_ORDER) self.ECH_NAME = ech_name self.NAME = name elif 'MultiSlit' in self.PYPELINE: # Spat name = naming_model['spat'] if self['SPAT_PIXPOS'] is None: name += '----' else: name += '{:04d}'.format(int(np.rint(self.SPAT_PIXPOS))) # Slit name += '-' + naming_model['slit'] name += '{:04d}'.format(self.SLITID) sdet = parse.get_dnum(self.DET, prefix=False) name += '-{:s}{:s}'.format(naming_model['det'], sdet) self.NAME = name elif 'IFU' in self.PYPELINE: # Spat name = naming_model['spat'] if self['SPAT_PIXPOS'] is None: name += '----' else: name += '{:04d}'.format(int(np.rint(self.SPAT_PIXPOS))) # Slit name += '-' + naming_model['slit'] name += '{:04d}'.format(self.SLITID) sdet = parse.get_dnum(self.DET, prefix=False) name += '-{:s}{:s}'.format(naming_model['det'], sdet) self.NAME = name else: msgs.error("Bad PYPELINE")
flat_model = np.ones_like(flat) flat_model[thismask] = twod_model[thismask]*np.fmax(illumflat[thismask],0.05)*np.fmax(spec_model[thismask],1.0) pixelflat[thismask] = flat[thismask]/flat_model[thismask] # ToDo Add some code here to treat the edges and places where fits go bad? return (pixelflat[thismask], illumflat[thismask], flat_model[thismask]) ''' type = 'ESI' devpath = os.getenv('PYPEIT_DEV') if type == 'LRIS_red': det = 1 sdet = parse.get_dnum(det, prefix=False) rawpath = devpath + '/RAW_DATA/Keck_LRIS_red/multi_400_8500_d560/' masterpath = devpath + '/REDUX_OUT/Keck_LRIS_red/multi_400_8500_d560/MF_keck_lris_red/' # Read in the msbias for bias subtraction biasfile = masterpath + 'MasterBias_A_' + sdet + '_aa.fits' msbias = fits.getdata(biasfile) # Read in and process flat field images pixflat_image_files = np.core.defchararray.add(rawpath, ['r170320_2057.fits','r170320_2058.fits','r170320_2059.fits']).tolist() spectro_name = 'keck_lris_red' spectrograph = load_spectrograph(spectrograph=spectro_name) par = spectrograph.default_pypeit_par() flatField = flatfield.FlatField(spectrograph, file_list=pixflat_image_files,det=det, par=par['calibrations']['pixelflatframe'] , msbias = msbias) flatimg = flatField.build_pixflat() # Read in the tilts
def main(args): tstart = time.time() # Parse the files sort by MJD files = np.array([os.path.join(args.full_rawpath, file) for file in args.files]) nfiles = len(files) # Read in the spectrograph, config the parset spectrograph = load_spectrograph('vlt_fors2') #spectrograph_def_par = spectrograph.default_pypeit_par() spectrograph_cfg_lines = spectrograph.config_specific_par(files[0]).to_config() parset = par.PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=config_lines(args)) science_path = os.path.join(parset['rdx']['redux_path'], parset['rdx']['scidir']) target = spectrograph.get_meta_value(files[0], 'target') mjds = np.zeros(nfiles) for ifile, file in enumerate(files): mjds[ifile] = spectrograph.get_meta_value(file, 'mjd', ignore_bad_header=True, no_fussing=True) files = files[np.argsort(mjds)] # Calibration Master directory #TODO hardwired for now master_dir ='./' #master_dir = resource_filename('pypeit', 'data/QL_MASTERS') \ # if args.master_dir is None else args.master_dir if not os.path.isdir(master_dir): msgs.error(f'{master_dir} does not exist! You must install the QL_MASTERS ' 'directory; download the data from the PypeIt dev-suite Google Drive and ' 'either define a QL_MASTERS environmental variable or use the ' 'pypeit_install_ql_masters script.') # Define some hard wired master files here to be later parsed out of the directory fors2_grism = spectrograph.get_meta_value(files[0], 'dispname') fors2_masters = os.path.join(master_dir, 'FORS2_MASTERS', fors2_grism) bias_masterframe_name = \ utils.find_single_file(os.path.join(fors2_masters, "MasterBias*")) slit_masterframe_name \ = utils.find_single_file(os.path.join(fors2_masters, "MasterSlits*")) tilts_masterframe_name \ = utils.find_single_file(os.path.join(fors2_masters, "MasterTilts*")) wvcalib_masterframe_name \ = utils.find_single_file(os.path.join(fors2_masters, 'MasterWaveCalib*')) std_spec1d_file = utils.find_single_file(os.path.join(fors2_masters, 'spec1d_*')) sensfunc_masterframe_name = utils.find_single_file(os.path.join(fors2_masters, 'sens_*')) # TODO make and impelement sensfunc if (bias_masterframe_name is None or not os.path.isfile(bias_masterframe_name)) or \ (slit_masterframe_name is None or not os.path.isfile(slit_masterframe_name)) or \ (tilts_masterframe_name is None or not os.path.isfile(tilts_masterframe_name)) or \ (std_spec1d_file is None or not os.path.isfile(std_spec1d_file)): # or (sensfunc_masterframe_name is None or not os.path.isfile(sensfunc_masterframe_name)): msgs.error('Master frames not found. Check that environment variable QL_MASTERS ' 'points at the Master Calibs') # We need the platescale # Get detector (there's only one) #det = 1 # MOSFIRE has a single detector #detector = spectrograph.get_detector_par(det) #detname = detector.name # We need the platescale det_container = spectrograph.get_detector_par(1, hdu=fits.open(files[0])) binspectral, binspatial = parse_binning(det_container['binning']) platescale = det_container['platescale']*binspatial # Parse the offset information out of the headers. _, _, offset_arcsec = spectrograph.parse_dither_pattern(files) # Print out a report on the offsets msg_string = msgs.newline() + '*******************************************************' msg_string += msgs.newline() + ' Summary of offsets for target {:s}: ' msg_string += msgs.newline() + '*******************************************************' msg_string += msgs.newline() + ' filename arcsec pixels ' msg_string += msgs.newline() + '----------------------------------------------------' for iexp, file in enumerate(files): msg_string += msgs.newline() + ' {:s} {:6.2f} {:6.2f}'.format( os.path.basename(file), offset_arcsec[iexp], offset_arcsec[iexp] / platescale) msg_string += msgs.newline() + '********************************************************' msgs.info(msg_string) ## Read in the master frames that we need ## det = 1 # Currently CHIP1 is supported if std_spec1d_file is not None: # Get the standard trace if need be sobjs = specobjs.SpecObjs.from_fitsfile(std_spec1d_file) this_det = sobjs.DET == det if np.any(this_det): sobjs_det = sobjs[this_det] sobjs_std = sobjs_det.get_std() std_trace = None if sobjs_std is None else sobjs_std.TRACE_SPAT.flatten() else: std_trace = None else: std_trace = None # Read in the bias msbias = buildimage.BiasImage.from_file(bias_masterframe_name) # Read in the msbpm sdet = get_dnum(det, prefix=False) msbpm = spectrograph.bpm(files[0], det) # Read in the slits slits = slittrace.SlitTraceSet.from_file(slit_masterframe_name) # Reset the bitmask slits.mask = slits.mask_init.copy() # Read in the wv_calib wv_calib = wavecalib.WaveCalib.from_file(wvcalib_masterframe_name) # wv_calib.is_synced(slits) slits.mask_wvcalib(wv_calib) # Read in the tilts tilts_obj = wavetilts.WaveTilts.from_file(tilts_masterframe_name) tilts_obj.is_synced(slits) slits.mask_wavetilts(tilts_obj) # Build the Calibrate object caliBrate = calibrations.Calibrations(None, parset['calibrations'], spectrograph, None) caliBrate.msbias = msbias caliBrate.msbpm = msbpm caliBrate.slits = slits caliBrate.wavetilts = tilts_obj caliBrate.wv_calib = wv_calib # Find the unique offsets. This is a bit of a kludge, i.e. we are considering offsets within # 0.1 arcsec of each other to be the same throw, but I should like to be able to specify a tolerance here, # but then I need a version of unique that accepts a tolerance uniq_offsets, uni_indx = np.unique(np.around(offset_arcsec), return_inverse=True) nuniq = uniq_offsets.size spec2d_list = [] offset_ref = offset_arcsec[0] offsets_dith_pix = [] # Generalize to a multiple slits, doing one slit at a time? islit = 0 # Loop over the unique throws and create a spec2d_A and spec2D_B for # each, which are then fed into coadd2d with the correct offsets # TODO Rework the logic here so that we can print out a unified report # on what was actually reduced. for iuniq in range(nuniq): indx = uni_indx == iuniq files_uni = files[indx] offsets = offset_arcsec[indx] msgs.info('Reducing images for offset = {:}'.format(offsets[0])) spec2DObj = run(files_uni, caliBrate, spectrograph, det, parset, show=args.show, std_trace=std_trace) spec2d_list += [spec2DObj] offsets_dith_pix += [np.mean(offsets)/platescale] offsets_dith_pix = np.array(offsets_dith_pix) if args.offset is not None: offsets_pixels = np.array([0.0, args.offset]) msgs.info('Using user specified offsets instead: {:5.2f}'.format(args.offset)) else: offsets_pixels = offsets_dith_pix # Instantiate Coadd2d coadd = coadd2d.CoAdd2D.get_instance(spec2d_list, spectrograph, parset, det=det, offsets=offsets_pixels, weights='uniform', spec_samp_fact=args.spec_samp_fact, spat_samp_fact=args.spat_samp_fact, ir_redux=True, debug=args.show) # Coadd the slits # TODO implement only_slits later coadd_dict_list = coadd.coadd(only_slits=None, interp_dspat=False) # Create the pseudo images pseudo_dict = coadd.create_pseudo_image(coadd_dict_list) # Multiply in a sensitivity function to flux the 2d image if args.flux: # Load the sensitivity function # wave_sens, sfunc, _, _, _ = sensfunc.SensFunc.load(sensfunc_masterframe_name) sens = sensfunc.SensFunc.from_file(sensfunc_masterframe_name) # Interpolate the sensitivity function onto the wavelength grid of # the data. Since the image is rectified this is trivial and we # don't need to do a 2d interpolation exptime = spectrograph.get_meta_value(files[0], 'exptime') sens_factor = flux_calib.get_sensfunc_factor(pseudo_dict['wave_mid'][:, islit], sens.wave, sens.zeropoint, exptime, extrap_sens=parset['fluxcalib']['extrap_sens']) # Compute the median sensitivity and set the sensitivity to zero at # locations 100 times the median. This prevents the 2d image from # blowing up where the sens_factor explodes because there is no # throughput sens_gpm = sens_factor < 100.0 * np.median(sens_factor) sens_factor_masked = sens_factor * sens_gpm sens_factor_img = np.repeat(sens_factor_masked[:, np.newaxis], pseudo_dict['nspat'], axis=1) imgminsky = sens_factor_img * pseudo_dict['imgminsky'] imgminsky_gpm = sens_gpm[:, np.newaxis] & pseudo_dict['inmask'] else: imgminsky = pseudo_dict['imgminsky'] imgminsky_gpm = pseudo_dict['inmask'] ########################## # Now display the images # ########################## if not args.no_gui: display.connect_to_ginga(raise_err=True, allow_new=True) # TODO: Bug in ginga prevents me from using cuts here for some # reason mean, med, sigma = sigma_clipped_stats(imgminsky[imgminsky_gpm], sigma_lower=3.0, sigma_upper=3.0) chname_skysub = 'fluxed-skysub-det{:s}'.format(sdet) \ if args.flux else 'skysub-det{:s}'.format(sdet) cuts_skysub = (med - 3.0 * sigma, med + 3.0 * sigma) cuts_resid = (-5.0, 5.0) # fits.writeto('/Users/joe/ginga_test.fits',imgminsky, overwrite=True) # fits.writeto('/Users/joe/ginga_mask.fits',imgminsky_gpm.astype(float), overwrite=True) # embed() # Clear all channels at the beginning # TODO: JFH For some reason Ginga crashes when I try to put cuts in here. viewer, ch_skysub = display.show_image(imgminsky, chname=chname_skysub, waveimg=pseudo_dict['waveimg'], clear=True, cuts=cuts_skysub) slit_left, slit_righ, _ = pseudo_dict['slits'].select_edges() slit_id = slits.slitord_id[0] display.show_slits(viewer, ch_skysub, slit_left, slit_righ, slit_ids=slit_id) # SKRESIDS chname_skyresids = 'sky_resid-det{:s}'.format(sdet) # sky residual map image = pseudo_dict['imgminsky'] * np.sqrt(pseudo_dict['sciivar']) * pseudo_dict['inmask'] viewer, ch_skyresids = display.show_image(image, chname_skyresids, waveimg=pseudo_dict['waveimg'], cuts=cuts_resid) display.show_slits(viewer, ch_skyresids, slit_left, slit_righ, slit_ids=slits.slitord_id[0]) shell = viewer.shell() out = shell.start_global_plugin('WCSMatch') out = shell.call_global_plugin_method('WCSMatch', 'set_reference_channel', [chname_skysub], {}) # TODO extract along a spatial position if args.writefits: head0 = fits.getheader(files[0]) # TODO use meta tools for the object name in the future. outfile = target + '_specXspat_{:3.2f}X{:3.2f}.fits'.format(args.spec_samp_fact, args.spat_samp_fact) hdu = fits.PrimaryHDU(imgminsky, header=head0) hdu_resid = fits.ImageHDU(pseudo_dict['imgminsky'] \ * np.sqrt(pseudo_dict['sciivar']) * pseudo_dict['inmask']) hdu_wave = fits.ImageHDU(pseudo_dict['waveimg']) hdul = fits.HDUList([hdu, hdu_resid, hdu_wave]) msgs.info('Writing sky subtracted image to {:s}'.format(outfile)) hdul.writeto(outfile, overwrite=True) msgs.info(utils.get_time_string(time.time()-tstart)) if args.embed: embed() return 0
def main(args): # List only? hdu = fits.open(args.file) head0 = hdu[0].header if args.list: hdu.info() return # Init sdet = get_dnum(args.det, prefix=False) # One detector, sky sub for now names = [hdu[i].name for i in range(len(hdu))] try: exten = names.index('DET{:s}-PROCESSED'.format(sdet)) except: # Backwards compatability msgs.error('Requested detector {:s} was not processed.\n' 'Maybe you chose the wrong one to view?\n' 'Set with --det= or check file contents with --list'.format(sdet)) sciimg = hdu[exten].data try: exten = names.index('DET{:s}-SKY'.format(sdet)) except: # Backwards compatability msgs.error('Requested detector {:s} has no sky model.\n' 'Maybe you chose the wrong one to view?\n' 'Set with --det= or check file contents with --list'.format(sdet)) skymodel = hdu[exten].data try: exten = names.index('DET{:s}-MASK'.format(sdet)) except ValueError: # Backwards compatability msgs.error('Requested detector {:s} has no bit mask.\n' 'Maybe you chose the wrong one to view?\n' 'Set with --det= or check file contents with --list'.format(sdet)) mask = hdu[exten].data frame = (sciimg - skymodel) * (mask == 0) mdir = head0['PYPMFDIR'] if not os.path.exists(mdir): mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir)) msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base)) mdir = mdir_base trace_key = '{0}_{1:02d}'.format(head0['TRACMKEY'], args.det) trc_file = os.path.join(mdir, MasterFrame.construct_file_name('Trace', trace_key)) # TODO -- Remove this once the move to Edges is complete if args.old: tslits_dict = TraceSlits.load_from_file(trc_file)[0] else: trc_file = trc_file.replace('Trace', 'Edges')+'.gz' tslits_dict = edgetrace.EdgeTraceSet.from_file(trc_file).convert_to_tslits_dict() shape = (tslits_dict['nspec'], tslits_dict['nspat']) slit_ids = [trace_slits.get_slitid(shape, tslits_dict['slit_left'], tslits_dict['slit_righ'], ii)[0] for ii in range(tslits_dict['slit_left'].shape[1])] # Object traces spec1d_file = args.file.replace('spec2d', 'spec1d') det_nm = 'DET{:s}'.format(sdet) if os.path.isfile(spec1d_file): hdulist_1d = fits.open(spec1d_file) else: hdulist_1d = [] msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) + msgs.newline() + ' No objects were extracted.') tslits_dict['objtrc'] = parse_traces(hdulist_1d, det_nm) # TODO :: Need to include standard star trace in the spec2d files std_trace = None # Extract some trace models fwhm = 2 # Start with some default value # Brightest object on slit trace_model_obj = None trace_model_dict = dict() if len(tslits_dict['objtrc']['pkflux']) > 0: smash_peakflux = tslits_dict['objtrc']['pkflux'] ibri = smash_peakflux.argmax() trace_model_obj = tslits_dict['objtrc']['traces'][ibri] fwhm = tslits_dict['objtrc']['fwhm'][ibri] trace_model_dict['object'] = dict(trace_model=trace_model_obj, fwhm=fwhm) # Standard star trace trace_model_dict['std'] = dict(trace_model=std_trace, fwhm=fwhm) # Trace of the slit edge trace_model_dict['slit'] = dict(trace_model=tslits_dict['slit_left'].copy(), fwhm=fwhm) tslits_dict['trace_model'] = trace_model_dict # Finally, initialise the GUI gui_object_find.initialise(args.det, frame, tslits_dict, None, printout=True, slit_ids=slit_ids)
def load_coadd2d_stacks(self, spec2d): """ Routine to read in required images for 2d coadds given a list of spec2d files. Args: spec2d_files: list List of spec2d filenames det: int detector in question Returns: dict: Dictionary containing all the images and keys required for perfomring 2d coadds. """ # Get the detector string sdet = parse.get_dnum(self.det, prefix=False) # Get the master dir redux_path = os.getcwd() # Grab the files #head2d_list = [] specobjs_list = [] slits_list = [] nfiles =len(spec2d) detectors_list = [] for ifile, f in enumerate(spec2d): if isinstance(f, spec2dobj.Spec2DObj): # If spec2d is a list of objects s2dobj = f else: # If spec2d is a list of files, option to also use spec1ds s2dobj = spec2dobj.Spec2DObj.from_file(f, self.det) spec1d_file = f.replace('spec2d', 'spec1d') if os.path.isfile(spec1d_file): sobjs = specobjs.SpecObjs.from_fitsfile(spec1d_file) this_det = sobjs.DET == self.det specobjs_list.append(sobjs[this_det]) # TODO the code should run without a spec1d file, but we need to implement that slits_list.append(s2dobj.slits) detectors_list.append(s2dobj.detector) if ifile == 0: sciimg_stack = np.zeros((nfiles,) + s2dobj.sciimg.shape, dtype=float) waveimg_stack = np.zeros_like(sciimg_stack, dtype=float) tilts_stack = np.zeros_like(sciimg_stack, dtype=float) skymodel_stack = np.zeros_like(sciimg_stack, dtype=float) sciivar_stack = np.zeros_like(sciimg_stack, dtype=float) mask_stack = np.zeros_like(sciimg_stack, dtype=float) slitmask_stack = np.zeros_like(sciimg_stack, dtype=int) sciimg_stack[ifile, :, :] = s2dobj.sciimg waveimg_stack[ifile, :, :] = s2dobj.waveimg skymodel_stack[ifile, :, :] = s2dobj.skymodel sciivar_stack[ifile, :, :] = s2dobj.ivarmodel mask_stack[ifile, :, :] = s2dobj.bpmmask # TODO -- Set back after done testing slitmask_stack[ifile, :, :] = s2dobj.slits.slit_img(flexure=s2dobj.sci_spat_flexure) #slitmask_stack[ifile, :, :] = spec2DObj.slits.slit_img(flexure=0.) _spat_flexure = 0. if s2dobj.sci_spat_flexure is None else s2dobj.sci_spat_flexure #_tilt_flexure_shift = _spat_flexure - spec2DObj.tilts.spat_flexure if spec2DObj.tilts.spat_flexure is not None else _spat_flexure tilts_stack[ifile,:,:] = s2dobj.tilts #.fit2tiltimg(slitmask_stack[ifile, :, :], flexure=_tilt_flexure_shift) return dict(specobjs_list=specobjs_list, slits_list=slits_list, slitmask_stack=slitmask_stack, sciimg_stack=sciimg_stack, sciivar_stack=sciivar_stack, skymodel_stack=skymodel_stack, mask_stack=mask_stack, tilts_stack=tilts_stack, waveimg_stack=waveimg_stack, redux_path=redux_path, detectors=detectors_list, spectrograph=self.spectrograph.name, pypeline=self.spectrograph.pypeline)
def main(args): # List only? if args.list: hdu = fits.open(args.file) hdu.info() return # Load it up -- NOTE WE ALLOW *OLD* VERSIONS TO GO FORTH spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, args.det, chk_version=False) # Setup for PypeIt imports msgs.reset(verbosity=2) # Init # TODO: get_dnum needs to be deprecated... sdet = get_dnum(args.det, prefix=False) # Grab the slit edges slits = spec2DObj.slits if spec2DObj.sci_spat_flexure is not None: msgs.info("Offseting slits by {}".format(spec2DObj.sci_spat_flexure)) all_left, all_right, mask = slits.select_edges( flexure=spec2DObj.sci_spat_flexure) # TODO -- This may be too restrictive, i.e. ignore BADFLTCALIB?? gpm = mask == 0 left = all_left[:, gpm] right = all_right[:, gpm] slid_IDs = spec2DObj.slits.slitord_id[gpm] bitMask = ImageBitMask() # Object traces from spec1d file spec1d_file = args.file.replace('spec2d', 'spec1d') if args.file[-2:] == 'gz': spec1d_file = spec1d_file[:-3] if os.path.isfile(spec1d_file): sobjs = specobjs.SpecObjs.from_fitsfile(spec1d_file) else: sobjs = None msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) + msgs.newline() + ' No objects were extracted.') display.connect_to_ginga(raise_err=True, allow_new=True) # Now show each image to a separate channel # Show the bitmask? mask_in = None if args.showmask: viewer, ch = display.show_image(spec2DObj.bpmmask, chname="BPM", waveimg=spec2DObj.waveimg, clear=True) #bpm, crmask, satmask, minmask, offslitmask, nanmask, ivar0mask, ivarnanmask, extractmask \ # SCIIMG image = spec2DObj.sciimg # Processed science image mean, med, sigma = sigma_clipped_stats(image[spec2DObj.bpmmask == 0], sigma_lower=5.0, sigma_upper=5.0) cut_min = mean - 1.0 * sigma cut_max = mean + 4.0 * sigma chname_skysub = 'sciimg-det{:s}'.format(sdet) # Clear all channels at the beginning viewer, ch = display.show_image(image, chname=chname_skysub, waveimg=spec2DObj.waveimg, clear=True) if sobjs is not None: show_trace(sobjs, args.det, viewer, ch) display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs) # SKYSUB if args.ignore_extract_mask: # TODO -- Is there a cleaner way to do this? gpm = (spec2DObj.bpmmask == 0) | (spec2DObj.bpmmask == 2** bitMask.bits['EXTRACT']) else: gpm = spec2DObj.bpmmask == 0 image = (spec2DObj.sciimg - spec2DObj.skymodel ) * gpm #(spec2DObj.mask == 0) # sky subtracted image mean, med, sigma = sigma_clipped_stats(image[spec2DObj.bpmmask == 0], sigma_lower=5.0, sigma_upper=5.0) cut_min = mean - 1.0 * sigma cut_max = mean + 4.0 * sigma chname_skysub = 'skysub-det{:s}'.format(sdet) # Clear all channels at the beginning # TODO: JFH For some reason Ginga crashes when I try to put cuts in here. viewer, ch = display.show_image( image, chname=chname_skysub, waveimg=spec2DObj.waveimg, bitmask=bitMask, mask=mask_in) #, cuts=(cut_min, cut_max),wcs_match=True) if not args.removetrace and sobjs is not None: show_trace(sobjs, args.det, viewer, ch) display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs) # SKRESIDS chname_skyresids = 'sky_resid-det{:s}'.format(sdet) image = (spec2DObj.sciimg - spec2DObj.skymodel) * np.sqrt( spec2DObj.ivarmodel ) * gpm #(spec2DObj.bpmmask == 0) # sky residual map viewer, ch = display.show_image(image, chname_skyresids, waveimg=spec2DObj.waveimg, cuts=(-5.0, 5.0), bitmask=bitMask, mask=mask_in) if not args.removetrace and sobjs is not None: show_trace(sobjs, args.det, viewer, ch) display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs) # RESIDS chname_resids = 'resid-det{:s}'.format(sdet) # full model residual map image = (spec2DObj.sciimg - spec2DObj.skymodel - spec2DObj.objmodel ) * np.sqrt(spec2DObj.ivarmodel) * (spec2DObj.bpmmask == 0) viewer, ch = display.show_image(image, chname=chname_resids, waveimg=spec2DObj.waveimg, cuts=(-5.0, 5.0), bitmask=bitMask, mask=mask_in) if not args.removetrace and sobjs is not None: show_trace(sobjs, args.det, viewer, ch) display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs) # After displaying all the images sync up the images with WCS_MATCH shell = viewer.shell() shell.start_global_plugin('WCSMatch') shell.call_global_plugin_method('WCSMatch', 'set_reference_channel', [chname_resids], {}) if args.embed: embed()
def load_coadd2d_stacks(spec2d_files, det): """ Args: spec2d_files: list List of spec2d filenames det: int detector in question Returns: stack_dict: dict Dictionary containing all the images and keys required for perfomring 2d coadds. """ # Get the detector string sdet = parse.get_dnum(det, prefix=False) # Get the master dir head0 = fits.getheader(spec2d_files[0]) master_dir = os.path.basename(head0['PYPMFDIR']) redux_path = os.getcwd() master_path = os.path.join(redux_path, master_dir) # Grab the files head2d_list=[] tracefiles = [] waveimgfiles = [] tiltfiles = [] spec1d_files = [] for f in spec2d_files: head = fits.getheader(f) trace_key = '{0}_{1:02d}'.format(head['TRACMKEY'], det) wave_key = '{0}_{1:02d}'.format(head['ARCMKEY'], det) head2d_list.append(head) spec1d_files.append(f.replace('spec2d', 'spec1d')) tracefiles.append(os.path.join(master_path, MasterFrame.construct_file_name('Trace', trace_key))) waveimgfiles.append(os.path.join(master_path, MasterFrame.construct_file_name('Wave', wave_key))) tiltfiles.append(os.path.join(master_path, MasterFrame.construct_file_name('Tilts', wave_key))) nfiles = len(spec2d_files) specobjs_list = [] head1d_list=[] # TODO Sort this out with the correct detector extensions etc. # Read in the image stacks for ifile in range(nfiles): waveimg = WaveImage.load_from_file(waveimgfiles[ifile]) tilts = WaveTilts.load_from_file(tiltfiles[ifile]) hdu = fits.open(spec2d_files[ifile]) # One detector, sky sub for now names = [hdu[i].name for i in range(len(hdu))] # science image try: exten = names.index('DET{:s}-PROCESSED'.format(sdet)) except: # Backwards compatability det_error_msg(exten, sdet) sciimg = hdu[exten].data # skymodel try: exten = names.index('DET{:s}-SKY'.format(sdet)) except: # Backwards compatability det_error_msg(exten, sdet) skymodel = hdu[exten].data # Inverse variance model try: exten = names.index('DET{:s}-IVARMODEL'.format(sdet)) except ValueError: # Backwards compatability det_error_msg(exten, sdet) sciivar = hdu[exten].data # Mask try: exten = names.index('DET{:s}-MASK'.format(sdet)) except ValueError: # Backwards compatability det_error_msg(exten, sdet) mask = hdu[exten].data if ifile == 0: # the two shapes accomodate the possibility that waveimg and tilts are binned differently shape_wave = (nfiles,waveimg.shape[0],waveimg.shape[1]) shape_sci = (nfiles,sciimg.shape[0],sciimg.shape[1]) waveimg_stack = np.zeros(shape_wave,dtype=float) tilts_stack = np.zeros(shape_wave,dtype=float) sciimg_stack = np.zeros(shape_sci,dtype=float) skymodel_stack = np.zeros(shape_sci,dtype=float) sciivar_stack = np.zeros(shape_sci,dtype=float) mask_stack = np.zeros(shape_sci,dtype=float) waveimg_stack[ifile,:,:] = waveimg tilts_stack[ifile,:,:] = tilts['tilts'] sciimg_stack[ifile,:,:] = sciimg sciivar_stack[ifile,:,:] = sciivar mask_stack[ifile,:,:] = mask skymodel_stack[ifile,:,:] = skymodel sobjs, head = load.load_specobjs(spec1d_files[ifile]) head1d_list.append(head) specobjs_list.append(sobjs) # Right now we assume there is a single tslits_dict for all images and read in the first one # TODO this needs to become a tslits_dict for each file to accomodate slits defined by flats taken on different # nights tslits_dict, _ = TraceSlits.load_from_file(tracefiles[0]) spectrograph = util.load_spectrograph(tslits_dict['spectrograph']) slitmask = pixels.tslits2mask(tslits_dict) slitmask_stack = np.einsum('i,jk->ijk', np.ones(nfiles), slitmask) # Fill the master key dict head2d = head2d_list[0] master_key_dict = {} master_key_dict['frame'] = head2d['FRAMMKEY'] + '_{:02d}'.format(det) master_key_dict['bpm'] = head2d['BPMMKEY'] + '_{:02d}'.format(det) master_key_dict['bias'] = head2d['BIASMKEY'] + '_{:02d}'.format(det) master_key_dict['arc'] = head2d['ARCMKEY'] + '_{:02d}'.format(det) master_key_dict['trace'] = head2d['TRACMKEY'] + '_{:02d}'.format(det) master_key_dict['flat'] = head2d['FLATMKEY'] + '_{:02d}'.format(det) stack_dict = dict(specobjs_list=specobjs_list, tslits_dict=tslits_dict, slitmask_stack=slitmask_stack, sciimg_stack=sciimg_stack, sciivar_stack=sciivar_stack, skymodel_stack=skymodel_stack, mask_stack=mask_stack, tilts_stack=tilts_stack, waveimg_stack=waveimg_stack, head1d_list = head1d_list, head2d_list=head2d_list, redux_path=redux_path, master_path=master_path, master_dir=master_dir, master_key_dict=master_key_dict, spectrograph = tslits_dict['spectrograph']) return stack_dict
def save_2d_images(sci_output, raw_header, spectrograph, master_key_dict, mfdir, outfile, clobber=True, update_det=None): """ Write 2D images to the hard drive Args: sci_output (OrderedDict): raw_header (astropy.fits.Header or dict): master_key_dict (str): mfdir (str): outfile (str): clobber: bool, optional Returns: """ if os.path.isfile(outfile) and update_det is not None: hdus, prihdu = init_hdus(update_det, outfile) else: # Primary HDU for output prihdu = fits.PrimaryHDU() # Update with original header, skipping a few keywords hdus = [prihdu] hdukeys = [ 'BUNIT', 'COMMENT', '', 'BITPIX', 'NAXIS', 'NAXIS1', 'NAXIS2', 'HISTORY', 'EXTEND', 'DATASEC' ] for key in raw_header.keys(): # Use new ones if key in hdukeys: continue # Update unused ones prihdu.header[key] = raw_header[key] # History if 'HISTORY' in raw_header.keys(): # Strip \n tmp = str(raw_header['HISTORY']).replace('\n', ' ') prihdu.header.add_history(str(tmp)) # PYPEIT # TODO Should the spectrograph be written to the header? prihdu.header['PIPELINE'] = str('PYPEIT') prihdu.header['PYPELINE'] = spectrograph.pypeline prihdu.header['SPECTROG'] = spectrograph.spectrograph prihdu.header['DATE-RDX'] = str( datetime.date.today().strftime('%Y-%b-%d')) prihdu.header['FRAMMKEY'] = master_key_dict['frame'][:-3] prihdu.header['BPMMKEY'] = master_key_dict['bpm'][:-3] prihdu.header['BIASMKEY'] = master_key_dict['bias'][:-3] prihdu.header['ARCMKEY'] = master_key_dict['arc'][:-3] prihdu.header['TRACMKEY'] = master_key_dict['trace'][:-3] prihdu.header['FLATMKEY'] = master_key_dict['flat'][:-3] prihdu.header['PYPMFDIR'] = str(mfdir) if sci_output['meta']['ir_redux']: prihdu.header['SKYSUB'] = 'DIFF' else: prihdu.header['SKYSUB'] = 'MODEL' # Fill in the images ext = len(hdus) - 1 for key in sci_output.keys(): if key in ['meta']: continue else: det = key sdet = parse.get_dnum(det, caps=True) # e.g. DET02 if 'sciimg' not in sci_output[det]: continue # Specified detector number? #if settings.argflag['reduce']['detnum'] is not None: # if det not in map(int, settings.argflag['reduce']['detnum']): # continue # else: # msgs.warn("Restricting the reduction to detector {:d}".format(det)) # Processed frame ext += 1 keywd = 'EXT{:04d}'.format(ext) prihdu.header[keywd] = '{:s}-Processed'.format(sdet) hdu = fits.ImageHDU(sci_output[det]['sciimg']) #slf._sciframe[det-1]) hdu.name = prihdu.header[keywd] hdus.append(hdu) # Raw Inverse Variance ext += 1 keywd = 'EXT{:04d}'.format(ext) prihdu.header[keywd] = '{:s}-IVARRAW'.format(sdet) hdu = fits.ImageHDU( sci_output[det]['sciivar']) #slf._modelvarframe[det-1]) hdu.name = prihdu.header[keywd] hdus.append(hdu) # Background model ext += 1 keywd = 'EXT{:04d}'.format(ext) prihdu.header[keywd] = '{:s}-SKY'.format(sdet) hdu = fits.ImageHDU( sci_output[det]['skymodel']) #slf._modelvarframe[det-1]) hdu.name = prihdu.header[keywd] hdus.append(hdu) # Object model ext += 1 keywd = 'EXT{:04d}'.format(ext) prihdu.header[keywd] = '{:s}-OBJ'.format(sdet) hdu = fits.ImageHDU( sci_output[det]['objmodel']) #slf._modelvarframe[det-1]) hdu.name = prihdu.header[keywd] hdus.append(hdu) # Inverse Variance model ext += 1 keywd = 'EXT{:04d}'.format(ext) prihdu.header[keywd] = '{:s}-IVARMODEL'.format(sdet) hdu = fits.ImageHDU( sci_output[det]['ivarmodel']) # slf._modelvarframe[det-1]) hdu.name = prihdu.header[keywd] hdus.append(hdu) # Final mask ext += 1 keywd = 'EXT{:04d}'.format(ext) prihdu.header[keywd] = '{:s}-MASK'.format(sdet) hdu = fits.ImageHDU( sci_output[det]['outmask']) # slf._modelvarframe[det-1]) hdu.name = prihdu.header[keywd] hdus.append(hdu) # Finish hdulist = fits.HDUList(hdus) hdulist.writeto(outfile, overwrite=clobber) msgs.info("Wrote: {:s}".format(outfile))