def ech_generate_sensfunc(stdframe,spectrograph=None, telluric=True, star_type=None, star_mag=None, ra=None, dec=None, std_file = None, BALM_MASK_WID=5., nresln=None,debug=False): if spectrograph is None: std_specobjs, std_header = ech_load_specobj(stdframe, order=0) spectrograph = std_header['INSTRUME'] msgs.info('You are working on {:s}'.format(spectrograph)) ext_final = fits.getheader(stdframe, -1) norder = 5 #norder = ext_final['ORDER'] + 1 sens_dicts = {} for iord in range(norder): std_specobjs, std_header = ech_load_specobj(stdframe, order=iord) std_idx = flux.find_standard(std_specobjs) std = std_specobjs[std_idx] wavemask = std.boxcar['WAVE']>1000.0*units.AA wave,counts,ivar = std.boxcar['WAVE'][wavemask],std.boxcar['COUNTS'][wavemask],std.boxcar['COUNTS_IVAR'][wavemask] sens_dict = flux.generate_sensfunc(wave,counts,ivar,std_header['AIRMASS'],std_header['EXPTIME'], spectrograph,star_type=star_type,star_mag=star_mag, telluric=telluric,ra=ra,dec=dec,BALM_MASK_WID=BALM_MASK_WID, nresln=nresln,std_file=std_file,debug=debug) sens_dict['ech_orderindx'] = iord sens_dicts[str(iord)] = sens_dict sens_dicts['norder'] = norder return sens_dicts
def load_master(self, filename, force=False): # Does the master file exist? if not os.path.isfile(filename): # msgs.warn("No Master frame found of type {:s}: {:s}".format(self.frametype, filename)) msgs.warn("No Master frame found of {:s}".format(filename)) if force: msgs.error("Crashing out because reduce-masters-force=True:" + msgs.newline() + filename) return None else: # msgs.info("Loading a pre-existing master calibration frame of type: {:}".format(self.frametype) + " from filename: {:}".format(filename)) msgs.info("Loading a pre-existing master calibration frame of SENSFUNC from filename: {:}".format(filename)) hdu = fits.open(filename) norder = hdu[0].header['NORDER'] sens_dicts = {} for iord in range(norder): head = hdu[iord + 1].header tbl = hdu['SENSFUNC-ORDER{0:04}'.format(iord)].data sens_dict = {} sens_dict['wave'] = tbl['WAVE'] sens_dict['sensfunc'] = tbl['SENSFUNC'] for key in ['wave_min', 'wave_max', 'exptime', 'airmass', 'std_file', 'std_ra', 'std_dec', 'std_name', 'cal_file', 'ech_orderindx']: try: sens_dict[key] = head[key.upper()] except: pass sens_dicts[str(iord)] = sens_dict sens_dicts['norder'] = norder return sens_dicts
def determine_bias_mode(self, prev_build=False): """ Determine the bias mode to use in this reduction - None -- No bias subtraction - 'overscan' -- Overscan subtract - msbias -- Use a generated bias image Args: prev_build (bool, optional): Load the master frame if it exists and was built on this run. Returns: ndarray, str or None: :attr:`msbias` str, np.ndarray or None """ # How are we treating biases? # 1) No bias subtraction if self.par['useframe'].lower() == 'none': msgs.info("Will not perform bias/dark subtraction") self.msbias = None # 2) Use overscan elif self.par['useframe'] == 'overscan': self.msbias = 'overscan' # 3) User wants bias subtractions, use a Master biasframe? elif self.par['useframe'] in ['bias', 'dark']: # Load the MasterFrame if it exists and user requested one to load it self.msbias, _ = self.master(prev_build=prev_build) return self.msbias
def load_spec_order(fname,objid=None,order=None,extract='OPT',flux=True): """ Loading single order spectrum from a PypeIt 1D specctrum fits file :param file: :param objid: :param order: :param extract: :param flux: :return: """ if objid is None: objid = 0 if order is None: msgs.error('Please specify which order you want to load') # read extension name into a list primary_header = fits.getheader(fname, 0) nspec = primary_header['NSPEC'] extnames = [primary_header['EXT0001']] * nspec for kk in range(nspec): extnames[kk] = primary_header['EXT' + '{0:04}'.format(kk + 1)] extnameroot = extnames[0] # Figure out which extension is the required data ordername = '{0:04}'.format(order) extname = extnameroot.replace('OBJ0000', objid) extname = extname.replace('ORDER0000', 'ORDER' + ordername) try: exten = extnames.index(extname) + 1 msgs.info("Loading extension {:s} of spectrum {:s}".format(extname, fname)) except: msgs.error("Spectrum {:s} does not contain {:s} extension".format(fname, extname)) spectrum = load.load_1dspec(fname, exten=exten, extract=extract, flux=flux) # Polish a bit -- Deal with NAN, inf, and *very* large values that will exceed # the floating point precision of float32 for var which is sig**2 (i.e. 1e38) bad_flux = np.any([np.isnan(spectrum.flux), np.isinf(spectrum.flux), np.abs(spectrum.flux) > 1e30, spectrum.sig ** 2 > 1e10, ], axis=0) # Sometimes Echelle spectra have zero wavelength bad_wave = spectrum.wavelength < 1000.0*units.AA bad_all = bad_flux + bad_wave ## trim bad part wave_out,flux_out,sig_out = spectrum.wavelength[~bad_all],spectrum.flux[~bad_all],spectrum.sig[~bad_all] spectrum_out = XSpectrum1D.from_tuple((wave_out,flux_out,sig_out), verbose=False) #if np.sum(bad_flux): # msgs.warn("There are some bad flux values in this spectrum. Will zero them out and mask them (not ideal)") # spectrum.data['flux'][spectrum.select][bad_flux] = 0. # spectrum.data['sig'][spectrum.select][bad_flux] = 0. return spectrum_out
def ech_load_spec(files,objid=None,order=None,extract='OPT',flux=True): """ files: A list of file names objid: extract: flux: """ nfiles = len(files) if objid is None: objid = ['OBJ0000'] * nfiles elif len(objid) == 1: objid = objid * nfiles elif len(objid) != nfiles: msgs.error('The length of objid should be either 1 or equal to the number of spectra files.') fname = files[0] ext_final = fits.getheader(fname, -1) norder = ext_final['ORDER'] + 1 msgs.info('spectrum {:s} has {:d} orders'.format(fname, norder)) if norder <= 1: msgs.error('The number of orders have to be greater than one for echelle. Longslit data?') # Load spectra spectra_list = [] for ii, fname in enumerate(files): if order is None: msgs.info('Loading all orders into a gaint spectra') for iord in range(norder): spectrum = load_spec_order(fname,objid=objid[ii],order=iord,extract=extract,flux=flux) # Append spectra_list.append(spectrum) elif order >= norder: msgs.error('order number cannot greater than the total number of orders') else: spectrum = load_spec_order(fname, objid=objid[ii], order=order, extract=extract, flux=flux) # Append spectra_list.append(spectrum) # Join into one XSpectrum1D object spectra = collate(spectra_list) # Return return spectra
def ech_flux_science(sci_specobjs,sens_dicts,sci_header,spectrograph=None): """ Flux the internal list of sci_specobjs Wrapper to flux.apply_sensfunc() Returns ------- """ norder = sens_dicts['norder'] if spectrograph is None: spectrograph = sci_header['INSTRUME'] msgs.info('You are working on {:s}'.format(spectrograph)) for iord in range(norder): sens_dict = sens_dicts[str(iord)] for sci_obj in sci_specobjs: if sci_obj.ech_orderindx == iord: flux.apply_sensfunc(sci_obj, sens_dict, sci_header['AIRMASS'], sci_header['EXPTIME'], spectrograph)
def main(args): # List only? if args.list: hdu = io.fits_open(args.file) hdu.info() return # Load it up -- NOTE WE ALLOW *OLD* VERSIONS TO GO FORTH spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, args.det, chk_version=False) # Setup for PypeIt imports msgs.reset(verbosity=2) # Init # TODO: get_dnum needs to be deprecated... sdet = get_dnum(args.det, prefix=False) # Grab the slit edges slits = spec2DObj.slits if spec2DObj.sci_spat_flexure is not None: msgs.info("Offseting slits by {}".format(spec2DObj.sci_spat_flexure)) all_left, all_right, mask = slits.select_edges( flexure=spec2DObj.sci_spat_flexure) # TODO -- This may be too restrictive, i.e. ignore BADFLTCALIB?? gpm = mask == 0 left = all_left[:, gpm] right = all_right[:, gpm] slid_IDs = spec2DObj.slits.slitord_id[gpm] maskdef_id = spec2DObj.slits.maskdef_id[ gpm] if spec2DObj.slits.maskdef_id is not None else None bitMask = ImageBitMask() # Object traces from spec1d file spec1d_file = args.file.replace('spec2d', 'spec1d') if args.file[-2:] == 'gz': spec1d_file = spec1d_file[:-3] if os.path.isfile(spec1d_file): sobjs = specobjs.SpecObjs.from_fitsfile(spec1d_file) else: sobjs = None msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) + msgs.newline() + ' No objects were extracted.') display.connect_to_ginga(raise_err=True, allow_new=True) # Now show each image to a separate channel # Show the bitmask? mask_in = None if args.showmask: viewer, ch = display.show_image(spec2DObj.bpmmask, chname="BPM", waveimg=spec2DObj.waveimg, clear=True) #bpm, crmask, satmask, minmask, offslitmask, nanmask, ivar0mask, ivarnanmask, extractmask \ # SCIIMG image = spec2DObj.sciimg # Processed science image mean, med, sigma = sigma_clipped_stats(image[spec2DObj.bpmmask == 0], sigma_lower=5.0, sigma_upper=5.0) cut_min = mean - 1.0 * sigma cut_max = mean + 4.0 * sigma chname_skysub = 'sciimg-det{:s}'.format(sdet) # Clear all channels at the beginning viewer, ch = display.show_image(image, chname=chname_skysub, waveimg=spec2DObj.waveimg, clear=True) if sobjs is not None: show_trace(sobjs, args.det, viewer, ch) display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs, maskdef_ids=maskdef_id) # SKYSUB if args.ignore_extract_mask: # TODO -- Is there a cleaner way to do this? gpm = (spec2DObj.bpmmask == 0) | (spec2DObj.bpmmask == 2** bitMask.bits['EXTRACT']) else: gpm = spec2DObj.bpmmask == 0 image = (spec2DObj.sciimg - spec2DObj.skymodel ) * gpm #(spec2DObj.mask == 0) # sky subtracted image mean, med, sigma = sigma_clipped_stats(image[spec2DObj.bpmmask == 0], sigma_lower=5.0, sigma_upper=5.0) cut_min = mean - 1.0 * sigma cut_max = mean + 4.0 * sigma chname_skysub = 'skysub-det{:s}'.format(sdet) # Clear all channels at the beginning # TODO: JFH For some reason Ginga crashes when I try to put cuts in here. viewer, ch = display.show_image( image, chname=chname_skysub, waveimg=spec2DObj.waveimg, bitmask=bitMask, mask=mask_in) #, cuts=(cut_min, cut_max),wcs_match=True) if not args.removetrace and sobjs is not None: show_trace(sobjs, args.det, viewer, ch) display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs, maskdef_ids=maskdef_id) # SKRESIDS chname_skyresids = 'sky_resid-det{:s}'.format(sdet) image = (spec2DObj.sciimg - spec2DObj.skymodel) * np.sqrt( spec2DObj.ivarmodel ) * gpm #(spec2DObj.bpmmask == 0) # sky residual map viewer, ch = display.show_image(image, chname_skyresids, waveimg=spec2DObj.waveimg, cuts=(-5.0, 5.0), bitmask=bitMask, mask=mask_in) if not args.removetrace and sobjs is not None: show_trace(sobjs, args.det, viewer, ch) display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs, maskdef_ids=maskdef_id) # RESIDS chname_resids = 'resid-det{:s}'.format(sdet) # full model residual map image = (spec2DObj.sciimg - spec2DObj.skymodel - spec2DObj.objmodel ) * np.sqrt(spec2DObj.ivarmodel) * (spec2DObj.bpmmask == 0) viewer, ch = display.show_image(image, chname=chname_resids, waveimg=spec2DObj.waveimg, cuts=(-5.0, 5.0), bitmask=bitMask, mask=mask_in) if not args.removetrace and sobjs is not None: show_trace(sobjs, args.det, viewer, ch) display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs, maskdef_ids=maskdef_id) # After displaying all the images sync up the images with WCS_MATCH shell = viewer.shell() shell.start_global_plugin('WCSMatch') shell.call_global_plugin_method('WCSMatch', 'set_reference_channel', [chname_resids], {}) if args.embed: embed()
def run(self, doqa=True, debug=False, show=False): """ Main driver for tracing arc lines Code flow: #. Extract an arc spectrum down the center of each slit/order #. Loop on slits/orders #. Trace and fit the arc lines (This is done twice, once with trace_crude as the tracing crutch, then again with a PCA model fit as the crutch). #. Repeat trace. #. 2D Fit to the offset from slitcen #. Save Args: doqa (bool): debug (bool): show (bool): Returns: :class:`WaveTilts`: """ # Extract the arc spectra for all slits self.arccen, self.arccen_bpm = self.extract_arcs() # TODO: Leave for now. Used for debugging # self.par['rm_continuum'] = True # debug = True # show = True # Subtract arc continuum _mstilt = self.mstilt.image.copy() if self.par['rm_continuum']: continuum = self.model_arc_continuum(debug=debug) _mstilt -= continuum if debug: # TODO: Put this into a function vmin, vmax = visualization.ZScaleInterval().get_limits(_mstilt) w, h = plt.figaspect(1) fig = plt.figure(figsize=(3 * w, h)) ax = fig.add_axes([0.15 / 3, 0.1, 0.8 / 3, 0.8]) ax.imshow(self.mstilt.image, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) ax.set_title('MasterArc') ax = fig.add_axes([1.15 / 3, 0.1, 0.8 / 3, 0.8]) ax.imshow(continuum, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) ax.set_title('Continuum') ax = fig.add_axes([2.15 / 3, 0.1, 0.8 / 3, 0.8]) ax.imshow(_mstilt, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) ax.set_title('MasterArc - Continuum') plt.show() # Final tilts image self.final_tilts = np.zeros(self.shape_science, dtype=float) max_spat_dim = (np.asarray(self.par['spat_order']) + 1).max() max_spec_dim = (np.asarray(self.par['spec_order']) + 1).max() self.coeffs = np.zeros((max_spec_dim, max_spat_dim, self.slits.nslits)) self.spat_order = np.zeros(self.slits.nslits, dtype=int) self.spec_order = np.zeros(self.slits.nslits, dtype=int) # TODO sort out show methods for debugging if show: viewer, ch = display.show_image(self.mstilt.image * (self.slitmask > -1), chname='tilts') # Loop on all slits for slit_idx, slit_spat in enumerate(self.slits.spat_id): if self.tilt_bpm[slit_idx]: continue #msgs.info('Computing tilts for slit {0}/{1}'.format(slit, self.slits.nslits-1)) msgs.info('Computing tilts for slit {0}/{1}'.format( slit_idx, self.slits.nslits)) # Identify lines for tracing tilts msgs.info('Finding lines for tilt analysis') self.lines_spec, self.lines_spat \ = self.find_lines(self.arccen[:,slit_idx], self.slitcen[:,slit_idx], slit_idx, bpm=self.arccen_bpm[:,slit_idx], debug=debug) if self.lines_spec is None: self.slits.mask[slit_idx] = self.slits.bitmask.turn_on( self.slits.mask[slit_idx], 'BADTILTCALIB') continue thismask = self.slitmask == slit_spat # Performs the initial tracing of the line centroids as a # function of spatial position resulting in 1D traces for # each line. msgs.info('Trace the tilts') self.trace_dict = self.trace_tilts(_mstilt, self.lines_spec, self.lines_spat, thismask, self.slitcen[:, slit_idx]) # TODO: Show the traces before running the 2D fit if show: display.show_tilts(viewer, ch, self.trace_dict) self.spat_order[slit_idx] = self._parse_param( self.par, 'spat_order', slit_idx) self.spec_order[slit_idx] = self._parse_param( self.par, 'spec_order', slit_idx) # 2D model of the tilts, includes construction of QA # NOTE: This also fills in self.all_fit_dict and self.all_trace_dict coeff_out = self.fit_tilts(self.trace_dict, thismask, self.slitcen[:, slit_idx], self.spat_order[slit_idx], self.spec_order[slit_idx], slit_idx, doqa=doqa, show_QA=show) self.coeffs[:self.spec_order[slit_idx] + 1, :self.spat_order[slit_idx] + 1, slit_idx] = coeff_out # TODO: Need a way to assess the success of fit_tilts and # flag the slit if it fails # Tilts are created with the size of the original slitmask, # which corresonds to the same binning as the science # images, trace images, and pixelflats etc. self.tilts = tracewave.fit2tilts(self.slitmask_science.shape, coeff_out, self.par['func2d']) # Save to final image thismask_science = self.slitmask_science == slit_spat self.final_tilts[thismask_science] = self.tilts[thismask_science] if debug: # TODO: Add this to the show method? vmin, vmax = visualization.ZScaleInterval().get_limits(_mstilt) plt.imshow(_mstilt, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) for slit_idx, slit_spat in enumerate(self.slits.spat_id): spat = self.all_trace_dict[slit_idx]['tilts_spat'] spec = self.all_trace_dict[slit_idx]['tilts'] spec_fit = self.all_trace_dict[slit_idx]['tilts_fit'] in_fit = self.all_trace_dict[slit_idx]['tot_mask'] not_fit = np.invert(in_fit) & (spec > 0) fit_rej = in_fit & np.invert( self.all_trace_dict[slit_idx]['fit_mask']) fit_keep = in_fit & self.all_trace_dict[slit_idx]['fit_mask'] plt.scatter(spat[not_fit], spec[not_fit], color='C1', marker='.', s=30, lw=0) plt.scatter(spat[fit_rej], spec[fit_rej], color='C3', marker='.', s=30, lw=0) plt.scatter(spat[fit_keep], spec[fit_keep], color='k', marker='.', s=30, lw=0) with_fit = np.invert(np.all(np.invert(fit_keep), axis=0)) for t in range(in_fit.shape[1]): if not with_fit[t]: continue l, r = np.nonzero(in_fit[:, t])[0][[0, -1]] plt.plot(spat[l:r + 1, t], spec_fit[l:r + 1, t], color='k') plt.show() # Record the Mask bpmtilts = np.zeros_like(self.slits.mask, dtype=self.slits.bitmask.minimum_dtype()) for flag in ['BADTILTCALIB']: bpm = self.slits.bitmask.flagged(self.slits.mask, flag) if np.any(bpm): bpmtilts[bpm] = self.slits.bitmask.turn_on(bpmtilts[bpm], flag) # Build and return DataContainer tilts_dict = { 'coeffs': self.coeffs, 'func2d': self.par['func2d'], 'nslit': self.slits.nslits, 'spat_order': self.spat_order, 'spec_order': self.spec_order, 'spat_id': self.slits.spat_id, 'bpmtilts': bpmtilts, 'spat_flexure': self.spat_flexure, 'PYP_SPEC': self.spectrograph.name } return WaveTilts(**tilts_dict)
def run(self, setup_only=False, calibration_check=False, use_header_id=False, sort_dir=None, write_bkg_pairs=False): """ Once instantiated, this is the main method used to construct the object. The code flow is as follows:: - Build the fitstbl from an input file_list (optional) - Type the files (bias, arc, etc.) - Match calibration files to the science files - Generate the setup_dict - Write group info to disk - Write calib info to disk (if main run) It is expected that a user will run this three times if they're being careful. Once with `setup_only=True` to confirm the images are properly typed and grouped together for calibration. A second time with `calibration_check=True` to confirm the appropriate calibrations frames are available. And a third time to do the actual setup before proceeding with the reductions. Args: setup_only (:obj:`bool`, optional): Only this setup will be performed. Pypit is expected to execute in a way that ends after this class is fully instantiated such that the user can inspect the results before proceeding. This has the effect of providing more output describing the success of the setup and how to proceed, and provides warnings (instead of errors) for issues that may cause the reduction itself to fail. calibration_check (obj:`bool`, optional): Only check that the calibration frames are appropriately setup and exist on disk. Pypit is expected to execute in a way that ends after this class is fully instantiated such that the user can inspect the results before proceeding. use_header_id (:obj:`bool`, optional): Allow setup to use the frame types drawn from single, instrument-specific header keywords set to `idname` in the metadata table (:attr:`fitstbl`). sort_dir (:obj:`str`, optional): The directory to put the '.sorted' file. Returns: :class:`pypeit.par.pypeitpar.PypeItPar`, :class:`pypeit.spectrographs.spectrograph.Spectrograph`, :class:`astropy.table.Table`: Returns the attributes :attr:`par`, :attr:`spectrograph`, :attr:`fitstbl` If running with `setup_only` or `calibrations_check`, these are all returned as `None` values. """ # Kludge pypeit_file = '' if self.pypeit_file is None else self.pypeit_file # Build fitstbl if self.fitstbl is None: self.build_fitstbl(strict=not setup_only) #, bkg_pairs=bkg_pairs) # File typing self.get_frame_types(flag_unknown=setup_only or calibration_check, use_header_id=use_header_id) # Determine the configurations and assign each frame to the # specified configuration ignore_frames = ['bias', 'dark'] cfgs = self.fitstbl.unique_configurations(ignore_frames=ignore_frames) self.fitstbl.set_configurations(cfgs, ignore_frames=ignore_frames) # Assign frames to calibration groups self.fitstbl.set_calibration_groups(global_frames=['bias', 'dark']) # Set default comb_id (only done if needed) self.fitstbl.set_defaults() # Assign science IDs based on the calibrations groups (to be # deprecated) #self.fitstbl.calib_to_science() self.fitstbl['failures'] = False # TODO: placeholder # if self.par['scienceimage'] is not None and self.par['scienceimage']['nodding']: # self.match_ABBA() if setup_only: # Collate all matching files and write .sorted Table (on pypeit_setup only) sorted_file = self.spectrograph.spectrograph + '.sorted' \ if pypeit_file is None or len(pypeit_file) == 0 \ else pypeit_file.replace('.pypeit', '.sorted') if sort_dir is not None: sorted_file = os.path.join(sort_dir, os.path.split(sorted_file)[1]) self.fitstbl.write_sorted(sorted_file, write_bkg_pairs=write_bkg_pairs) msgs.info("Wrote sorted file data to {:s}".format(sorted_file)) else: # Write the calib file calib_file = self.spectrograph.spectrograph + '.calib' \ if pypeit_file is None or len(pypeit_file) == 0 \ else pypeit_file.replace('.pypeit', '.calib') if sort_dir is not None: calib_file = os.path.join(sort_dir, os.path.split(calib_file)[1]) self.fitstbl.write_calib(calib_file) # Finish (depends on PypeIt run mode) # TODO: Do we need this functionality? if calibration_check: msgs.info("Inspect the .calib file: {:s}".format(calib_file)) msgs.info( "*********************************************************") msgs.info("Calibration check complete and successful!") # msgs.info("Set 'run calcheck False' to continue with data reduction") msgs.info( "*********************************************************") #TODO: Why should this not return the same as when setup_only is True if setup_only: for idx in np.where(self.fitstbl['failures'])[0]: msgs.warn( "No Arc found: Skipping object {:s} with file {:s}".format( self.fitstbl['target'][idx], self.fitstbl['filename'][idx])) msgs.info("Setup is complete.") msgs.info("Inspect the .sorted file") return None, None, None return self.par, self.spectrograph, self.fitstbl
def slit_match(x_det, x_model, step=1, xlag_range=[-50, 50], sigrej=3, print_matches=False, edge=None): """ Script that perform the slit edges matching. This method uses :func:`discrete_correlate_match` to find the indices of x_model that match x_det. Taken from DEEP2/spec2d/pro/deimos_slit_match.pro Parameters ---------- x_det: `numpy.ndarray`_ 1D array of slit edge spatial positions found from image. x_model: `numpy.ndarray`_ 1D array of slit edge spatial positions predicted by the optical model. step: :obj:`int`, optional Step size in pixels used to generate a list of possible offsets within the `offsets_range`. xlag_range: :obj:`list`, optional Range of offsets in pixels allowed between the slit positions predicted by the mask design and the traced slit positions. sigrej: :obj:`float`, optional Reject slit matches larger than this number of sigma in the match residuals. print_matches: :obj:`bool`, optional Print the result of the matching. edge: :obj:`str`, optional String that indicates which edges are being plotted, i.e., left of right. Ignored if ``print_matches`` is False. Returns ------- ind: `numpy.ndarray`_ 1D array of indices for `x_model`, which defines the matches to `x_det`, i.e., `x_det` matches `x_model[ind]` dupl: `numpy.ndarray`_ 1D array of `bool` that flags which `ind` are duplicates. coeff: `numpy.ndarray`_ pypeitFit coefficients of the fitted relation between `x_det` and `x_model[ind]` sigres: :obj:`float` RMS residual for the fitted relation between `x_det` and `x_model[ind]` """ # Determine the indices of `x_model` that match `x_det` ind = discrete_correlate_match(x_det, np.ma.masked_equal(x_model, -1), step=step, xlag_range=xlag_range) # Define the weights for the fitting residual = (x_det - x_model[ind]) - np.median(x_det - x_model[ind]) weights = np.zeros(residual.size, dtype=int) weights[np.abs(residual) < 100.] = 1 if weights.sum() == 0: weights = np.ones(residual.size, dtype=int) # Fit between `x_det` and `x_model[ind]` pypeitFit = fitting.robust_fit(x_model[ind], x_det, 1, maxiter=100, weights=weights, lower=3, upper=3) coeff = pypeitFit.fitc yfit = pypeitFit.eval(x_model[ind]) # compute residuals res = yfit - x_det sigres = sigma_clipped_stats(res, sigma=sigrej)[2] # RMS residuals # flag the matches that have residuals > `sigrej` times the RMS, or if res>5 cut = 5 if res.size < 5 else sigrej * sigres out = np.abs(res) > cut # check for duplicate indices dupl = np.ones(ind.size, dtype=bool) # If there are duplicates of `ind`, for now we keep only the first one. We don't remove the others yet dupl[np.unique(ind, return_index=True)[1]] = False wdupl = np.where(dupl)[0] # Iterate over the duplicates flagged as bad if wdupl.size > 0: for i in range(wdupl.size): duplind = ind[wdupl[i]] # Where are the other duplicates of this `ind`? w = np.where(ind == duplind)[0] # set those to be bad (for the moment) dupl[w] = True # Among the duplicates of this particular `ind`, which one has the smallest residual? wdif = np.argmin(np.abs(res[w])) # The one with the smallest residuals, is then set to not bad dupl[w[wdif]] = False # Both duplicates and matches with high RMS are considered bad dupl = dupl | out if edge is not None: msgs.warn('{} duplicate match(es) for {} edges'.format( dupl[dupl == 1].size, edge)) else: msgs.warn('{} duplicate match(es)'.format(dupl[dupl == 1].size)) # I commented the 3 lines below because I don't really need to trim the duplicate matches. I just # propagate the flag. # good = dupl == 0 # ind = ind[good] # x_det=x_det[good] if print_matches: if edge is not None: msgs.info('-----------------------------------------------') msgs.info(' {} slit edges '.format(edge)) msgs.info('-----------------------------------------------') msgs.info('Index omodel_edge spat_edge ') msgs.info('-----------------------------------------------') for i in range(ind.size): msgs.info('{} {} {}'.format(ind[i], x_model[ind][i], x_det[i])) msgs.info('-----------------------------------------------') return ind, dupl, coeff, sigres
def find_objects_pypeline(self, image, std=False, std_trace=None, maskslits=None, manual_extract_dict=None, show_peaks=False, show_fits=False, show_trace=False, show=False, debug=False): """ Find objects in the slits. This is currently setup only for ARMS Wrapper to extract.objfind Parameters ---------- tslits_dict: dict Dictionary containing information on the slits traced for this image Optional Parameters ------------------- SHOW_PEAKS: bool Generate QA showing peaks identified by object finding SHOW_FITS: bool Generate QA showing fits to traces SHOW_TRACE: bool Generate QA showing traces identified. Requires an open ginga RC modules window Returns ------- specobjs : Specobjs object Container holding Specobj objects nobj: Number of objects identified self.skymask : ndarray Boolean image indicating which pixels are useful for global sky subtraction """ self.maskslits = self.maskslits if maskslits is None else maskslits gdslits = np.where(np.invert(self.maskslits))[0] # create the ouptut image for skymask skymask = np.zeros_like(image, dtype=bool) # Instantiate the specobjs container sobjs = specobjs.SpecObjs() # Loop on slits for slit in gdslits: qa_title = "Finding objects on slit # {:d}".format(slit) msgs.info(qa_title) thismask = (self.slitmask == slit) inmask = (self.sciImg.mask == 0) & thismask # Find objects specobj_dict = { 'setup': self.setup, 'slitid': slit, 'orderindx': 999, 'det': self.det, 'objtype': self.objtype, 'pypeline': self.pypeline } # TODO we need to add QA paths and QA hooks. QA should be # done through objfind where all the relevant information # is. This will be a png file(s) per slit. sig_thresh = 30.0 if std else self.redux_par['sig_thresh'] # sobjs_slit, skymask[thismask] = \ extract.objfind(image, thismask, self.tslits_dict['slit_left'][:,slit], self.tslits_dict['slit_righ'][:,slit], inmask=inmask, ncoeff=self.redux_par['trace_npoly'], std_trace=std_trace, sig_thresh=sig_thresh, hand_extract_dict=manual_extract_dict, specobj_dict=specobj_dict, show_peaks=show_peaks, show_fits=show_fits, show_trace=show_trace, trim_edg=self.redux_par['find_trim_edge'], qa_title=qa_title, nperslit=self.redux_par['maxnumber']) sobjs.add_sobj(sobjs_slit) # Steps self.steps.append(inspect.stack()[0][3]) if show: self.show('image', image=image * (self.sciImg.mask == 0), chname='objfind', sobjs=sobjs, slits=True) # Return return sobjs, len(sobjs), skymask
def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None, show=False, redux_path=None, calib_only=False): # Set up logging self.logname = logname self.verbosity = verbosity self.pypeit_file = pypeit_file self.msgs_reset() # Load cfg_lines, data_files, frametype, usrdata, setups, _ \ = parse_pypeit_file(pypeit_file, runtime=True) self.calib_only = calib_only # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] self.spectrograph = load_spectrograph(spectrograph_name) msgs.info('Loaded spectrograph {0}'.format(self.spectrograph.name)) # -------------------------------------------------------------- # Get the full set of PypeIt parameters # - Grab a science or standard file for configuration specific parameters config_specific_file = None for idx, row in enumerate(usrdata): if ('science' in row['frametype']) or ('standard' in row['frametype']): config_specific_file = data_files[idx] # search for arcs, trace if no scistd was there if config_specific_file is None: for idx, row in enumerate(usrdata): if ('arc' in row['frametype']) or ('trace' in row['frametype']): config_specific_file = data_files[idx] if config_specific_file is not None: msgs.info( 'Setting configuration-specific parameters using {0}'.format( os.path.split(config_specific_file)[1])) spectrograph_cfg_lines = self.spectrograph.config_specific_par( config_specific_file).to_config() # - Build the full set, merging with any user-provided # parameters self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) msgs.info('Built full PypeIt parameter set.') # Check the output paths are ready if redux_path is not None: self.par['rdx']['redux_path'] = redux_path # TODO: Write the full parameter set here? # -------------------------------------------------------------- # -------------------------------------------------------------- # Build the meta data # - Re-initilize based on the file data msgs.info('Compiling metadata') self.fitstbl = PypeItMetaData(self.spectrograph, self.par, files=data_files, usrdata=usrdata, strict=True) # - Interpret automated or user-provided data from the PypeIt # file self.fitstbl.finalize_usr_build(frametype, setups[0]) # -------------------------------------------------------------- # - Write .calib file (For QA naming amongst other things) calib_file = pypeit_file.replace('.pypeit', '.calib') self.fitstbl.write_calib(calib_file) # Other Internals self.overwrite = overwrite # Currently the runtime argument determines the behavior for # reuse_masters. self.reuse_masters = reuse_masters self.show = show # Set paths self.calibrations_path = os.path.join( self.par['rdx']['redux_path'], self.par['calibrations']['master_dir']) # Check for calibrations if not self.calib_only: calibrations.check_for_calibs( self.par, self.fitstbl, raise_error=self.par['calibrations']['raise_chk_error']) # Report paths msgs.info('Setting reduction path to {0}'.format( self.par['rdx']['redux_path'])) msgs.info('Master calibration data output to: {0}'.format( self.calibrations_path)) msgs.info('Science data output to: {0}'.format(self.science_path)) msgs.info('Quality assessment plots output to: {0}'.format( self.qa_path)) # TODO: Is anything written to the qa dir or only to qa/PNGs? # Should we have separate calibration and science QA # directories? # An html file wrapping them all too # Init # TODO: I don't think this ever used self.det = None self.tstart = None self.basename = None self.sciI = None self.obstime = None
def lacosmic(det, sciframe, saturation, nonlinear, varframe=None, maxiter=1, grow=1.5, remove_compact_obj=True, sigclip=5.0, sigfrac=0.3, objlim=5.0): """ Identify cosmic rays using the L.A.Cosmic algorithm U{http://www.astro.yale.edu/dokkum/lacosmic/} (article : U{http://arxiv.org/abs/astro-ph/0108003}) This routine is mostly courtesy of Malte Tewes Args: det: sciframe: saturation: nonlinear: varframe: maxiter: grow: remove_compact_obj: sigclip: sigfrac: objlim: Returns: ndarray: mask of cosmic rays (0=no CR, 1=CR) """ dnum = parse.get_dnum(det) msgs.info("Detecting cosmic rays with the L.A.Cosmic algorithm") # msgs.work("Include these parameters in the settings files to be adjusted by the user") # Set the settings scicopy = sciframe.copy() crmask = np.cast['bool'](np.zeros(sciframe.shape)) sigcliplow = sigclip * sigfrac # Determine if there are saturated pixels satpix = np.zeros_like(sciframe) # satlev = settings_det['saturation']*settings_det['nonlinear'] satlev = saturation * nonlinear wsat = np.where(sciframe >= satlev) if wsat[0].size == 0: satpix = None else: satpix[wsat] = 1.0 satpix = np.cast['bool'](satpix) # Define the kernels laplkernel = np.array([[0.0, -1.0, 0.0], [-1.0, 4.0, -1.0], [0.0, -1.0, 0.0]]) # Laplacian kernal growkernel = np.ones((3, 3)) for i in range(1, maxiter + 1): msgs.info("Convolving image with Laplacian kernel") # Subsample, convolve, clip negative values, and rebin to original size subsam = utils.subsample(scicopy) conved = signal.convolve2d(subsam, laplkernel, mode="same", boundary="symm") cliped = conved.clip(min=0.0) lplus = utils.rebin_evlist(cliped, np.array(cliped.shape) / 2.0) msgs.info("Creating noise model") # Build a custom noise map, and compare this to the laplacian m5 = ndimage.filters.median_filter(scicopy, size=5, mode='mirror') if varframe is None: noise = np.sqrt(np.abs(m5)) else: noise = np.sqrt(varframe) msgs.info("Calculating Laplacian signal to noise ratio") # Laplacian S/N s = lplus / (2.0 * noise ) # Note that the 2.0 is from the 2x2 subsampling # Remove the large structures sp = s - ndimage.filters.median_filter(s, size=5, mode='mirror') msgs.info("Selecting candidate cosmic rays") # Candidate cosmic rays (this will include HII regions) candidates = sp > sigclip nbcandidates = np.sum(candidates) msgs.info("{0:5d} candidate pixels".format(nbcandidates)) # At this stage we use the saturated stars to mask the candidates, if available : if satpix is not None: msgs.info("Masking saturated pixels") candidates = np.logical_and(np.logical_not(satpix), candidates) nbcandidates = np.sum(candidates) msgs.info( "{0:5d} candidate pixels not part of saturated stars".format( nbcandidates)) msgs.info("Building fine structure image") # We build the fine structure image : m3 = ndimage.filters.median_filter(scicopy, size=3, mode='mirror') m37 = ndimage.filters.median_filter(m3, size=7, mode='mirror') f = m3 - m37 f /= noise f = f.clip(min=0.01) msgs.info("Removing suspected compact bright objects") # Now we have our better selection of cosmics : if remove_compact_obj: cosmics = np.logical_and(candidates, sp / f > objlim) else: cosmics = candidates nbcosmics = np.sum(cosmics) msgs.info("{0:5d} remaining candidate pixels".format(nbcosmics)) # What follows is a special treatment for neighbors, with more relaxed constains. msgs.info("Finding neighboring pixels affected by cosmic rays") # We grow these cosmics a first time to determine the immediate neighborhod : growcosmics = np.cast['bool'](signal.convolve2d( np.cast['float32'](cosmics), growkernel, mode="same", boundary="symm")) # From this grown set, we keep those that have sp > sigmalim # so obviously not requiring sp/f > objlim, otherwise it would be pointless growcosmics = np.logical_and(sp > sigclip, growcosmics) # Now we repeat this procedure, but lower the detection limit to sigmalimlow : finalsel = np.cast['bool'](signal.convolve2d( np.cast['float32'](growcosmics), growkernel, mode="same", boundary="symm")) finalsel = np.logical_and(sp > sigcliplow, finalsel) # Unmask saturated pixels: if satpix is not None: msgs.info("Masking saturated stars") finalsel = np.logical_and(np.logical_not(satpix), finalsel) ncrp = np.sum(finalsel) msgs.info("{0:5d} pixels detected as cosmics".format(ncrp)) # We find how many cosmics are not yet known : newmask = np.logical_and(np.logical_not(crmask), finalsel) nnew = np.sum(newmask) # We update the mask with the cosmics we have found : crmask = np.logical_or(crmask, finalsel) msgs.info( "Iteration {0:d} -- {1:d} pixels identified as cosmic rays ({2:d} new)" .format(i, ncrp, nnew)) if ncrp == 0: break # Additional algorithms (not traditionally implemented by LA cosmic) to remove some false positives. msgs.work( "The following algorithm would be better on the rectified, tilts-corrected image" ) filt = ndimage.sobel(sciframe, axis=1, mode='constant') filty = ndimage.sobel(filt / np.sqrt(np.abs(sciframe)), axis=0, mode='constant') filty[np.where(np.isnan(filty))] = 0.0 sigimg = cr_screen(filty) sigsmth = ndimage.filters.gaussian_filter(sigimg, 1.5) sigsmth[np.where(np.isnan(sigsmth))] = 0.0 sigmask = np.cast['bool'](np.zeros(sciframe.shape)) sigmask[np.where(sigsmth > sigclip)] = True crmask = np.logical_and(crmask, sigmask) msgs.info("Growing cosmic ray mask by 1 pixel") crmask = grow_masked(crmask.astype(np.float), grow, 1.0) return crmask.astype(bool)
def fit_tilts(trc_tilt_dict, thismask, slit_cen, spat_order=3, spec_order=4, maxdev = 0.2, maxrej = None, maxiter = 100, sigrej = 3.0, pad_spec = 30, pad_spat =5, func2d='legendre2d', doqa=True, master_key='test', slit = 0, show_QA=False, out_dir=None, debug=False): """ Parameters ---------- trc_tilt_dict: dict Diciontary containing tilt info Optional Parameters ------------------- slit: all_tilts: order: yorder: func2D: maskval: setup: doqa: show_QA: out_dir: Returns: """ nspec = trc_tilt_dict['nspec'] nspat = trc_tilt_dict['nspat'] fwhm = trc_tilt_dict['fwhm'] maxdev_pix = maxdev*fwhm xnspecmin1 = float(nspec-1) xnspatmin1 = float(nspat-1) nspat = trc_tilt_dict['nspat'] use_tilt = trc_tilt_dict['use_tilt'] # mask for good/bad tilts, based on aggregate fit, frac good pixels nuse = np.sum(use_tilt) tilts = trc_tilt_dict['tilts'] # legendre polynomial fit #JFH Before we were fitting the fits. Now we fit the actual flux weighted centroided tilts. tilts_err = trc_tilt_dict['tilts_err'] # flux weighted centroidding error tilts_dspat = trc_tilt_dict['tilts_dspat'] # spatial offset from the central trace #tilts_spat = trc_tilt_dict['tilts_dspat'][:,use_tilt] # spatial offset from the central trace tilts_spec = trc_tilt_dict['tilts_spec'] # line spectral pixel position from legendre fit evaluated at slit center tilts_mask = trc_tilt_dict['tilts_mask'] # Reflects if trace is on the slit tilts_mad = trc_tilt_dict['tilts_mad'] # quantitfies aggregate error of this tilt use_mask = np.outer(np.ones(nspat,dtype=bool),use_tilt) tot_mask = tilts_mask & (tilts_err < 900) & use_mask fitxy = [spec_order, spat_order] # Fit the inverted model with a 2D polynomial msgs.info("Fitting tilts with a low order, 2D {:s}".format(func2d)) adderr = 0.03 tilts_sigma = ((tilts_mad < 100.0) & (tilts_mad > 0.0))*np.sqrt(np.abs(tilts_mad)**2 + adderr**2) fitmask, coeff2 = utils.robust_polyfit_djs(tilts_spec.flatten()/xnspecmin1, (tilts.flatten() - tilts_spec.flatten())/xnspecmin1, fitxy, x2=tilts_dspat.flatten()/xnspatmin1, inmask = tot_mask.flatten(), sigma=tilts_sigma.flatten()/xnspecmin1, function=func2d, maxiter=maxiter, lower=sigrej, upper=sigrej, maxdev=maxdev_pix/xnspecmin1,minx=-0.0, maxx=1.0, minx2=-1.0, maxx2=1.0, use_mad=False, sticky=False) fitmask = fitmask.reshape(tilts_dspat.shape) # Compute a rejection mask that we will use later. These are locations that were fit but were rejected rej_mask = tot_mask & np.invert(fitmask) # Compute and store the 2d tilts fit delta_tilt_1 = xnspecmin1*utils.func_val(coeff2, tilts_spec[tilts_mask]/xnspecmin1, func2d, x2=tilts_dspat[tilts_mask]/xnspatmin1, minx=0.0, maxx=1.0, minx2=-1.0, maxx2=1.0) delta_tilt = np.zeros_like(tilts_dspat) tilts_2dfit = np.zeros_like(tilts_dspat) delta_tilt[tilts_mask] = delta_tilt_1 tilts_2dfit[tilts_mask] = tilts_spec[tilts_mask] + delta_tilt[tilts_mask] # Add the 2d fit to the tracetilt dictionary trc_tilt_dict_out = copy.deepcopy(trc_tilt_dict) trc_tilt_dict_out['tilt_2dfit'] = tilts_2dfit # Report the residuals in pixels res_fit = tilts[fitmask] - tilts_2dfit[fitmask] rms_fit = np.std(res_fit) msgs.info("Residuals: 2D Legendre Fit") msgs.info("RMS (pixels): {}".format(rms_fit)) msgs.info("RMS/FWHM: {}".format(rms_fit/fwhm)) msgs.info('Inverting the fit to generate the tilts image') spec_vec = np.arange(nspec) spat_vec = np.arange(nspat) spat_img, spec_img = np.meshgrid(spat_vec, spec_vec) # We do some padding here to guarantee that the tilts arc lines falling off the image get tilted onto the image spec_vec_pad = np.arange(-pad_spec, nspec + pad_spec) spat_vec_pad = np.arange(-pad_spat, nspat + pad_spat) spat_img_pad, spec_img_pad = np.meshgrid(np.arange(-pad_spat, nspat + pad_spat),np.arange(-pad_spec, nspec + pad_spec)) slit_cen_pad = (scipy.interpolate.interp1d(spec_vec, slit_cen, bounds_error=False, fill_value='extrapolate'))(spec_vec_pad) thismask_pad = np.zeros_like(spec_img_pad, dtype=bool) ind_spec, ind_spat = np.where(thismask) slit_cen_img_pad = np.outer(slit_cen_pad, np.ones(nspat + 2 * pad_spat)) # center of the slit replicated spatially # Normalized spatial offset image (from central trace) dspat_img_nrm = (spat_img_pad - slit_cen_img_pad) / xnspatmin1 # normalized spec image spec_img_nrm = spec_img_pad / xnspecmin1 # Embed the old thismask in the new larger padded thismask thismask_pad[ind_spec + pad_spec, ind_spat + pad_spat] = thismask[ind_spec, ind_spat] # Now grow the thismask_pad kernel = np.ones((2*pad_spec, 2*pad_spat)) / float(4 * pad_spec * pad_spat) thismask_grow = scipy.ndimage.convolve(thismask_pad.astype(float), kernel, mode='nearest') > 0.0 # Evaluate the tilts on the padded image grid tiltpix = spec_img_pad[thismask_grow] + xnspecmin1 * utils.func_val(coeff2, spec_img_nrm[thismask_grow], func2d, x2=dspat_img_nrm[thismask_grow], minx=0.0, maxx=1.0, minx2=-1.0, maxx2=1.0) # Now do one last fit to invert the function above to obtain the final tilts model in normalized image coordinates inmask = np.isfinite(tiltpix) sigma = np.full_like(spec_img_pad, 10.0) # JFH What I find confusing is that this last fit was actually what Burles was doing on the raw tilts, so why was that failing? fitmask_tilts, coeff2_tilts = utils.robust_polyfit_djs(tiltpix/xnspecmin1, spec_img_pad[thismask_grow]/xnspecmin1, fitxy, x2=spat_img_pad[thismask_grow]/xnspatmin1, sigma=sigma[thismask_grow]/xnspecmin1, upper=5.0, lower=5.0, maxdev=10.0/xnspecmin1, inmask=inmask, function=func2d, maxiter=20, minx=0.0, maxx=1.0, minx2=0.0, maxx2=1.0, use_mad=False) irej = np.invert(fitmask_tilts) & inmask msgs.info('Rejected {:d}/{:d} pixels in final inversion tilts image fit'.format(np.sum(irej),np.sum(inmask))) # normalized tilts image #tilts_img = utils.func_val(coeff2_tilts, spec_img/xnspecmin1, func2d, x2=spat_img/xnspatmin1,minx=0.0, maxx=1.0, minx2=0.0, maxx2=1.0) #tilts_img = np.fmax(np.fmin(tilts_img, 1.2),-0.2) tilt_fit_dict = dict(nspec = nspec, nspat = nspat, ngood_lines=np.sum(use_tilt), npix_fit = np.sum(tot_mask), npix_rej = np.sum(fitmask == False), coeff2=coeff2_tilts, spec_order = spec_order, spat_order = spat_order, minx = 0.0, maxx = 1.0, minx2 = 0.0, maxx2 = 1.0, func=func2d) # Now do some QA if doqa: plot_tilt_2d(tilts_dspat, tilts, tilts_2dfit, tot_mask, rej_mask, spat_order, spec_order, rms_fit, fwhm, slit=slit, setup=master_key, show_QA=show_QA, out_dir=out_dir) plot_tilt_spat(tilts_dspat, tilts, tilts_2dfit, tilts_spec, tot_mask, rej_mask, spat_order, spec_order, rms_fit, fwhm, slit=slit, setup=master_key, show_QA=show_QA, out_dir=out_dir) plot_tilt_spec(tilts_spec, tilts, tilts_2dfit, tot_mask, rej_mask, rms_fit, fwhm, slit=slit, setup = master_key, show_QA=show_QA, out_dir=out_dir) return tilt_fit_dict, trc_tilt_dict_out
def trace_tilts(arcimg, lines_spec, lines_spat, thismask, slit_cen, inmask=None, gauss=False, fwhm=4.0,spat_order=5, maxdev_tracefit=0.2, sigrej_trace=3.0, max_badpix_frac=0.20, tcrude_nave = 5, npca = 1, coeff_npoly_pca = 2, sigrej_pca = 2.0,debug_pca = False, show_tracefits=False): """ Use a PCA model to determine the best object (or slit edge) traces for echelle spectrographs. Parameters ---------- arcimg: ndarray, float (nspec, nspat) Image of arc or sky that will be used for tracing tilts. lines_spec: ndarray, float (nlines,) Array containing arc line centroids along the center of the slit for each arc line that will be traced. This is in pixels in image coordinates. lines_spat: ndarray, float (nlines,) Array contianing the spatial position of the center of the slit along which the arc was extracted. This is is in pixels in image coordinates. thismask: ndarray, boolean (nspec, nsapt) Boolean mask image specifying the pixels which lie on the slit/order to search for objects on. The convention is: True = on the slit/order, False = off the slit/order. This must be the same size as the arcimg. Optional Parameters ------------------- inmask: float ndarray, default = None Input mask image. gauss: bool, default = False If true the code will trace the arc lines usign Gaussian weighted centroiding (trace_gweight) instead of the default, which is flux weighted centroiding (trace_fweight) fwhm: float Expected FWHM of the arc lines. spat_order: int, default = None Order of the legendre polynomial that will be fit to the tilts. maxdev_tracefit: float, default = 1.0 Maximum absolute deviation for the arc tilt fits during iterative trace fitting expressed in units of the fwhm. sigrej_trace: float, default = 3.0 From each line we compute a median absolute deviation of the trace from the polynomial fit. We then analyze the distribution of maximxum absolute deviations (MADs) for all the lines, and reject sigrej_trace outliers from that distribution. max_badpix_frac: float, default = 0.20 Maximum fraction of total pixels that can be masked by the trace_gweight algorithm (because the residuals are too large) to still be usable for tilt fitting. tcrude_nave: int, default = 5 Trace crude is used to determine the initial arc line tilts, which are then iteratively fit. Trace crude can optionally boxcar smooth the image (along the spatial direction of the image, i.e. roughly along the arc line tilts) to improve the tracing. npca: int, default = 1 Tilts are initially traced and then a PCA is performed. The PCA is used to determine better crutches for a second round of improved tilt tracing. This parameter is the order of that PCA and determined how much the tilts behavior is being compressed. npca = 0 would be just using the mean tilt. This PCA is only an intermediate step to improve the crutches and is an attempt to make the tilt tracing that goes into the final fit more robust. coeff_npoly_pca: int, default = 1 Order of polynomial fits used for PCA coefficients fitting for the PCA described above. sigrej_pca: float, default = 2.0 Significance threhsold for rejection of outliers from fits to PCA coefficients for the PCA described above. show_tracefits: bool, default = False If true the fits will be shown to each arc line trace by iter_fitting.py Returns: -------- """ tcrude_maxerr = fwhm/4.0 tcrude_maxshift = 3.0*fwhm/4.0 tcrude_maxshift0 = fwhm trace_dict0 = trace_tilts_work( arcimg, lines_spec, lines_spat, thismask, slit_cen, inmask=inmask, gauss=gauss, tilts_guess=None, fwhm=fwhm, spat_order=spat_order, maxdev_tracefit=maxdev_tracefit,sigrej_trace=sigrej_trace, max_badpix_frac=max_badpix_frac, tcrude_maxerr=tcrude_maxerr, tcrude_maxshift=tcrude_maxshift,tcrude_maxshift0=tcrude_maxshift0, tcrude_nave=tcrude_nave, show_tracefits=show_tracefits) # TODO THE PCA may not be necessary. It appears to improve the results though for some instruments where the # tracing is problematic. We could consider making this optional to speed things up. debug_pca_fit = False if debug_pca_fit: # !!!! FOR TESTING ONLY!!!! Evaluate the model fit to the tilts for all of our lines msgs.info('TESTING: Performing an initial fit before PCA.') # JFH Note spec_order is hard wired here as we don't pass it in tilt_fit_dict0 = fit_tilts(trace_dict0, spat_order=spat_order, spec_order=6, debug=True, maxdev=0.2, sigrej=3.0,doqa=True, setup='test', slit=0, show_QA=True) # Do a PCA fit, which rejects some outliers iuse = trace_dict0['use_tilt'] nuse = np.sum(iuse) msgs.info('PCA modeling {:d} good tilts'.format(nuse)) pca_fit, poly_fit_dict, pca_mean, pca_vectors = extract.pca_trace( trace_dict0['tilts_sub_fit'], predict=np.invert(iuse), npca=npca, coeff_npoly=coeff_npoly_pca, lower=sigrej_pca, upper=sigrej_pca, order_vec=lines_spec, xinit_mean=lines_spec, minv=0.0, maxv=float(trace_dict0['nsub'] - 1), debug=debug_pca) # Now trace again with the PCA predictions as the starting crutches trace_dict1 = trace_tilts_work(arcimg, lines_spec, lines_spat, thismask, slit_cen, inmask=inmask, gauss=gauss, tilts_guess=pca_fit, fwhm=fwhm, spat_order=spat_order, maxdev_tracefit=maxdev_tracefit,sigrej_trace=sigrej_trace, max_badpix_frac=max_badpix_frac,show_tracefits=show_tracefits) return trace_dict1
def tilts_find_lines(arc_spec, slit_cen, tracethresh=10.0, sig_neigh=5.0, nfwhm_neigh=3.0, only_these_lines=None, fwhm=4.0, nonlinear_counts=1e10, fit_frac_fwhm=1.25, cont_frac_fwhm=1.0, max_frac_fwhm=2.0, cont_samp=30, niter_cont=3, debug_lines=False, debug_peaks=False): """ I can't believe this method has no docs Args: arc_spec: slit_cen: tracethresh: sig_neigh: nfwhm_neigh: only_these_lines: fwhm: nonlinear_counts: fit_frac_fwhm: cont_frac_fwhm: max_frac_fwhm: cont_samp: niter_cont: debug_lines: debug_peaks: Returns: (np.ndarray, np.ndarray) or (None,None): """ nspec = arc_spec.size spec_vec = np.arange(nspec) # Find peaks with a liberal threshold of sigdetect = 5.0 tampl_tot, tampl_cont_tot, tcent_tot, twid_tot, _, wgood, arc_cont_sub, nsig_tot = arc.detect_lines( arc_spec, sigdetect=np.min([sig_neigh,tracethresh]), fwhm=fwhm, fit_frac_fwhm=fit_frac_fwhm, cont_frac_fwhm=cont_frac_fwhm, max_frac_fwhm=max_frac_fwhm, cont_samp=cont_samp, niter_cont=niter_cont, nonlinear_counts=nonlinear_counts, debug=debug_peaks) # Good lines arcdet = tcent_tot[wgood] arc_ampl = tampl_cont_tot[wgood] nsig = nsig_tot[wgood] npix_neigh = nfwhm_neigh*fwhm # Determine the best lines to use to trace the tilts aduse = np.zeros(arcdet.size, dtype=np.bool) # Which lines should be used to trace the tilts w = np.where(nsig >= tracethresh) aduse[w] = 1 # Remove lines that are within npix_neigh pixels. # #ToDO replce this with a near-neighbor based approach, where # we identify groups and take the brightest line in a given group? nuse = np.sum(aduse) detuse = arcdet[aduse] idxuse = np.arange(arcdet.size)[aduse] olduse = aduse.copy() for s in range(nuse): w = np.where((np.abs(arcdet - detuse[s]) <= npix_neigh) & (np.abs(arcdet - detuse[s]) >= 1.0) & (nsig > sig_neigh))[0] for u in range(w.size): if nsig[w[u]] > nsig[olduse][s]: aduse[idxuse[s]] = False break # Restricted to ID lines? [introduced to avoid LRIS ghosts] if only_these_lines is not None: ids_pix = np.array(only_these_lines) idxuse = np.arange(arcdet.size)[aduse] for s in idxuse: if np.min(np.abs(arcdet[s] - ids_pix)) > 2.0: msgs.info("Ignoring line at spectral position={:6.1f} which was not identified".format(arcdet[s])) aduse[s] = False # Final spectral positions of arc lines we will trace lines_spec = arcdet[aduse] nlines = len(lines_spec) if nlines == 0: msgs.warn('No arc lines were deemed usable on this slit. Cannot compute tilts. Try lowering tracethresh.') msgs.warn('Or, more likely, this was a bad slit (which you might remove)') return None, None else: msgs.info('Modelling arc line tilts with {:d} arc lines'.format(nlines)) if debug_lines: xrng = np.arange(nspec) plt.figure(figsize=(14, 6)) plt.plot(xrng, arc_cont_sub, color='black', drawstyle='steps-mid', lw=3, label='arc', linewidth=1.0) plt.plot(arcdet[~aduse], arc_ampl[~aduse], 'r+', markersize=6.0, label='bad for tilts') plt.plot(arcdet[aduse], arc_ampl[aduse], 'g+', markersize=6.0, label='good for tilts') if nonlinear_counts < 1e9: plt.hlines(nonlinear_counts, xrng.min(), xrng.max(), color='orange', linestyle='--', linewidth=2.0, label='nonlinear', zorder=10) plt.title('Good Lines = {:d}'.format(np.sum(aduse)) + ', Bad Lines = {:d}'.format(np.sum(~aduse))) plt.ylim(arc_cont_sub.min(), 1.5 * arc_cont_sub.max()) plt.legend() plt.show() # Spatial position of line, i.e. the central trace interpolated onto the spectral pixel of the line lines_spat = np.interp(lines_spec, spec_vec, slit_cen) return lines_spec, lines_spat
def ech_coadd(files,objids=None,extract='OPT',flux=True,giantcoadd=False, wave_grid_method='velocity', niter=5,wave_grid_min=None, wave_grid_max=None,v_pix=None, scale_method='auto', do_offset=False, sigrej_final=3.,do_var_corr=False, qafile=None, outfile=None,do_cr=True, debug=False,**kwargs): nfile = len(files) if nfile <=1: msgs.info('Only one spectrum exits coadding...') return fname = files[0] ext_final = fits.getheader(fname, -1) norder = ext_final['ORDER'] + 1 msgs.info('spectrum {:s} has {:d} orders'.format(fname, norder)) if norder <= 1: msgs.error('The number of orders have to be greater than one for echelle. Longslit data?') if giantcoadd: msgs.info('Coadding all orders and exposures at once') spectra = ech_load_spec(files, objid=objids,order=None, extract=extract, flux=flux) wave_grid = np.zeros((2,spectra.nspec)) for i in range(spectra.nspec): wave_grid[0, i] = spectra[i].wvmin.value wave_grid[1, i] = spectra[i].wvmax.value ech_kwargs = {'echelle': True, 'wave_grid_min': np.min(wave_grid), 'wave_grid_max': np.max(wave_grid), 'v_pix': v_pix} kwargs.update(ech_kwargs) # Coadding spec1d = coadd.coadd_spectra(spectra, wave_grid_method=wave_grid_method, niter=niter, scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final, do_var_corr=do_var_corr, qafile=qafile, outfile=outfile, do_cr=do_cr, debug=debug,**kwargs) else: msgs.info('Coadding individual orders first and then merge order') spectra_list = [] # Keywords for Table rsp_kwargs = {} rsp_kwargs['wave_tag'] = '{:s}_WAVE'.format(extract) rsp_kwargs['flux_tag'] = '{:s}_FLAM'.format(extract) rsp_kwargs['sig_tag'] = '{:s}_FLAM_SIG'.format(extract) wave_grid = np.zeros((2,norder)) for iord in range(norder): spectra = ech_load_spec(files, objid=objids, order=iord, extract=extract, flux=flux) ech_kwargs = {'echelle': False, 'wave_grid_min': spectra.wvmin.value, 'wave_grid_max': spectra.wvmax.value, 'v_pix': v_pix} wave_grid[0,iord] = spectra.wvmin.value wave_grid[1,iord] = spectra.wvmax.value kwargs.update(ech_kwargs) # Coadding the individual orders if qafile is not None: qafile_iord = qafile+'_%s'%str(iord) else: qafile_iord = None spec1d_iord = coadd.coadd_spectra(spectra, wave_grid_method=wave_grid_method, niter=niter, scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final, do_var_corr=do_var_corr, qafile=qafile_iord, outfile=outfile, do_cr=do_cr, debug=debug, **kwargs) spectrum = spec_from_array(spec1d_iord.wavelength, spec1d_iord.flux, spec1d_iord.sig,**rsp_kwargs) spectra_list.append(spectrum) # Join into one XSpectrum1D object spectra_coadd = collate(spectra_list) kwargs['echelle'] = True kwargs['wave_grid_min'] = np.min(wave_grid) kwargs['wave_grid_max'] = np.max(wave_grid) # ToDo: Currently I'm not using the first order due to some problem. Need to add it back after fix the problem. spec1d = coadd.coadd_spectra(spectra_coadd[1:], wave_grid_method=wave_grid_method, niter=niter, scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final, do_var_corr=do_var_corr, qafile=qafile, outfile=outfile, do_cr=do_cr, debug=debug, **kwargs) return spec1d
def write_to_fits(self, subheader, outfile, overwrite=True, update_det=None, slitspatnum=None, debug=False): """ Write the set of SpecObj objects to one multi-extension FITS file Args: subheader (:obj:`dict`): outfile (str): overwrite (bool, optional): slitspatnum (:obj:`str` or :obj:`list`, optional): Restricted set of slits for reduction update_det (int or list, optional): If provided, do not clobber the existing file but only update the indicated detectors. Useful for re-running on a subset of detectors """ if os.path.isfile(outfile) and (not overwrite): msgs.warn("Outfile exists. Set overwrite=True to clobber it") return # If the file exists and update_det (and slit_spat_num) is provided, use the existing header # and load up all the other hdus so that we only over-write the ones # we are updating if os.path.isfile(outfile) and (update_det is not None or slitspatnum is not None): _specobjs = SpecObjs.from_fitsfile(outfile) mask = np.ones(_specobjs.nobj, dtype=bool) # Update_det if update_det is not None: # Pop out those with this detector (and slit if slit_spat_num is provided) for det in np.atleast_1d(update_det): mask[_specobjs.DET == det] = False elif slitspatnum is not None: # slitspatnum dets, spat_ids = slittrace.parse_slitspatnum(slitspatnum) for det, spat_id in zip(dets, spat_ids): mask[(_specobjs.DET == det) & (_specobjs.SLITID == spat_id)] = False _specobjs = _specobjs[mask] # Add in the new for sobj in self.specobjs: _specobjs.add_sobj(sobj) else: _specobjs = self.specobjs # Build up the Header header = io.initialize_header(primary=True) for key in subheader.keys(): header[key.upper()] = subheader[key] # Init prihdu = fits.PrimaryHDU() hdus = [prihdu] prihdu.header = header # Add class info prihdu.header['DMODCLS'] = (self.__class__.__name__, 'Datamodel class') prihdu.header['DMODVER'] = (self.version, 'Datamodel version') detector_hdus = {} nspec, ext = 0, 0 # Loop on the SpecObj objects for sobj in _specobjs: if sobj is None: continue # HDUs if debug: import pdb pdb.set_trace() shdul = sobj.to_hdu() if len(shdul) == 2: # Detector? detector_hdus[sobj['DET']] = shdul[1] shdu = [shdul[0]] elif len(shdul) == 1: # Detector? shdu = shdul else: msgs.error("Should not get here...") # Check -- If sobj had only 1 array, the BinTableHDU test will fail assert len(shdu) == 1, 'Bad data model!!' assert isinstance(shdu[0], fits.hdu.table.BinTableHDU), 'Bad data model2' #shdu[0].header['DMODCLS'] = (self.__class__.__name__, 'Datamodel class') #shdu[0].header['DMODVER'] = (self.version, 'Datamodel version') # Name shdu[0].name = sobj.NAME # Extension keywd = 'EXT{:04d}'.format(ext) prihdu.header[keywd] = sobj.NAME ext += 1 nspec += 1 # Append hdus += shdu # Deal with Detectors for key, item in detector_hdus.items(): # TODO - Add EXT to the primary header for these?? prefix = specobj.det_hdu_prefix(key) # Name if prefix not in item.name: # In case we are re-loading item.name = specobj.det_hdu_prefix(key) + item.name # Append hdus += [item] # A few more for the header prihdu.header['NSPEC'] = nspec # Code versions io.initialize_header(hdr=prihdu.header) # Finish hdulist = fits.HDUList(hdus) if debug: import pdb pdb.set_trace() hdulist.writeto(outfile, overwrite=overwrite) msgs.info("Wrote 1D spectra to {:s}".format(outfile)) return
def get_rawimage(self, raw_file, det): """ Read a raw DEIMOS data frame (one or more detectors). Data are unpacked from the multi-extension HDU. Function is based :func:`pypeit.spectrographs.keck_lris.read_lris`, which was based on the IDL procedure ``readmhdufits.pro``. Parameters ---------- raw_file : str Filename Returns ------- array : ndarray Combined image hdu: HDUList sections : tuple List of datasec, oscansec sections """ # Check for file; allow for extra .gz, etc. suffix fil = glob.glob(raw_file + '*') if len(fil) != 1: msgs.error('Found {0} files matching {1}'.format( len(fil), raw_file + '*')) # Read try: msgs.info("Reading DEIMOS file: {:s}".format(fil[0])) except AttributeError: print("Reading DEIMOS file: {:s}".format(fil[0])) hdu = fits.open(fil[0]) head0 = hdu[0].header # Get post, pre-pix values postpix = head0['POSTPIX'] detlsize = head0['DETLSIZE'] x0, x_npix, y0, y_npix = np.array( parse.load_sections(detlsize)).flatten() # Create final image if det is None: image = np.zeros((x_npix, y_npix + 4 * postpix)) rawdatasec_img = np.zeros_like(image, dtype=int) oscansec_img = np.zeros_like(image, dtype=int) # get the x and y binning factors... binning = head0['BINNING'] if binning != '1,1': msgs.error( "This binning for DEIMOS might not work. But it might..") # DEIMOS detectors nchip = 8 if det is None: chips = range(nchip) else: chips = [det - 1] # Indexing starts at 0 here # Loop for tt in chips: data, oscan = deimos_read_1chip(hdu, tt + 1) # One detector?? if det is not None: image = np.zeros( (data.shape[0], data.shape[1] + oscan.shape[1])) rawdatasec_img = np.zeros_like(image, dtype=int) oscansec_img = np.zeros_like(image, dtype=int) # Indexing x1, x2, y1, y2, o_x1, o_x2, o_y1, o_y2 = indexing(tt, postpix, det=det) # Fill image[y1:y2, x1:x2] = data rawdatasec_img[y1:y2, x1:x2] = 1 # Amp image[o_y1:o_y2, o_x1:o_x2] = oscan oscansec_img[o_y1:o_y2, o_x1:o_x2] = 1 # Amp # Return exptime = hdu[self.meta['exptime']['ext']].header[self.meta['exptime'] ['card']] return image, hdu, exptime, rawdatasec_img, oscansec_img
def get_tiltimg(self): """ Load or generate the Tilt image Requirements: master_key, det, par Args: Returns: ndarray: :attr:`mstilt` image """ # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep tilt_rows = self.fitstbl.find_frames('tilt', calib_ID=self.calib_ID, index=True) if len(tilt_rows) == 0: msgs.error('Must identify tilt frames to construct tilt image.') self.tilt_files = self.fitstbl.frame_paths(tilt_rows) self.master_key_dict['tilt'] \ = self.fitstbl.master_key(tilt_rows[0] if len(tilt_rows) > 0 else self.frame, det=self.det) if self._cached('tiltimg', self.master_key_dict['tilt']): # Previously calculated self.mstilt = self.calib_dict[ self.master_key_dict['tilt']]['tiltimg'] return self.mstilt # Instantiate with everything needed to generate the image (in case we do) self.tiltImage = tiltimage.TiltImage( self.spectrograph, files=self.tilt_files, det=self.det, msbias=self.msbias, par=self.par['tiltframe'], master_key=self.master_key_dict['tilt'], master_dir=self.master_dir, reuse_masters=self.reuse_masters) # Load the MasterFrame (if it exists and is desired)? self.mstilt = self.tiltImage.load() if self.mstilt is None: # Otherwise build it msgs.info("Preparing a master {0:s} frame".format( self.tiltImage.frametype)) self.mstilt = self.tiltImage.build_image(bias=self.msbias, bpm=self.msbpm) # JFH Add a cr_masking option here. The image processing routines are not ready for it yet. # Save to Masters if self.save_masters: self.tiltImage.save() # Save & return self._update_cache('tilt', 'tiltimg', self.mstilt) # TODO in the future add in a tilt_inmask #self._update_cache('tilt', 'tilt_inmask', self.mstilt_inmask) return self.mstilt
#slit_ids = [trace_slits.get_slitid(flat.shape, tslits_dict['lcen'], tslits_dict['rcen'], ii)[0] # for ii in range(tslits_dict['lcen'].shape[1])] #viewer, ch = ginga.show_image(flat, cuts = (1000.0,30000.0),clear=True) #ginga.show_slits(viewer, ch, tslits_dict['lcen'], tslits_dict['rcen'], slit_ids) # , args.det) maskslits = np.zeros(tslits_dict['lcen'].shape[1], dtype=bool) gdslits = np.where(~maskslits)[0] pixelflat = np.ones_like(flatimg) illumflat = np.ones_like(flatimg) flat_model = np.zeros_like(flatimg) debug = True # Loop on slits for slit in gdslits: msgs.info("Computing flat field image for slit: {:d}".format(slit + 1)) slit_left = tslits_dict['lcen'][:, slit] slit_righ = tslits_dict['rcen'][:, slit] thismask = (tslits_dict['slitpix'] == slit + 1) inmask = None # in the future set this to the bpm sys.exit(-1) pixelflat[thismask], illumflat[thismask], flat_model[thismask] = flat.fit_flat(flatimg, mstilts, thismask, slit_left, slit_righ,inmask=inmask, debug = debug) ginga.show_image(pixelflat,cuts = (0.9,1.1),chname='pixeflat', wcs_match=True, clear=True) ginga.show_image(illumflat,cuts = (0.9,1.1), chname='illumflat', wcs_match=True) ginga.show_image(flatimg,chname='flat', wcs_match=True) ginga.show_image(flat_model,chname='flat_model',wcs_match = True)
def get_flats(self): """ Load or generate a normalized pixel flat and slit illumination flat. Requires :attr:`tslits_dict`, :attr:`tilts_dict`, :attr:`det`, :attr:`par`. Returns: `numpy.ndarray`_: Two arrays are returned, the normalized pixel flat image (:attr:`mspixelflat`) and the slit illumination flat (:attr:`msillumflat`). If the user requested the field flattening be skipped (`FlatFieldPar['method'] == 'skip'`) or if the slit and tilt traces are not provided, the function returns two None objects instead. """ # Check for existing data if not self._chk_objs(['msarc', 'msbpm', 'tslits_dict', 'wv_calib']): msgs.error('dont have all the objects') if self.par['flatfield']['method'] is 'skip': # User does not want to flat-field self.mspixelflat = None self.msillumflat = None msgs.warning( 'Parameter calibrations.flatfield.method is set to skip. You are NOT ' 'flatfielding your data!!!') # TODO: Why does this not return unity arrays, like what's # done below? return self.mspixelflat, self.msillumflat # Slit and tilt traces are required to flat-field the data if not self._chk_objs(['tslits_dict', 'tilts_dict']): msgs.warning( 'Flats were requested, but there are quantities missing necessary to ' 'create flats. Proceeding without flat fielding....') # User cannot flat-field self.mspixelflat = None self.msillumflat = None # TODO: Why does this not return unity arrays, like what's # done below? return self.mspixelflat, self.msillumflat # Check internals self._chk_set(['det', 'calib_ID', 'par']) pixflat_rows = self.fitstbl.find_frames('pixelflat', calib_ID=self.calib_ID, index=True) # TODO: Why aren't these set to self # KBW: They're kept in self.flatField.files pixflat_image_files = self.fitstbl.frame_paths(pixflat_rows) # Allow for user-supplied file (e.g. LRISb) self.master_key_dict['flat'] \ = self.fitstbl.master_key(pixflat_rows[0] if len(pixflat_rows) > 0 else self.frame, det=self.det) # Return already generated data if self._cached('pixelflat', self.master_key_dict['flat']) \ and self._cached('illumflat', self.master_key_dict['flat']): self.mspixelflat = self.calib_dict[ self.master_key_dict['flat']]['pixelflat'] self.msillumflat = self.calib_dict[ self.master_key_dict['flat']]['illumflat'] return self.mspixelflat, self.msillumflat # Instantiate # TODO: This should automatically attempt to load and instatiate # from a file if it exists. self.flatField = flatfield.FlatField( self.spectrograph, self.par['pixelflatframe'], files=pixflat_image_files, det=self.det, master_key=self.master_key_dict['flat'], master_dir=self.master_dir, reuse_masters=self.reuse_masters, flatpar=self.par['flatfield'], msbias=self.msbias, # TODO: msbpm is not passed? tslits_dict=self.tslits_dict, tilts_dict=self.tilts_dict) # --- Pixel flats # 1) Try to load master files from disk (MasterFrame)? _, self.mspixelflat, self.msillumflat = self.flatField.load() # 2) Did the user specify a flat? If so load it in (e.g. LRISb with pixel flat)? # TODO: We need to document this format for the user! if self.par['flatfield']['frame'] != 'pixelflat': # - Name is explicitly correct? if os.path.isfile(self.par['flatfield']['frame']): flat_file = self.par['flatfield']['frame'] # - Is it in the master directory? elif os.path.isfile( os.path.join(self.flatField.master_dir, self.par['flatfield']['frame'])): flat_file = os.path.join(self.flatField.master_dir, self.par['flatfield']['frame']) else: msgs.error( 'Could not find user-defined flatfield file: {0}'.format( self.par['flatfield']['frame'])) msgs.info('Using user-defined file: {0}'.format(flat_file)) with fits.open(flat_file) as hdu: self.mspixelflat = hdu[self.det].data self.msillumflat = None # 3) there is no master or no user supplied flat, generate the flat if self.mspixelflat is None and len(pixflat_image_files) != 0: # Run self.mspixelflat, self.msillumflat = self.flatField.run( show=self.show, maskslits=self.tslits_dict['maskslits']) # If we tweaked the slits, update the tilts_dict and # tslits_dict to reflect new slit edges if self.par['flatfield']['tweak_slits']: msgs.info( 'Using slit boundary tweaks from IllumFlat and updated tilts image' ) self.tslits_dict = self.flatField.tslits_dict self.tilts_dict = self.flatField.tilts_dict # Save to Masters if self.save_masters: self.flatField.save() # If we tweaked the slits update the master files for tilts and slits # TODO: These should be saved separately if self.par['flatfield']['tweak_slits']: msgs.info( 'Updating MasterTrace and MasterTilts using tweaked slit boundaries' ) self.edges.update_using_tslits_dict( self.flatField.tslits_dict) self.edges.save() # Write the final_tilts using the new slit boundaries to the MasterTilts file self.waveTilts.final_tilts = self.flatField.tilts_dict[ 'tilts'] self.waveTilts.tilts_dict = self.flatField.tilts_dict self.waveTilts.save() # 4) If either of the two flats are still None, use unity # everywhere and print out a warning # TODO: These will barf if self.tilts_dict['tilts'] isn't # defined. if self.mspixelflat is None: self.mspixelflat = np.ones_like(self.tilts_dict['tilts']) msgs.warn('You are not pixel flat fielding your data!!!') if self.msillumflat is None: self.msillumflat = np.ones_like(self.tilts_dict['tilts']) msgs.warn('You are not illumination flat fielding your data!') # Save & return self._update_cache('flat', ('pixelflat', 'illumflat'), (self.mspixelflat, self.msillumflat)) return self.mspixelflat, self.msillumflat
def global_skysub(self, std=False, skymask=None, update_crmask=True, maskslits=None, show_fit=False, show=False, show_objs=False): """ Perform global sky subtraction, slit by slit Wrapper to skysub.global_skysub Parameters ---------- tslits_dict: dict Dictionary containing information on the slits traced for this image Optional Parameters ------------------- bspline_spaceing: (float): Break-point spacing for bspline use_skymask: (bool, optional): Mask objects using self.skymask if object finding has been run (This requires they were found previously, i.e. that find_objects was already run) Returns: global_sky: (numpy.ndarray) image of the the global sky model """ # Prep self.global_sky = np.zeros_like(self.sciImg.image) if (std and not self.redux_par['global_sky_std']): msgs.info('Skipping global sky-subtraction for standard star.') return self.global_sky if std: sigrej = 7.0 update_crmask = False else: sigrej = 3.0 self.maskslits = self.maskslits if maskslits is None else maskslits gdslits = np.where(np.invert(self.maskslits))[0] # Mask objects using the skymask? If skymask has been set by objfinding, and masking is requested, then do so skymask_now = skymask if (skymask is not None) else np.ones_like( self.sciImg.image, dtype=bool) # Loop on slits for slit in gdslits: msgs.info("Global sky subtraction for slit: {:d}".format(slit)) thismask = (self.slitmask == slit) inmask = (self.sciImg.mask == 0) & thismask & skymask_now # Find sky self.global_sky[thismask] = skysub.global_skysub( self.sciImg.image, self.sciImg.ivar, self.tilts, thismask, self.tslits_dict['slit_left'][:, slit], self.tslits_dict['slit_righ'][:, slit], inmask=inmask, sigrej=sigrej, bsp=self.redux_par['bspline_spacing'], no_poly=self.redux_par['no_poly'], pos_mask=(not self.ir_redux), show_fit=show_fit) # Mask if something went wrong if np.sum(self.global_sky[thismask]) == 0.: self.maskslits[slit] = True if update_crmask: self.sciImg.update_mask_cr(subtract_img=self.global_sky) #self.crmask = procimg.lacosmic(self.det, self.sciimg-self.global_sky, # self.spectrograph.detector[self.det-1]['saturation'], # self.spectrograph.detector[self.det-1]['nonlinear'], # varframe=utils.calc_ivar(self.sciivar), # maxiter=self.proc_par['lamaxiter'], # grow=self.proc_par['grow'], # remove_compact_obj=self.proc_par['rmcompact'], # sigclip=self.proc_par['sigclip'], # sigfrac=self.proc_par['sigfrac'], # objlim=self.proc_par['objlim']) # Rebuild the mask with this new crmask #self.mask = procimg.update_mask_cr(self.sciImg.bitmask, self.mask, self.crmask) # Step self.steps.append(inspect.stack()[0][3]) if show: sobjs_show = None if show_objs else self.sobjs_obj # Global skysub is the first step in a new extraction so clear the channels here self.show('global', slits=True, sobjs=sobjs_show, clear=False) # Return return self.global_sky
def get_wv_calib(self): """ Load or generate the 1D wavelength calibrations Requirements: msarc, msbpm, tslits_dict det, par Returns: dict, ndarray: :attr:`wv_calib` calibration dict and the updated slit mask array """ # Check for existing data if not self._chk_objs(['msarc', 'msbpm', 'tslits_dict']): msgs.error('dont have all the objects') # Check internals self._chk_set(['det', 'calib_ID', 'par']) if 'arc' not in self.master_key_dict.keys(): msgs.error('Arc master key not set. First run get_arc.') # Return existing data if self._cached('wavecalib', self.master_key_dict['arc']) \ and self._cached('wvmask', self.master_key_dict['arc']): self.wv_calib = self.calib_dict[ self.master_key_dict['arc']]['wavecalib'] self.wv_maskslits = self.calib_dict[ self.master_key_dict['arc']]['wvmask'] self.tslits_dict['maskslits'] += self.wv_maskslits return self.wv_calib # No wavelength calibration requested if self.par['wavelengths']['reference'] == 'pixel': msgs.info("A wavelength calibration will not be performed") self.wv_calib = None self.wv_maskslits = np.zeros_like(self.maskslits, dtype=bool) self.tslits_dict['maskslits'] += self.wv_maskslits return self.wv_calib # Grab arc binning (may be different from science!) arc_rows = self.fitstbl.find_frames('arc', calib_ID=self.calib_ID, index=True) self.arc_files = self.fitstbl.frame_paths(arc_rows) binspec, binspat = parse.parse_binning( self.spectrograph.get_meta_value(self.arc_files[0], 'binning')) # Instantiate self.waveCalib = wavecalib.WaveCalib( self.msarc, self.tslits_dict, self.spectrograph, self.par['wavelengths'], binspectral=binspec, det=self.det, master_key=self.master_key_dict['arc'], master_dir=self.master_dir, reuse_masters=self.reuse_masters, qa_path=self.qa_path, msbpm=self.msbpm) # Load from disk (MasterFrame)? self.wv_calib = self.waveCalib.load() if self.wv_calib is None: self.wv_calib, _ = self.waveCalib.run(skip_QA=(not self.write_qa)) # Save to Masters if self.save_masters: self.waveCalib.save() # Create the mask (needs to be done here in case wv_calib was loaded from Masters) # TODO: This should either be done here or save as part of the # master frame file. As it is, if not loaded from the master # frame file, mask_maskslits is run twice, once in run above and # once here... self.wv_maskslits = self.waveCalib.make_maskslits( self.tslits_dict['slit_left'].shape[1]) self.tslits_dict['maskslits'] += self.wv_maskslits # Save & return self._update_cache('arc', ('wavecalib', 'wvmask'), (self.wv_calib, self.wv_maskslits)) # Return return self.wv_calib
def local_skysub_extract(self, waveimg, global_sky, sobjs, spat_pix=None, maskslits=None, model_noise=True, std=False, show_profile=False, show=False): """ Perform local sky subtraction, profile fitting, and optimal extraction slit by slit Wrapper to skysub.local_skysub_extract Parameters ---------- sobjs: object Specobjs object containing Specobj objects containing information about objects found. waveimg: ndarray, shape (nspec, nspat) Wavelength map Optional Parameters ------------------- Returns: global_sky: (numpy.ndarray) image of the the global sky model """ self.waveimg = waveimg self.global_sky = global_sky # get the good slits and assign self.maskslits self.maskslits = self.maskslits if maskslits is None else maskslits gdslits = np.where(np.invert(self.maskslits))[0] # Allocate the images that are needed # Initialize to mask in case no objects were found self.outmask = np.copy(self.sciImg.mask) # Initialize to input mask in case no objects were found self.extractmask = (self.sciImg.mask == 0) # Initialize to zero in case no objects were found self.objmodel = np.zeros_like(self.sciImg.image) # Set initially to global sky in case no objects were found self.skymodel = np.copy(self.global_sky) # Set initially to sciivar in case no obects were found. self.ivarmodel = np.copy(self.sciImg.ivar) # Could actually create a model anyway here, but probably # overkill since nothing is extracted self.sobjs = sobjs.copy() # Loop on slits for slit in gdslits: msgs.info( "Local sky subtraction and extraction for slit: {:d}".format( slit)) thisobj = (self.sobjs.slitid == slit ) # indices of objects for this slit if np.any(thisobj): thismask = (self.slitmask == slit) # pixels for this slit # True = Good, False = Bad for inmask inmask = (self.sciImg.mask == 0) & thismask # Local sky subtraction and extraction self.skymodel[thismask], self.objmodel[thismask], self.ivarmodel[thismask], \ self.extractmask[thismask] = skysub.local_skysub_extract( self.sciImg.image, self.sciImg.ivar, self.tilts, self.waveimg, self.global_sky, self.sciImg.rn2img, thismask, self.tslits_dict['slit_left'][:,slit], self.tslits_dict['slit_righ'][:, slit], self.sobjs[thisobj], spat_pix=spat_pix, model_full_slit=self.redux_par['model_full_slit'], box_rad=self.redux_par['boxcar_radius']/self.spectrograph.detector[self.det-1]['platescale'], sigrej=self.redux_par['sky_sigrej'], model_noise=model_noise, std=std, bsp=self.redux_par['bspline_spacing'], sn_gauss=self.redux_par['sn_gauss'], inmask=inmask, show_profile=show_profile) # Set the bit for pixels which were masked by the extraction. # For extractmask, True = Good, False = Bad iextract = (self.sciImg.mask == 0) & (self.extractmask == False) self.outmask[iextract] = self.sciImg.bitmask.turn_on( self.outmask[iextract], 'EXTRACT') # Step self.steps.append(inspect.stack()[0][3]) if show: self.show('local', sobjs=self.sobjs, slits=True) self.show('resid', sobjs=self.sobjs, slits=True) # Return return self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs
def iterative_fitting(spec, tcent, ifit, IDs, llist, disp, match_toler = 2.0, func = 'legendre', n_first=2, sigrej_first=2.0, n_final=4, sigrej_final=3.0, input_only=False, weights=None, plot_fil=None, verbose=False): """ Routine for iteratively fitting wavelength solutions. Parameters ---------- spec : ndarray, shape = (nspec,) arcline spectrum tcent : ndarray Centroids in pixels of lines identified in spec ifit : ndarray Indices of the lines that will be fit IDs: ndarray wavelength IDs of the lines that will be fit (I think?) llist: dict Linelist dictionary disp: float dispersion Optional Parameters ------------------- match_toler: float, default = 3.0 Matching tolerance when searching for new lines. This is the difference in pixels between the wavlength assigned to an arc line by an iteration of the wavelength solution to the wavelength in the line list. func: str, default = 'legendre' Name of function used for the wavelength solution n_first: int, default = 2 Order of first guess to the wavelength solution. sigrej_first: float, default = 2.0 Number of sigma for rejection for the first guess to the wavelength solution. n_final: int, default = 4 Order of the final wavelength solution fit sigrej_final: float, default = 3.0 Number of sigma for rejection for the final fit to the wavelength solution. input_only: bool If True, the routine will only perform a robust polyfit to the input IDs. If False, the routine will fit the input IDs, and then include additional lines in the linelist that are a satisfactory fit. weights: ndarray Weights to be used? verbose : bool If True, print out more information. plot_fil: Filename for plotting some QA? Returns ------- final_fit: :class:`pypeit.core.wavecal.wv_fitting.WaveFit` """ #TODO JFH add error checking here to ensure that IDs and ifit have the same size! if weights is None: weights = np.ones(tcent.size) nspec = spec.size xnspecmin1 = float(nspec-1) # Setup for fitting sv_ifit = list(ifit) # Keep the originals all_ids = -999.*np.ones(len(tcent)) all_idsion = np.array(['UNKNWN']*len(tcent)) all_ids[ifit] = IDs # Fit n_order = n_first flg_continue = True flg_penultimate = False fmin, fmax = 0.0, 1.0 # Note the number of parameters is actually n_order and not n_order+1 while flg_continue: if flg_penultimate: flg_continue = False # Fit with rejection xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit] maxiter = xfit.size - n_order - 2 # if xfit.size == 0: msgs.warn("All points rejected !!") return None # Fit pypeitFit = fitting.robust_fit(xfit/xnspecmin1, yfit, n_order, function=func, maxiter=maxiter, lower=sigrej_first, upper=sigrej_first, maxrej=1, sticky=True, minx=fmin, maxx=fmax, weights=wfit) # Junk fit? if pypeitFit is None: msgs.warn("Bad fit!!") return None rms_ang = pypeitFit.calc_fit_rms(apply_mask=True) rms_pix = rms_ang/disp if verbose: msgs.info('n_order = {:d}'.format(n_order) + ': RMS = {:g}'.format(rms_pix)) # Reject but keep originals (until final fit) ifit = list(ifit[pypeitFit.gpm == 1]) + sv_ifit if not input_only: # Find new points from the linelist (should we allow removal of the originals?) twave = pypeitFit.eval(tcent/xnspecmin1)#, func, minx=fmin, maxx=fmax) for ss, iwave in enumerate(twave): mn = np.min(np.abs(iwave-llist['wave'])) if mn/disp < match_toler: imn = np.argmin(np.abs(iwave-llist['wave'])) #if verbose: # print('Adding {:g} at {:g}'.format(llist['wave'][imn],tcent[ss])) # Update and append all_ids[ss] = llist['wave'][imn] all_idsion[ss] = llist['ion'][imn] ifit.append(ss) # Keep unique ones ifit = np.unique(np.array(ifit, dtype=int)) # Increment order? if n_order < n_final: n_order += 1 else: flg_penultimate = True # Final fit (originals can now be rejected) xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit] pypeitFit = fitting.robust_fit(xfit/xnspecmin1, yfit, n_order, function=func, lower=sigrej_final, upper=sigrej_final, maxrej=1, sticky=True, minx=fmin, maxx=fmax, weights=wfit)#, debug=True) irej = np.where(np.logical_not(pypeitFit.bool_gpm))[0] if len(irej) > 0: xrej = xfit[irej] yrej = yfit[irej] if verbose: for kk, imask in enumerate(irej): wave = pypeitFit.eval(xrej[kk]/xnspecmin1)#, func, minx=fmin, maxx=fmax) msgs.info('Rejecting arc line {:g}; {:g}'.format(yfit[imask], wave)) else: xrej = [] yrej = [] ions = all_idsion[ifit] # Final RMS rms_ang = pypeitFit.calc_fit_rms(apply_mask=True) rms_pix = rms_ang/disp # Pack up fit spec_vec = np.arange(nspec) wave_soln = pypeitFit.eval(spec_vec/xnspecmin1) cen_wave = pypeitFit.eval(float(nspec)/2/xnspecmin1) cen_wave_min1 = pypeitFit.eval((float(nspec)/2 - 1.0)/xnspecmin1) cen_disp = cen_wave - cen_wave_min1 # Ions bit ion_bits = np.zeros(len(ions), dtype=WaveFit.bitmask.minimum_dtype()) for kk,ion in enumerate(ions): ion_bits[kk] = WaveFit.bitmask.turn_on(ion_bits[kk], ion.replace(' ', '')) # DataContainer time # spat_id is set to an arbitrary -1 here and is updated in wavecalib.py final_fit = WaveFit(-1, pypeitfit=pypeitFit, pixel_fit=xfit, wave_fit=yfit, ion_bits=ion_bits, xnorm=xnspecmin1, cen_wave=cen_wave, cen_disp=cen_disp, spec=spec, wave_soln = wave_soln, sigrej=sigrej_final, shift=0., tcent=tcent, rms=rms_pix) # QA if plot_fil is not None: autoid.arc_fit_qa(final_fit, plot_fil) # Return return final_fit
def gen_pixloc(frame_shape, xgap=0, ygap=0, ysize=1., gen=True): """ Generate an array of physical pixel coordinates Parameters ---------- frame : ndarray uniformly illuminated and normalized flat field frame xgap : int (optional) ygap : int (optional) ysize : float (optional) gen : bool, optional Only allows True right now Returns ------- locations : ndarray A 3D array containing the x center, y center, x width and y width of each pixel. The returned array has a shape: frame.shape + (4,) """ #dnum = settings.get_dnum(det) msgs.info("Deriving physical pixel locations on the detector") locations = np.zeros((frame_shape[0],frame_shape[1],4)) if gen: msgs.info("Pixel gap in the dispersion direction = {0:4.3f}".format(xgap)) msgs.info("Pixel size in the dispersion direction = {0:4.3f}".format(1.0)) xs = np.arange(frame_shape[0]*1.0)*xgap xt = 0.5 + np.arange(frame_shape[0]*1.0) + xs msgs.info("Pixel gap in the spatial direction = {0:4.3f}".format(ygap)) msgs.info("Pixel size in the spatial direction = {0:4.3f}".format(ysize)) ys = np.arange(frame_shape[1])*ygap*ysize yt = ysize*(0.5 + np.arange(frame_shape[1]*1.0)) + ys xloc, yloc = np.meshgrid(xt, yt) # xwid, ywid = np.meshgrid(xs,ys) msgs.info("Saving pixel locations") locations[:,:,0] = xloc.T locations[:,:,1] = yloc.T locations[:,:,2] = 1.0 locations[:,:,3] = ysize else: msgs.error("Have not yet included an algorithm to automatically generate pixel locations") return locations
def main(pargs): from astropy.io import fits # Load the file config_lines, spec1dfiles = read_flexfile(pargs.flex_file) # Read in spectrograph from spec1dfile header header = fits.getheader(spec1dfiles[0]) spectrograph = load_spectrograph(header['PYP_SPEC']) # Parameters spectrograph_def_par = spectrograph.default_pypeit_par() par = pypeitpar.PypeItPar.from_cfg_lines( cfg_lines=spectrograph_def_par.to_config(), merge_with=config_lines) # Loop to my loop for filename in spec1dfiles: # Instantiate mdFlex = flexure.MultiSlitFlexure(s1dfile=filename) # Initalize msgs.info("Setup") mdFlex.init(spectrograph, par['flexure']) # INITIAL SKY LINE STUFF msgs.info("Measuring sky lines") mdFlex.measure_sky_lines() # FIT SURFACES msgs.info("Fitting the surface") mdFlex.fit_mask_surfaces() # Apply msgs.info("Applying flexure correction") mdFlex.update_fit() # REFIT FOR QA PLOTS msgs.info("Generate QA") mask = header['TARGET'].strip() fnames = header['FILENAME'].split('.') root = mask + '_' + fnames[2] mdFlex.qa_plots('./', root) # Write msgs.info("Write to disk") mdFlex.to_file(pargs.outroot + root + '.fits', overwrite=pargs.clobber) # Apply?? print("All done!!")
def apply_flux_calib(self, sens_dict, exptime, telluric_correct=False, extinct_correct=False, airmass=None, longitude=None, latitude=None): """ Apply a sensitivity function to our spectrum FLAM, FLAM_SIG, and FLAM_IVAR are generated Args: sens_dict (dict): Sens Function dict exptime (float): telluric_correct: extinct_correct: airmass (float, optional): longitude (float, optional): longitude in degree for observatory latitude: latitude in degree for observatory Used for extinction correction """ # Loop on extraction modes for attr in ['BOX', 'OPT']: if attr + '_WAVE' not in self._data.keys(): continue msgs.info("Fluxing {:s} extraction for:".format(attr) + msgs.newline() + "{}".format(self)) # #try: # wave = np.copy(np.array(extract['WAVE_GRID'])) #except KeyError: wave = self[attr + '_WAVE'] wave_sens = sens_dict['wave'] sensfunc = sens_dict['sensfunc'].copy() # Did the user request a telluric correction from the same file? if telluric_correct and 'telluric' in sens_dict.keys(): # This assumes there is a separate telluric key in this dict. telluric = sens_dict['telluric'] msgs.info('Applying telluric correction') sensfunc = sensfunc * (telluric > 1e-10) / (telluric + (telluric < 1e-10)) sensfunc_obs = interpolate.interp1d(wave_sens, sensfunc, bounds_error=False, fill_value='extrapolate')(wave) if extinct_correct: if longitude is None or latitude is None: msgs.error( 'You must specify longitude and latitude if we are extinction correcting' ) # Apply Extinction if optical bands msgs.info("Applying extinction correction") msgs.warn( "Extinction correction applyed only if the spectra covers <10000Ang." ) extinct = flux_calib.load_extinction_data(longitude, latitude) ext_corr = flux_calib.extinction_correction( wave * units.AA, airmass, extinct) senstot = sensfunc_obs * ext_corr else: senstot = sensfunc_obs.copy() flam = self[attr + '_COUNTS'] * senstot / exptime flam_sig = (senstot / exptime) / (np.sqrt( self[attr + '_COUNTS_IVAR'])) flam_var = self[attr + '_COUNTS_IVAR'] / (senstot / exptime)**2 # Mask bad pixels msgs.info(" Masking bad pixels") msk = np.zeros_like(senstot).astype(bool) msk[senstot <= 0.] = True msk[self[attr + '_COUNTS_IVAR'] <= 0.] = True flam[msk] = 0. flam_sig[msk] = 0. flam_var[msk] = 0. # Finish self[attr + '_FLAM'] = flam self[attr + '_FLAM_SIG'] = flam_sig self[attr + '_FLAM_IVAR'] = flam_var
def print_end_time(self): """ Print the elapsed time """ # Capture the end time and print it to user msgs.info(utils.get_time_string(time.time() - self.tstart))
def splice(self, wave): """ Routine to splice together sensitivity functions into one global sensitivity function for spectrographs with multiple detectors extending across the wavelength direction. Parameters ---------- wave: ndarray, shape (nspec, norddet) Returns ------- wave_splice: ndarray, shape (nspec_splice,) sensfunc_splice: ndarray, shape (nspec_splice,) """ msgs.info('Merging sensfunc for {:d} detectors {:}'.format( self.norderdet, self.par['multi_spec_det'])) wave_splice_min = wave.min() wave_splice_max = wave.max() wave_splice, _, _ = coadd.get_wave_grid(wave, wave_method='linear', wave_grid_min=wave_splice_min, wave_grid_max=wave_splice_max, samp_fact=1.0) sensfunc_splice = np.zeros_like(wave_splice) for idet in range(self.norderdet): wave_min = self.out_table['WAVE_MIN'][idet] wave_max = self.out_table['WAVE_MAX'][idet] if idet == 0: # If this is the bluest detector, extrapolate to wave_extrap_min wave_mask_min = wave_splice_min wave_mask_max = wave_max elif idet == (self.norderdet - 1): # If this is the reddest detector, extrapolate to wave_extrap_max wave_mask_min = wave_min wave_mask_max = wave_splice_max else: wave_mask_min = wave_min wave_mask_max = wave_max splice_wave_mask = (wave_splice >= wave_mask_min) & (wave_splice <= wave_mask_max) sensfunc_splice[splice_wave_mask] = self.eval_sensfunc( wave_splice[splice_wave_mask], idet) # Interpolate over gaps zeros = sensfunc_splice == 0. if np.any(zeros): msgs.info( "Interpolating over gaps (and extrapolating with fill_value=1, if need be)" ) interp_func = scipy.interpolate.interp1d( wave_splice[np.invert(zeros)], sensfunc_splice[np.invert(zeros)], kind='nearest', fill_value=0., bounds_error=False) # #kind='nearest', fill_value='extrapoloate', bounds_error=False) # extrapolate fails for JXP, even on 1.4.1 zero_values = interp_func(wave_splice[zeros]) sensfunc_splice[zeros] = zero_values self.steps.append(inspect.stack()[0][3]) return wave_splice, sensfunc_splice
def calib_all(self, run=True): """ Create calibrations for all setups This will not crash if not all of the standard set of files are not provided Args: run (bool, optional): If False, only print the calib names and do not actually run. Only used with the pypeit_parse_calib_id script Returns: dict: A simple dict summarizing the calibration names """ calib_dict = {} self.tstart = time.time() # Frame indices frame_indx = np.arange(len(self.fitstbl)) for i in range(self.fitstbl.n_calib_groups): # 1-indexed calib number calib_grp = str(i + 1) # Find all the frames in this calibration group in_grp = self.fitstbl.find_calib_group(i) grp_frames = frame_indx[in_grp] # Find the detectors to reduce # detectors = PypeIt.select_detectors(detnum=self.par['rdx']['detnum'], # ndet=self.spectrograph.ndet) detectors = self.spectrograph.select_detectors( subset=self.par['rdx']['detnum']) calib_dict[calib_grp] = {} # Loop on Detectors for self.det in detectors: # Instantiate Calibrations class self.caliBrate = calibrations.Calibrations.get_instance( self.fitstbl, self.par['calibrations'], self.spectrograph, self.calibrations_path, qadir=self.qa_path, reuse_masters=self.reuse_masters, show=self.show, user_slits=slittrace.merge_user_slit( self.par['rdx']['slitspatnum'], self.par['rdx']['maskIDs'])) # Do it # TODO: Why isn't set_config part of the Calibrations.__init__ method? self.caliBrate.set_config(grp_frames[0], self.det, self.par['calibrations']) # Allow skipping the run (e.g. parse_calib_id.py script) if run: self.caliBrate.run_the_steps() key = self.caliBrate.master_key_dict['frame'] calib_dict[calib_grp][key] = {} for step in self.caliBrate.steps: if step in ['bpm', 'slits', 'wv_calib', 'tilts', 'flats']: continue elif step == 'tiltimg': # Annoying kludge step = 'tilt' # Prep raw_files, self.caliBrate.master_key_dict[ step] = self.caliBrate._prep_calibrations(step) masterframe_name = masterframe.construct_file_name( buildimage.frame_image_classes[step], self.caliBrate.master_key_dict[step], master_dir=self.caliBrate.master_dir) # Add to dict if len(raw_files) > 0: calib_dict[calib_grp][key][step] = {} calib_dict[calib_grp][key][step][ 'master_key'] = self.caliBrate.master_key_dict[ step] calib_dict[calib_grp][key][step][ 'master_name'] = os.path.basename(masterframe_name) calib_dict[calib_grp][key][step]['raw_files'] = [ os.path.basename(ifile) for ifile in raw_files ] # Print the results print(json.dumps(calib_dict, sort_keys=True, indent=4)) # Write msgs.info('Writing calib file') calib_file = self.pypeit_file.replace('.pypeit', '.calib_ids') ltu.savejson(calib_file, calib_dict, overwrite=True, easy_to_read=True) # Finish self.print_end_time() # Return return calib_dict
def main(args): """ Executes 2d coadding """ msgs.warn('PATH =' + os.getcwd()) # Load the file if args.file is not None: spectrograph, config_lines, spec2d_files = read_coadd2d_file(args.file) # Parameters # TODO: Shouldn't this reinstantiate the same parameters used in # the PypeIt run that extracted the objects? Why are we not # just passing the pypeit file? # JFH: The reason is that the coadd2dfile may want different reduction parameters spectrograph_def_par = spectrograph.default_pypeit_par() parset = par.PypeItPar.from_cfg_lines( cfg_lines=spectrograph_def_par.to_config(), merge_with=config_lines) elif args.obj is not None: # TODO: We should probably be reading the pypeit file and using those parameters here rather than using the # default parset. # TODO: This needs to define the science path spec2d_files = glob.glob('./Science/spec2d_*' + args.obj + '*') head0 = fits.getheader(spec2d_files[0]) spectrograph_name = head0['SPECTROG'] spectrograph = load_spectrograph(spectrograph_name) parset = spectrograph.default_pypeit_par() else: msgs.error( 'You must either input a coadd2d file with --file or an object name with --obj' ) # If detector was passed as an argument override whatever was in the coadd2d_file if args.det is not None: msgs.info("Restricting reductions to detector={}".format(args.det)) parset['rdx']['detnum'] = int(args.det) # Get headers (if possible) and base names spec1d_files = [ files.replace('spec2d', 'spec1d') for files in spec2d_files ] head1d = None for spec1d_file in spec1d_files: if os.path.isfile(spec1d_file): head1d = fits.getheader(spec1d_file) break if head1d is None: msgs.warn("No 1D spectra so am generating a dummy header for output") head1d = io.initialize_header() head2d = fits.getheader(spec2d_files[0]) if args.basename is None: filename = os.path.basename(spec2d_files[0]) basename = filename.split('_')[2] else: basename = args.basename # Write the par to disk par_outfile = basename + '_coadd2d.par' print("Writing the parameters to {}".format(par_outfile)) parset.to_config(par_outfile) # Now run the coadds skysub_mode = head2d['SKYSUB'] ir_redux = True if 'DIFF' in skysub_mode else False # Print status message msgs_string = 'Reducing target {:s}'.format(basename) + msgs.newline() msgs_string += 'Performing coadd of frames reduce with {:s} imaging'.format( skysub_mode) msgs_string += msgs.newline( ) + 'Combining frames in 2d coadd:' + msgs.newline() for file in spec2d_files: msgs_string += '{0:s}'.format(os.path.basename(file)) + msgs.newline() msgs.info(msgs_string) # TODO: This needs to be added to the parameter list for rdx redux_path = os.getcwd() master_dirname = os.path.basename(head2d['PYPMFDIR']) + '_coadd' master_dir = os.path.join(redux_path, master_dirname) # Make the new Master dir if not os.path.isdir(master_dir): msgs.info( 'Creating directory for Master output: {0}'.format(master_dir)) os.makedirs(master_dir) # Instantiate the sci_dict sci_dict = OrderedDict() # This needs to be ordered sci_dict['meta'] = {} sci_dict['meta']['vel_corr'] = 0. sci_dict['meta']['ir_redux'] = ir_redux # Find the detectors to reduce detectors = PypeIt.select_detectors(detnum=parset['rdx']['detnum'], ndet=spectrograph.ndet) if len(detectors) != spectrograph.ndet: msgs.warn('Not reducing detectors: {0}'.format(' '.join([ str(d) for d in set(np.arange(spectrograph.ndet) + 1) - set(detectors) ]))) # Loop on detectors for det in detectors: msgs.info("Working on detector {0}".format(det)) sci_dict[det] = {} # Instantiate Coadd2d coadd = coadd2d.instantiate_me(spec2d_files, spectrograph, det=det, offsets=parset['coadd2d']['offsets'], weights=parset['coadd2d']['weights'], par=parset, ir_redux=ir_redux, debug_offsets=args.debug_offsets, debug=args.debug, samp_fact=args.samp_fact) # Coadd the slits coadd_dict_list = coadd.coadd( only_slits=None) # TODO implement only_slits later # Create the psuedo images psuedo_dict = coadd.create_psuedo_image(coadd_dict_list) # Reduce msgs.info('Running the extraction') sci_dict[det]['sciimg'], sci_dict[det]['sciivar'], sci_dict[det]['skymodel'], sci_dict[det]['objmodel'], \ sci_dict[det]['ivarmodel'], sci_dict[det]['outmask'], sci_dict[det]['specobjs'] = coadd.reduce( psuedo_dict, show = args.show, show_peaks = args.peaks) # Save psuedo image master files coadd.save_masters(master_dir) # Make the new Science dir # TODO: This needs to be defined by the user scipath = os.path.join(redux_path, 'Science_coadd') if not os.path.isdir(scipath): msgs.info('Creating directory for Science output: {0}'.format(scipath)) os.makedirs(scipath) # Save the results save.save_all(sci_dict, coadd.stack_dict['master_key_dict'], master_dir, spectrograph, head1d, head2d, scipath, basename) #, binning=coadd.binning)
def reduce_all(self): """ Main driver of the entire reduction Calibration and extraction via a series of calls to reduce_exposure() """ # Validate the parameter set self.par.validate_keys(required=[ 'rdx', 'calibrations', 'scienceframe', 'reduce', 'flexure' ]) self.tstart = time.time() # Find the standard frames is_standard = self.fitstbl.find_frames('standard') # Find the science frames is_science = self.fitstbl.find_frames('science') # Frame indices frame_indx = np.arange(len(self.fitstbl)) # Standard Star(s) Loop # Iterate over each calibration group and reduce the standards for i in range(self.fitstbl.n_calib_groups): # Find all the frames in this calibration group in_grp = self.fitstbl.find_calib_group(i) # Find the indices of the standard frames in this calibration group: grp_standards = frame_indx[is_standard & in_grp] # Reduce all the standard frames, loop on unique comb_id u_combid_std = np.unique(self.fitstbl['comb_id'][grp_standards]) for j, comb_id in enumerate(u_combid_std): frames = np.where(self.fitstbl['comb_id'] == comb_id)[0] bg_frames = np.where(self.fitstbl['bkg_id'] == comb_id)[0] if not self.outfile_exists(frames[0]) or self.overwrite: # Build history to document what contributd to the reduced # exposure history = History(self.fitstbl.frame_paths(frames[0])) history.add_reduce(i, self.fitstbl, frames, bg_frames) std_spec2d, std_sobjs = self.reduce_exposure( frames, bg_frames=bg_frames) # TODO come up with sensible naming convention for save_exposure for combined files self.save_exposure(frames[0], std_spec2d, std_sobjs, self.basename, history) else: msgs.info( 'Output file: {:s} already exists'.format( self.fitstbl.construct_basename(frames[0])) + '. Set overwrite=True to recreate and overwrite.') # Science Frame(s) Loop # Iterate over each calibration group again and reduce the science frames for i in range(self.fitstbl.n_calib_groups): # Find all the frames in this calibration group in_grp = self.fitstbl.find_calib_group(i) # Find the indices of the science frames in this calibration group: grp_science = frame_indx[is_science & in_grp] # Associate standards (previously reduced above) for this setup std_outfile = self.get_std_outfile(frame_indx[is_standard]) # Reduce all the science frames; keep the basenames of the science frames for use in flux calibration science_basename = [None] * len(grp_science) # Loop on unique comb_id u_combid = np.unique(self.fitstbl['comb_id'][grp_science]) for j, comb_id in enumerate(u_combid): frames = np.where(self.fitstbl['comb_id'] == comb_id)[0] # Find all frames whose comb_id matches the current frames bkg_id. bg_frames = np.where((self.fitstbl['comb_id'] == self.fitstbl['bkg_id'][frames][0]) & (self.fitstbl['comb_id'] >= 0))[0] # JFH changed the syntax below to that above, which allows frames to be used more than once # as a background image. The syntax below would require that we could somehow list multiple # numbers for the bkg_id which is impossible without a comma separated list # bg_frames = np.where(self.fitstbl['bkg_id'] == comb_id)[0] if not self.outfile_exists(frames[0]) or self.overwrite: # Build history to document what contributd to the reduced # exposure history = History(self.fitstbl.frame_paths(frames[0])) history.add_reduce(i, self.fitstbl, frames, bg_frames) # TODO -- Should we reset/regenerate self.slits.mask for a new exposure sci_spec2d, sci_sobjs = self.reduce_exposure( frames, bg_frames=bg_frames, std_outfile=std_outfile) science_basename[j] = self.basename # TODO come up with sensible naming convention for save_exposure for combined files self.save_exposure(frames[0], sci_spec2d, sci_sobjs, self.basename, history) else: msgs.warn( 'Output file: {:s} already exists'.format( self.fitstbl.construct_basename(frames[0])) + '. Set overwrite=True to recreate and overwrite.') msgs.info('Finished calibration group {0}'.format(i)) # Finish self.print_end_time()
def find_standard(self): """ Identify the standard star from the list of all spectra in the specobjs Wrapper to flux.find_standard which simply takes the brightest Returns ------- self.std : SpecObj Corresponds to the chosen spectrum """ if self.par['std_obj_id'] is not None: _ = self._set_std_obj() return if self.multi_det is not None: sv_stds = [] # Find the standard in each detector for det in self.multi_det: stds = [sobj for sobj in self.std_specobjs if sobj.det == det] if len(stds) == 0: debugger.set_trace() idx = flux.find_standard(stds) sv_stds.append(stds[idx]) msgs.info("Using standard {} for det={}".format( stds[idx], det)) # Now splice msgs.info( "Splicing the standards -- The name will be for the first detector" ) std_splice = sv_stds[0].copy() # Append for ostd in sv_stds[1:]: try: std_splice.optimal['WAVE_GRID'] = np.append( std_splice.optimal['WAVE_GRID'].value, ostd.optimal['WAVE_GRID'].value) * units.AA except KeyError: std_splice.optimal['WAVE'] = np.append( std_splice.optimal['WAVE'].value, ostd.optimal['WAVE'].value) * units.AA for key in ['COUNTS', 'COUNTS_IVAR']: std_splice.optimal[key] = np.append( std_splice.optimal[key], ostd.optimal[key]) self.std = std_splice elif self.spectrograph.pypeline == 'Echelle': # Find brightest object in each order std_brightest = self.std_specobjs[flux.find_standard( self.std_specobjs)] std_objid = std_brightest['idx'].split('-')[0] self.std_idx = np.zeros(len(self.std_specobjs), dtype=bool) for ii in range(len(self.std_specobjs)): if std_objid in self.std_specobjs[ii]['idx']: self.std_idx[ii] = True # Set internal self.std = self.std_specobjs[self.std_idx] # Step self.steps.append(inspect.stack()[0][3]) # Return return self.std else: # Find brightest object in the exposures # Searches over all slits (over all detectors), and all objects self.std_idx = flux.find_standard(self.std_specobjs) # Set internal self.std = self.std_specobjs[self.std_idx] # Step self.steps.append(inspect.stack()[0][3]) # Return return self.std
def save_master(self, sens_dicts, outfile=None): """ Parameters ---------- outfile : str, optional Use this input instead of the 'proper' (or unattainable) MasterFrame name Returns ------- """ # Step self.steps.append(inspect.stack()[0][3]) # Allow one to over-ride output name if outfile is None: outfile = self.ms_name # Add steps self.sens_dict['steps'] = self.steps norder = self.sens_dict['norder'] # Do it prihdu = fits.PrimaryHDU() hdus = [prihdu] for iord in range(norder): sens_dict_iord = self.sens_dict[str(iord)] cols = [] cols += [fits.Column(array=sens_dict_iord['wave'], name=str('WAVE'), format=sens_dict_iord['wave'].dtype)] cols += [ fits.Column(array=sens_dict_iord['sensfunc'], name=str('SENSFUNC'), format=sens_dict_iord['sensfunc'].dtype)] # Finish coldefs = fits.ColDefs(cols) tbhdu = fits.BinTableHDU.from_columns(coldefs) tbhdu.name = 'SENSFUNC-ORDER{0:04}'.format(iord) # Add critical keys from sens_dict to header for key in ['wave_min', 'wave_max', 'exptime', 'airmass', 'std_file', 'std_ra', 'std_dec', 'std_name', 'cal_file', 'ech_orderindx']: try: tbhdu.header[key.upper()] = sens_dict_iord[key].value except AttributeError: tbhdu.header[key.upper()] = sens_dict_iord[key] except KeyError: pass # Will not require all of these hdus += [tbhdu] # Add critical keys from sens_dict to primary header for key in ['exptime', 'airmass', 'std_file', 'std_ra', 'std_dec', 'std_name', 'cal_file']: try: prihdu.header[key.upper()] = sens_dict_iord[key].value except AttributeError: prihdu.header[key.upper()] = sens_dict_iord[key] except KeyError: pass # Will not require all of these prihdu.header['NORDER'] = norder # Finish hdulist = fits.HDUList(hdus) hdulist.writeto(outfile, overwrite=True) # Finish msgs.info("Wrote sensfunc to MasterFrame: {:s}".format(outfile))
def reduce_exposure(self, frames, bg_frames=None, std_outfile=None): """ Reduce a single exposure Args: frames (:obj:`list`): List of 0-indexed rows in :attr:`fitstbl` with the frames to reduce. bg_frames (:obj:`list`, optional): List of frame indices for the background. std_outfile (:obj:`str`, optional): File with a previously reduced standard spectrum from PypeIt. Returns: dict: The dictionary containing the primary outputs of extraction. """ # if show is set, clear the ginga channels at the start of each new sci_ID if self.show: # TODO: Put this in a try/except block? display.clear_all(allow_new=True) has_bg = True if bg_frames is not None and len( bg_frames) > 0 else False # Is this an b/g subtraction reduction? if has_bg: self.bkg_redux = True # The default is to find_negative objects if the bg_frames are classified as "science", and to not find_negative # objects if the bg_frames are classified as "sky". This can be explicitly overridden if # par['reduce']['findobj']['find_negative'] is set to something other than the default of None. self.find_negative = (('science' in self.fitstbl['frametype'][bg_frames[0]]) | ('standard' in self.fitstbl['frametype'][bg_frames[0]]))\ if self.par['reduce']['findobj']['find_negative'] is None else self.par['reduce']['findobj']['find_negative'] else: self.bkg_redux = False self.find_negative = False # Container for all the Spec2DObj all_spec2d = spec2dobj.AllSpec2DObj() all_spec2d['meta']['bkg_redux'] = self.bkg_redux all_spec2d['meta']['find_negative'] = self.find_negative # TODO -- Should we reset/regenerate self.slits.mask for a new exposure # container for specobjs during first loop (objfind) all_specobjs_objfind = specobjs.SpecObjs() # container for specobjs during second loop (extraction) all_specobjs_extract = specobjs.SpecObjs() # list of global_sky obtained during objfind and used in extraction initial_sky_list = [] # list of sciImg sciImg_list = [] # List of detectors with successful calibration calibrated_det = [] # list of successful MasterSlits calibrations to be used in the extraction loop calib_slits = [] # List of objFind objects objFind_list = [] # Print status message msgs_string = 'Reducing target {:s}'.format( self.fitstbl['target'][frames[0]]) + msgs.newline() # TODO: Print these when the frames are actually combined, # backgrounds are used, etc? msgs_string += 'Combining frames:' + msgs.newline() for iframe in frames: msgs_string += '{0:s}'.format( self.fitstbl['filename'][iframe]) + msgs.newline() msgs.info(msgs_string) if has_bg: bg_msgs_string = '' for iframe in bg_frames: bg_msgs_string += '{0:s}'.format( self.fitstbl['filename'][iframe]) + msgs.newline() bg_msgs_string = msgs.newline( ) + 'Using background from frames:' + msgs.newline( ) + bg_msgs_string msgs.info(bg_msgs_string) # Find the detectors to reduce subset = self.par['rdx']['slitspatnum'] if self.par['rdx']['slitspatnum'] is not None \ else self.par['rdx']['detnum'] detectors = self.spectrograph.select_detectors(subset=subset) msgs.info(f'Detectors to work on: {detectors}') # Loop on Detectors # TODO: Attempt to put in a multiprocessing call here? # objfind for self.det in detectors: msgs.info("Working on detector {0}".format(self.det)) # run calibration self.caliBrate = self.calib_one(frames, self.det) if not self.caliBrate.success: msgs.warn( f'Calibrations for detector {self.det} were unsuccessful! The step ' f'that failed was {self.caliBrate.failed_step}. Continuing by ' f'skipping this detector.') continue # we save only the detectors that had a successful calibration, # and we use only those in the extract loop below calibrated_det.append(self.det) # we also save the successful MasterSlits calibrations because they are used and modified # in the slitmask stuff in between the two loops calib_slits.append(self.caliBrate.slits) # global_sky, skymask and sciImg are needed in the extract loop initial_sky, sobjs_obj, sciImg, objFind = self.objfind_one( frames, self.det, bg_frames, std_outfile=std_outfile) if len(sobjs_obj) > 0: all_specobjs_objfind.add_sobj(sobjs_obj) initial_sky_list.append(initial_sky) sciImg_list.append(sciImg) objFind_list.append(objFind) # slitmask stuff if self.par['reduce']['slitmask']['assign_obj']: # get object positions from slitmask design and slitmask offsets for all the detectors spat_flexure = np.array([ss.spat_flexure for ss in sciImg_list]) # Grab platescale with binning bin_spec, bin_spat = parse.parse_binning(self.binning) platescale = np.array( [ss.detector.platescale * bin_spat for ss in sciImg_list]) # get the dither offset if available if self.par['reduce']['slitmask']['use_dither_offset']: dither = self.spectrograph.parse_dither_pattern( [self.fitstbl.frame_paths(frames[0])]) dither_off = dither[2][0] if dither is not None else None else: dither_off = None calib_slits = slittrace.get_maskdef_objpos_offset_alldets( all_specobjs_objfind, calib_slits, spat_flexure, platescale, self.par['calibrations']['slitedges']['det_buffer'], self.par['reduce']['slitmask'], dither_off=dither_off) # determine if slitmask offsets exist and compute an average offsets over all the detectors calib_slits = slittrace.average_maskdef_offset( calib_slits, platescale[0], self.spectrograph.list_detectors( mosaic='MSC' in calib_slits[0].detname)) # slitmask design matching and add undetected objects all_specobjs_objfind = slittrace.assign_addobjs_alldets( all_specobjs_objfind, calib_slits, spat_flexure, platescale, self.par['reduce']['slitmask'], self.par['reduce']['findobj']['find_fwhm']) # Extract for i, self.det in enumerate(calibrated_det): # re-run (i.e., load) calibrations self.caliBrate = self.calib_one(frames, self.det) self.caliBrate.slits = calib_slits[i] detname = sciImg_list[i].detector.name # TODO: pass back the background frame, pass in background # files as an argument. extract one takes a file list as an # argument and instantiates science within if all_specobjs_objfind.nobj > 0: all_specobjs_on_det = all_specobjs_objfind[ all_specobjs_objfind.DET == detname] else: all_specobjs_on_det = all_specobjs_objfind # Extract all_spec2d[detname], tmp_sobjs \ = self.extract_one(frames, self.det, sciImg_list[i], objFind_list[i], initial_sky_list[i], all_specobjs_on_det) # Hold em if tmp_sobjs.nobj > 0: all_specobjs_extract.add_sobj(tmp_sobjs) # JFH TODO write out the background frame? # TODO -- Save here? Seems like we should. Would probably need to use update_det=True # Return return all_spec2d, all_specobjs_extract
def fit2darc(all_wv, all_pix, t, nspec, nycoeff=3, nocoeff=5, sigmarjct=3.0, debug=True): """Routine to obtain the 2D wavelength solution for an echelle spectrograph. This is calculated from the y-centroid and the order number of identified arc lines. The fit is a simple least-squares with one round of rejections. This is a direct porting of the XIDL code: x_fit2darc.pro Parameters ---------- all_wv: np.array wavelength of the identified lines all_pix: np.array y-centroid position of the identified lines t: np.array order number of the identified lines nycoeff : np.int order of the fitting along the pixel direction for each order nocoeff : np.int order of the fitting in the order direction sigmarjct: np.float sigma level for the rejection debug: boolean Extra plots to check the status of the procedure Returns: ------- """ # To use the legendre polynomial pixels and orders # need to be normalized in the -1,+1 range # Normalize pixels mnx = 0 #np.min(all_pix) mxx = float(nspec - 1) #np.max(all_pix) nrmp = np.array([0.5 * (mnx + mxx), mxx - mnx]) pix_nrm = 2. * (all_pix - nrmp[0])/nrmp[1] # Normalize orders mnx = np.min(t) mxx = np.max(t) nrmt = np.array([0.5 * (mnx + mxx), mxx - mnx]) t_nrm = 2. * (t - nrmt[0])/nrmt[1] if debug: # set some plotting parameters prettyplot() plt.figure(figsize=(7,5)) msgs.info("Plot identified lines") cm = plt.cm.get_cmap('RdYlBu_r') sc = plt.scatter(t_nrm, pix_nrm, c=all_wv/10000., cmap=cm) cbar = plt.colorbar(sc) cbar.set_label(r'Wavelength [$\mu$m]', rotation=270, labelpad=20) plt.xlabel(r'Normalized Orders') plt.ylabel(r'Normalized Pixels') plt.title(r'Location of the identified lines') plt.show() msgs.info("First iteration") # all lines have the same weight invvar = np.ones(len(all_wv), dtype=np.float64) all_wv_order = all_wv * t work2d = np.zeros((nycoeff*nocoeff, len(all_wv)), dtype=np.float64) worky = pydl.flegendre(pix_nrm, nycoeff) workt = pydl.flegendre(t_nrm, nocoeff) for i in range(nocoeff): for j in range(nycoeff): work2d[j*nocoeff+i,:] = worky[j,:] * workt[i,:] work2di = np.transpose(work2d * np.outer(np.ones(nocoeff*nycoeff, dtype=np.float64), invvar)) alpha = work2d.dot(work2di) beta = all_wv_order.dot(work2di) res = np.linalg.solve(alpha,beta) wv_mod = res.dot(work2d) if debug: # set some plotting parameters prettyplot() plt.figure(figsize=(7,5)) plt.axhline(y=np.average(wv_mod / t - all_wv), color='r', linestyle='--') plt.axhline(y=+np.std(wv_mod / t - all_wv), color='r', linestyle=':') plt.axhline(y=-np.std(wv_mod / t - all_wv), color='r', linestyle=':') plt.scatter(all_wv/10000., wv_mod / t - all_wv, marker="v") plt.text(np.min(all_wv/10000), np.average(wv_mod/t-all_wv), r'Average={0:.1f}$\AA$'.format(np.average(wv_mod/t-all_wv)), ha="left", va="bottom", bbox=dict(boxstyle="square", ec=(1., 0.5, 0.5), fc=(1., 0.8, 0.8), alpha=0.7, )) plt.text(np.max(all_wv/10000), np.std(wv_mod/t-all_wv), r'Sigma={0:.1f}$\AA$'.format(np.std(wv_mod/t-all_wv)), ha="right", va="bottom", bbox=dict(boxstyle="square", ec=(1., 0.5, 0.5), fc=(1., 0.8, 0.8), alpha=0.7, )) plt.title(r'Residuals after 1st iteration') plt.xlabel(r'Wavelength [$\mu$m]') plt.ylabel(r'Residuals [$\AA$]') plt.show() msgs.info("Second iteration") # Mask Values # msk = True means a bad value msk = sigma_clip(wv_mod-all_wv_order, sigma=sigmarjct, cenfunc=np.ma.mean).mask if np.any(msk): msgs.info("Rejecting: {} of {} lines.".format(len(msk[np.where(msk == True)]),len(msk))) invvar[msk] = 0. work2di = np.transpose(work2d * np.outer(np.ones(nocoeff*nycoeff, dtype=np.float64), invvar)) alpha = work2d.dot(work2di) beta = all_wv_order.dot(work2di) res = np.linalg.solve(alpha,beta) wv_mod = res.dot(work2d) if debug: prettyplot() plt.figure(figsize=(7,5)) plt.axhline(y=np.average(wv_mod[~msk] / t[~msk] - all_wv[~msk]), color='r', linestyle='--') plt.axhline(y=+np.std(wv_mod[~msk] / t[~msk] - all_wv[~msk]), color='r', linestyle=':') plt.axhline(y=-np.std(wv_mod[~msk] / t[~msk] - all_wv[~msk]), color='r', linestyle=':') plt.scatter(all_wv[msk]/10000., wv_mod[msk] / t[msk] - all_wv[msk], marker="v", label=r'Rejected values') plt.scatter(all_wv[~msk]/10000., wv_mod[~msk] / t[~msk] - all_wv[~msk], marker="v", label=r'Good values') plt.text(np.min(all_wv/10000), np.average(wv_mod[~msk]/t[~msk]-all_wv[~msk]), r'Average={0:.1f}$\AA$'.format(np.average(wv_mod[~msk]/t[~msk]-all_wv[~msk])), ha="left", va="bottom", bbox=dict(boxstyle="square", ec=(1., 0.5, 0.5), fc=(1., 0.8, 0.8), alpha=0.7, )) plt.text(np.max(all_wv/10000), np.std(wv_mod[~msk]/t[~msk]-all_wv[~msk]), r'Sigma={0:.1f}$\AA$'.format(np.std(wv_mod[~msk]/t[~msk]-all_wv[~msk])), ha="right", va="bottom", bbox=dict(boxstyle="square", ec=(1., 0.5, 0.5), fc=(1., 0.8, 0.8), alpha=0.7, )) plt.legend() plt.title(r'Residuals after 2nd iteration') plt.xlabel(r'Wavelength [$\mu$m]') plt.ylabel(r'Residuals [$\AA$]') plt.show() else: msgs.info("No line rejected") # Check quality gd_wv = invvar > 0. resid = (wv_mod[gd_wv]-all_wv_order[gd_wv]) fin_rms = np.sqrt(np.mean(resid**2)) msgs.info("RMS: {0:.5f} Ang*Order#".format(fin_rms)) # Plot QA # Full plot all_pix_qa = np.arange(np.min(all_pix),np.max(all_pix),1) pix_nrm_qa = 2. * (all_pix_qa - nrmp[0])/nrmp[1] worky_qa = pydl.flegendre(pix_nrm_qa, nycoeff) mn, mx = np.min(wv_mod/t), np.max(wv_mod/t) order = np.arange(np.min(t),np.max(t)+1,1) prettyplot() plt.figure(figsize=(7,5)) plt.title(r'Arc 2D FIT, nx={0:.0f}, ny={1:.0f}, RMS={2:.5f} Ang*Order#'.format(nocoeff, nycoeff,fin_rms)) plt.xlabel(r'Wavelength [$\AA$]') plt.ylabel(r'Row [pixel]') for ii in order: # define the color rr = (ii-np.max(order))/(np.min(order)-np.max(order)) gg = 0.0 bb = (ii-np.min(order))/(np.max(order)-np.min(order)) tsub = np.ones_like(len(all_pix_qa),dtype=np.float64) * ii t_nrm_qa = 2. * (tsub - nrmt[0])/nrmt[1] work2d_qa = np.zeros((nycoeff*nocoeff, len(all_pix_qa)), dtype=np.float64) workt_qa = pydl.flegendre(t_nrm_qa, nocoeff) for i in range(nocoeff): for j in range(nycoeff): work2d_qa[j*nocoeff+i,:] = worky_qa[j,:] * workt_qa[i,:] wv_mod_qa = res.dot(work2d_qa) plt.plot(wv_mod_qa/ii, all_pix_qa, color=(rr,gg,bb), linestyle='-') # Residuals resid_qa = (wv_mod[t == ii]-all_wv_order[t == ii])/t[t == ii] plt.scatter(wv_mod[t == ii]/t[t == ii]+100*resid_qa, all_pix[t == ii], color=(rr,gg,bb)) plt.text(mx,np.max(all_pix), r'residuals $\times$100', ha="right", va="top",) plt.show() # Individual plots prettyplot() nrow = np.int(2) ncol = np.int(np.ceil(len(order)/2.)) fig, ax = plt.subplots(nrow,ncol,figsize=(4*ncol,4*nrow)) for ii_row in np.arange(nrow): for ii_col in np.arange(ncol): if (ii_row*(nrow+1))+ii_col < len(order): from IPython import embed embed() ii = order[(ii_row*(nrow+1))+ii_col] rr = (ii-np.max(order))/(np.min(order)-np.max(order)) gg = 0.0 bb = (ii-np.min(order))/(np.max(order)-np.min(order)) tsub = np.ones_like(len(all_pix_qa),dtype=np.float64) * ii t_nrm_qa = 2. * (tsub - nrmt[0])/nrmt[1] work2d_qa = np.zeros((nycoeff*nocoeff, len(all_pix_qa)), dtype=np.float64) workt_qa = pydl.flegendre(t_nrm_qa, nocoeff) for i in range(nocoeff): for j in range(nycoeff): work2d_qa[j*nocoeff+i,:] = worky_qa[j,:] * workt_qa[i,:] wv_mod_qa = res.dot(work2d_qa) ax[ii_row,ii_col].plot(all_pix_qa, wv_mod_qa/ii/10000., color=(rr,gg,bb), linestyle='-') ax[ii_row,ii_col].set_title('Order = {0:0.0f}'.format(ii)) # Residuals resid_qa = (wv_mod[t == ii]-all_wv_order[t == ii])/t[t == ii] resid_qa_mask = (wv_mod[gd_wv][t[gd_wv] == ii]-all_wv_order[gd_wv][t[gd_wv] == ii])/t[gd_wv][t[gd_wv] == ii] rms_qa = np.sqrt(np.mean(resid_qa_mask**2)) dwl=(wv_mod_qa[-1]-wv_mod_qa[0])/ii/(all_pix_qa[-1]-all_pix_qa[0]) ax[ii_row,ii_col].scatter(all_pix[t == ii], wv_mod[t == ii]/t[t == ii]/10000.+100.*resid_qa/10000., color=(rr,gg,bb)) ax[ii_row,ii_col].text(0.9,0.9, r'RMS={0:.2f} Pixel'.format(rms_qa/np.abs(dwl)), ha="right", va="top", transform = ax[ii_row,ii_col].transAxes) ax[ii_row,ii_col].text(0.9,0.8, r'$\Delta\lambda$={0:.2f} $\AA$/Pixel'.format(np.abs(dwl)), ha="right", va="top", transform = ax[ii_row,ii_col].transAxes) else: ax[ii_row,ii_col].axis('off') fig.text(0.5, 0.04, r'Row [pixel]', ha='center', size='large') fig.text(0.04, 0.5, r'Wavelength [$\mu$m]', va='center', rotation='vertical', size='large') fig.suptitle(r'Arc 2D FIT, nx={0:.0f}, ny={1:.0f}, RMS={2:.5f} Ang*Order#, residuals $\times$100'.format(nocoeff, nycoeff,fin_rms)) plt.show() return wv_mod
def __init__(self, std_spec1d_file=None, sci_spec1d_file=None, sens_file=None, std_specobjs=None, std_header=None, spectrograph=None, telluric=False, setup=None, master_dir=None, mode=None, star_type=None, star_mag=None, BALM_MASK_WID=5.0, nresln=None, debug=False): # Load standard files std_spectro = None self.std_spec1d_file = std_spec1d_file # Need to unwrap these (sometimes).. self.std_specobjs = std_specobjs self.std_header = std_header if self.std_spec1d_file is not None: self.std_specobjs, self.std_header = load.ech_load_specobj(self.std_spec1d_file) msgs.info('Loaded {0} spectra from the spec1d standard star file: {1}'.format( len(self.std_specobjs), self.std_spec1d_file)) std_spectro = self.std_header['INSTRUME'] try: self.std_ra = self.std_header['RA'] except: self.std_ra = None try: self.std_dec = self.std_header['DEC'] except: self.std_dec = None try: self.std_file = self.std_header['FILENAME'] except: self.std_file = None # Load the science files sci_spectro = None self.sci_spec1d_file = sci_spec1d_file self.sci_specobjs = [] self.sci_header = None if self.sci_spec1d_file is not None: self.sci_specobjs, self.sci_header = load.ech_load_specobj(self.sci_spec1d_file) msgs.info('Loaded {0} spectra from the spec1d science file: {1}'.format( len(self.sci_specobjs), self.sci_spec1d_file)) sci_spectro = self.sci_header['INSTRUME'] # Compare instruments if they exist if std_spectro is not None and sci_spectro is not None and std_spectro != sci_spectro: msgs.error('Standard spectra are not the same instrument as science!!') # Instantiate the spectrograph _spectrograph = spectrograph if _spectrograph is None: _spectrograph = std_spectro if _spectrograph is not None: msgs.info("Spectrograph set to {0} from standard file".format(_spectrograph)) if _spectrograph is None: _spectrograph = sci_spectro if _spectrograph is not None: msgs.info("Spectrograph set to {0} from science file".format(_spectrograph)) self.spectrograph = load_spectrograph(_spectrograph) # MasterFrame masterframe.MasterFrame.__init__(self, self.frametype, setup, master_dir=master_dir, mode=mode) # Get the extinction data self.extinction_data = None if self.spectrograph is not None: self.extinction_data \ = flux.load_extinction_data(self.spectrograph.telescope['longitude'], self.spectrograph.telescope['latitude']) elif self.sci_header is not None and 'LON-OBS' in self.sci_header.keys(): self.extinction_data \ = flux.load_extinction_data(self.sci_header['LON-OBS'], self.sci_header['LAT-OBS']) # Once the spectrograph is instantiated, can also set the # extinction data # Parameters self.sens_file = sens_file # Set telluric option self.telluric = telluric # Main outputs self.sens_dict = None if self.sens_file is None \ else self.load_master(self.sens_file) # Attributes self.steps = [] # Key Internals self.std = None # Standard star spectrum (SpecObj object) self.std_idx = None # Nested indices for the std_specobjs list that corresponds # to the star! # Echelle key self.star_type = star_type self.star_mag = star_mag self.BALM_MASK_WID = BALM_MASK_WID self.nresln = nresln self.debug = debug
def objfind_one(self, frames, det, bg_frames, std_outfile=None): """ Reduce + Find Objects in a single exposure/detector pair sci_ID and det need to have been set internally prior to calling this method Parameters ---------- frames : :obj:`list` List of frames to extract; stacked if more than one is provided det : :obj:`int` Detector number (1-indexed) bg_frames : :obj:`list` List of frames to use as the background. Can be empty. std_outfile : :obj:`str`, optional Filename for the standard star spec1d file. Passed directly to :func:`get_std_trace`. Returns ------- global_sky : `numpy.ndarray`_ Initial global sky model sobjs_obj : :class:`~pypeit.specobjs.SpecObjs` List of objects found sciImg : :class:`~pypeit.images.pypeitimage.PypeItImage` Science image objFind : :class:`~pypeit.find_objects.FindObjects` Object finding speobject """ # Grab some meta-data needed for the reduction from the fitstbl self.objtype, self.setup, self.obstime, self.basename, self.binning \ = self.get_sci_metadata(frames[0], det) msgs.info("Object finding begins for {} on det={}".format( self.basename, det)) # Is this a standard star? self.std_redux = 'standard' in self.objtype frame_par = self.par['calibrations'][ 'standardframe'] if self.std_redux else self.par['scienceframe'] # Get the standard trace if need be if self.std_redux is False and std_outfile is not None: std_trace = specobjs.get_std_trace( self.spectrograph.get_det_name(det), std_outfile) else: std_trace = None # Build Science image sci_files = self.fitstbl.frame_paths(frames) sciImg = buildimage.buildimage_fromlist( self.spectrograph, det, frame_par, sci_files, bias=self.caliBrate.msbias, bpm=self.caliBrate.msbpm, dark=self.caliBrate.msdark, flatimages=self.caliBrate.flatimages, slits=self.caliBrate.slits, # For flexure correction ignore_saturation=False) # Background Image? if len(bg_frames) > 0: bg_file_list = self.fitstbl.frame_paths(bg_frames) sciImg = sciImg.sub( buildimage.buildimage_fromlist( self.spectrograph, det, frame_par, bg_file_list, bpm=self.caliBrate.msbpm, bias=self.caliBrate.msbias, dark=self.caliBrate.msdark, flatimages=self.caliBrate.flatimages, slits=self.caliBrate.slits, # For flexure correction ignore_saturation=False), frame_par['process']) # Deal with manual extraction row = self.fitstbl[frames[0]] manual_obj = ManualExtractionObj.by_fitstbl_input( row['filename'], row['manual'], self.spectrograph) if len(row['manual'].strip()) > 0 else None # Instantiate Reduce object # Required for pypeline specific object # At instantiaton, the fullmask in self.sciImg is modified objFind = find_objects.FindObjects.get_instance( sciImg, self.spectrograph, self.par, self.caliBrate, self.objtype, bkg_redux=self.bkg_redux, manual=manual_obj, find_negative=self.find_negative, std_redux=self.std_redux, show=self.show, basename=self.basename) # Do it initial_sky, sobjs_obj = objFind.run(std_trace=std_trace, show_peaks=self.show) # Return return initial_sky, sobjs_obj, sciImg, objFind
def ech_objfind(image, ivar, ordermask, slit_left, slit_righ,inmask=None,plate_scale=0.2,npca=2,ncoeff = 5,min_snr=0.0,nabove_min_snr=0, pca_percentile=20.0,snr_pca=3.0,box_radius=2.0,show_peaks=False,show_fits=False,show_trace=False): if inmask is None: inmask = (ordermask > 0) frameshape = image.shape nspec = frameshape[0] norders = slit_left.shape[1] if isinstance(plate_scale,(float, int)): plate_scale_ord = np.full(norders, plate_scale) # 0.12 binned by 3 spatially for HIRES elif isinstance(plate_scale,(np.ndarray, list, tuple)): if len(plate_scale) == norders: plate_scale_ord = plate_scale elif len(plate_scale) == 1: plate_scale_ord = np.full(norders, plate_scale[0]) else: msgs.error('Invalid size for plate_scale. It must either have one element or norders elements') else: msgs.error('Invalid type for plate scale') specmid = nspec // 2 slit_width = slit_righ - slit_left spec_vec = np.arange(nspec) slit_spec_pos = nspec/2.0 slit_spat_pos = np.zeros((norders, 2)) for iord in range(norders): slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec, slit_left[:,iord]), np.interp(slit_spec_pos, spec_vec, slit_righ[:,iord])) # Loop over orders and find objects sobjs = specobjs.SpecObjs() show_peaks=True show_fits=True # ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx! for iord in range(norders): msgs.info('Finding objects on slit # {:d}'.format(iord + 1)) thismask = ordermask == (iord + 1) inmask_iord = inmask & thismask specobj_dict = {'setup': 'HIRES', 'slitid': iord + 1, 'scidx': 0,'det': 1, 'objtype': 'science'} sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \ extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks, show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0) # ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord for spec in sobjs_slit: spec.ech_orderindx = iord sobjs.add_sobj(sobjs_slit) nfound = len(sobjs) # Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0" FOFSEP = 1.0 # separation of FOF algorithm in arcseconds FOF_frac = FOFSEP/(np.median(slit_width)*np.median(plate_scale_ord)) # Feige: made the code also works for only one object found in one order # Run the FOF. We use fake coordinaes fracpos = sobjs.spat_fracpos ra_fake = fracpos/1000.0 # Divide all angles by 1000 to make geometry euclidian dec_fake = 0.0*fracpos if nfound>1: (ingroup, multgroup, firstgroup, nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac/1000.0) group = ingroup.copy() uni_group, uni_ind = np.unique(group, return_index=True) nobj = len(uni_group) msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects') elif nfound==1: group = np.zeros(1,dtype='int') uni_group, uni_ind = np.unique(group, return_index=True) nobj = len(group) msgs.warn('Only find one object no FOF matching is needed') gfrac = np.zeros(nfound) for jj in range(nobj): this_group = group == uni_group[jj] gfrac[this_group] = np.median(fracpos[this_group]) uni_frac = gfrac[uni_ind] sobjs_align = sobjs.copy() # Now fill in the missing objects and their traces for iobj in range(nobj): for iord in range(norders): # Is there an object on this order that grouped into the current group in question? on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx == iord) if not np.any(on_slit): # Add this to the sobjs_align, and assign required tags thisobj = specobjs.SpecObj(frameshape, slit_spat_pos[iord,:], slit_spec_pos, det = sobjs_align[0].det, setup = sobjs_align[0].setup, slitid = (iord + 1), scidx = sobjs_align[0].scidx, objtype=sobjs_align[0].objtype) thisobj.ech_orderindx = iord thisobj.spat_fracpos = uni_frac[iobj] thisobj.trace_spat = slit_left[:,iord] + slit_width[:,iord]*uni_frac[iobj] # new trace thisobj.trace_spec = spec_vec thisobj.spat_pixpos = thisobj.trace_spat[specmid] thisobj.set_idx() # Use the real detections of this objects for the FWHM this_group = group == uni_group[iobj] # Assign to the fwhm of the nearest detected order imin = np.argmin(np.abs(sobjs_align[this_group].ech_orderindx - iord)) thisobj.fwhm = sobjs_align[imin].fwhm thisobj.maskwidth = sobjs_align[imin].maskwidth thisobj.ech_fracpos = uni_frac[iobj] thisobj.ech_group = uni_group[iobj] thisobj.ech_usepca = True sobjs_align.add_sobj(thisobj) group = np.append(group, uni_group[iobj]) gfrac = np.append(gfrac, uni_frac[iobj]) else: # ToDo fix specobjs to get rid of these crappy loops! for spec in sobjs_align[on_slit]: spec.ech_fracpos = uni_frac[iobj] spec.ech_group = uni_group[iobj] spec.ech_usepca = False # Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order # respectively sobjs_sort = specobjs.SpecObjs() for iobj in range(nobj): this_group = group == uni_group[iobj] this_sobj = sobjs_align[this_group] sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)]) # Loop over the objects and perform a quick and dirty extraction to assess S/N. varimg = utils.calc_ivar(ivar) flux_box = np.zeros((nspec, norders, nobj)) ivar_box = np.zeros((nspec, norders, nobj)) mask_box = np.zeros((nspec, norders, nobj)) SNR_arr = np.zeros((norders, nobj)) for iobj in range(nobj): for iord in range(norders): indx = (sobjs_sort.ech_group == uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord) spec = sobjs_sort[indx] thismask = ordermask == (iord + 1) inmask_iord = inmask & thismask box_rad_pix = box_radius/plate_scale_ord[iord] flux_tmp = extract.extract_boxcar(image*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) var_tmp = extract.extract_boxcar(varimg*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) ivar_tmp = utils.calc_ivar(var_tmp) pixtot = extract.extract_boxcar(ivar*0 + 1.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) mask_tmp = (extract.extract_boxcar(ivar*inmask_iord == 0.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) != pixtot) flux_box[:,iord,iobj] = flux_tmp*mask_tmp ivar_box[:,iord,iobj] = np.fmax(ivar_tmp*mask_tmp,0.0) mask_box[:,iord,iobj] = mask_tmp (mean, med_sn, stddev) = sigma_clipped_stats(flux_box[mask_tmp,iord,iobj]*np.sqrt(ivar_box[mask_tmp,iord,iobj]), sigma_lower=5.0,sigma_upper=5.0) SNR_arr[iord,iobj] = med_sn # Purge objects with low SNR and that don't show up in enough orders keep_obj = np.zeros(nobj,dtype=bool) sobjs_trim = specobjs.SpecObjs() uni_group_trim = np.array([],dtype=int) uni_frac_trim = np.array([],dtype=float) for iobj in range(nobj): if (np.sum(SNR_arr[:,iobj] > min_snr) >= nabove_min_snr): keep_obj[iobj] = True ikeep = sobjs_sort.ech_group == uni_group[iobj] sobjs_trim.add_sobj(sobjs_sort[ikeep]) uni_group_trim = np.append(uni_group_trim, uni_group[iobj]) uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj]) else: msgs.info('Purging object #{:d}'.format(iobj) + ' which does not satisfy min_snr > {:5.2f}'.format(min_snr) + ' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) + ' orders') nobj_trim = np.sum(keep_obj) if nobj_trim == 0: return specobjs.SpecObjs() SNR_arr_trim = SNR_arr[:,keep_obj] # Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA for iobj in range(nobj_trim): SNR_now = SNR_arr_trim[:,iobj] indx = (sobjs_trim.ech_group == uni_group_trim[iobj]) # PCA interp/extrap if: # (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca) # OR # (if this order was not originally traced by the object finding, see above) usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) & (SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca # ToDo fix specobjs to get rid of these crappy loops! for iord, spec in enumerate(sobjs_trim[indx]): spec.ech_usepca = usepca[iord] if usepca[iord]: msgs.info('Using PCA to predict trace for object #{:d}'.format(iobj) + ' on order #{:d}'.format(iord)) sobjs_final = sobjs_trim.copy() # Loop over the objects one by one and adjust/predict the traces npoly_cen = 3 pca_fits = np.zeros((nspec, norders, nobj_trim)) for iobj in range(nobj_trim): igroup = sobjs_final.ech_group == uni_group_trim[iobj] # PCA predict the masked orders which were not traced pca_fits[:,:,iobj] = pca_trace((sobjs_final[igroup].trace_spat).T, usepca = None, npca = npca, npoly_cen = npoly_cen) # usepca = sobjs_final[igroup].ech_usepca, # Perform iterative flux weighted centroiding using new PCA predictions xinit_fweight = pca_fits[:,:,iobj].copy() inmask_now = inmask & (ordermask > 0) xfit_fweight = extract.iter_tracefit(image, xinit_fweight, ncoeff, inmask = inmask_now, show_fits=show_fits) # Perform iterative Gaussian weighted centroiding xinit_gweight = xfit_fweight.copy() xfit_gweight = extract.iter_tracefit(image, xinit_gweight, ncoeff, inmask = inmask_now, gweight=True,show_fits=show_fits) # Assign the new traces for iord, spec in enumerate(sobjs_final[igroup]): spec.trace_spat = xfit_gweight[:,iord] spec.spat_pixpos = spec.trace_spat[specmid] # Set the IDs sobjs_final.set_idx() if show_trace: viewer, ch = ginga.show_image(objminsky*(ordermask > 0)) for iobj in range(nobj_trim): for iord in range(norders): ginga.show_trace(viewer, ch, pca_fits[:,iord, iobj], str(uni_frac[iobj]), color='yellow') for spec in sobjs_trim: color = 'green' if spec.ech_usepca else 'magenta' ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color) #for spec in sobjs_final: # color = 'red' if spec.ech_usepca else 'green' # ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color) return sobjs_final
def extract_one(self, frames, det, sciImg, objFind, initial_sky, sobjs_obj): """ Extract Objects in a single exposure/detector pair sci_ID and det need to have been set internally prior to calling this method Args: frames (:obj:`list`): List of frames to extract; stacked if more than one is provided det (:obj:`int`): Detector number (1-indexed) sciImg (:class:`PypeItImage`): Data container that holds a single image from a single detector its related images (e.g. ivar, mask) objFind : :class:`~pypeit.find_objects.FindObjects` Object finding object initial_sky (`numpy.ndarray`_): Initial global sky model sobjs_obj (:class:`pypeit.specobjs.SpecObjs`): List of objects found during `run_objfind` Returns: tuple: Returns six `numpy.ndarray`_ objects and a :class:`pypeit.specobjs.SpecObjs` object with the extracted spectra from this exposure/detector pair. The six `numpy.ndarray`_ objects are (1) the science image, (2) its inverse variance, (3) the sky model, (4) the object model, (5) the model inverse variance, and (6) the mask. """ # Grab some meta-data needed for the reduction from the fitstbl self.objtype, self.setup, self.obstime, self.basename, self.binning \ = self.get_sci_metadata(frames[0], det) # Is this a standard star? self.std_redux = 'standard' in self.objtype # Update the skymask skymask = objFind.create_skymask(sobjs_obj) # Update the global sky if 'standard' in self.fitstbl['frametype'][frames[0]] or \ self.par['reduce']['findobj']['skip_final_global'] or \ self.par['reduce']['skysub']['load_mask'] or \ self.par['reduce']['skysub']['user_regions'] is not None: final_global_sky = initial_sky else: final_global_sky = objFind.global_skysub(previous_sky=initial_sky, skymask=skymask, show=self.show) scaleImg = objFind.scaleimg # update here slits.mask since global_skysub modify reduce_bpm and we need to propagate it into extraction flagged_slits = np.where(objFind.reduce_bpm)[0] if len(flagged_slits) > 0: self.caliBrate.slits.mask[flagged_slits] = \ self.caliBrate.slits.bitmask.turn_on(self.caliBrate.slits.mask[flagged_slits], 'BADREDUCE') msgs.info("Extraction begins for {} on det={}".format( self.basename, det)) # Instantiate Reduce object # Required for pypeline specific object # At instantiaton, the fullmask in self.sciImg is modified # TODO Are we repeating steps in the init for FindObjects and Extract?? self.exTract = extraction.Extract.get_instance( sciImg, sobjs_obj, self.spectrograph, self.par, self.caliBrate, self.objtype, bkg_redux=self.bkg_redux, return_negative=self.par['reduce']['extraction'] ['return_negative'], std_redux=self.std_redux, show=self.show, basename=self.basename) if not self.par['reduce']['extraction']['skip_extraction']: skymodel, objmodel, ivarmodel, outmask, sobjs, waveImg, \ tilts = self.exTract.run(final_global_sky, ra=self.fitstbl["ra"][frames[0]], dec=self.fitstbl["dec"][frames[0]], obstime=self.obstime) else: # Although exrtaction is not performed, still need to prepare some masks and the tilts self.exTract.prepare_extraction() # Since the extraction was not performed, fill the arrays with the best available information skymodel = final_global_sky objmodel = np.zeros_like(self.exTract.sciImg.image) ivarmodel = np.copy(self.exTract.sciImg.ivar) outmask = self.exTract.sciImg.fullmask waveImg = self.exTract.waveimg tilts = self.exTract.tilts sobjs = sobjs_obj # TODO -- Do this upstream # Tack on detector and wavelength RMS for sobj in sobjs: sobj.DETECTOR = sciImg.detector iwv = np.where( self.caliBrate.wv_calib.spat_ids == sobj.SLITID)[0][0] sobj.WAVE_RMS = self.caliBrate.wv_calib.wv_fits[iwv].rms # Construct table of spectral flexure spec_flex_table = Table() spec_flex_table['spat_id'] = self.caliBrate.slits.spat_id spec_flex_table['sci_spec_flexure'] = self.exTract.slitshift # pull out maskdef_designtab from caliBrate.slits maskdef_designtab = self.caliBrate.slits.maskdef_designtab slits = copy.deepcopy(self.caliBrate.slits) slits.maskdef_designtab = None # Construct the Spec2DObj spec2DObj = spec2dobj.Spec2DObj( sciimg=sciImg.image, ivarraw=sciImg.ivar, skymodel=skymodel, objmodel=objmodel, ivarmodel=ivarmodel, scaleimg=scaleImg, waveimg=waveImg, bpmmask=outmask, detector=sciImg.detector, sci_spat_flexure=sciImg.spat_flexure, sci_spec_flexure=spec_flex_table, vel_corr=self.exTract.vel_corr, vel_type=self.par['calibrations']['wavelengths']['refframe'], tilts=tilts, slits=slits, maskdef_designtab=maskdef_designtab) spec2DObj.process_steps = sciImg.process_steps # QA spec2DObj.gen_qa() # Return return spec2DObj, sobjs