def load_master(self, filename, force=False): # Does the master file exist? if not os.path.isfile(filename): # msgs.warn("No Master frame found of type {:s}: {:s}".format(self.frametype, filename)) msgs.warn("No Master frame found of {:s}".format(filename)) if force: msgs.error("Crashing out because reduce-masters-force=True:" + msgs.newline() + filename) return None else: # msgs.info("Loading a pre-existing master calibration frame of type: {:}".format(self.frametype) + " from filename: {:}".format(filename)) msgs.info("Loading a pre-existing master calibration frame of SENSFUNC from filename: {:}".format(filename)) hdu = fits.open(filename) norder = hdu[0].header['NORDER'] sens_dicts = {} for iord in range(norder): head = hdu[iord + 1].header tbl = hdu['SENSFUNC-ORDER{0:04}'.format(iord)].data sens_dict = {} sens_dict['wave'] = tbl['WAVE'] sens_dict['sensfunc'] = tbl['SENSFUNC'] for key in ['wave_min', 'wave_max', 'exptime', 'airmass', 'std_file', 'std_ra', 'std_dec', 'std_name', 'cal_file', 'ech_orderindx']: try: sens_dict[key] = head[key.upper()] except: pass sens_dicts[str(iord)] = sens_dict sens_dicts['norder'] = norder return sens_dicts
def write_science(sci_specobjs, sci_header, outfile): """ Write the flux-calibrated science spectra Parameters ---------- outfile : str Returns ------- """ if len(sci_specobjs) == 0: msgs.warn("No science spectra to write to disk!") # if 'VEL-TYPE' in sci_header.keys(): helio_dict = dict(refframe=sci_header['VEL-TYPE'], vel_correction=sci_header['VEL']) else: helio_dict = None telescope=None if 'LON-OBS' in sci_header.keys(): telescope = TelescopePar(longitude=sci_header['LON-OBS'], latitude=sci_header['LAT-OBS'], elevation=sci_header['ALT-OBS']) # KLUDGE ME if isinstance(sci_specobjs, list): specObjs = specobjs.SpecObjs(sci_specobjs) elif isinstance(sci_specobjs, specobjs.SpecObjs): specObjs = sci_specobjs else: msgs.error("BAD INPUT") save.save_1d_spectra_fits(specObjs, sci_header,'ECHELLE', outfile, helio_dict=helio_dict, telescope=telescope, overwrite=True)
def show_sensfunc(self): """ Plot the sensitivity function """ if self.sens_dict is None: msgs.warn("You need to generate the sensfunc first!") return None plt.rcdefaults() plt.rcParams["xtick.top"] = True plt.rcParams["ytick.right"] = True plt.rcParams["xtick.minor.visible"] = True plt.rcParams["ytick.minor.visible"] = True plt.rcParams["ytick.direction"] = 'in' plt.rcParams["xtick.direction"] = 'in' plt.rcParams["xtick.labelsize"] = 13 plt.rcParams["ytick.labelsize"] = 13 plt.rcParams['font.family'] = 'times new roman' norder = self.sens_dict['norder'] for iord in range(norder): sens_dict_iord = self.sens_dict[str(iord)] plt.plot(sens_dict_iord['wave'], sens_dict_iord['sensfunc']) plt.xlabel('Wavelength [ang]', fontsize=14) plt.ylabel('Sensfunc', fontsize=14) plt.ylim([0., 100.0]) plt.show()
def subtract_bias(self, bias_image, force=False): """ Perform bias subtraction Args: bias_image (np.ndarray): Bias image force (bool, optional): Force the processing even if the image was already processed """ step = inspect.stack()[0][3] # Check if already trimmed if self.steps[step] and (not force): msgs.warn("Image was already bias subtracted. Returning the current image") return self.image.copy() # Do it self.image -= bias_image self.steps[step] = True
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) if ftype in ['pinhole', 'bias']: # No pinhole or bias frames return np.zeros(len(fitstbl), dtype=bool) if ftype in ['pixelflat', 'trace']: return good_exp & (fitstbl['idname'] == 'PixFlat') if ftype == 'standard': return good_exp & (fitstbl['idname'] == 'Telluric') if ftype == 'science': return good_exp & (fitstbl['idname'] == 'Science') if ftype in ['arc', 'tilt']: return good_exp & (fitstbl['idname'] == 'Science') msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) if ftype in ['science', 'standard']: return good_exp & (fitstbl['idname'] == 'OBJECT') if ftype == 'bias': return good_exp & (fitstbl['idname'] == 'BIAS') if ftype in ['pixelflat', 'trace', 'illumflat']: return good_exp & (fitstbl['idname'] == 'FLAT,LAMP') if ftype in ['pinhole', 'dark']: # Don't type pinhole or dark frames return np.zeros(len(fitstbl), dtype=bool) if ftype in ['arc', 'tilt']: return good_exp & (fitstbl['idname'] == 'WAVE,LAMP') msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def subtract_dark(self, dark_image, force=False): """ Perform dark image subtraction Args: dark_image (PypeItImage): Dark image """ step = inspect.stack()[0][3] # Check if already bias subtracted if self.steps[step] and not force: msgs.warn( "Image was already dark subtracted. Returning the current image" ) return self.image.copy() # Do it self.image -= dark_image.image self.steps[step] = True
def spec_from_array(wave,flux,sig,**kwargs): """ return spectrum from arrays of wave, flux and sigma """ ituple = (wave, flux, sig) spectrum = XSpectrum1D.from_tuple(ituple, **kwargs) # Polish a bit -- Deal with NAN, inf, and *very* large values that will exceed # the floating point precision of float32 for var which is sig**2 (i.e. 1e38) bad_flux = np.any([np.isnan(spectrum.flux), np.isinf(spectrum.flux), np.abs(spectrum.flux) > 1e30, spectrum.sig ** 2 > 1e10, ], axis=0) if np.sum(bad_flux): msgs.warn("There are some bad flux values in this spectrum. Will zero them out and mask them (not ideal)") spectrum.data['flux'][spectrum.select][bad_flux] = 0. spectrum.data['sig'][spectrum.select][bad_flux] = 0. return spectrum
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) if ftype in ['science', 'standard']: return good_exp & (fitstbl['lampstat01'] == 'Off') & (fitstbl['idname'] == 'object') if ftype == 'bias': return good_exp & (fitstbl['idname'] == 'zero') if ftype in ['pixelflat', 'trace']: return good_exp & (fitstbl['lampstat01'] == 'W') & (fitstbl['idname'] == 'flat') if ftype in ['pinhole', 'dark']: # Don't type pinhole or dark frames return np.zeros(len(fitstbl), dtype=bool) if ftype == 'arc': return good_exp & (fitstbl['lampstat01'] == 'CuNe+CuAr') & (fitstbl['idname'] == 'arc') msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def generate_sensfunc(self): """ Generate the senstivity function Wrapper to flux.generate_sensfunc Requires self.std has been set Returns ------- self.sensfunc : dict """ # Check internals # if self.std is None: # msgs.warn('First identify the star first (with find_standard).') # return None if self.std_header is None: msgs.warn('First set std_header with a dict-like object holding RA, DEC, ' 'AIRMASS, EXPTIME.') return None ext_final = fits.getheader(self.std_spec1d_file, -1) norder = ext_final['ORDER'] + 1 self.sens_dict = {} for iord in range(norder): std_specobjs, std_header = load.ech_load_specobj(self.std_spec1d_file, order=iord) std_idx = flux.find_standard(std_specobjs) std = std_specobjs[std_idx] wavemask = std.boxcar['WAVE'] > 1000.0 * units.AA wave, counts, ivar = std.boxcar['WAVE'][wavemask], std.boxcar['COUNTS'][wavemask], \ std.boxcar['COUNTS_IVAR'][wavemask] sens_dict_iord = flux.generate_sensfunc(wave, counts, ivar, std_header['AIRMASS'], std_header['EXPTIME'], self.spectrograph, star_type=self.star_type, star_mag=self.star_mag, telluric=self.telluric, ra=self.std_ra, dec=self.std_dec, BALM_MASK_WID=self.BALM_MASK_WID, nresln=self.nresln, std_file=self.std_file, debug=self.debug) sens_dict_iord['ech_orderindx'] = iord self.sens_dict[str(iord)] = sens_dict_iord self.sens_dict['norder'] = norder # Step self.steps.append(inspect.stack()[0][3]) # Return return self.sens_dict
def get_arc(self): """ Load or generate the Arc image Requirements: master_key, det, par Args: Returns: `numpy.ndarray`_: :attr:`msarc` image """ # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep arc_files, self.master_key_dict['arc'] = self._prep_calibrations('arc') masterframe_name = masterframe.construct_file_name( buildimage.ArcImage, self.master_key_dict['arc'], master_dir=self.master_dir) # Reuse master frame? if os.path.isfile(masterframe_name) and self.reuse_masters: self.msarc = buildimage.ArcImage.from_file(masterframe_name) elif len(arc_files) == 0: msgs.warn("No frametype=arc files to build arc") return else: # Build it msgs.info("Preparing a master {0:s} frame".format( buildimage.ArcImage.master_type)) self.msarc = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['arcframe'], arc_files, bias=self.msbias, bpm=self.msbpm, dark=self.msdark) # Save self.msarc.to_master_file(masterframe_name) # Return return self.msarc
def _chk_objs(self, items): """ Check that the input items exist internally as attributes Args: items (list): Returns: bool: True if all exist """ for obj in items: if getattr(self, obj) is None: msgs.warn("You need to generate {:s} prior to this calibration..".format(obj)) # Strip ms iobj = obj[2:] if obj[0:2] == 'ms' else obj msgs.warn("Use get_{:s}".format(iobj)) return False return True
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. Args: ftype (:obj:`str`): Type of frame to check. Must be a valid frame type; see frame-type :ref:`frame_type_defs`. fitstbl (`astropy.table.Table`_): The table with the metadata for one or more frames to check. exprng (:obj:`list`, optional): Range in the allowed exposure time for a frame of type ``ftype``. See :func:`pypeit.core.framematch.check_frame_exptime`. Returns: `numpy.ndarray`_: Boolean array with the flags selecting the exposures in ``fitstbl`` that are ``ftype`` type frames. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) if ftype in ['science', 'standard']: return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 0) \ & (fitstbl['idname'] == 'object') if ftype in ['bias', 'dark']: return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 0) \ & (fitstbl['idname'] == 'dark') if ftype in ['pixelflat', 'trace']: # Flats and trace frames are typed together return good_exp & self.lamps(fitstbl, 'dome') & (fitstbl['hatch'] == 1) \ & (fitstbl['idname'] == 'flatlamp') if ftype == 'pinhole': # Don't type pinhole frames return np.zeros(len(fitstbl), dtype=bool) if ftype in ['arc', 'tilt']: # TODO: This is a kludge. Allow science frames to also be # classified as arcs is_arc = self.lamps(fitstbl, 'arcs') & (fitstbl['hatch'] == 1) \ & (fitstbl['idname'] == 'arclamp') is_obj = self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 0) \ & (fitstbl['idname'] == 'object') return good_exp & (is_arc | is_obj) msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def save(self, outfile=None, overwrite=True): """ Save the wavelength tilts data to a master frame Args: outfile (:obj:`str`, optional): Name for the output file. Defaults to :attr:`file_path`. overwrite (:obj:`bool`, optional): Overwrite any existing file. """ _outfile = self.file_path if outfile is None else outfile # Check if it exists if os.path.exists(_outfile) and not overwrite: msgs.warn('Master file exists: {0}'.format(_outfile) + msgs.newline() + 'Set overwrite=True to overwrite it.') return # Log msgs.info('Saving master frame to {0}'.format(_outfile)) # Build the header hdr = fits.Header() # - Set the master frame type hdr['FRAMETYP'] = (self.master_type, 'PypeIt: Master calibration frame type') # - List the completed steps hdr['STEPS'] = (','.join(self.steps), 'Completed reduction steps') # - Tilts metadata hdr['FUNC2D'] = self.tilts_dict['func2d'] hdr['NSLIT'] = self.tilts_dict['nslit'] # Write the fits file fits.HDUList([ fits.PrimaryHDU(header=hdr), fits.ImageHDU(data=self.tilts_dict['tilts'], name='TILTS'), fits.ImageHDU(data=self.tilts_dict['coeffs'], name='COEFFS'), fits.ImageHDU(data=self.tilts_dict['slitcen'], name='SLITCEN'), fits.ImageHDU(data=self.tilts_dict['spat_order'], name='SPAT_ORDER'), fits.ImageHDU(data=self.tilts_dict['spec_order'], name='SPEC_ORDER') ]).writeto(_outfile, overwrite=True)
def orient(self, force=False): """ Orient the image in the PypeIt format with spectra running blue (down) to red (up). Args: force (bool, optional): Force the processing even if the image was already processed """ step = inspect.stack()[0][3] # Orient the image to have blue/red run bottom to top # Check if already oriented if self.steps[step] and not force: msgs.warn("Image was already oriented. Returning current image") return self.image.copy() # Orient me self.image = self.spectrograph.orient_image(self.image, self.det) self.steps[step] = True
def connect_to_ginga(host='localhost', port=9000, raise_err=False, allow_new=False): """ Connect to a RC Ginga. Args: host (:obj:`str`, optional): Host name. port (:obj:`int`, optional): Probably should remain at 9000 raise_err (:obj:`bool`, optional): Raise an error if no connection is made, otherwise just raise a warning and continue allow_new (:obj:`bool`, optional): Allow a subprocess to be called to execute a new ginga viewer if one is not already running. Returns: RemoteClient: connection to ginga viewer. """ # Start viewer = grc.RemoteClient(host, port) # Test ginga = viewer.shell() try: tmp = ginga.get_current_workspace() except: if allow_new: subprocess.Popen(['ginga', '--modules=RC']) time.sleep(3) return grc.RemoteClient(host, port) if raise_err: raise ValueError else: msgs.warn( 'Problem connecting to Ginga. Launch an RC Ginga viewer and ' 'then continue: \n ginga --modules=RC') # Return return viewer
def load(self, ifile=None, return_header=False): """ Load the trace slits data. Reads both the trace data and the trace image data. This is largely a wrapper for :func:`pypeit.traceslits.TraceSlits.load_from_file`. Args: ifile (:obj:`str`, optional): Name of the master frame file. Defaults to :attr:`file_path`. return_header (:obj:`bool`, optional): Return the header, which will include the TraceImage metadata if available. Returns: tuple: Returns the trace slits dictionary and the trace slit image. If return_header is true, the primary header is also returned. If nothing is loaded, either because :attr:`reuse_masters` is `False` or the file does not exist, everything is returned as None (one per expected return object). """ # Format the input and set the tuple for an empty return _ifile = self.file_path if ifile is None else ifile empty_return = (None, None, None) if return_header else (None, None) if not self.reuse_masters: # User does not want to load masters msgs.warn('PypeIt will not reuse masters!') return empty_return if not os.path.isfile(_ifile): # Master file doesn't exist msgs.warn('No Master {0} frame found: {1}'.format( self.master_type, self.file_path)) return empty_return # Read and return msgs.info('Loading Master {0} frame: {1}'.format( self.master_type, _ifile)) return self.load_from_file(_ifile, return_header=return_header)
def save_master(self, tilts_dict, outfile=None, steps=None, overwrite=True): """ Save the tilts dict to a MasterFile Args: tilts_dict (dict): To be saved outfile (str): Returns: """ _outfile = self.ms_name if outfile is None else outfile # Additional keywords for the Header keywds = None if steps is None else dict(steps=','.join(steps)) # Check for existing if os.path.exists(_outfile) and (not overwrite): msgs.warn("This file already exists. Use overwrite=True to overwrite it") return # msgs.info("Saving master {0:s} frame as:".format(self.frametype) + msgs.newline() + _outfile) hdu0 = fits.PrimaryHDU(tilts_dict['tilts']) hdu0.name='TILTS' hdu0.header['FUNC2D'] = tilts_dict['func2d'] hdu0.header['NSLIT'] = tilts_dict['nslit'] hdul = [hdu0] hdu_coeff = fits.ImageHDU(tilts_dict['coeffs']) hdu_coeff.name='COEFFS' hdul.append(hdu_coeff) hdu_slitcen = fits.ImageHDU(tilts_dict['slitcen']) hdu_slitcen.name = 'SLITCEN' hdul.append(hdu_slitcen) hdu_spat_order = fits.ImageHDU(tilts_dict['spat_order']) hdu_spat_order.name = 'SPAT_ORDER' hdul.append(hdu_spat_order) hdu_spec_order = fits.ImageHDU(tilts_dict['spec_order']) hdu_spec_order.name = 'SPEC_ORDER' hdul.append(hdu_spec_order) # Finish hdulist = fits.HDUList(hdul) hdulist.writeto(_outfile, clobber=True)
def get_tiltimg(self): """ Load or generate the Tilt image Requirements: master_key, det, par Args: Returns: ndarray: :attr:`mstilt` image """ # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep tilt_files, self.master_key_dict['tilt'] = self._prep_calibrations('tilt') masterframe_name = masterframe.construct_file_name( buildimage.TiltImage, self.master_key_dict['tilt'], master_dir=self.master_dir) # Reuse master frame? if os.path.isfile(masterframe_name) and self.reuse_masters: self.mstilt = buildimage.TiltImage.from_file(masterframe_name) elif len(tilt_files) == 0: msgs.warn("No frametype=tilt files to build tiltimg") return else: # Build msgs.info("Preparing a master {0:s} frame".format(buildimage.TiltImage.master_type)) self.mstilt = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['tiltframe'], tilt_files, bias=self.msbias, bpm=self.msbpm, slits=self.slits) # For flexure # Save to Masters self.mstilt.to_master_file(masterframe_name) # TODO in the future add in a tilt_inmask #self._update_cache('tilt', 'tilt_inmask', self.mstilt_inmask) # Return return self.mstilt
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) if ftype == 'science': return good_exp & (fitstbl['idname'] == 'OBJECT') if ftype == 'standard': return good_exp & (fitstbl['idname'] == 'OBJECT') if ftype == 'pixelflat' or ftype == 'trace': # Flats and trace frames are typed together return good_exp & (fitstbl['idname'] == 'FLAT') if ftype == 'pinhole' or ftype == 'dark' or ftype == 'bias': # Don't type pinhole, dark, or bias frames return np.zeros(len(fitstbl), dtype=bool) if ftype == 'arc': return good_exp & (fitstbl['idname'] == 'ARC') msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def get_result(self): """Save a mask containing the skysub regions, and print information for what the user should include in their .pypeit file """ # Only do this if the user wishes to save the result if self._use_updates: # Generate the mask inmask = skysub.generate_mask(self.pypeline, self._skyreg, self.slits, self.slits_left, self.slits_right) # Save the mask outfil = self._outname if os.path.exists(self._outname) and not self._overwrite: outfil = 'temp.fits' msgs.warn("File exists:\n{0:s}\nSaving regions to 'temp.fits'") self._overwrite = True msskyreg = buildimage.SkyRegions(image=inmask.astype(np.float), PYP_SPEC=self.spectrograph) msskyreg.to_master_file(master_filename=outfil) return
def subtract_overscan(self, force=False): """ Analyze and subtract the overscan from the image Args: force (bool, optional): Force the processing even if the image was already processed """ step = inspect.stack()[0][3] # Check if already trimmed if self.steps[step] and (not force): msgs.warn("Image was already trimmed") temp = procimg.subtract_overscan(self.image, self.rawdatasec_img, self.oscansec_img, method=self.par['overscan'], params=self.par['overscan_par']) # Fill self.steps[step] = True self.image = temp
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) if ftype == 'science': return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'open') if ftype == 'bias': return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'closed') if ftype in ['pixelflat', 'trace']: # Flats and trace frames are typed together return good_exp & self.lamps(fitstbl, 'dome') & (fitstbl['hatch'] == 'open') if ftype in ['pinhole', 'dark']: # Don't type pinhole or dark frames return np.zeros(len(fitstbl), dtype=bool) if ftype in ['arc', 'tilt']: return good_exp & self.lamps(fitstbl, 'arcs') & (fitstbl['hatch'] == 'closed') msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) if ftype in ['science', 'standard']: return good_exp & (fitstbl['idname'] == 'OBJECT') & (fitstbl['lamps'] == 'Parking') \ & (fitstbl['dispname'] != 'OPEN') if ftype == 'bias': return good_exp & (fitstbl['dispname'] == 'OPEN') if ftype in ['pixelflat', 'trace']: return good_exp & (fitstbl['idname'] == 'CALIB') & (fitstbl['lamps'] == 'Halogen') \ & (fitstbl['dispname'] != 'OPEN') if ftype in ['pinhole', 'dark']: # Don't type pinhole or dark frames return np.zeros(len(fitstbl), dtype=bool) if ftype in ['arc', 'tilt']: return good_exp & (fitstbl['idname'] == 'arc') & (fitstbl['lamps'] == 'Ne+Hg') \ & (fitstbl['dispname'] != 'OPEN') msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) if ftype == 'science': return good_exp & (fitstbl['lampstat01'] == 'off') & ( fitstbl['lampstat02'] == 'stowed') & (fitstbl['exptime'] > 100.0) if ftype == 'standard': return good_exp & (fitstbl['lampstat01'] == 'off') & ( fitstbl['lampstat02'] == 'stowed') & (fitstbl['exptime'] <= 100.0) if ftype in ['arc', 'tilt']: return good_exp & (fitstbl['lampstat01'] == 'on') if ftype in ['pixelflat', 'trace']: return good_exp & (fitstbl['lampstat01'] == 'off') & ( fitstbl['lampstat02'] == 'deployed') msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) if ftype in ['science', 'standard']: return good_exp & (fitstbl['idname'] == 'OBJECT') & (fitstbl['ra'] != 'none') \ & (fitstbl['dispname'] != 'Flat') if ftype == 'bias': return good_exp & (fitstbl['idname'] == 'BIAS') if ftype in ['pixelflat', 'trace']: # Flats and trace frames are typed together return good_exp & (fitstbl['idname'] == 'FLAT') & (fitstbl['decker'] != 'Imaging') if ftype in ['pinhole', 'dark']: # Don't type pinhole or dark frames return np.zeros(len(fitstbl), dtype=bool) if ftype in ['arc', 'tilt']: return good_exp & (fitstbl['idname'] == 'COMP') & (fitstbl['dispname'] != 'Flat') msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def find_single_file(file_pattern): """Find a single file matching a wildcard pattern. Args: file_pattern (str): A filename pattern, see the python 'glob' module. Returns: str: A file name, or None if no filename was found. This will give a warning if multiple files are found and return the first one. """ files = glob(file_pattern) if len(files) == 1: return files[0] elif len(files) == 0: return None else: msgs.warn( f'Found multiple files matching {file_pattern}; using the first one.' ) return files[0]
def _parse(cls, hdu, ext=None, transpose_table_arrays=False, debug=False, hdu_prefix=None): # Grab everything but the bspline's _d, dm_version_passed, dm_type_passed, parsed_hdus = super( WaveCalib, cls)._parse(hdu) # Now the wave_fits list_of_wave_fits = [] spat_ids = [] for ihdu in hdu: if 'WAVEFIT' in ihdu.name: # Allow for empty if len(ihdu.data) == 0: iwavefit = wv_fitting.WaveFit(ihdu.header['SPAT_ID']) else: iwavefit = wv_fitting.WaveFit.from_hdu(ihdu) parsed_hdus += ihdu.name if iwavefit.version != wv_fitting.WaveFit.version: msgs.warn("Your WaveFit is out of date!!") # Grab PypeItFit (if it exists) hdname = ihdu.name.replace('WAVEFIT', 'PYPEITFIT') if hdname in [khdu.name for khdu in hdu]: iwavefit.pypeitfit = fitting.PypeItFit.from_hdu( hdu[hdname]) parsed_hdus += hdname list_of_wave_fits.append(iwavefit) # Grab SPAT_ID for checking spat_ids.append(iwavefit.spat_id) elif ihdu.name == 'PYPEITFIT': # 2D fit _d['wv_fit2d'] = fitting.PypeItFit.from_hdu(ihdu) parsed_hdus += ihdu.name # Check if spat_ids != _d['spat_ids'].tolist(): msgs.error("Bad parsing of the MasterFlat") # Finish _d['wv_fits'] = np.asarray(list_of_wave_fits) return _d, dm_version_passed, dm_type_passed, parsed_hdus
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. Args: ftype (:obj:`str`): Type of frame to check. Must be a valid frame type; see frame-type :ref:`frame_type_defs`. fitstbl (`astropy.table.Table`_): The table with the metadata for one or more frames to check. exprng (:obj:`list`, optional): Range in the allowed exposure time for a frame of type ``ftype``. See :func:`pypeit.core.framematch.check_frame_exptime`. Returns: `numpy.ndarray`_: Boolean array with the flags selecting the exposures in ``fitstbl`` that are ``ftype`` type frames. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) # TODO: Allow for 'sky' frame type, for now include sky in # 'science' category if ftype == 'science': return good_exp & (fitstbl['idname'] == 'Object') if ftype == 'standard': return good_exp & ((fitstbl['idname'] == 'Std') | (fitstbl['idname'] == 'Object')) if ftype == 'bias': return good_exp & (fitstbl['idname'] == 'Bias') if ftype == 'dark': return good_exp & (fitstbl['idname'] == 'Dark') if ftype in ['pixelflat', 'trace']: # Flats and trace frames are typed together return good_exp & ((fitstbl['idname'] == 'Flat') | (fitstbl['idname'] == 'IntFlat')) if ftype in ['arc', 'tilt']: return good_exp & (fitstbl['idname'] == 'Line') msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def load_extinction_data(longitude, latitude, toler=5. * units.deg): """ Find the best extinction file to use, based on longitude and latitude Loads it and returns a Table Parameters ---------- toler : Angle, optional Tolerance for matching detector to site (5 deg) Returns ------- ext_file : Table astropy Table containing the 'wavelength', 'extinct' data for AM=1. """ # Mosaic coord mosaic_coord = coordinates.SkyCoord(longitude, latitude, frame='gcrs', unit=units.deg) # Read list extinct_path = resource_filename('pypeit', '/data/extinction/') extinct_summ = extinct_path + 'README' extinct_files = table.Table.read(extinct_summ, comment='#', format='ascii') # Coords ext_coord = coordinates.SkyCoord(extinct_files['Lon'], extinct_files['Lat'], frame='gcrs', unit=units.deg) # Match idx, d2d, d3d = coordinates.match_coordinates_sky(mosaic_coord, ext_coord, nthneighbor=1) if d2d < toler: extinct_file = extinct_files[int(idx)]['File'] msgs.info("Using {:s} for extinction corrections.".format(extinct_file)) else: msgs.warn("No file found for extinction corrections. Applying none") msgs.warn("You should generate a site-specific file") return None # Read extinct = table.Table.read(extinct_path + extinct_file, comment='#', format='ascii', names=('iwave', 'mag_ext')) wave = table.Column(np.array(extinct['iwave']) * units.AA, name='wave') extinct.add_column(wave) # Return return extinct[['wave', 'mag_ext']]
def save(self, outfile=None, overwrite=True): """ Save the bias master data. Args: outfile (:obj:`str`, optional): Name for the output file. Defaults to :attr:`file_path`. overwrite (:obj:`bool`, optional): Overwrite any existing file. """ # Some checks if self.pypeitImage is None: msgs.warn('No MasterBias to save!') return if not self.pypeitImage.validate(): msgs.warn('MasterBias is not a proper image.') return # Proceed _outfile = self.master_file_path if outfile is None else outfile # Check if it exists if os.path.exists(_outfile) and not overwrite: msgs.warn('Master file exists: {0}'.format(_outfile) + msgs.newline() + 'Set overwrite=True to overwrite it.') return # Save hdr = self.build_master_header(steps=self.process_steps, raw_files=self.file_list) self.pypeitImage.write(_outfile, hdr=hdr, iext='BIAS') msgs.info('Master frame written to {0}'.format(_outfile))
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. Args: ftype (:obj:`str`): Type of frame to check. Must be a valid frame type; see frame-type :ref:`frame_type_defs`. fitstbl (`astropy.table.Table`_): The table with the metadata for one or more frames to check. exprng (:obj:`list`, optional): Range in the allowed exposure time for a frame of type ``ftype``. See :func:`pypeit.core.framematch.check_frame_exptime`. Returns: `numpy.ndarray`_: Boolean array with the flags selecting the exposures in ``fitstbl`` that are ``ftype`` type frames. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) if ftype == 'bias': return fitstbl['idname'] == 'zero' if ftype in ['science', 'standard']: return good_exp & (fitstbl['lampstat01'] == 'off') & (fitstbl['idname'] == 'object') if ftype in ['arc', 'tilt']: # should flesh this out to include all valid arc lamp combos return good_exp & (fitstbl['lampstat01'] != 'off') & ( fitstbl['idname'] == 'comp') & (fitstbl['decker'] != '5.0x180') if ftype in ['trace', 'pixelflat']: # i think the bright lamp, BC, is the only one ever used for this. imagetyp should always be set to flat. return good_exp & (fitstbl['lampstat01'] == 'BC') & (fitstbl['idname'] == 'flat') if ftype in ['illumflat']: # these can be set to flat or object depending if they're twilight or dark sky return good_exp & (fitstbl['idname'] in ['flat', 'object']) & ( fitstbl['lampstat01'] == 'off') msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. Args: ftype (:obj:`str`): Type of frame to check. Must be a valid frame type; see frame-type :ref:`frame_type_defs`. fitstbl (`astropy.table.Table`_): The table with the metadata for one or more frames to check. exprng (:obj:`list`, optional): Range in the allowed exposure time for a frame of type ``ftype``. See :func:`pypeit.core.framematch.check_frame_exptime`. Returns: `numpy.ndarray`_: Boolean array with the flags selecting the exposures in ``fitstbl`` that are ``ftype`` type frames. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) if ftype == 'science': return good_exp & (fitstbl['idname'] == 'OBJECT') if ftype == 'standard': return good_exp & (fitstbl['idname'] == 'OBJECT') if ftype in ['pixelflat', 'trace']: # Flats and trace frames are typed together return good_exp & (fitstbl['idname'] == 'FLAT') if ftype in ['pinhole', 'dark', 'bias']: # Don't type pinhole, dark, or bias frames return np.zeros(len(fitstbl), dtype=bool) if ftype in ['arc', 'tilt']: ## FW ToDo: self.dispname does not work yet. need to replace the following later. if '32/mm' in fitstbl['dispname'][0]: return good_exp & (fitstbl['idname'] == 'OBJECT') elif '10/mmLBSX' in fitstbl['dispname'][0]: return good_exp & (fitstbl['idname'] == 'ARC') msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool)
def parse_binning(binning): """ Parse input binning into binspectral, binspatial Note that for some instruments, the meaning will be swapped if parsed directly from the Header. The developer needs to react accordingly.. Args: binning (str, ndarray or tuple): Returns: int,int: binspectral, binspatial """ # comma separated format if isinstance(binning, str): if ',' in binning: binspectral, binspatial = [ int(item) for item in binning.split(',') ] # Keck standard, I think elif 'x' in binning: binspectral, binspatial = [ int(item) for item in binning.split('x') ] # LRIS elif binning == 'None': msgs.warn("Assuming unbinned, i.e. 1x1") binspectral, binspatial = 1, 1 else: binspectral, binspatial = [ int(item) for item in binning.strip().split(' ') ] # Gemini elif isinstance(binning, tuple): binspectral, binspatial = binning elif isinstance(binning, np.ndarray): binspectral, binspatial = binning else: msgs.error("Unable to parse input binning: {}".format(binning)) # Return return binspectral, binspatial
def xcorr_shift_stretch(inspec1, inspec2, smooth = 5.0, shift_mnmx = (-0.05,0.05), stretch_mnmx = (0.9,1.1), debug = True): nspec = inspec1.size y1 = scipy.ndimage.filters.gaussian_filter(inspec1, smooth) y2 = scipy.ndimage.filters.gaussian_filter(inspec2, smooth) # Do the cross-correlation first and determine the shift_cc, cc_val = xcorr_shift(y1, y2, debug = debug) bounds = [(shift_cc + nspec*shift_mnmx[0],shift_cc + nspec*shift_mnmx[1]), stretch_mnmx] #guess = np.array([shift_cc,1.0]) #result = scipy.optimize.minimize(xcorr_shift_stretch, guess, args=(y1,y2), bounds=bounds) result = scipy.optimize.differential_evolution(zerolag_shift_stretch, args=(y1,y2), tol = 1e-4, bounds=bounds, disp=False, polish=True) #rranges = (slice(bounds[0][0], bounds[0][1], 0.1), slice(bounds[1][0], bounds[1][1], 1e-4)) #from IPython import embed #embed() #result = scipy.optimize.brute(xcorr_shift_stretch, rranges, args=(y1,y2), finish = scipy.optimize.fmin, disp=True) #corr_val = xcorr_stretch((shift,stretch), y1,y2) #corr = scipy.signal.correlate(y1, y2, mode='same') if not result.success: msgs.warn('Fit for shift and stretch did not converge!') if debug: x1 = np.arange(nspec) inspec2_trans = shift_and_stretch(inspec2, result.x[0], result.x[1]) plt.plot(x1,inspec1, 'k-', drawstyle='steps', label ='inspec1') plt.plot(x1,inspec2_trans, 'r-', drawstyle='steps', label = 'inspec2') plt.title('shift= {:5.3f}'.format(result.x[0]) + ', stretch = {:7.5f}'.format(result.x[1]) + ', corr = {:5.3f}'.format(-result.fun)) plt.legend() plt.show() return result.success, result.x[0], result.x[1], -result.fun, shift_cc, cc_val
def ech_load_specobj(fname,order=None): """ Load a spec1d file into a list of SpecObjExp objects Parameters ---------- fname : str Returns ------- specObjs : list of SpecObjExp head0 """ #if order is None: # msgs.warn('You did not specify an order. Return specObjs with all orders.') # specObjs, head0 = load.load_specobj(fname) # return specObjs, head0 speckeys = ['WAVE', 'SKY', 'MASK', 'FLAM', 'FLAM_IVAR', 'FLAM_SIG', 'COUNTS_IVAR', 'COUNTS'] # specObjs = [] hdulist = fits.open(fname) head0 = hdulist[0].header for hdu in hdulist: if hdu.name == 'PRIMARY': continue #elif hdu.name[8:17] != 'ORDER'+'{0:04}'.format(order): # continue # Parse name idx = hdu.name objp = idx.split('-') if objp[-1][0:3] == 'DET': det = int(objp[-1][3:]) else: det = int(objp[-1][1:]) if objp[-2][:5] == 'ORDER': iord = int(objp[-2][5:]) else: msgs.warn('Loading longslit data ?') iord = int(-1) # if order is not None and iord !=order then do not return this extenction # if order is None return all extensions # if order is not None and iord ==order then only return the specific order you want. if (order is not None) and (iord !=order): continue # Load data spec = Table(hdu.data) shape = (len(spec), 1024) # 2nd number is dummy # New and wrong try: specobj = specobjs.SpecObj(shape, None, None, idx = idx) except: debugger.set_trace() msgs.error("BUG ME") # Add order number specobj.ech_orderindx = iord # ToDo: need to changed to the real order number? specobj.ech_order = iord # Add trace try: specobj.trace_spat = spec['TRACE'] except: # KLUDGE! specobj.trace_spat = np.arange(len(spec['BOX_WAVE'])) # Add spectrum if 'BOX_COUNTS' in spec.keys(): for skey in speckeys: try: specobj.boxcar[skey] = spec['BOX_{:s}'.format(skey)].data except KeyError: pass # Add units on wave specobj.boxcar['WAVE'] = specobj.boxcar['WAVE'] * units.AA if 'OPT_COUNTS' in spec.keys(): for skey in speckeys: try: specobj.optimal[skey] = spec['OPT_{:s}'.format(skey)].data except KeyError: pass # Add units on wave specobj.optimal['WAVE'] = specobj.optimal['WAVE'] * units.AA # Append specObjs.append(specobj) # Return return specObjs, head0
def pca_trace(xcen, usepca = None, npca = 2, npoly_cen = 3, debug=True): nspec = xcen.shape[0] norders = xcen.shape[1] if usepca is None: usepca = np.zeros(norders,dtype=bool) # use_order = True orders used to predict the usepca = True bad orders use_order = np.invert(usepca) ngood = np.sum(use_order) if ngood < npca: msgs.warn('Not enough good traces for a PCA fit: ngood = {:d}'.format(ngood) + ' is < npca = {:d}'.format(npca)) msgs.warn('Using the input trace for now') return xcen pca = PCA(n_components=npca) xcen_use = (xcen[:,use_order] - np.mean(xcen[:,use_order],0)).T pca_coeffs_use = pca.fit_transform(xcen_use) pca_vectors = pca.components_ # Fit first pca dimension (with largest variance) with a higher order npoly depending on number of good orders. # Fit all higher dimensions (with lower variance) with a line npoly = int(np.fmin(np.fmax(np.floor(3.3*ngood/norders),1.0),3.0)) npoly_vec = np.full(npca, npoly) order_vec = np.arange(norders,dtype=float) # pca_coeffs = np.zeros((norders, npca)) pca_coeffs_new = np.zeros((norders, npca)) # Now loop over the dimensionality of the compression and perform a polynomial fit to for idim in range(npca): # ToDO robust_polyfit is garbage remove it entirely from PypeIT! xfit = order_vec[use_order] yfit = pca_coeffs_use[:,idim] norder = npoly_vec[idim] # msk, poly_coeff = utils.robust_polyfit(xfit, yfit, norder, sigma = 3.0, function='polynomial') # pca_coeffs[:,idim] = utils.func_val(poly_coeff, order_vec, 'polynomial') # TESTING traceset fitting xtemp = xfit.reshape(1, xfit.size) ytemp = yfit.reshape(1, yfit.size) tset = pydl.xy2traceset(xtemp, ytemp, ncoeff=norder,func='polynomial') #tset_yfit = tset.yfit.reshape(tset.yfit.shape[1]) ## Test new robust fitting with djs_reject msk_new, poly_coeff_new = utils.robust_polyfit_djs(xfit, yfit, norder, \ function='polynomial', minv=None, maxv=None, bspline_par=None, \ guesses=None, maxiter=10, inmask=None, sigma=None, invvar=None, \ lower=5, upper=5, maxdev=None, maxrej=None, groupdim=None, groupsize=None, \ groupbadpix=False, grow=0, sticky=False) pca_coeffs_new[:,idim] = utils.func_val(poly_coeff_new, order_vec, 'polynomial') if debug: # Evaluate the fit xvec = np.linspace(order_vec.min(),order_vec.max(),num=100) (_,tset_fit) = tset.xy(xpos=xvec.reshape(1,xvec.size)) yfit_tset = tset_fit[0,:] #robust_mask = msk == 0 robust_mask_new = msk_new == 1 tset_mask = tset.outmask[0,:] plt.plot(xfit, yfit, 'ko', mfc='None', markersize=8.0, label='pca coeff') #plt.plot(xfit[~robust_mask], yfit[~robust_mask], 'ms', mfc='None', markersize=10.0,label='robust_polyfit rejected') #plt.plot(xfit[~robust_mask_new], yfit[~robust_mask_new], 'r+', markersize=20.0,label='robust_polyfit_djs rejected') plt.plot(xfit[~tset_mask],yfit[~tset_mask], 'bo', markersize = 10.0, label = 'traceset rejected') #plt.plot(xvec, utils.func_val(poly_coeff, xvec, 'polynomial'),ls='--', color='m', label='robust polyfit') plt.plot(xvec, utils.func_val(poly_coeff_new, xvec, 'polynomial'),ls='-.', color='r', label='new robust polyfit') plt.plot(xvec, yfit_tset,ls=':', color='b',label='traceset') plt.legend() plt.show() #ToDo should we be masking the bad orders here and interpolating/extrapolating? spat_mean = np.mean(xcen,0) msk_spat, poly_coeff_spat = utils.robust_polyfit(order_vec, spat_mean, npoly_cen, sigma = 3.0, function = 'polynomial') ibad = np.where(msk_spat == 1) spat_mean[ibad] = utils.func_val(poly_coeff_spat,order_vec[ibad],'polynomial') #pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer(pca.mean_,np.ones(norders)) + (np.dot(pca_coeffs, pca_vectors)).T pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer(pca.mean_,np.ones(norders)) + (np.dot(pca_coeffs_new, pca_vectors)).T return pca_fit
def ech_objfind(image, ivar, ordermask, slit_left, slit_righ,inmask=None,plate_scale=0.2,npca=2,ncoeff = 5,min_snr=0.0,nabove_min_snr=0, pca_percentile=20.0,snr_pca=3.0,box_radius=2.0,show_peaks=False,show_fits=False,show_trace=False): if inmask is None: inmask = (ordermask > 0) frameshape = image.shape nspec = frameshape[0] norders = slit_left.shape[1] if isinstance(plate_scale,(float, int)): plate_scale_ord = np.full(norders, plate_scale) # 0.12 binned by 3 spatially for HIRES elif isinstance(plate_scale,(np.ndarray, list, tuple)): if len(plate_scale) == norders: plate_scale_ord = plate_scale elif len(plate_scale) == 1: plate_scale_ord = np.full(norders, plate_scale[0]) else: msgs.error('Invalid size for plate_scale. It must either have one element or norders elements') else: msgs.error('Invalid type for plate scale') specmid = nspec // 2 slit_width = slit_righ - slit_left spec_vec = np.arange(nspec) slit_spec_pos = nspec/2.0 slit_spat_pos = np.zeros((norders, 2)) for iord in range(norders): slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec, slit_left[:,iord]), np.interp(slit_spec_pos, spec_vec, slit_righ[:,iord])) # Loop over orders and find objects sobjs = specobjs.SpecObjs() show_peaks=True show_fits=True # ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx! for iord in range(norders): msgs.info('Finding objects on slit # {:d}'.format(iord + 1)) thismask = ordermask == (iord + 1) inmask_iord = inmask & thismask specobj_dict = {'setup': 'HIRES', 'slitid': iord + 1, 'scidx': 0,'det': 1, 'objtype': 'science'} sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \ extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks, show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0) # ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord for spec in sobjs_slit: spec.ech_orderindx = iord sobjs.add_sobj(sobjs_slit) nfound = len(sobjs) # Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0" FOFSEP = 1.0 # separation of FOF algorithm in arcseconds FOF_frac = FOFSEP/(np.median(slit_width)*np.median(plate_scale_ord)) # Feige: made the code also works for only one object found in one order # Run the FOF. We use fake coordinaes fracpos = sobjs.spat_fracpos ra_fake = fracpos/1000.0 # Divide all angles by 1000 to make geometry euclidian dec_fake = 0.0*fracpos if nfound>1: (ingroup, multgroup, firstgroup, nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac/1000.0) group = ingroup.copy() uni_group, uni_ind = np.unique(group, return_index=True) nobj = len(uni_group) msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects') elif nfound==1: group = np.zeros(1,dtype='int') uni_group, uni_ind = np.unique(group, return_index=True) nobj = len(group) msgs.warn('Only find one object no FOF matching is needed') gfrac = np.zeros(nfound) for jj in range(nobj): this_group = group == uni_group[jj] gfrac[this_group] = np.median(fracpos[this_group]) uni_frac = gfrac[uni_ind] sobjs_align = sobjs.copy() # Now fill in the missing objects and their traces for iobj in range(nobj): for iord in range(norders): # Is there an object on this order that grouped into the current group in question? on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx == iord) if not np.any(on_slit): # Add this to the sobjs_align, and assign required tags thisobj = specobjs.SpecObj(frameshape, slit_spat_pos[iord,:], slit_spec_pos, det = sobjs_align[0].det, setup = sobjs_align[0].setup, slitid = (iord + 1), scidx = sobjs_align[0].scidx, objtype=sobjs_align[0].objtype) thisobj.ech_orderindx = iord thisobj.spat_fracpos = uni_frac[iobj] thisobj.trace_spat = slit_left[:,iord] + slit_width[:,iord]*uni_frac[iobj] # new trace thisobj.trace_spec = spec_vec thisobj.spat_pixpos = thisobj.trace_spat[specmid] thisobj.set_idx() # Use the real detections of this objects for the FWHM this_group = group == uni_group[iobj] # Assign to the fwhm of the nearest detected order imin = np.argmin(np.abs(sobjs_align[this_group].ech_orderindx - iord)) thisobj.fwhm = sobjs_align[imin].fwhm thisobj.maskwidth = sobjs_align[imin].maskwidth thisobj.ech_fracpos = uni_frac[iobj] thisobj.ech_group = uni_group[iobj] thisobj.ech_usepca = True sobjs_align.add_sobj(thisobj) group = np.append(group, uni_group[iobj]) gfrac = np.append(gfrac, uni_frac[iobj]) else: # ToDo fix specobjs to get rid of these crappy loops! for spec in sobjs_align[on_slit]: spec.ech_fracpos = uni_frac[iobj] spec.ech_group = uni_group[iobj] spec.ech_usepca = False # Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order # respectively sobjs_sort = specobjs.SpecObjs() for iobj in range(nobj): this_group = group == uni_group[iobj] this_sobj = sobjs_align[this_group] sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)]) # Loop over the objects and perform a quick and dirty extraction to assess S/N. varimg = utils.calc_ivar(ivar) flux_box = np.zeros((nspec, norders, nobj)) ivar_box = np.zeros((nspec, norders, nobj)) mask_box = np.zeros((nspec, norders, nobj)) SNR_arr = np.zeros((norders, nobj)) for iobj in range(nobj): for iord in range(norders): indx = (sobjs_sort.ech_group == uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord) spec = sobjs_sort[indx] thismask = ordermask == (iord + 1) inmask_iord = inmask & thismask box_rad_pix = box_radius/plate_scale_ord[iord] flux_tmp = extract.extract_boxcar(image*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) var_tmp = extract.extract_boxcar(varimg*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) ivar_tmp = utils.calc_ivar(var_tmp) pixtot = extract.extract_boxcar(ivar*0 + 1.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) mask_tmp = (extract.extract_boxcar(ivar*inmask_iord == 0.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) != pixtot) flux_box[:,iord,iobj] = flux_tmp*mask_tmp ivar_box[:,iord,iobj] = np.fmax(ivar_tmp*mask_tmp,0.0) mask_box[:,iord,iobj] = mask_tmp (mean, med_sn, stddev) = sigma_clipped_stats(flux_box[mask_tmp,iord,iobj]*np.sqrt(ivar_box[mask_tmp,iord,iobj]), sigma_lower=5.0,sigma_upper=5.0) SNR_arr[iord,iobj] = med_sn # Purge objects with low SNR and that don't show up in enough orders keep_obj = np.zeros(nobj,dtype=bool) sobjs_trim = specobjs.SpecObjs() uni_group_trim = np.array([],dtype=int) uni_frac_trim = np.array([],dtype=float) for iobj in range(nobj): if (np.sum(SNR_arr[:,iobj] > min_snr) >= nabove_min_snr): keep_obj[iobj] = True ikeep = sobjs_sort.ech_group == uni_group[iobj] sobjs_trim.add_sobj(sobjs_sort[ikeep]) uni_group_trim = np.append(uni_group_trim, uni_group[iobj]) uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj]) else: msgs.info('Purging object #{:d}'.format(iobj) + ' which does not satisfy min_snr > {:5.2f}'.format(min_snr) + ' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) + ' orders') nobj_trim = np.sum(keep_obj) if nobj_trim == 0: return specobjs.SpecObjs() SNR_arr_trim = SNR_arr[:,keep_obj] # Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA for iobj in range(nobj_trim): SNR_now = SNR_arr_trim[:,iobj] indx = (sobjs_trim.ech_group == uni_group_trim[iobj]) # PCA interp/extrap if: # (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca) # OR # (if this order was not originally traced by the object finding, see above) usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) & (SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca # ToDo fix specobjs to get rid of these crappy loops! for iord, spec in enumerate(sobjs_trim[indx]): spec.ech_usepca = usepca[iord] if usepca[iord]: msgs.info('Using PCA to predict trace for object #{:d}'.format(iobj) + ' on order #{:d}'.format(iord)) sobjs_final = sobjs_trim.copy() # Loop over the objects one by one and adjust/predict the traces npoly_cen = 3 pca_fits = np.zeros((nspec, norders, nobj_trim)) for iobj in range(nobj_trim): igroup = sobjs_final.ech_group == uni_group_trim[iobj] # PCA predict the masked orders which were not traced pca_fits[:,:,iobj] = pca_trace((sobjs_final[igroup].trace_spat).T, usepca = None, npca = npca, npoly_cen = npoly_cen) # usepca = sobjs_final[igroup].ech_usepca, # Perform iterative flux weighted centroiding using new PCA predictions xinit_fweight = pca_fits[:,:,iobj].copy() inmask_now = inmask & (ordermask > 0) xfit_fweight = extract.iter_tracefit(image, xinit_fweight, ncoeff, inmask = inmask_now, show_fits=show_fits) # Perform iterative Gaussian weighted centroiding xinit_gweight = xfit_fweight.copy() xfit_gweight = extract.iter_tracefit(image, xinit_gweight, ncoeff, inmask = inmask_now, gweight=True,show_fits=show_fits) # Assign the new traces for iord, spec in enumerate(sobjs_final[igroup]): spec.trace_spat = xfit_gweight[:,iord] spec.spat_pixpos = spec.trace_spat[specmid] # Set the IDs sobjs_final.set_idx() if show_trace: viewer, ch = ginga.show_image(objminsky*(ordermask > 0)) for iobj in range(nobj_trim): for iord in range(norders): ginga.show_trace(viewer, ch, pca_fits[:,iord, iobj], str(uni_frac[iobj]), color='yellow') for spec in sobjs_trim: color = 'green' if spec.ech_usepca else 'magenta' ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color) #for spec in sobjs_final: # color = 'red' if spec.ech_usepca else 'green' # ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color) return sobjs_final