def test_assign_maskinfo(): # Spectrograph keck_deimos = KeckDEIMOSSpectrograph() par = keck_deimos.default_pypeit_par() # working only on detector 3 det = 3 # Built trace image traceImage = buildimage.buildimage_fromlist( keck_deimos, det, par['calibrations']['traceframe'], deimos_flat_files()) msbpm = keck_deimos.bpm(traceImage.files[0], det) # load specific config parameters par = keck_deimos.config_specific_par(traceImage.files[0]) trace_par = par['calibrations']['slitedges'] # Run edge trace edges = edgetrace.EdgeTraceSet(traceImage, keck_deimos, trace_par, bpm=msbpm, auto=True, debug=False, show_stages=False, qa_path=None) slits = edges.get_slits() # Test that the maskfile is saved properly hdul = fits.open(slits.maskfile) det_par = keck_deimos.get_detector_par(hdul, det=det) specobjs_file = os.path.join( os.getenv('PYPEIT_DEV'), 'Cooked', 'Science', 'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_2010Sep13T061231.334.fits') # specobjs_file = os.path.join(os.getenv('PYPEIT_DEV'), 'REDUX_OUT', 'keck_deimos', # '830G_M_8500', 'Science', # 'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_2010Sep13T061231.334.fits') sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file) # Init at null for sobj in sobjs: sobj.MASKDEF_OBJNAME = None sobj.RA = None sobj.DEC = None # Run me slits.assign_maskinfo(sobjs, det_par['platescale']) # Test assert sobjs[sobjs.SLITID == 496].MASKDEF_OBJNAME == 'ero89', 'Wrong MASKDEF_OBJNAME' assert sobjs[sobjs.SLITID == 496].RA == 352.27471667, 'Wrong object RA' assert sobjs[sobjs.SLITID == 496].DEC == -3.09223056, 'Wrong object DEC' # Write sobjs sobjs.write_to_fits({}, data_path('tst_sobjs.fits')) os.remove(data_path('tst_sobjs.fits'))
def get_slits(self): """ Load or generate the definition of the slit boundaries. Internals that must be available are :attr:`fitstbl`, :attr:`calib_ID`, :attr:`det`. Returns: :class:`pypeit.slittrace.SlitTraceSet`: Traces of the slit edges; also kept internally as :attr:`slits`. """ # Check for existing data if not self._chk_objs(['msbpm']): return # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep trace_image_files, self.master_key_dict[ 'trace'] = self._prep_calibrations('trace') # Reuse master frame? slit_masterframe_name = masterframe.construct_file_name( slittrace.SlitTraceSet, self.master_key_dict['trace'], master_dir=self.master_dir) if os.path.isfile(slit_masterframe_name) and self.reuse_masters: self.slits = slittrace.SlitTraceSet.from_file( slit_masterframe_name) # Reset the bitmask self.slits.mask = self.slits.mask_init.copy() else: # Slits don't exist or we're not resusing them edge_masterframe_name = masterframe.construct_file_name( edgetrace.EdgeTraceSet, self.master_key_dict['trace'], master_dir=self.master_dir) # Reuse master frame? if os.path.isfile(edge_masterframe_name) and self.reuse_masters: self.edges = edgetrace.EdgeTraceSet.from_file( edge_masterframe_name) elif len(trace_image_files) == 0: msgs.warn("No frametype=trace files to build slits") return None else: # Build the trace image self.traceImage = buildimage.buildimage_fromlist( self.spectrograph, self.det, self.par['traceframe'], trace_image_files, bias=self.msbias, bpm=self.msbpm, dark=self.msdark) self.edges = edgetrace.EdgeTraceSet(self.traceImage, self.spectrograph, self.par['slitedges'], bpm=self.msbpm, auto=True) self.edges.save(edge_masterframe_name, master_dir=self.master_dir, master_key=self.master_key_dict['trace']) # Show the result if requested if self.show: self.edges.show(in_ginga=True) # Get the slits from the result of the edge tracing, delete # the edges object, and save the slits, if requested self.slits = self.edges.get_slits() self.edges = None self.slits.to_master_file(slit_masterframe_name) # User mask? if self.slitspat_num is not None: self.slits.user_mask(self.det, self.slitspat_num) return self.slits
def main(args): import time import os import numpy as np from pypeit.spectrographs.util import load_spectrograph from pypeit import traceimage, edgetrace, biasframe from pypeit.pypeit import PypeIt from pypeit.core import parse from IPython import embed if args.pypeit_file is not None: pypeit_file = args.pypeit_file if not os.path.isfile(pypeit_file): raise FileNotFoundError( 'File does not exist: {0}'.format(pypeit_file)) pypeit_file = os.path.abspath(pypeit_file) redux_path = os.path.abspath( os.path.split(pypeit_file)[0] if args.redux_path is None else args. redux_path) rdx = PypeIt(pypeit_file, redux_path=redux_path) # Save the spectrograph spec = rdx.spectrograph # Get the calibration group to use group = np.unique( rdx.fitstbl['calib'])[0] if args.group is None else args.group if group not in np.unique(rdx.fitstbl['calib']): raise ValueError( 'Not a valid calibration group: {0}'.format(group)) # Find the rows in the metadata table with trace frames in the # specified calibration group tbl_rows = rdx.fitstbl.find_frames('trace', calib_ID=int(group), index=True) # Master keyword master_key_base = '_'.join( rdx.fitstbl.master_key(tbl_rows[0]).split('_')[:2]) # Save the binning binning = rdx.fitstbl['binning'][tbl_rows[0]] # Save the full file paths files = rdx.fitstbl.frame_paths(tbl_rows) # Trace image processing parameters proc_par = rdx.caliBrate.par['traceframe'] # Slit tracing parameters trace_par = rdx.caliBrate.par['slitedges'] # Get the bias files, if requested bias_rows = rdx.fitstbl.find_frames('bias', calib_ID=int(group), index=True) bias_files = rdx.fitstbl.frame_paths(bias_rows) bias_par = rdx.caliBrate.par['biasframe'] # Set the QA path qa_path = rdx.qa_path else: spec = load_spectrograph(args.spectrograph) master_key_base = 'A_1' binning = '1,1' if args.binning is None else args.binning if not os.path.isfile(args.trace_file): raise FileNotFoundError('File does not exist: {0}'.format( args.trace_file)) files = [os.path.abspath(args.trace_file)] redux_path = os.path.abspath( os.path.split(files[0])[0] if args.redux_path is None else args. redux_path) par = spec.default_pypeit_par() proc_par = par['calibrations']['traceframe'] trace_par = par['calibrations']['slitedges'] bias_files = None bias_par = None # Set the QA path qa_path = os.path.join(os.path.abspath(os.path.split(files[0])[0]), 'QA') detectors = np.arange(spec.ndet) + 1 if args.detector is None else [ args.detector ] master_dir = os.path.join(redux_path, args.master_dir) for det in detectors: # Master keyword for output file name master_key = '{0}_{1}'.format(master_key_base, str(det).zfill(2)) # Get the bias frame if requested if bias_files is None: proc_par['process']['bias'] = 'skip' msbias = None else: biasFrame = biasframe.BiasFrame(spec, files=bias_files, det=det, par=bias_par, master_key=master_key, master_dir=master_dir) msbias = biasFrame.build_image() msbpm = spec.bpm(files[0], det) # Build the trace image traceImage = traceimage.TraceImage(spec, files=files, det=det, par=proc_par, bias=msbias) traceImage.build_image(bias=msbias, bpm=msbpm) # Trace the slit edges t = time.perf_counter() edges = edgetrace.EdgeTraceSet(spec, trace_par, master_key=master_key, master_dir=master_dir, img=traceImage, det=det, bpm=msbpm, auto=True, debug=args.debug, show_stages=args.show, qa_path=qa_path) print('Tracing for detector {0} finished in {1} s.'.format( det, time.perf_counter() - t)) edges.save() return 0
def test_assign_maskinfo_add_missing(): instr_names = ['keck_deimos', 'keck_mosfire'] for name in instr_names: # Spectrograph instrument = load_spectrograph(name) par = instrument.default_pypeit_par() # working only on detector 3 (det=3 for DEIMOS. For MOSFIRE does not matter because we have only one det) det = 3 if name == 'keck_deimos' else 1 # Built trace image traceImage = buildimage.buildimage_fromlist( instrument, det, par['calibrations']['traceframe'], flat_files(instr=name)) # load specific config parameters par = instrument.config_specific_par(traceImage.files[0]) # Run edge trace edges = edgetrace.EdgeTraceSet(traceImage, instrument, par['calibrations']['slitedges'], auto=True, debug=False, show_stages=False, qa_path=None) slits = edges.get_slits() # Test that the maskfile is saved properly hdul = fits.open(slits.maskfile) det_par = instrument.get_detector_par(det, hdu=hdul) if name == 'keck_deimos': specobjs_file = os.path.join( os.getenv('PYPEIT_DEV'), 'Cooked', 'Science', 'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_20100913T061231.334.fits' ) sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file) # correct value slitid = sobjs[sobjs.MASKDEF_OBJNAME == 'ero89'].SLITID[0] true_maskdef_objname = sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[0] true_ra = round(sobjs[sobjs.SLITID == slitid].RA[0], 6) true_dec = round(sobjs[sobjs.SLITID == slitid].DEC[0], 6) true_spat_pixpos = round( sobjs[sobjs.MASKDEF_OBJNAME == 'ero884'].SPAT_PIXPOS[0]) true_spat_pixpos_2 = round( sobjs[sobjs.MASKDEF_OBJNAME == 'ero191'].SPAT_PIXPOS[0]) elif name == 'keck_mosfire': specobjs_file = os.path.join( os.getenv('PYPEIT_DEV'), 'Cooked', 'Science', 'spec1d_m191014_0170-2M2228_12_MOSFIRE_20191014T095212.598.fits' ) sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file) # correct value slitid = sobjs[sobjs.MASKDEF_OBJNAME == '18'].SLITID[0] true_maskdef_objname = sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[0] true_ra = round(sobjs[sobjs.SLITID == slitid].RA[0], 6) true_dec = round(sobjs[sobjs.SLITID == slitid].DEC[0], 6) true_spat_pixpos = round( sobjs[sobjs.MASKDEF_OBJNAME == '7'].SPAT_PIXPOS[0]) # Init at null and remove the force extraction idx_remove = [] for i, sobj in enumerate(sobjs): if sobj.MASKDEF_EXTRACT: idx_remove.append(i) else: sobj.MASKDEF_ID = None sobj.MASKDEF_OBJNAME = None sobj.RA = None sobj.DEC = None sobj.MASKDEF_EXTRACT = None sobjs.remove_sobj(idx_remove) # get the dither offset if available if name == 'keck_deimos': dither_off = None elif name == 'keck_mosfire': dither_off = instrument.parse_dither_pattern([ os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'keck_mosfire', 'J_multi', 'm191014_0170.fits') ])[2][0] # get object positions from slitmask design and slitmask offsets calib_slits = slittrace.get_maskdef_objpos_offset_alldets( sobjs, [slits], [None], [det_par['platescale']], par['calibrations']['slitedges']['det_buffer'], par['reduce']['slitmask'], dither_off=dither_off) # determine if slitmask offsets exist and compute an average offsets over all the detectors calib_slits = slittrace.average_maskdef_offset( calib_slits, det_par['platescale'], instrument.list_detectors()) # slitmask design matching and add undetected objects sobjs = slittrace.assign_addobjs_alldets( sobjs, calib_slits, [None], [det_par['platescale']], par['reduce']['slitmask'], par['reduce']['findobj']['find_fwhm']) # Test if name == 'keck_deimos': # Check if recover the maskdef assignment assert sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[ 0] == true_maskdef_objname, 'Wrong DEIMOS MASKDEF_OBJNAME' assert round(sobjs[sobjs.SLITID == slitid].RA[0], 6) == true_ra, 'Wrong object DEIMOS RA' assert round(sobjs[sobjs.SLITID == slitid].DEC[0], 6) == true_dec, 'Wrong object DEIMOS DEC' # Test that undetected objects are found at the correct location (the correct location is # verified by visual inspection) assert round(sobjs[sobjs.MASKDEF_OBJNAME == 'ero884'].SPAT_PIXPOS[0]) == true_spat_pixpos, \ 'Wrong object (ero884) location on the DEIMOS slit' assert round(sobjs[sobjs.MASKDEF_OBJNAME == 'ero191'].SPAT_PIXPOS[0]) == true_spat_pixpos_2, \ 'Wrong object (ero191) location on the DEIMOS slit' elif name == 'keck_mosfire': # Check if recover the maskdef assignment assert sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[ 0] == true_maskdef_objname, 'Wrong MOSFIRE MASKDEF_OBJNAME' assert round(sobjs[sobjs.SLITID == slitid].RA[0], 6) == true_ra, 'Wrong object MOSFIRE RA' assert round(sobjs[sobjs.SLITID == slitid].DEC[0], 6) == true_dec, 'Wrong object MOSFIRE DEC' # Test that undetected object are found at the correct location (the correct location is # verified by visual inspection) assert round(sobjs[sobjs.MASKDEF_OBJNAME == '7'].SPAT_PIXPOS[0]) == true_spat_pixpos, \ 'Wrong object (7) location on the MOSFIRE slit' # Write sobjs sobjs.write_to_fits({}, data_path('tst_sobjs.fits')) os.remove(data_path('tst_sobjs.fits'))
def get_slits(self, redo=False, write_qa=True): """ Load or generate the definition of the slit boundaries. Internals that must be available are :attr:`fitstbl`, :attr:`calib_ID`, :attr:`det`. Args: redo (bool): Redo write_qa (bool, optional): Generate the QA? Turn off for testing.. Returns: Returns the trace-slits dictionary (also kept internally as :attr:`tslits_dict`) and the slit mask array (numpy.ndarray; also kept internally as :attr:`maskslits`) """ # Check for existing data if not self._chk_objs(['msbpm']): self.tslits_dict = None return self.tslits_dict # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep trace_rows = self.fitstbl.find_frames('trace', calib_ID=self.calib_ID, index=True) self.trace_image_files = self.fitstbl.frame_paths(trace_rows) self.master_key_dict['trace'] \ = self.fitstbl.master_key(trace_rows[0] if len(trace_rows) > 0 else self.frame, det=self.det) # Return already generated data if self._cached('trace', self.master_key_dict['trace']) and not redo: self.tslits_dict = self.calib_dict[ self.master_key_dict['trace']]['trace'] return self.tslits_dict # Instantiate self.edges = edgetrace.EdgeTraceSet( self.spectrograph, self.par['slitedges'], master_key=self.master_key_dict['trace'], master_dir=self.master_dir, qa_path=self.qa_path if write_qa else None) if self.reuse_masters and self.edges.exists(): self.edges.load() self.tslits_dict = self.edges.convert_to_tslits_dict() else: # Build the trace image self.traceImage = traceimage.TraceImage( self.spectrograph, files=self.trace_image_files, det=self.det, par=self.par['traceframe'], bias=self.msbias) self.traceImage.build_image(bias=self.msbias, bpm=self.msbpm) try: self.edges.auto_trace( self.traceImage, bpm=self.msbpm, det=self.det, save=self.save_masters) #, debug=True, show_stages=True) except: self.edges.save() msgs.error( 'Crashed out of finding the slits. Have saved the work done to disk ' 'but it needs fixing.') return None # Show the result if requested if self.show: self.edges.show(thin=10, in_ginga=True) # TODO: Stop-gap until we can get rid of tslits_dict self.tslits_dict = self.edges.convert_to_tslits_dict() # Save, initialize maskslits, and return self._update_cache('trace', 'trace', self.tslits_dict) return self.tslits_dict
def main(args): import time import os import numpy as np from pypeit.spectrographs.util import load_spectrograph from pypeit import edgetrace from pypeit import slittrace from pypeit.pypeit import PypeIt from pypeit.images import buildimage from pypeit import masterframe from IPython import embed if args.pypeit_file is not None: pypeit_file = args.pypeit_file if not os.path.isfile(pypeit_file): raise FileNotFoundError('File does not exist: {0}'.format(pypeit_file)) pypeit_file = os.path.abspath(pypeit_file) redux_path = os.path.abspath(os.path.split(pypeit_file)[0] if args.redux_path is None else args.redux_path) rdx = PypeIt(pypeit_file, redux_path=redux_path) detectors = rdx.par['rdx']['detnum'] if args.detector is None else args.detector # Save the spectrograph spec = rdx.spectrograph # Get the calibration group to use group = np.unique(rdx.fitstbl['calib'])[0] if args.group is None else args.group if group not in np.unique(rdx.fitstbl['calib']): raise ValueError('Not a valid calibration group: {0}'.format(group)) # Find the rows in the metadata table with trace frames in the # specified calibration group tbl_rows = rdx.fitstbl.find_frames('trace', calib_ID=int(group), index=True) # Master keyword master_key_base = '_'.join(rdx.fitstbl.master_key(tbl_rows[0]).split('_')[:2]) # Save the binning binning = rdx.fitstbl['binning'][tbl_rows[0]] # Save the full file paths files = rdx.fitstbl.frame_paths(tbl_rows) # Trace image processing parameters proc_par = rdx.par['calibrations']['traceframe'] # Slit tracing parameters trace_par = rdx.par['calibrations']['slitedges'] # Get the bias files, if requested bias_rows = rdx.fitstbl.find_frames('bias', calib_ID=int(group), index=True) bias_files = rdx.fitstbl.frame_paths(bias_rows) bias_par = rdx.par['calibrations']['biasframe'] if len(bias_files) == 0: bias_files = None # Get the dark files, if requested dark_rows = rdx.fitstbl.find_frames('dark', calib_ID=int(group), index=True) dark_files = rdx.fitstbl.frame_paths(dark_rows) dark_par = rdx.par['calibrations']['darkframe'] if len(dark_files) == 0: dark_files = None # Set the QA path qa_path = rdx.qa_path else: detectors = args.detector spec = load_spectrograph(args.spectrograph) master_key_base = 'A_1' binning = '1,1' if args.binning is None else args.binning if not os.path.isfile(args.trace_file): raise FileNotFoundError('File does not exist: {0}'.format(args.trace_file)) files = [os.path.abspath(args.trace_file)] redux_path = os.path.abspath(os.path.split(files[0])[0] if args.redux_path is None else args.redux_path) par = spec.default_pypeit_par() proc_par = par['calibrations']['traceframe'] trace_par = par['calibrations']['slitedges'] bias_files = None bias_par = None dark_files = None dark_par = None # Set the QA path qa_path = os.path.join(os.path.abspath(os.path.split(files[0])[0]), 'QA') if detectors is None: detectors = np.arange(spec.ndet)+1 elif isinstance(detectors, (int, tuple)): detectors = [detectors] elif any([isinstance(d,str) for d in detectors]): detectors = [eval(d) for d in detectors] master_dir = os.path.join(redux_path, args.master_dir) for det in detectors: # Master keyword for output file name master_key = f'{master_key_base}_{spec.get_det_name(det)}' # Get the bias frame if requested if bias_files is None: proc_par['process']['use_biasimage'] = False msbias = None else: msbias = buildimage.buildimage_fromlist(spec, det, bias_par, bias_files) # Get the dark frame if requested if dark_files is None: proc_par['process']['use_darkimage'] = False msdark = None else: msdark = buildimage.buildimage_fromlist(spec, det, dark_par, dark_files) msbpm = spec.bpm(files[0], det) # Build the trace image traceImage = buildimage.buildimage_fromlist(spec, det, proc_par, files, bias=msbias, bpm=msbpm, dark=msdark) # Trace the slit edges t = time.perf_counter() edges = edgetrace.EdgeTraceSet(traceImage, spec, trace_par, auto=True, debug=args.debug, show_stages=args.show, qa_path=qa_path) print('Tracing for detector {0} finished in {1} s.'.format(det, time.perf_counter()-t)) # Write the MasterEdges file edge_masterframe_name = masterframe.construct_file_name(edgetrace.EdgeTraceSet, master_key, master_dir=master_dir) edges.to_master_file(edge_masterframe_name) # Write the MasterSlits file slit_masterframe_name = masterframe.construct_file_name(slittrace.SlitTraceSet, master_key, master_dir=master_dir) edges.get_slits().to_master_file(slit_masterframe_name) return 0