def test_proc_diff(nires_sci_files, nires_bg_files): """ Run on near-IR frames """ # Setup det = 1 bpm = np.zeros((2048, 1024)) pixelflat = np.ones_like(bpm) # Sci image flatImages = flatfield.FlatImages(pixelflat_norm=pixelflat) sciImg = buildimage.buildimage_fromlist(keck_nires, det, nires_par['scienceframe'], nires_sci_files, bias=None, bpm=bpm, flatimages=flatImages) # Bg image bgImg = buildimage.buildimage_fromlist(keck_nires, det, nires_par['scienceframe'], nires_bg_files, bias=None, bpm=bpm, flatimages=flatImages) # Difference sciImg = sciImg.sub(bgImg, nires_par['scienceframe']['process']) # Test assert isinstance(sciImg, pypeitimage.PypeItImage)
def test_proc_diff(nires_sci_files, nires_bg_files): """ Run on near-IR frames """ # Setup det = 1 bpm = np.zeros((2048, 1024), dtype=int) pixelflat = np.ones(bpm.shape, dtype=float) nires_par = keck_nires.default_pypeit_par() # Sci image flatImages = flatfield.FlatImages(pixelflat_norm=pixelflat) nires_par['scienceframe']['process']['use_illumflat'] = False nires_par['scienceframe']['process']['use_specillum'] = False sciImg = buildimage.buildimage_fromlist(keck_nires, det, nires_par['scienceframe'], nires_sci_files, bias=None, bpm=bpm, flatimages=flatImages) # Bg image bgImg = buildimage.buildimage_fromlist(keck_nires, det, nires_par['scienceframe'], nires_bg_files, bias=None, bpm=bpm, flatimages=flatImages) # Difference sciImg = sciImg.sub(bgImg, nires_par['scienceframe']['process']) # Test assert isinstance(sciImg, pypeitimage.PypeItImage)
def test_maskdef_id(): # Load instrument keck_deimos = load_spectrograph('keck_deimos') par = keck_deimos.default_pypeit_par() # working only on detector 1 det = 1 # Built trace image traceImage = buildimage.buildimage_fromlist( keck_deimos, det, par['calibrations']['traceframe'], deimos_flat_files()) msbpm = keck_deimos.bpm(traceImage.files[0], det) # load specific config parameters par = keck_deimos.config_specific_par(traceImage.files[0]) trace_par = par['calibrations']['slitedges'] # Run edge trace edges = EdgeTraceSet(traceImage, keck_deimos, trace_par, bpm=msbpm, auto=True, debug=False, show_stages=False, qa_path=None) slits = edges.get_slits() # Check that the `maskdef_id` assigned to the first and last slits is correct assert slits.maskdef_id[0] == 1098247, 'maskdef_id not found or wrong' assert slits.maskdef_id[-1] == 1098226, 'maskdef_id not found or wrong'
def test_maskdef_id(): instr_names = ['keck_deimos', 'keck_mosfire'] for name in instr_names: # Load instrument instrument = load_spectrograph(name) par = instrument.default_pypeit_par() # working only on detector 1 det=1 # Built trace image traceImage = buildimage.buildimage_fromlist(instrument, det, par['calibrations']['traceframe'], flat_files(instr=name)) # load specific config parameters par = instrument.config_specific_par(traceImage.files[0]) trace_par = par['calibrations']['slitedges'] # Run edge trace edges = EdgeTraceSet(traceImage, instrument, trace_par, auto=True, debug=False, show_stages=False,qa_path=None) slits = edges.get_slits() # Check that the `maskdef_id` assigned to the first and last slits is correct if name == 'keck_deimos': assert slits.maskdef_id[0] == 1098247, 'DEIMOS maskdef_id not found or wrong' assert slits.maskdef_id[-1] == 1098226, 'DEIMOS maskdef_id not found or wrong' # `maskdef_id` is the slit_id ("dSlitId") from the DEIMOS slit-mask design. If the matching is done # correctly these are the slit_id values that the first and last slits should have. These values are # coming from a reduction run with DEEP2 IDL-based pipeline. elif name == 'keck_mosfire': assert slits.maskdef_id[0] == 11, 'MOSFIRE maskdef_id not found or wrong' assert slits.maskdef_id[-1] == 1, 'MOSFIRE maskdef_id not found or wrong'
def test_io(kast_blue_bias_files): # outfile = masterframe.construct_file_name(buildimage.BiasImage, master_key, master_dir=master_dir) if os.path.isfile(outfile): os.remove(outfile) # Build msbias = buildimage.buildimage_fromlist(shane_kast_blue, 1, frame_par, kast_blue_bias_files) # Save as a master frame master_filename = masterframe.construct_file_name(msbias, master_key, master_dir=master_dir) msbias.to_master_file(master_filename)#master_dir, master_key, # Naming #shane_kast_blue.spectrograph, # Header #steps=msbias.process_steps, #raw_files=kast_blue_bias_files) assert os.path.isfile(outfile), 'Error writing MasterBias' # Load master frame biasImage = buildimage.BiasImage.from_file(outfile) assert np.array_equal(biasImage.image, msbias.image) # Instantiate from master frame #bias_frame2 = biasframe.BiasFrame.from_master_file(bias_frame.master_file_path) #assert np.array_equal(pypeitImage.image, bias_frame2.pypeitImage.image) # Clean up os.remove(outfile)
def get_align(self): """ Load or generate the alignment frame Requires: :attr:`slits`, :attr:`det`, :attr:`par` Returns: :class:`pypeit.alignframe.Alignments`: """ # Check for existing data if not self._chk_objs(['msbpm', 'slits']): msgs.error('Must have the bpm and slits to make the alignments!') # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep align_files, self.master_key_dict['align'] = self._prep_calibrations( 'align') masterframe_filename = masterframe.construct_file_name( alignframe.Alignments, self.master_key_dict['align'], master_dir=self.master_dir) # Reuse master frame? if os.path.isfile(masterframe_filename) and self.reuse_masters: self.alignments = alignframe.Alignments.from_file( masterframe_filename) self.alignments.is_synced(self.slits) return self.alignments msalign = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['alignframe'], align_files, bias=self.msbias, bpm=self.msbpm, dark=self.msdark) # Extract some header info needed by the algorithm binning = self.spectrograph.get_meta_value(align_files[0], 'binning') # Instantiate # TODO: From JFH: Do we need the bpm here? Check that this was in the previous code. alignment = alignframe.TraceAlignment(msalign, self.slits, self.spectrograph, self.par['alignment'], det=self.det, binning=binning, qa_path=self.qa_path, msbpm=self.msbpm) # Run self.alignments = alignment.run(show=self.show) # Save to Masters self.alignments.to_master_file(masterframe_filename) return self.alignments
def test_overlapped_slits(): # Load flats frames that have overlapping alignment slits. deimos_flats = [os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'keck_deimos', '1200G_alignslit', ifile) for ifile in ['DE.20110529.11732.fits', 'DE.20110529.11813.fits', 'DE.20110529.11895.fits']] # Load instrument keck_deimos = load_spectrograph('keck_deimos') par = keck_deimos.default_pypeit_par() # working only on detector 2 det = 2 # Built trace image traceImage = buildimage.buildimage_fromlist(keck_deimos, det, par['calibrations']['traceframe'], deimos_flats) # load specific config parameters par = keck_deimos.config_specific_par(traceImage.files[0]) trace_par = par['calibrations']['slitedges'] # Run edge trace edges = EdgeTraceSet(traceImage, keck_deimos, trace_par, auto=True, debug=False, show_stages=False, qa_path=None) slits = edges.get_slits() # Check that the total number of expected slits and the number of alignment slits are correct. assert len(slits.maskdef_designtab['MASKDEF_ID'].data) == 22, \ 'wrong number of slits for this detector' assert np.sum(slits.maskdef_designtab['ALIGN'].data) == 3, 'wrong number of alignment slits'
def test_assign_maskinfo(): # Spectrograph keck_deimos = KeckDEIMOSSpectrograph() par = keck_deimos.default_pypeit_par() # working only on detector 3 det = 3 # Built trace image traceImage = buildimage.buildimage_fromlist( keck_deimos, det, par['calibrations']['traceframe'], deimos_flat_files()) msbpm = keck_deimos.bpm(traceImage.files[0], det) # load specific config parameters par = keck_deimos.config_specific_par(traceImage.files[0]) trace_par = par['calibrations']['slitedges'] # Run edge trace edges = edgetrace.EdgeTraceSet(traceImage, keck_deimos, trace_par, bpm=msbpm, auto=True, debug=False, show_stages=False, qa_path=None) slits = edges.get_slits() # Test that the maskfile is saved properly hdul = fits.open(slits.maskfile) det_par = keck_deimos.get_detector_par(hdul, det=det) specobjs_file = os.path.join( os.getenv('PYPEIT_DEV'), 'Cooked', 'Science', 'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_2010Sep13T061231.334.fits') # specobjs_file = os.path.join(os.getenv('PYPEIT_DEV'), 'REDUX_OUT', 'keck_deimos', # '830G_M_8500', 'Science', # 'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_2010Sep13T061231.334.fits') sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file) # Init at null for sobj in sobjs: sobj.MASKDEF_OBJNAME = None sobj.RA = None sobj.DEC = None # Run me slits.assign_maskinfo(sobjs, det_par['platescale']) # Test assert sobjs[sobjs.SLITID == 496].MASKDEF_OBJNAME == 'ero89', 'Wrong MASKDEF_OBJNAME' assert sobjs[sobjs.SLITID == 496].RA == 352.27471667, 'Wrong object RA' assert sobjs[sobjs.SLITID == 496].DEC == -3.09223056, 'Wrong object DEC' # Write sobjs sobjs.write_to_fits({}, data_path('tst_sobjs.fits')) os.remove(data_path('tst_sobjs.fits'))
def test_process_multidet(): files = glob.glob( os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'gemini_gmos', 'GN_HAM_R400_885', 'N20190205S024*.fits')) files.sort() spec = load_spectrograph('gemini_gmos_north_ham') frame_par = spec.default_pypeit_par()['calibrations']['biasframe'] det = 1 bias_img_det1 = buildimage.buildimage_fromlist(spec, det, frame_par, files) det = (1, 2, 3) bias_img = buildimage.buildimage_fromlist(spec, det, frame_par, files) assert np.array_equal(bias_img_det1.image, bias_img.image[0]) \ and not np.array_equal(bias_img_det1.image, bias_img.image[1]) \ and not np.array_equal(bias_img_det1.image, bias_img.image[2]), \ 'Bad multi-detector processing'
def test_process(deimos_flat_files): keck_deimos = load_spectrograph('keck_deimos') par = keck_deimos.default_pypeit_par() par['calibrations']['traceframe']['process']['use_biasimage'] = False # Instantiate traceImage = buildimage.buildimage_fromlist(keck_deimos, 1, par['calibrations']['traceframe'], deimos_flat_files) # Run assert isinstance(traceImage.image, np.ndarray) for key in ['subtract_overscan', 'apply_gain']: assert key in traceImage.process_steps
def test_combine(deimos_flat_files): spectograph = load_spectrograph('keck_deimos') par = spectograph.default_pypeit_par() par['calibrations']['pixelflatframe']['process']['use_biasimage'] = False # DEIMOS deimos_flat = buildimage.buildimage_fromlist( spectograph, 3, par['calibrations']['pixelflatframe'], deimos_flat_files) # Process steps # Test assert deimos_flat.image.shape == (4096, 2048)
def get_tiltimg(self): """ Load or generate the Tilt image Requirements: master_key, det, par Args: Returns: `numpy.ndarray`_: :attr:`mstilt` image """ # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep tilt_files, self.master_key_dict['tilt'] = self._prep_calibrations( 'tilt') masterframe_name = masterframe.construct_file_name( buildimage.TiltImage, self.master_key_dict['tilt'], master_dir=self.master_dir) # Reuse master frame? if os.path.isfile(masterframe_name) and self.reuse_masters: self.mstilt = buildimage.TiltImage.from_file(masterframe_name) elif len(tilt_files) == 0: msgs.warn("No frametype=tilt files to build tiltimg") return else: # Build msgs.info("Preparing a master {0:s} frame".format( buildimage.TiltImage.master_type)) # NOTE: Slits passed for the spatial flexure correction self.mstilt = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['tiltframe'], tilt_files, bias=self.msbias, bpm=self.msbpm, dark=self.msdark, slits=self.slits) # Save to Masters self.mstilt.to_master_file(masterframe_name) # TODO in the future add in a tilt_inmask #self._update_cache('tilt', 'tilt_inmask', self.mstilt_inmask) # Return return self.mstilt
def get_dark(self): """ Load or generate the dark image Returns: :class:`~pypeit.images.buildimage.DarkImage`: The combined dark image. """ # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep dark_files, self.master_key_dict['dark'] = self._prep_calibrations( 'dark') # Construct the name, in case we need it masterframe_name = masterframe.construct_file_name( buildimage.DarkImage, self.master_key_dict['dark'], master_dir=self.master_dir) # Try to load? if os.path.isfile(masterframe_name) and self.reuse_masters: self.msdark = buildimage.DarkImage.from_file(masterframe_name) elif len(dark_files) == 0: self.msdark = None else: # TODO: If a bias has been constructed and it will be subtracted # from the science images, it should also be subtracted from this # image. If it isn't, subtracting the dark will effectively lead to # subtracting the bias twice. # TODO: The order is such that the bpm doesn't exist yet. But # calling buildimage_fromlist will create the bpm if it isn't # passed. So calling get_dark then get_bpm unnecessarily creates # the bpm twice. Is there any reason why creation of the bpm should # come after the dark, or can we change the order? # Build and save it self.msdark = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['darkframe'], dark_files, bias=self.msbias) self.msdark.to_master_file(masterframe_name) # Return return self.msdark
def test_lris_red_biases(): path = pathlib.Path(os.getenv('PYPEIT_DEV')).absolute() / 'RAW_DATA' / 'keck_lris_red' \ / 'multi_600_5000_d560' files = [ str(path / f) for f in [ 'LR.20170324.06731.fits.gz', 'LR.20170324.06791.fits.gz', 'LR.20170324.06849.fits.gz', 'LR.20170324.06908.fits.gz', 'LR.20170324.06967.fits.gz' ] ] spectrograph = load_spectrograph('keck_lris_red') par = spectrograph.default_pypeit_par() bias = buildimage.buildimage_fromlist(spectrograph, 1, par['calibrations']['biasframe'], files)
def get_arc(self): """ Load or generate the Arc image Requirements: master_key, det, par Args: Returns: `numpy.ndarray`_: :attr:`msarc` image """ # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep arc_files, self.master_key_dict['arc'] = self._prep_calibrations('arc') masterframe_name = masterframe.construct_file_name( buildimage.ArcImage, self.master_key_dict['arc'], master_dir=self.master_dir) # Reuse master frame? if os.path.isfile(masterframe_name) and self.reuse_masters: self.msarc = buildimage.ArcImage.from_file(masterframe_name) elif len(arc_files) == 0: msgs.warn("No frametype=arc files to build arc") return else: # Build it msgs.info("Preparing a master {0:s} frame".format( buildimage.ArcImage.master_type)) self.msarc = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['arcframe'], arc_files, bias=self.msbias, bpm=self.msbpm, dark=self.msdark) # Save self.msarc.to_master_file(masterframe_name) # Return return self.msarc
def test_combine_deimos_flats(): deimos_flat_files = [ os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'keck_deimos', '830G_L_8400', ifile) for ifile in ['d0914_0014.fits.gz', 'd0914_0015.fits.gz'] ] assert len(deimos_flat_files) == 2, 'Incorrect number of files.' spectrograph = load_spectrograph('keck_deimos') par = spectrograph.default_pypeit_par() par['calibrations']['pixelflatframe']['process']['use_biasimage'] = False # DEIMOS deimos_flat = buildimage.buildimage_fromlist( spectrograph, 3, par['calibrations']['pixelflatframe'], deimos_flat_files) # Process steps # Test assert deimos_flat.image.shape == (4096, 2048)
def get_bias(self): """ Load or generate the bias frame/command Requirements: master_key, det, par Returns: :class:`pypeit.images.buildimage.BiasImage`: """ # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep bias_files, self.master_key_dict['bias'] = self._prep_calibrations( 'bias') # Construct the name, in case we need it masterframe_name = masterframe.construct_file_name( buildimage.BiasImage, self.master_key_dict['bias'], master_dir=self.master_dir) if self.par['biasframe']['useframe'] is not None: msgs.error("Not ready to load from disk") # Try to load? if os.path.isfile(masterframe_name) and self.reuse_masters: self.msbias = buildimage.BiasImage.from_file(masterframe_name) elif len(bias_files) == 0: self.msbias = None else: # Build it self.msbias = buildimage.buildimage_fromlist( self.spectrograph, self.det, self.par['biasframe'], bias_files) # Save it? self.msbias.to_master_file(masterframe_name) # Return return self.msbias
def test_from_list(shane_kast_blue_sci_files): """ Run on two frames """ det = 1 # Load calibrations pixelflat = load_kast_blue_masters(pixflat=True)[0] bpm = kast_blue.empty_bpm(shane_kast_blue_sci_files[0], det) # Do it flatImages = flatfield.FlatImages(pixelflat_norm=pixelflat) kast_par['scienceframe']['process']['use_illumflat'] = False kast_par['scienceframe']['process']['use_biasimage'] = False sciImg = buildimage.buildimage_fromlist(kast_blue, det, kast_par['scienceframe'], shane_kast_blue_sci_files, bpm=bpm, bias=None, flatimages=flatImages) # Test assert isinstance(sciImg, pypeitimage.PypeItImage)
def get_dark(self): """ Load or generate the dark image Requirements: master_key, det, par Returns: :class:`pypeit.images.buildimage.DarkImage`: """ # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep dark_files, self.master_key_dict['dark'] = self._prep_calibrations( 'dark') # Construct the name, in case we need it masterframe_name = masterframe.construct_file_name( buildimage.DarkImage, self.master_key_dict['dark'], master_dir=self.master_dir) # Try to load? if os.path.isfile(masterframe_name) and self.reuse_masters: self.msdark = buildimage.DarkImage.from_file(masterframe_name) elif len(dark_files) == 0: self.msdark = None else: # TODO: Should this include the bias? # Build it self.msdark = buildimage.buildimage_fromlist( self.spectrograph, self.det, self.par['darkframe'], dark_files) # Save it? self.msdark.to_master_file(masterframe_name) # Return return self.msdark
def test_mosaic_io(): outfile = data_path('test_bias_mosaic.fits') if os.path.isfile(outfile): os.remove(outfile) files = glob.glob( os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'gemini_gmos', 'GN_HAM_R400_885', 'N20190205S024*.fits')) files.sort() spec = load_spectrograph('gemini_gmos_north_ham') frame_par = spec.default_pypeit_par()['calibrations']['biasframe'] det = (1, 2, 3) bias = buildimage.buildimage_fromlist(spec, det, frame_par, files) bias.to_file(outfile) _bias = buildimage.BiasImage.from_file(outfile) assert isinstance(_bias.detector, Mosaic), 'Bad detector type' assert _bias.detector.ndet == bias.detector.ndet, 'Bad number of detectors' assert np.array_equal(_bias.detector.tform, bias.detector.tform), 'Bad number of detectors' # Clean up os.remove(outfile)
def test_io(kast_blue_bias_files): # outfile = masterframe.construct_file_name(buildimage.BiasImage, master_key, master_dir=master_dir) if os.path.isfile(outfile): os.remove(outfile) # Build msbias = buildimage.buildimage_fromlist(shane_kast_blue, 1, frame_par, kast_blue_bias_files) # Save as a master frame master_filename = masterframe.construct_file_name(msbias, master_key, master_dir=master_dir) msbias.to_master_file(master_filename) assert os.path.isfile(outfile), 'Error writing MasterBias' # Load master frame biasImage = buildimage.BiasImage.from_file(outfile) assert np.array_equal(biasImage.image, msbias.image), 'Image changed' assert np.array_equal(biasImage.ivar, msbias.ivar), 'Inverse-variance changed' # Clean up os.remove(outfile)
def get_flats(self): """ Load or generate a normalized pixel flat and slit illumination flat. Requires :attr:`slits`, :attr:`wavetilts`, :attr:`det`, :attr:`par`. Constructs :attr:`flatimages`. """ # Check for existing data if not self._chk_objs(['msarc', 'msbpm', 'slits', 'wv_calib']): msgs.warn( 'Must have the arc, bpm, slits, and wv_calib defined to make flats! Skipping and may crash down the line' ) self.flatimages = flatfield.FlatImages() return # Slit and tilt traces are required to flat-field the data if not self._chk_objs(['slits', 'wavetilts']): # TODO: Why doesn't this fault? msgs.warn( 'Flats were requested, but there are quantities missing necessary to ' 'create flats. Proceeding without flat fielding....') self.flatimages = flatfield.FlatImages() return # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep illum_image_files, self.master_key_dict[ 'flat'] = self._prep_calibrations('illumflat') pixflat_image_files, self.master_key_dict[ 'flat'] = self._prep_calibrations('pixelflat') masterframe_filename = masterframe.construct_file_name( flatfield.FlatImages, self.master_key_dict['flat'], master_dir=self.master_dir) # The following if-elif-else does: # 1. Try to load a MasterFrame (if reuse_masters is True). If successful, pass it back # 2. Build from scratch # 3. Load any user-supplied images to over-ride any built # Load MasterFrame? if os.path.isfile(masterframe_filename) and self.reuse_masters: flatimages = flatfield.FlatImages.from_file(masterframe_filename) flatimages.is_synced(self.slits) # Load user defined files if self.par['flatfield']['pixelflat_file'] is not None: # Load msgs.info( 'Using user-defined file: {0}'.format('pixelflat_file')) with fits.open(self.par['flatfield']['pixelflat_file']) as hdu: nrm_image = flatfield.FlatImages( pixelflat_norm=hdu[self.det].data) flatimages = flatfield.merge(flatimages, nrm_image) self.flatimages = flatimages # update slits self.slits.mask_flats(self.flatimages) return self.flatimages # Generate the image pixelflatImages, illumflatImages = None, None # Check if the image files are the same pix_is_illum = Counter(illum_image_files) == Counter( pixflat_image_files) if len(pixflat_image_files) > 0: pixel_flat = buildimage.buildimage_fromlist( self.spectrograph, self.det, self.par['pixelflatframe'], pixflat_image_files, dark=self.msdark, bias=self.msbias, bpm=self.msbpm) # Initialise the pixel flat pixelFlatField = flatfield.FlatField(pixel_flat, self.spectrograph, self.par['flatfield'], self.slits, self.wavetilts, self.wv_calib) # Generate pixelflatImages = pixelFlatField.run(show=self.show) # Only build illum_flat if the input files are different from the pixel flat if (not pix_is_illum) and len(illum_image_files) > 0: illum_flat = buildimage.buildimage_fromlist( self.spectrograph, self.det, self.par['illumflatframe'], illum_image_files, dark=self.msdark, bias=self.msbias, bpm=self.msbpm) # Initialise the pixel flat illumFlatField = flatfield.FlatField(illum_flat, self.spectrograph, self.par['flatfield'], self.slits, self.wavetilts, self.wv_calib, spat_illum_only=True) # Generate illumflatImages = illumFlatField.run(show=self.show) # Merge the illum flat with the pixel flat if pixelflatImages is not None: # Combine the pixelflat and illumflat parameters into flatimages. # This will merge the attributes of pixelflatImages that are not None # with the attributes of illflatImages that are not None. Default is # to take pixelflatImages. flatimages = flatfield.merge(pixelflatImages, illumflatImages) else: # No pixel flat, but there might be an illumflat. This will mean that # the attributes prefixed with 'pixelflat_' will all be None. flatimages = illumflatImages # Save flat images if flatimages is not None: flatimages.to_master_file(masterframe_filename) # Save slits too, in case they were tweaked self.slits.to_master_file() # 3) Load user-supplied images # NOTE: This is the *final* images, not just a stack # And it will over-ride what is generated below (if generated) if self.par['flatfield']['pixelflat_file'] is not None: # Load msgs.info('Using user-defined file: {0}'.format('pixelflat_file')) with fits.open(self.par['flatfield']['pixelflat_file']) as hdu: flatimages = flatfield.merge( flatimages, flatfield.FlatImages(pixelflat_norm=hdu[self.det].data)) self.flatimages = flatimages # Return return self.flatimages
def test_assign_maskinfo_add_missing(): instr_names = ['keck_deimos', 'keck_mosfire'] for name in instr_names: # Spectrograph instrument = load_spectrograph(name) par = instrument.default_pypeit_par() # working only on detector 3 (det=3 for DEIMOS. For MOSFIRE does not matter because we have only one det) det = 3 if name == 'keck_deimos' else 1 # Built trace image traceImage = buildimage.buildimage_fromlist( instrument, det, par['calibrations']['traceframe'], flat_files(instr=name)) # load specific config parameters par = instrument.config_specific_par(traceImage.files[0]) # Run edge trace edges = edgetrace.EdgeTraceSet(traceImage, instrument, par['calibrations']['slitedges'], auto=True, debug=False, show_stages=False, qa_path=None) slits = edges.get_slits() # Test that the maskfile is saved properly hdul = fits.open(slits.maskfile) det_par = instrument.get_detector_par(det, hdu=hdul) if name == 'keck_deimos': specobjs_file = os.path.join( os.getenv('PYPEIT_DEV'), 'Cooked', 'Science', 'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_20100913T061231.334.fits' ) sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file) # correct value slitid = sobjs[sobjs.MASKDEF_OBJNAME == 'ero89'].SLITID[0] true_maskdef_objname = sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[0] true_ra = round(sobjs[sobjs.SLITID == slitid].RA[0], 6) true_dec = round(sobjs[sobjs.SLITID == slitid].DEC[0], 6) true_spat_pixpos = round( sobjs[sobjs.MASKDEF_OBJNAME == 'ero884'].SPAT_PIXPOS[0]) true_spat_pixpos_2 = round( sobjs[sobjs.MASKDEF_OBJNAME == 'ero191'].SPAT_PIXPOS[0]) elif name == 'keck_mosfire': specobjs_file = os.path.join( os.getenv('PYPEIT_DEV'), 'Cooked', 'Science', 'spec1d_m191014_0170-2M2228_12_MOSFIRE_20191014T095212.598.fits' ) sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file) # correct value slitid = sobjs[sobjs.MASKDEF_OBJNAME == '18'].SLITID[0] true_maskdef_objname = sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[0] true_ra = round(sobjs[sobjs.SLITID == slitid].RA[0], 6) true_dec = round(sobjs[sobjs.SLITID == slitid].DEC[0], 6) true_spat_pixpos = round( sobjs[sobjs.MASKDEF_OBJNAME == '7'].SPAT_PIXPOS[0]) # Init at null and remove the force extraction idx_remove = [] for i, sobj in enumerate(sobjs): if sobj.MASKDEF_EXTRACT: idx_remove.append(i) else: sobj.MASKDEF_ID = None sobj.MASKDEF_OBJNAME = None sobj.RA = None sobj.DEC = None sobj.MASKDEF_EXTRACT = None sobjs.remove_sobj(idx_remove) # get the dither offset if available if name == 'keck_deimos': dither_off = None elif name == 'keck_mosfire': dither_off = instrument.parse_dither_pattern([ os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'keck_mosfire', 'J_multi', 'm191014_0170.fits') ])[2][0] # get object positions from slitmask design and slitmask offsets calib_slits = slittrace.get_maskdef_objpos_offset_alldets( sobjs, [slits], [None], [det_par['platescale']], par['calibrations']['slitedges']['det_buffer'], par['reduce']['slitmask'], dither_off=dither_off) # determine if slitmask offsets exist and compute an average offsets over all the detectors calib_slits = slittrace.average_maskdef_offset( calib_slits, det_par['platescale'], instrument.list_detectors()) # slitmask design matching and add undetected objects sobjs = slittrace.assign_addobjs_alldets( sobjs, calib_slits, [None], [det_par['platescale']], par['reduce']['slitmask'], par['reduce']['findobj']['find_fwhm']) # Test if name == 'keck_deimos': # Check if recover the maskdef assignment assert sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[ 0] == true_maskdef_objname, 'Wrong DEIMOS MASKDEF_OBJNAME' assert round(sobjs[sobjs.SLITID == slitid].RA[0], 6) == true_ra, 'Wrong object DEIMOS RA' assert round(sobjs[sobjs.SLITID == slitid].DEC[0], 6) == true_dec, 'Wrong object DEIMOS DEC' # Test that undetected objects are found at the correct location (the correct location is # verified by visual inspection) assert round(sobjs[sobjs.MASKDEF_OBJNAME == 'ero884'].SPAT_PIXPOS[0]) == true_spat_pixpos, \ 'Wrong object (ero884) location on the DEIMOS slit' assert round(sobjs[sobjs.MASKDEF_OBJNAME == 'ero191'].SPAT_PIXPOS[0]) == true_spat_pixpos_2, \ 'Wrong object (ero191) location on the DEIMOS slit' elif name == 'keck_mosfire': # Check if recover the maskdef assignment assert sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[ 0] == true_maskdef_objname, 'Wrong MOSFIRE MASKDEF_OBJNAME' assert round(sobjs[sobjs.SLITID == slitid].RA[0], 6) == true_ra, 'Wrong object MOSFIRE RA' assert round(sobjs[sobjs.SLITID == slitid].DEC[0], 6) == true_dec, 'Wrong object MOSFIRE DEC' # Test that undetected object are found at the correct location (the correct location is # verified by visual inspection) assert round(sobjs[sobjs.MASKDEF_OBJNAME == '7'].SPAT_PIXPOS[0]) == true_spat_pixpos, \ 'Wrong object (7) location on the MOSFIRE slit' # Write sobjs sobjs.write_to_fits({}, data_path('tst_sobjs.fits')) os.remove(data_path('tst_sobjs.fits'))
def main(args): # Build the fitstable since we currently need it for output. This should not be the case! A_files = [os.path.join(args.full_rawpath, file) for file in args.Afiles] B_files = [os.path.join(args.full_rawpath, file) for file in args.Bfiles] data_files = A_files + B_files ps = pypeitsetup.PypeItSetup(A_files, path='./', spectrograph_name='keck_mosfire') ps.build_fitstbl() fitstbl = ps.fitstbl # Read in the spectrograph, config the parset spectrograph = load_spectrograph('keck_mosfire') spectrograph_def_par = spectrograph.default_pypeit_par() parset = par.PypeItPar.from_cfg_lines( cfg_lines=spectrograph_def_par.to_config(), merge_with=config_lines(args)) science_path = os.path.join(parset['rdx']['redux_path'], parset['rdx']['scidir']) # Calibration Master directory if args.master_dir is None: msgs.error( "You need to set an Environmental variable MOSFIRE_MASTERS that points at the Master Calibs" ) # Define some hard wired master files here to be later parsed out of the directory slit_masterframe_name = os.path.join(args.master_dir, 'MasterSlits_E_15_01.fits.gz') tilts_masterframe_name = os.path.join(args.master_dir, 'MasterTilts_E_1_01.fits') wvcalib_masterframe_name = os.path.join(args.master_dir, 'MasterWaveCalib_E_1_01.fits') # For now don't require a standard std_outfile = None #std_outfile = os.path.join('/Users/joe/Dropbox/PypeIt_Redux/MOSFIRE/Nov19/quicklook/Science/', # 'spec1d_m191118_0064-GD71_MOSFIRE_2019Nov18T104704.507.fits') # make the get_std from pypeit a utility function or class method det = 1 # MOSFIRE has a single detector if std_outfile is not None: # Get the standard trace if need be sobjs = specobjs.SpecObjs.from_fitsfile(std_outfile) this_det = sobjs.DET == det if np.any(this_det): sobjs_det = sobjs[this_det] sobjs_std = sobjs_det.get_std() std_trace = None if sobjs_std is None else sobjs_std.TRACE_SPAT.flatten( ) else: std_trace = None else: std_trace = None # Read in the msbpm sdet = get_dnum(det, prefix=False) msbpm = spectrograph.bpm(A_files[0], det) # Read in the slits slits = slittrace.SlitTraceSet.from_file(slit_masterframe_name) # Reset the bitmask slits.mask = slits.mask_init.copy() # Read in the wv_calib wv_calib = wavecalib.WaveCalib.from_file(wvcalib_masterframe_name) wv_calib.is_synced(slits) slits.mask_wvcalib(wv_calib) # Read in the tilts tilts_obj = wavetilts.WaveTilts.from_file(tilts_masterframe_name) tilts_obj.is_synced(slits) slits.mask_wavetilts(tilts_obj) # Build Science image sciImg = buildimage.buildimage_fromlist(spectrograph, det, parset['scienceframe'], A_files, bpm=msbpm, slits=slits, ignore_saturation=False) # Background Image? sciImg = sciImg.sub( buildimage.buildimage_fromlist(spectrograph, det, parset['scienceframe'], B_files, bpm=msbpm, slits=slits, ignore_saturation=False), parset['scienceframe']['process']) # Build the Calibrate object caliBrate = calibrations.Calibrations(None, parset['calibrations'], spectrograph, None) caliBrate.slits = slits caliBrate.wavetilts = tilts_obj caliBrate.wv_calib = wv_calib # Instantiate Reduce object # Required for pypeline specific object # At instantiaton, the fullmask in self.sciImg is modified redux = reduce.Reduce.get_instance(sciImg, spectrograph, parset, caliBrate, 'science', ir_redux=True, show=args.show, det=det, std_outfile=std_outfile) manual_extract_dict = None skymodel, objmodel, ivarmodel, outmask, sobjs, waveImg, tilts = redux.run( std_trace=std_trace, return_negative=True, manual_extract_dict=manual_extract_dict, show_peaks=args.show) # TODO -- Do this upstream # Tack on detector for sobj in sobjs: sobj.DETECTOR = sciImg.detector # Construct the Spec2DObj with the positive image spec2DObj_A = spec2dobj.Spec2DObj(det=det, sciimg=sciImg.image, ivarraw=sciImg.ivar, skymodel=skymodel, objmodel=objmodel, ivarmodel=ivarmodel, waveimg=waveImg, bpmmask=outmask, detector=sciImg.detector, sci_spat_flexure=sciImg.spat_flexure, tilts=tilts, slits=copy.deepcopy(caliBrate.slits)) spec2DObj_A.process_steps = sciImg.process_steps all_spec2d = spec2dobj.AllSpec2DObj() all_spec2d['meta']['ir_redux'] = True all_spec2d[det] = spec2DObj_A # Save image A but with all the objects extracted, i.e. positive and negative #outfile2d, outfile1d = save_exposure(fitstbl, 0, spectrograph, science_path, parset, caliBrate, all_spec2d, sobjs) # Construct the Spec2DObj with the negative image spec2DObj_B = spec2dobj.Spec2DObj(det=det, sciimg=-sciImg.image, ivarraw=sciImg.ivar, skymodel=-skymodel, objmodel=-objmodel, ivarmodel=ivarmodel, waveimg=waveImg, bpmmask=outmask, detector=sciImg.detector, sci_spat_flexure=sciImg.spat_flexure, tilts=tilts, slits=copy.deepcopy(caliBrate.slits)) # Parse the offset information out of the headers. TODO in the future get this out of fitstable dither_pattern_A, dither_id_A, offset_arcsec_A = parse_dither_pattern( A_files, spectrograph.primary_hdrext) dither_pattern_B, dither_id_B, offset_arcsec_B = parse_dither_pattern( B_files, spectrograph.primary_hdrext) # Print out a report on the offsets msg_string = msgs.newline( ) + '****************************************************' msg_string += msgs.newline( ) + ' Summary of offsets for dither pattern: {:s}'.format( dither_pattern_A[0]) msg_string += msgs.newline( ) + '****************************************************' msg_string += msgs.newline( ) + 'Position filename arcsec pixels ' msg_string += msgs.newline( ) + '----------------------------------------------------' for iexp, file in enumerate(A_files): msg_string += msgs.newline( ) + ' A {:s} {:6.2f} {:6.2f}'.format( os.path.basename(file), offset_arcsec_A[iexp], offset_arcsec_A[iexp] / sciImg.detector.platescale) for iexp, file in enumerate(B_files): msg_string += msgs.newline( ) + ' B {:s} {:6.2f} {:6.2f}'.format( os.path.basename(file), offset_arcsec_B[iexp], offset_arcsec_B[iexp] / sciImg.detector.platescale) msg_string += msgs.newline( ) + '****************************************************' msgs.info(msg_string) #offset_dith_pix = offset_dith_pix = offset_arcsec_A[0]/sciImg.detector.platescale offsets_dith_pix = (np.array([ 0.0, np.mean(offset_arcsec_B) - np.mean(offset_arcsec_A) ])) / sciImg.detector.platescale if args.offset is not None: offsets_pixels = np.array([0.0, args.offset]) msgs.info('Using user specified offsets instead: {:5.2f}'.format( args.offset)) else: offsets_pixels = offsets_dith_pix spec2d_list = [spec2DObj_A, spec2DObj_B] # Instantiate Coadd2d coadd = coadd2d.CoAdd2D.get_instance(spec2d_list, spectrograph, parset, det=det, offsets=offsets_pixels, weights='uniform', ir_redux=True, debug=args.show, samp_fact=args.samp_fact) # Coadd the slits coadd_dict_list = coadd.coadd( only_slits=None, interp_dspat=False) # TODO implement only_slits later # Create the pseudo images pseudo_dict = coadd.create_pseudo_image(coadd_dict_list) ########################## # Now display the images # ########################## display.display.connect_to_ginga(raise_err=True, allow_new=True) # Bug in ginga prevents me from using cuts here for some reason #mean, med, sigma = sigma_clipped_stats(pseudo_dict['imgminsky'][pseudo_dict['inmask']], sigma_lower=5.0,sigma_upper=5.0) #cut_min = mean - 4.0 * sigma #cut_max = mean + 4.0 * sigma chname_skysub = 'skysub-det{:s}'.format(sdet) # Clear all channels at the beginning # TODO: JFH For some reason Ginga crashes when I try to put cuts in here. viewer, ch = ginga.show_image(pseudo_dict['imgminsky'], chname=chname_skysub, waveimg=pseudo_dict['waveimg'], clear=True) # cuts=(cut_min, cut_max), slit_left, slit_righ, _ = pseudo_dict['slits'].select_edges() slit_id = slits.slitord_id[0] ginga.show_slits(viewer, ch, slit_left, slit_righ, slit_ids=slit_id) # SKRESIDS chname_skyresids = 'sky_resid-det{:s}'.format(sdet) image = pseudo_dict['imgminsky'] * np.sqrt( pseudo_dict['sciivar']) * pseudo_dict['inmask'] # sky residual map viewer, ch = ginga.show_image( image, chname_skyresids, waveimg=pseudo_dict['waveimg'], cuts=(-5.0, 5.0), ) ginga.show_slits(viewer, ch, slit_left, slit_righ, slit_ids=slits.slitord_id[0]) shell = viewer.shell() out = shell.start_global_plugin('WCSMatch') out = shell.call_global_plugin_method('WCSMatch', 'set_reference_channel', [chname_skyresids], {}) if args.embed: embed() return 0
def get_flats(self): """ Load or generate a normalized pixel flat and slit illumination flat. Requires :attr:`slits`, :attr:`wavetilts`, :attr:`det`, :attr:`par`. Returns: :class:`pypeit.flatfield.FlatImages`: """ # Check for existing data if not self._chk_objs(['msarc', 'msbpm', 'slits', 'wv_calib']): msgs.warn( 'Must have the arc, bpm, slits, and wv_calib defined to make flats! Skipping and may crash down the line' ) self.flatimages = flatfield.FlatImages(None, None, None, None) return # Slit and tilt traces are required to flat-field the data if not self._chk_objs(['slits', 'wavetilts']): # TODO: Why doesn't this fault? msgs.warn( 'Flats were requested, but there are quantities missing necessary to ' 'create flats. Proceeding without flat fielding....') self.flatimages = flatfield.FlatImages(None, None, None, None) return # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep illum_image_files, self.master_key_dict[ 'flat'] = self._prep_calibrations('illumflat') pixflat_image_files, self.master_key_dict[ 'flat'] = self._prep_calibrations('pixelflat') masterframe_filename = masterframe.construct_file_name( flatfield.FlatImages, self.master_key_dict['flat'], master_dir=self.master_dir) # The following if-elif-else does: # 1. Try to load a MasterFrame (if reuse_masters is True). If successful, pass it back # 2. Build from scratch # 3. Load any user-supplied images to over-ride any built # Load MasterFrame? if os.path.isfile(masterframe_filename) and self.reuse_masters: self.flatimages = flatfield.FlatImages.from_file( masterframe_filename) self.flatimages.is_synced(self.slits) self.slits.mask_flats(self.flatimages) return self.flatimages # TODO -- Allow for separate pixelflat and illumflat images # Generate the image stacked_flat = None if len(illum_image_files) > 0: stacked_flat = buildimage.buildimage_fromlist( self.spectrograph, self.det, self.par['illumflatframe'], illum_image_files, dark=self.msdark, bias=self.msbias, bpm=self.msbpm) if stacked_flat is None and len(pixflat_image_files) > 0: stacked_flat = buildimage.buildimage_fromlist( self.spectrograph, self.det, self.par['pixelflatframe'], pixflat_image_files, dark=self.msdark, bias=self.msbias, bpm=self.msbpm) if stacked_flat is not None: # Create pixelflat and illumination flat from illumination flat stack flatField = flatfield.FlatField(stacked_flat, self.spectrograph, self.par['flatfield'], self.slits, self.wavetilts) # Run self.flatimages = flatField.run(show=self.show) # Save to Masters self.flatimages.to_master_file(masterframe_filename) # Save slits too, in case they were tweaked self.slits.to_master_file() else: self.flatimages = flatfield.FlatImages(None, None, None, None) # 3) Load user-supplied images # NOTE: This is the *final* images, not just a stack # And it will over-ride what is generated below (if generated) if self.par['flatfield']['pixelflat_file'] is not None: # - Name is explicitly correct? # THIS IS DONE IN PYPEITPAR #if os.path.isfile(self.par['flatfield']['pixelflat_file']): # flat_file = self.par['flatfield']['pixelflat_file'] #else: # msgs.error('Could not find user-defined flatfield file: {0}'.format( # self.par['flatfield']['pixelflat_file'])) # Load msgs.info('Using user-defined file: {0}'.format('pixelflat_file')) with fits.open(self.par['flatfield']['pixelflat_file']) as hdu: self.flatimages.pixelflat = hdu[self.det].data # Return return self.flatimages
def get_slits(self): """ Load or generate the definition of the slit boundaries. Internals that must be available are :attr:`fitstbl`, :attr:`calib_ID`, :attr:`det`. Returns: :class:`pypeit.slittrace.SlitTraceSet`: Traces of the slit edges; also kept internally as :attr:`slits`. """ # Check for existing data if not self._chk_objs(['msbpm']): return # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep trace_image_files, self.master_key_dict[ 'trace'] = self._prep_calibrations('trace') # Reuse master frame? slit_masterframe_name = masterframe.construct_file_name( slittrace.SlitTraceSet, self.master_key_dict['trace'], master_dir=self.master_dir) if os.path.isfile(slit_masterframe_name) and self.reuse_masters: self.slits = slittrace.SlitTraceSet.from_file( slit_masterframe_name) # Reset the bitmask self.slits.mask = self.slits.mask_init.copy() else: # Slits don't exist or we're not resusing them edge_masterframe_name = masterframe.construct_file_name( edgetrace.EdgeTraceSet, self.master_key_dict['trace'], master_dir=self.master_dir) # Reuse master frame? if os.path.isfile(edge_masterframe_name) and self.reuse_masters: self.edges = edgetrace.EdgeTraceSet.from_file( edge_masterframe_name) elif len(trace_image_files) == 0: msgs.warn("No frametype=trace files to build slits") return None else: # Build the trace image self.traceImage = buildimage.buildimage_fromlist( self.spectrograph, self.det, self.par['traceframe'], trace_image_files, bias=self.msbias, bpm=self.msbpm, dark=self.msdark) self.edges = edgetrace.EdgeTraceSet(self.traceImage, self.spectrograph, self.par['slitedges'], bpm=self.msbpm, auto=True) self.edges.save(edge_masterframe_name, master_dir=self.master_dir, master_key=self.master_key_dict['trace']) # Show the result if requested if self.show: self.edges.show(in_ginga=True) # Get the slits from the result of the edge tracing, delete # the edges object, and save the slits, if requested self.slits = self.edges.get_slits() self.edges = None self.slits.to_master_file(slit_masterframe_name) # User mask? if self.slitspat_num is not None: self.slits.user_mask(self.det, self.slitspat_num) return self.slits
def main(args): import time import os import numpy as np from pypeit.spectrographs.util import load_spectrograph from pypeit import edgetrace from pypeit import slittrace from pypeit.pypeit import PypeIt from pypeit.images import buildimage from pypeit import masterframe from IPython import embed if args.pypeit_file is not None: pypeit_file = args.pypeit_file if not os.path.isfile(pypeit_file): raise FileNotFoundError('File does not exist: {0}'.format(pypeit_file)) pypeit_file = os.path.abspath(pypeit_file) redux_path = os.path.abspath(os.path.split(pypeit_file)[0] if args.redux_path is None else args.redux_path) rdx = PypeIt(pypeit_file, redux_path=redux_path) detectors = rdx.par['rdx']['detnum'] if args.detector is None else args.detectors # Save the spectrograph spec = rdx.spectrograph # Get the calibration group to use group = np.unique(rdx.fitstbl['calib'])[0] if args.group is None else args.group if group not in np.unique(rdx.fitstbl['calib']): raise ValueError('Not a valid calibration group: {0}'.format(group)) # Find the rows in the metadata table with trace frames in the # specified calibration group tbl_rows = rdx.fitstbl.find_frames('trace', calib_ID=int(group), index=True) # Master keyword master_key_base = '_'.join(rdx.fitstbl.master_key(tbl_rows[0]).split('_')[:2]) # Save the binning binning = rdx.fitstbl['binning'][tbl_rows[0]] # Save the full file paths files = rdx.fitstbl.frame_paths(tbl_rows) # Trace image processing parameters proc_par = rdx.par['calibrations']['traceframe'] # Slit tracing parameters trace_par = rdx.par['calibrations']['slitedges'] # Get the bias files, if requested bias_rows = rdx.fitstbl.find_frames('bias', calib_ID=int(group), index=True) bias_files = rdx.fitstbl.frame_paths(bias_rows) bias_par = rdx.par['calibrations']['biasframe'] if len(bias_files) == 0: bias_files = None # Set the QA path qa_path = rdx.qa_path else: detectors = args.detector spec = load_spectrograph(args.spectrograph) master_key_base = 'A_1' binning = '1,1' if args.binning is None else args.binning if not os.path.isfile(args.trace_file): raise FileNotFoundError('File does not exist: {0}'.format(args.trace_file)) files = [os.path.abspath(args.trace_file)] redux_path = os.path.abspath(os.path.split(files[0])[0] if args.redux_path is None else args.redux_path) par = spec.default_pypeit_par() proc_par = par['calibrations']['traceframe'] trace_par = par['calibrations']['slitedges'] bias_files = None bias_par = None # Set the QA path qa_path = os.path.join(os.path.abspath(os.path.split(files[0])[0]), 'QA') if detectors is None: detectors = np.arange(spec.ndet)+1 if isinstance(detectors, int): detectors = [detectors] master_dir = os.path.join(redux_path, args.master_dir) for det in detectors: # Master keyword for output file name master_key = '{0}_{1}'.format(master_key_base, str(det).zfill(2)) # Get the bias frame if requested if bias_files is None: proc_par['process']['bias'] = 'skip' msbias = None else: #biasFrame = biasframe.BiasFrame(spec, files=bias_files, det=det, par=bias_par, # master_key=master_key, master_dir=master_dir) #msbias = biasFrame.build_image() msbias = buildimage.buildimage_fromlist(spec, det, bias_par, bias_files) msbpm = spec.bpm(files[0], det) # Build the trace image #traceImage = traceimage.TraceImage(spec, files=files, det=det, par=proc_par, bias=msbias) #traceImage.build_image(bias=msbias, bpm=msbpm) traceImage = buildimage.buildimage_fromlist(spec, det, proc_par, files, bias=msbias, bpm=msbpm) # Trace the slit edges t = time.perf_counter() edges = edgetrace.EdgeTraceSet(traceImage, spec, trace_par, det=det, bpm=msbpm, auto=True, debug=args.debug, show_stages=args.show, qa_path=qa_path) print('Tracing for detector {0} finished in {1} s.'.format(det, time.perf_counter()-t)) # Write the MasterEdges file edge_masterframe_name = masterframe.construct_file_name(edgetrace.EdgeTraceSet, master_key, master_dir=master_dir) edges.save(edge_masterframe_name, master_dir=master_dir, master_key=master_key) # Write the MasterSlits file slit_masterframe_name = masterframe.construct_file_name(slittrace.SlitTraceSet, master_key, master_dir=master_dir) edges.get_slits().to_master_file(slit_masterframe_name) #master_dir, master_key, # Naming #spec.spectrograph) # Header return 0
def get_align(self): """ Load or generate the alignment frame Requirements: master_key, det, par Returns: ndarray or str: :attr:`align` """ # Check for existing data if not self._chk_objs(['msbpm', 'tslits_dict']): msgs.error("Don't have all the objects") # Check internals self._chk_set(['det', 'calib_ID', 'par']) # Prep align_files = self._prep_calibrations('align') #align_rows = self.fitstbl.find_frames('align', calib_ID=self.calib_ID, index=True) #self.align_files = self.fitstbl.frame_paths(align_rows) #self.master_key_dict['align'] \ # = self.fitstbl.master_key(align_rows[0] if len(align_rows) > 0 else self.frame, # det=self.det) masterframe_name = masterframe.construct_file_name( buildimage.AlignImage, self.master_key_dict['align'], master_dir=self.master_dir) # Previously cahded? if self._cached('align', self.master_key_dict['align']): # Previously calculated self.msalign = self.calib_dict[self.master_key_dict['align']]['align'] elif os.path.isfile(masterframe_name) and self.reuse_masters: self.msalign = buildimage.AlignImage.from_file(masterframe_name) else: # Instantiate with everything needed to generate the image (in case we do) #self.alignFrame = alignframe.AlignFrame(self.spectrograph, files=self.align_files, # det=self.det, msbias=self.msbias, # par=self.par['alignframe'], # master_key=self.master_key_dict['align'], # master_dir=self.master_dir, # reuse_masters=self.reuse_masters) self.align = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['alignframe'], align_files, bias=self.msbias, bpm=self.msbpm) # Load the MasterFrame (if it exists and is desired)? #self.msalign = self.alignFrame.load() #if self.msalign is None: # Otherwise build it # msgs.info("Preparing a master {0:s} frame".format(self.alignFrame.master_type)) # self.msalign = self.alignFrame.build_image(bias=self.msbias, bpm=self.msbpm) # # Need to set head0 here, since a master align frame loaded from file will have head0 set. # self.msalign.head0 = self.alignFrame.build_master_header(steps=self.alignFrame.process_steps, # raw_files=self.alignFrame.file_list) # # Save to Masters # Save to Masters self.msalign.to_master_file(self.master_dir, self.master_key_dict['align'], # Naming self.spectrograph.spectrograph, # Header steps=self.msalign.process_steps, raw_files=align_files) # Store the alignment frame self._update_cache('align', 'align', self.msalign) # Check if the alignment dictionary exists if self._cached('align_dict', self.master_key_dict['align']) \ and self._cached('wtmask', self.master_key_dict['align']): self.align_dict = self.calib_dict[self.master_key_dict['align']]['align_dict'] else: # Extract some header info needed by the algorithm binning = self.spectrograph.get_meta_value(self.align_files[0], 'binning') # Instantiate self.alignment = alignframe.Alignment(self.msalign, self.tslits_dict, self.spectrograph, self.par['alignment'], det=self.det, binning=binning, master_key=self.master_key_dict['align'], master_dir=self.master_dir, reuse_masters=self.reuse_masters, qa_path=self.qa_path, msbpm=self.msbpm) # Master self.align_dict = self.alignment.load() if self.align_dict is None: self.align_dict = self.alignment.run(self.show) self.alignment.save() # Save & return self._update_cache('align', 'align_dict', self.align_dict) return self.msalign, self.align_dict
def test_add_missing_slits(): # Load instrument keck_deimos = load_spectrograph('keck_deimos') par = keck_deimos.default_pypeit_par() # working only on detector 1 det = 1 # Built trace image traceImage = buildimage.buildimage_fromlist( keck_deimos, det, par['calibrations']['traceframe'], deimos_flat_files()) msbpm = keck_deimos.bpm(traceImage.files[0], det) # load specific config parameters par = keck_deimos.config_specific_par(traceImage.files[0]) trace_par = par['calibrations']['slitedges'] # Running the EdgeTraceSet steps (one-by-one) # Initialize EdgeTraceSet edges = EdgeTraceSet(traceImage, keck_deimos, trace_par, bpm=msbpm, auto=False, debug=False, show_stages=False, qa_path=None) # Perform the initial edge detection and trace identification edges.initial_trace(bpm=msbpm) # Initial trace can result in no edges found if not edges.is_empty: # Refine the locations of the trace using centroids of the # features in the Sobel-filtered image. edges.centroid_refine() # Initial trace can result in no edges found, or centroid # refinement could have removed all traces (via `check_traces`) if not edges.is_empty: # Fit the trace locations with a polynomial edges.fit_refine(debug=False) # Use the fits to determine if there are any discontinous # trace centroid measurements that are actually components # of the same slit edge edges.merge_traces(debug=False) # Check if the PCA decomposition is possible; this should catch # long slits if edges.par['auto_pca'] and edges.can_pca(): # Use a PCA decomposition to parameterize the trace # functional forms edges.pca_refine(debug=False) # Use the results of the PCA decomposition to rectify and # detect peaks/troughs in the spectrally collapsed # Sobel-filtered image, then use those peaks to further # refine the edge traces edges.peak_refine(rebuild_pca=True, debug=False) # Check the values of the traces that will be removed # Two traces NOT from the same slit assert round(edges.edge_fit[edges.pca.reference_row, :] [8]) == 458, 'wrong left trace position' assert round(edges.edge_fit[edges.pca.reference_row, :] [-12]) == 1686, 'wrong right trace position' # Two traces from the same slit assert round(edges.edge_fit[edges.pca.reference_row, :] [10]) == 496, 'wrong left trace position' assert round(edges.edge_fit[edges.pca.reference_row, :] [11]) == 561, 'wrong right trace position' # Remove two left traces and two right traces # NOT form the same slit indx = np.zeros(edges.ntrace, dtype=bool) indx[8] = True indx[-12] = True # Form the same slit indx[10] = True indx[11] = True edges.remove_traces(edges.synced_selection(indx, mode='ignore'), rebuild_pca=True) # Run slitmask matching algorithm edges.maskdesign_matching(debug=False) # Sync left and right edges edges.sync() # Check the values of the traces that have been recovered # Two traces NOT from the same slit assert round(edges.edge_fit[edges.pca.reference_row, :] [8]) == 459, 'left trace position not recovered' assert round(edges.edge_fit[edges.pca.reference_row, :] [-12]) == 1690, 'right trace position not recovered' # Two traces from the same slit assert round(edges.edge_fit[edges.pca.reference_row, :] [10]) == 497, 'wrong left trace position' assert round(edges.edge_fit[edges.pca.reference_row, :] [11]) == 561, 'wrong right trace position'
def test_process(kast_blue_bias_files): # Instantiate bias_img = buildimage.buildimage_fromlist(shane_kast_blue, 1, frame_par, kast_blue_bias_files) assert isinstance(bias_img.image, np.ndarray)