def test_all2dobj_update_image(init_dict): # Build two spec2DObj1 = spec2dobj.Spec2DObj(**init_dict) spec2DObj2 = spec2dobj.Spec2DObj(**init_dict) spec2DObj2.det = 2 # allspec2D = spec2dobj.AllSpec2DObj() allspec2D['meta']['ir_redux'] = False allspec2D[1] = spec2DObj1 allspec2D[2] = spec2DObj2 # Write ofile = data_path('tst_allspec2d.fits') if os.path.isfile(ofile): os.remove(ofile) allspec2D.write_to_fits(ofile) # Update _allspec2D = spec2dobj.AllSpec2DObj() spec2DObj1.sciimg = spec2DObj1.sciimg.copy() * 2. _allspec2D['meta']['ir_redux'] = False _allspec2D[1] = spec2DObj1 _allspec2D.write_to_fits(ofile, update_det=1, overwrite=True) # Check allspec2D_2 = spec2dobj.AllSpec2DObj.from_fits(ofile) assert np.array_equal(allspec2D_2[1].sciimg, spec2DObj1.sciimg) os.remove(ofile)
def test_all2dobj_update_image(init_dict): allspec2D = spec2dobj.AllSpec2DObj() allspec2D['meta']['bkg_redux'] = False allspec2D['meta']['find_negative'] = False for i in range(2): d = load_spectrograph('keck_deimos').get_detector_par(i + 1) allspec2D[d.name] = spec2dobj.Spec2DObj(detector=d, **init_dict) # Write ofile = tstutils.data_path('tst_allspec2d.fits') if os.path.isfile(ofile): os.remove(ofile) allspec2D.write_to_fits(ofile) _allspec2D = spec2dobj.AllSpec2DObj() _allspec2D['meta']['bkg_redux'] = False _allspec2D['meta']['find_negative'] = False d = load_spectrograph('keck_deimos').get_detector_par(2) detname = d.name init_dict['sciimg'] = allspec2D[detname].sciimg.copy() * 2 _allspec2D[detname] = spec2dobj.Spec2DObj(detector=d, **init_dict) _allspec2D.write_to_fits(ofile, update_det=_allspec2D.detectors, overwrite=True) # Check allspec2D_2 = spec2dobj.AllSpec2DObj.from_fits(ofile) assert np.array_equal(allspec2D_2[detname].sciimg, _allspec2D[detname].sciimg), 'Bad update' assert np.array_equal(allspec2D_2[detname].sciimg, allspec2D[detname].sciimg * 2), 'Bad update' os.remove(ofile)
def test_all2dobj_write(init_dict): # Build one init_dict['detector'] = tstutils.get_kastb_detector() spec2DObj = spec2dobj.Spec2DObj(**init_dict) allspec2D = spec2dobj.AllSpec2DObj() allspec2D['meta']['bkg_redux'] = False allspec2D['meta']['find_negative'] = False detname = spec2DObj.detname allspec2D[detname] = spec2DObj # Write ofile = tstutils.data_path('tst_allspec2d.fits') if os.path.isfile(ofile): os.remove(ofile) allspec2D.write_to_fits(ofile) # Read _allspec2D = spec2dobj.AllSpec2DObj.from_fits(ofile) # Check assert allspec2D.detectors == _allspec2D.detectors, 'Bad read: detector mismatch' assert allspec2D['meta'] == _allspec2D['meta'], 'Bad read: meta mismatch' # Try to update it _allspec2D['meta']['bkg_redux'] = True _allspec2D[detname].vel_corr = 2. _allspec2D.write_to_fits(ofile, update_det='DET01') __allspec2D = spec2dobj.AllSpec2DObj.from_fits(ofile) assert __allspec2D['meta'] == _allspec2D['meta'], 'Bad read: meta mismatch' assert __allspec2D['meta'] != allspec2D['meta'], 'Bad read: meta mismatch' assert __allspec2D[detname].vel_corr == 2., 'Bad update' os.remove(ofile)
def test_all2dobj_hdr(init_dict): # Build one spec2DObj = spec2dobj.Spec2DObj(**init_dict) allspec2D = spec2dobj.AllSpec2DObj() allspec2D['meta']['ir_redux'] = False allspec2D[1] = spec2DObj # kast_file = data_path('b1.fits.gz') header = fits.getheader(kast_file) spectrograph = load_spectrograph('shane_kast_blue') # Do it hdr = allspec2D.build_primary_hdr(header, spectrograph, master_dir=data_path('')) # Test it assert hdr['SKYSUB'] == 'MODEL'
def test_all2dobj_hdr(init_dict): # Build one init_dict['detector'] = tstutils.get_kastb_detector() spec2DObj = spec2dobj.Spec2DObj(**init_dict) allspec2D = spec2dobj.AllSpec2DObj() allspec2D['meta']['bkg_redux'] = False allspec2D['meta']['find_negative'] = False allspec2D[spec2DObj.detname] = spec2DObj # kast_file = tstutils.data_path('b1.fits.gz') header = fits.getheader(kast_file) spectrograph = load_spectrograph('shane_kast_blue') # Do it hdr = allspec2D.build_primary_hdr(header, spectrograph, master_dir=tstutils.data_path('')) # Test it assert hdr['SKYSUB'] == 'MODEL'
def test_all2dobj_write(init_dict): # Build one spec2DObj = spec2dobj.Spec2DObj(**init_dict) allspec2D = spec2dobj.AllSpec2DObj() allspec2D['meta']['ir_redux'] = False allspec2D[1] = spec2DObj allspec2D[1].detector = tstutils.get_kastb_detector() # Write ofile = data_path('tst_allspec2d.fits') if os.path.isfile(ofile): os.remove(ofile) allspec2D.write_to_fits(ofile) # Read _allspec2D = spec2dobj.AllSpec2DObj.from_fits(ofile) # Write again os.remove(ofile) _allspec2D.write_to_fits(ofile) os.remove(ofile)
def main(args): """ Executes 2d coadding """ msgs.info('PATH =' + os.getcwd()) # Load the file if args.file is not None: spectrograph_name, config_lines, spec2d_files \ = io.read_spec2d_file(args.file, filetype="coadd2d") spectrograph = load_spectrograph(spectrograph_name) # Parameters # TODO: Shouldn't this reinstantiate the same parameters used in # the PypeIt run that extracted the objects? Why are we not # just passing the pypeit file? # JFH: The reason is that the coadd2dfile may want different reduction parameters # DP: I think config_specific_par() is more appropriate here. default_pypeit_par() # is included in config_specific_par() # NOTE `config_specific_par` works with the spec2d files because we construct the header # of those files to include all the relevant keywords from the raw file. spectrograph_cfg_lines = spectrograph.config_specific_par( spec2d_files[0]).to_config() parset = par.PypeItPar.from_cfg_lines( cfg_lines=spectrograph_cfg_lines, merge_with=config_lines) elif args.obj is not None: # TODO: We should probably be reading the pypeit file and using # those parameters here rather than using the default parset. # TODO: This needs to define the science path spec2d_files = glob.glob('./Science/spec2d_*' + args.obj + '*') head0 = fits.getheader(spec2d_files[0]) spectrograph_name = head0['PYP_SPEC'] spectrograph = load_spectrograph(spectrograph_name) # NOTE `config_specific_par` works with the spec2d files because we construct the header # of those files to include all the relevant keywords from the raw file. spectrograph_cfg_lines = spectrograph.config_specific_par( spec2d_files[0]).to_config() parset = par.PypeItPar.from_cfg_lines( cfg_lines=spectrograph_cfg_lines) else: return msgs.error( 'You must provide either a coadd2d file (--file) or an object name (--obj)' ) # If detector was passed as an argument override whatever was in the coadd2d_file if args.det is not None: msgs.info("Restricting reductions to detector={}".format(args.det)) # parset['rdx']['detnum'] = par.util.eval_tuple(args.det.split(',')) # TODO this needs to be adjusted if we want to pass (as inline command) mosaic detectors parset['rdx']['detnum'] = [int(d) for d in args.det.split(',')] # Get headers (if possible) and base names spec1d_files = [ files.replace('spec2d', 'spec1d') for files in spec2d_files ] head1d = None for spec1d_file in spec1d_files: if os.path.isfile(spec1d_file): head1d = fits.getheader(spec1d_file) break if head1d is None: msgs.warn( "No 1D spectra so am generating a dummy header for output") head1d = io.initialize_header() head2d = fits.getheader(spec2d_files[0]) if args.basename is None: #TODO Fix this, currently does not work if target names have - or _ filename_first = os.path.basename(spec2d_files[0]) filename_last = os.path.basename(spec2d_files[-1]) prefix_first = (filename_first.split('_')[1]).split('-')[0] prefix_last = (filename_last.split('_')[1]).split('-')[0] objname = (filename_first.split('-')[1]).split('_')[0] basename = '{:s}-{:s}-{:s}'.format(prefix_first, prefix_last, objname) else: basename = args.basename # TODO Heliocentric for coadd2d needs to be thought through. Currently turning it off. parset['calibrations']['wavelengths']['refframe'] = 'observed' # TODO Flexure correction for coadd2d needs to be thought through. Currently turning it off. parset['flexure']['spec_method'] = 'skip' # Write the par to disk par_outfile = basename + '_coadd2d.par' print("Writing the parameters to {}".format(par_outfile)) parset.to_config(par_outfile, exclude_defaults=True, include_descr=False) # Now run the coadds skysub_mode = head2d['SKYSUB'] findobj_mode = head2d['FINDOBJ'] bkg_redux = True if 'DIFF' in skysub_mode else False find_negative = True if 'NEG' in findobj_mode else False # Print status message msgs_string = 'Reducing target {:s}'.format(basename) + msgs.newline() msgs_string += 'Coadding frame sky-subtraced with {:s}'.format( skysub_mode) msgs_string += 'Searching for objects that are {:s}'.format( findobj_mode) msgs_string += msgs.newline( ) + 'Combining frames in 2d coadd:' + msgs.newline() for f, file in enumerate(spec2d_files): msgs_string += 'Exp {0}: {1:s}'.format( f, os.path.basename(file)) + msgs.newline() msgs.info(msgs_string) # TODO: This needs to be added to the parameter list for rdx redux_path = os.getcwd() master_dirname = os.path.basename(head2d['PYPMFDIR']) + '_coadd' master_dir = os.path.join(redux_path, master_dirname) # Make the new Master dir if not os.path.isdir(master_dir): msgs.info( 'Creating directory for Master output: {0}'.format(master_dir)) os.makedirs(master_dir) # Instantiate the sci_dict sci_dict = OrderedDict() # This needs to be ordered sci_dict['meta'] = {} sci_dict['meta']['vel_corr'] = 0. sci_dict['meta']['bkg_redux'] = bkg_redux sci_dict['meta']['find_negative'] = find_negative # Make QA coadd directory parset['rdx']['qadir'] += '_coadd' qa_path = os.path.join(parset['rdx']['redux_path'], parset['rdx']['qadir'], 'PNGs') if not os.path.isdir(qa_path): os.makedirs(qa_path) # Find the detectors to reduce # detectors = PypeIt.select_detectors(detnum=parset['rdx']['detnum'], ndet=spectrograph.ndet) detectors = spectrograph.select_detectors( subset=parset['rdx']['detnum']) # if len(detectors) != spectrograph.ndet: # msgs.warn('Not reducing detectors: {0}'.format(' '.join([str(d) for d in # set(np.arange(spectrograph.ndet) + 1) - set(detectors)]))) msgs.info(f'Detectors to work on: {detectors}') # Only_slits? if args.only_slits: parset['coadd2d']['only_slits'] = [ int(item) for item in args.only_slits.split(',') ] # container for specobjs all_specobjs = specobjs.SpecObjs() # container for spec2dobj all_spec2d = spec2dobj.AllSpec2DObj() # set some meta all_spec2d['meta']['bkg_redux'] = bkg_redux all_spec2d['meta']['find_negative'] = find_negative # Loop on detectors for det in detectors: msgs.info("Working on detector {0}".format(det)) # Instantiate Coadd2d coadd = coadd2d.CoAdd2D.get_instance( spec2d_files, spectrograph, parset, det=det, offsets=parset['coadd2d']['offsets'], weights=parset['coadd2d']['weights'], spec_samp_fact=args.spec_samp_fact, spat_samp_fact=args.spat_samp_fact, bkg_redux=bkg_redux, find_negative=find_negative, debug_offsets=args.debug_offsets, debug=args.debug) # TODO Add this stuff to a run method in coadd2d # Coadd the slits coadd_dict_list = coadd.coadd( only_slits=parset['coadd2d']['only_slits']) # Create the pseudo images pseudo_dict = coadd.create_pseudo_image(coadd_dict_list) # Reduce msgs.info('Running the extraction') # TODO -- This should mirror what is in pypeit.extract_one # TODO -- JFH :: This ought to return a Spec2DObj and SpecObjs which # would be slurped into AllSpec2DObj and all_specobsj, as below. # TODO -- JFH -- Check that the slits we are using are correct sci_dict[coadd.detname] = {} sci_dict[coadd.detname]['sciimg'], sci_dict[coadd.detname]['sciivar'], \ sci_dict[coadd.detname]['skymodel'], sci_dict[coadd.detname]['objmodel'], \ sci_dict[coadd.detname]['ivarmodel'], sci_dict[coadd.detname]['outmask'], \ sci_dict[coadd.detname]['specobjs'], sci_dict[coadd.detname]['detector'], \ sci_dict[coadd.detname]['slits'], sci_dict[coadd.detname]['tilts'], \ sci_dict[coadd.detname]['waveimg'] \ = coadd.reduce(pseudo_dict, show=args.show, show_peaks=args.peaks, basename=basename) # Tack on detector (similarly to pypeit.extract_one) for sobj in sci_dict[coadd.detname]['specobjs']: sobj.DETECTOR = sci_dict[coadd.detname]['detector'] # fill the specobjs container all_specobjs.add_sobj(sci_dict[coadd.detname]['specobjs']) # fill the spec2dobj container but first ... # pull out maskdef_designtab from sci_dict[det]['slits'] maskdef_designtab = sci_dict[ coadd.detname]['slits'].maskdef_designtab slits = copy.deepcopy(sci_dict[coadd.detname]['slits']) slits.maskdef_designtab = None # fill up all_spec2d[coadd.detname] = spec2dobj.Spec2DObj( sciimg=sci_dict[coadd.detname]['sciimg'], ivarraw=sci_dict[coadd.detname]['sciivar'], skymodel=sci_dict[coadd.detname]['skymodel'], objmodel=sci_dict[coadd.detname]['objmodel'], ivarmodel=sci_dict[coadd.detname]['ivarmodel'], scaleimg=np.array([1.0], dtype=np.float), bpmmask=sci_dict[coadd.detname]['outmask'], detector=sci_dict[coadd.detname]['detector'], slits=slits, waveimg=sci_dict[coadd.detname]['waveimg'], tilts=sci_dict[coadd.detname]['tilts'], sci_spat_flexure=None, sci_spec_flexure=None, vel_corr=None, vel_type=None, maskdef_designtab=maskdef_designtab) # Save pseudo image master files #coadd.save_masters() # SAVE TO DISK # Make the new Science dir # TODO: This needs to be defined by the user scipath = os.path.join(redux_path, 'Science_coadd') if not os.path.isdir(scipath): msgs.info( 'Creating directory for Science output: {0}'.format(scipath)) os.makedirs(scipath) # THE FOLLOWING MIMICS THE CODE IN pypeit.save_exposure() subheader = spectrograph.subheader_for_spec(head2d, head2d) # Write spec1D if all_specobjs.nobj > 0: outfile1d = os.path.join(scipath, 'spec1d_{:s}.fits'.format(basename)) all_specobjs.write_to_fits(subheader, outfile1d) # Info outfiletxt = os.path.join(scipath, 'spec1d_{:s}.txt'.format(basename)) sobjs = specobjs.SpecObjs.from_fitsfile(outfile1d, chk_version=False) sobjs.write_info(outfiletxt, spectrograph.pypeline) # Build header for spec2d outfile2d = os.path.join(scipath, 'spec2d_{:s}.fits'.format(basename)) pri_hdr = all_spec2d.build_primary_hdr( head2d, spectrograph, subheader=subheader, # TODO -- JFH :: Decide if we need any of these redux_path=None, master_key_dict=None, master_dir=None) # Write spec2d all_spec2d.write_to_fits(outfile2d, pri_hdr=pri_hdr)
def main(args): """ Executes 2d coadding """ msgs.warn('PATH =' + os.getcwd()) # Load the file if args.file is not None: spectrograph_name, config_lines, spec2d_files = io.read_spec2d_file( args.file, filetype="coadd2d") spectrograph = load_spectrograph(spectrograph_name) # Parameters # TODO: Shouldn't this reinstantiate the same parameters used in # the PypeIt run that extracted the objects? Why are we not # just passing the pypeit file? # JFH: The reason is that the coadd2dfile may want different reduction parameters spectrograph_def_par = spectrograph.default_pypeit_par() parset = par.PypeItPar.from_cfg_lines( cfg_lines=spectrograph_def_par.to_config(), merge_with=config_lines) elif args.obj is not None: # TODO: We should probably be reading the pypeit file and using those parameters here rather than using the # default parset. # TODO: This needs to define the science path spec2d_files = glob.glob('./Science/spec2d_*' + args.obj + '*') head0 = fits.getheader(spec2d_files[0]) spectrograph_name = head0['PYP_SPEC'] spectrograph = load_spectrograph(spectrograph_name) parset = spectrograph.default_pypeit_par() else: msgs.error( 'You must either input a coadd2d file with --file or an object name with --obj' ) # Update with configuration specific parameters (which requires science file) and initialize spectrograph spectrograph_cfg_lines = spectrograph.config_specific_par( spec2d_files[0]).to_config() parset = par.PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=parset.to_config()) # If detector was passed as an argument override whatever was in the coadd2d_file if args.det is not None: msgs.info("Restricting reductions to detector={}".format(args.det)) parset['rdx']['detnum'] = int(args.det) # Get headers (if possible) and base names spec1d_files = [ files.replace('spec2d', 'spec1d') for files in spec2d_files ] head1d = None for spec1d_file in spec1d_files: if os.path.isfile(spec1d_file): head1d = fits.getheader(spec1d_file) break if head1d is None: msgs.warn("No 1D spectra so am generating a dummy header for output") head1d = io.initialize_header() head2d = fits.getheader(spec2d_files[0]) if args.basename is None: filename = os.path.basename(spec2d_files[0]) basename = filename.split('_')[2] else: basename = args.basename # Write the par to disk par_outfile = basename + '_coadd2d.par' print("Writing the parameters to {}".format(par_outfile)) parset.to_config(par_outfile) # Now run the coadds skysub_mode = head2d['SKYSUB'] ir_redux = True if 'DIFF' in skysub_mode else False # Print status message msgs_string = 'Reducing target {:s}'.format(basename) + msgs.newline() msgs_string += 'Performing coadd of frames reduce with {:s} imaging'.format( skysub_mode) msgs_string += msgs.newline( ) + 'Combining frames in 2d coadd:' + msgs.newline() for file in spec2d_files: msgs_string += '{0:s}'.format(os.path.basename(file)) + msgs.newline() msgs.info(msgs_string) # TODO: This needs to be added to the parameter list for rdx redux_path = os.getcwd() master_dirname = os.path.basename(head2d['PYPMFDIR']) + '_coadd' master_dir = os.path.join(redux_path, master_dirname) # Make the new Master dir if not os.path.isdir(master_dir): msgs.info( 'Creating directory for Master output: {0}'.format(master_dir)) os.makedirs(master_dir) # Instantiate the sci_dict sci_dict = OrderedDict() # This needs to be ordered sci_dict['meta'] = {} sci_dict['meta']['vel_corr'] = 0. sci_dict['meta']['ir_redux'] = ir_redux # Find the detectors to reduce detectors = PypeIt.select_detectors(detnum=parset['rdx']['detnum'], ndet=spectrograph.ndet) if len(detectors) != spectrograph.ndet: msgs.warn('Not reducing detectors: {0}'.format(' '.join([ str(d) for d in set(np.arange(spectrograph.ndet) + 1) - set(detectors) ]))) # Loop on detectors for det in detectors: msgs.info("Working on detector {0}".format(det)) sci_dict[det] = {} # Instantiate Coadd2d coadd = coadd2d.CoAdd2D.get_instance( spec2d_files, spectrograph, parset, det=det, offsets=parset['coadd2d']['offsets'], weights=parset['coadd2d']['weights'], ir_redux=ir_redux, debug_offsets=args.debug_offsets, debug=args.debug, samp_fact=args.samp_fact) # Coadd the slits coadd_dict_list = coadd.coadd( only_slits=None) # TODO implement only_slits later # Create the pseudo images pseudo_dict = coadd.create_pseudo_image(coadd_dict_list) # Reduce msgs.info('Running the extraction') # TODO -- This should mirror what is in pypeit.extract_one # TODO -- JFH :: This ought to return a Spec2DObj and SpecObjs which would be slurped into # AllSpec2DObj and all_specobsj, as below. # TODO -- JFH -- Check that the slits we are using are correct sci_dict[det]['sciimg'], sci_dict[det]['sciivar'], sci_dict[det]['skymodel'], sci_dict[det]['objmodel'], \ sci_dict[det]['ivarmodel'], sci_dict[det]['outmask'], sci_dict[det]['specobjs'], sci_dict[det]['detector'], \ sci_dict[det]['slits'], sci_dict[det]['tilts'], sci_dict[det]['waveimg'] = coadd.reduce( pseudo_dict, show = args.show, show_peaks = args.peaks) # Save pseudo image master files #coadd.save_masters() # Make the new Science dir # TODO: This needs to be defined by the user scipath = os.path.join(redux_path, 'Science_coadd') if not os.path.isdir(scipath): msgs.info('Creating directory for Science output: {0}'.format(scipath)) os.makedirs(scipath) # THE FOLLOWING MIMICS THE CODE IN pypeit.save_exposure() # TODO -- These lines should be above once reduce() passes back something sensible all_specobjs = specobjs.SpecObjs() for det in detectors: all_specobjs.add_sobj(sci_dict[det]['specobjs']) # Write outfile1d = os.path.join(scipath, 'spec1d_{:s}.fits'.format(basename)) subheader = spectrograph.subheader_for_spec(head2d, head2d) all_specobjs.write_to_fits(subheader, outfile1d) # 2D spectra # TODO -- These lines should be above once reduce() passes back something sensible all_spec2d = spec2dobj.AllSpec2DObj() all_spec2d['meta']['ir_redux'] = ir_redux for det in detectors: all_spec2d[det] = spec2dobj.Spec2DObj( det=det, sciimg=sci_dict[det]['sciimg'], ivarraw=sci_dict[det]['sciivar'], skymodel=sci_dict[det]['skymodel'], objmodel=sci_dict[det]['objmodel'], ivarmodel=sci_dict[det]['ivarmodel'], scaleimg=np.array([1.0], dtype=np.float), bpmmask=sci_dict[det]['outmask'], detector=sci_dict[det]['detector'], slits=sci_dict[det]['slits'], waveimg=sci_dict[det]['waveimg'], tilts=sci_dict[det]['tilts'], sci_spat_flexure=None, sci_spec_flexure=None, vel_corr=None, vel_type=None) # Build header outfile2d = os.path.join(scipath, 'spec2d_{:s}.fits'.format(basename)) pri_hdr = all_spec2d.build_primary_hdr( head2d, spectrograph, subheader=subheader, # TODO -- JFH :: Decide if we need any of these redux_path=None, master_key_dict=None, master_dir=None) # Write all_spec2d.write_to_fits(outfile2d, pri_hdr=pri_hdr)
def main(args): # Build the fitstable since we currently need it for output. This should not be the case! A_files = [os.path.join(args.full_rawpath, file) for file in args.Afiles] B_files = [os.path.join(args.full_rawpath, file) for file in args.Bfiles] data_files = A_files + B_files ps = pypeitsetup.PypeItSetup(A_files, path='./', spectrograph_name='keck_mosfire') ps.build_fitstbl() fitstbl = ps.fitstbl # Read in the spectrograph, config the parset spectrograph = load_spectrograph('keck_mosfire') spectrograph_def_par = spectrograph.default_pypeit_par() parset = par.PypeItPar.from_cfg_lines( cfg_lines=spectrograph_def_par.to_config(), merge_with=config_lines(args)) science_path = os.path.join(parset['rdx']['redux_path'], parset['rdx']['scidir']) # Calibration Master directory if args.master_dir is None: msgs.error( "You need to set an Environmental variable MOSFIRE_MASTERS that points at the Master Calibs" ) # Define some hard wired master files here to be later parsed out of the directory slit_masterframe_name = os.path.join(args.master_dir, 'MasterSlits_E_15_01.fits.gz') tilts_masterframe_name = os.path.join(args.master_dir, 'MasterTilts_E_1_01.fits') wvcalib_masterframe_name = os.path.join(args.master_dir, 'MasterWaveCalib_E_1_01.fits') # For now don't require a standard std_outfile = None #std_outfile = os.path.join('/Users/joe/Dropbox/PypeIt_Redux/MOSFIRE/Nov19/quicklook/Science/', # 'spec1d_m191118_0064-GD71_MOSFIRE_2019Nov18T104704.507.fits') # make the get_std from pypeit a utility function or class method det = 1 # MOSFIRE has a single detector if std_outfile is not None: # Get the standard trace if need be sobjs = specobjs.SpecObjs.from_fitsfile(std_outfile) this_det = sobjs.DET == det if np.any(this_det): sobjs_det = sobjs[this_det] sobjs_std = sobjs_det.get_std() std_trace = None if sobjs_std is None else sobjs_std.TRACE_SPAT.flatten( ) else: std_trace = None else: std_trace = None # Read in the msbpm sdet = get_dnum(det, prefix=False) msbpm = spectrograph.bpm(A_files[0], det) # Read in the slits slits = slittrace.SlitTraceSet.from_file(slit_masterframe_name) # Reset the bitmask slits.mask = slits.mask_init.copy() # Read in the wv_calib wv_calib = wavecalib.WaveCalib.from_file(wvcalib_masterframe_name) wv_calib.is_synced(slits) slits.mask_wvcalib(wv_calib) # Read in the tilts tilts_obj = wavetilts.WaveTilts.from_file(tilts_masterframe_name) tilts_obj.is_synced(slits) slits.mask_wavetilts(tilts_obj) # Build Science image sciImg = buildimage.buildimage_fromlist(spectrograph, det, parset['scienceframe'], A_files, bpm=msbpm, slits=slits, ignore_saturation=False) # Background Image? sciImg = sciImg.sub( buildimage.buildimage_fromlist(spectrograph, det, parset['scienceframe'], B_files, bpm=msbpm, slits=slits, ignore_saturation=False), parset['scienceframe']['process']) # Build the Calibrate object caliBrate = calibrations.Calibrations(None, parset['calibrations'], spectrograph, None) caliBrate.slits = slits caliBrate.wavetilts = tilts_obj caliBrate.wv_calib = wv_calib # Instantiate Reduce object # Required for pypeline specific object # At instantiaton, the fullmask in self.sciImg is modified redux = reduce.Reduce.get_instance(sciImg, spectrograph, parset, caliBrate, 'science', ir_redux=True, show=args.show, det=det, std_outfile=std_outfile) manual_extract_dict = None skymodel, objmodel, ivarmodel, outmask, sobjs, waveImg, tilts = redux.run( std_trace=std_trace, return_negative=True, manual_extract_dict=manual_extract_dict, show_peaks=args.show) # TODO -- Do this upstream # Tack on detector for sobj in sobjs: sobj.DETECTOR = sciImg.detector # Construct the Spec2DObj with the positive image spec2DObj_A = spec2dobj.Spec2DObj(det=det, sciimg=sciImg.image, ivarraw=sciImg.ivar, skymodel=skymodel, objmodel=objmodel, ivarmodel=ivarmodel, waveimg=waveImg, bpmmask=outmask, detector=sciImg.detector, sci_spat_flexure=sciImg.spat_flexure, tilts=tilts, slits=copy.deepcopy(caliBrate.slits)) spec2DObj_A.process_steps = sciImg.process_steps all_spec2d = spec2dobj.AllSpec2DObj() all_spec2d['meta']['ir_redux'] = True all_spec2d[det] = spec2DObj_A # Save image A but with all the objects extracted, i.e. positive and negative #outfile2d, outfile1d = save_exposure(fitstbl, 0, spectrograph, science_path, parset, caliBrate, all_spec2d, sobjs) # Construct the Spec2DObj with the negative image spec2DObj_B = spec2dobj.Spec2DObj(det=det, sciimg=-sciImg.image, ivarraw=sciImg.ivar, skymodel=-skymodel, objmodel=-objmodel, ivarmodel=ivarmodel, waveimg=waveImg, bpmmask=outmask, detector=sciImg.detector, sci_spat_flexure=sciImg.spat_flexure, tilts=tilts, slits=copy.deepcopy(caliBrate.slits)) # Parse the offset information out of the headers. TODO in the future get this out of fitstable dither_pattern_A, dither_id_A, offset_arcsec_A = parse_dither_pattern( A_files, spectrograph.primary_hdrext) dither_pattern_B, dither_id_B, offset_arcsec_B = parse_dither_pattern( B_files, spectrograph.primary_hdrext) # Print out a report on the offsets msg_string = msgs.newline( ) + '****************************************************' msg_string += msgs.newline( ) + ' Summary of offsets for dither pattern: {:s}'.format( dither_pattern_A[0]) msg_string += msgs.newline( ) + '****************************************************' msg_string += msgs.newline( ) + 'Position filename arcsec pixels ' msg_string += msgs.newline( ) + '----------------------------------------------------' for iexp, file in enumerate(A_files): msg_string += msgs.newline( ) + ' A {:s} {:6.2f} {:6.2f}'.format( os.path.basename(file), offset_arcsec_A[iexp], offset_arcsec_A[iexp] / sciImg.detector.platescale) for iexp, file in enumerate(B_files): msg_string += msgs.newline( ) + ' B {:s} {:6.2f} {:6.2f}'.format( os.path.basename(file), offset_arcsec_B[iexp], offset_arcsec_B[iexp] / sciImg.detector.platescale) msg_string += msgs.newline( ) + '****************************************************' msgs.info(msg_string) #offset_dith_pix = offset_dith_pix = offset_arcsec_A[0]/sciImg.detector.platescale offsets_dith_pix = (np.array([ 0.0, np.mean(offset_arcsec_B) - np.mean(offset_arcsec_A) ])) / sciImg.detector.platescale if args.offset is not None: offsets_pixels = np.array([0.0, args.offset]) msgs.info('Using user specified offsets instead: {:5.2f}'.format( args.offset)) else: offsets_pixels = offsets_dith_pix spec2d_list = [spec2DObj_A, spec2DObj_B] # Instantiate Coadd2d coadd = coadd2d.CoAdd2D.get_instance(spec2d_list, spectrograph, parset, det=det, offsets=offsets_pixels, weights='uniform', ir_redux=True, debug=args.show, samp_fact=args.samp_fact) # Coadd the slits coadd_dict_list = coadd.coadd( only_slits=None, interp_dspat=False) # TODO implement only_slits later # Create the pseudo images pseudo_dict = coadd.create_pseudo_image(coadd_dict_list) ########################## # Now display the images # ########################## display.display.connect_to_ginga(raise_err=True, allow_new=True) # Bug in ginga prevents me from using cuts here for some reason #mean, med, sigma = sigma_clipped_stats(pseudo_dict['imgminsky'][pseudo_dict['inmask']], sigma_lower=5.0,sigma_upper=5.0) #cut_min = mean - 4.0 * sigma #cut_max = mean + 4.0 * sigma chname_skysub = 'skysub-det{:s}'.format(sdet) # Clear all channels at the beginning # TODO: JFH For some reason Ginga crashes when I try to put cuts in here. viewer, ch = ginga.show_image(pseudo_dict['imgminsky'], chname=chname_skysub, waveimg=pseudo_dict['waveimg'], clear=True) # cuts=(cut_min, cut_max), slit_left, slit_righ, _ = pseudo_dict['slits'].select_edges() slit_id = slits.slitord_id[0] ginga.show_slits(viewer, ch, slit_left, slit_righ, slit_ids=slit_id) # SKRESIDS chname_skyresids = 'sky_resid-det{:s}'.format(sdet) image = pseudo_dict['imgminsky'] * np.sqrt( pseudo_dict['sciivar']) * pseudo_dict['inmask'] # sky residual map viewer, ch = ginga.show_image( image, chname_skyresids, waveimg=pseudo_dict['waveimg'], cuts=(-5.0, 5.0), ) ginga.show_slits(viewer, ch, slit_left, slit_righ, slit_ids=slits.slitord_id[0]) shell = viewer.shell() out = shell.start_global_plugin('WCSMatch') out = shell.call_global_plugin_method('WCSMatch', 'set_reference_channel', [chname_skyresids], {}) if args.embed: embed() return 0
def reduce_exposure(self, frames, bg_frames=None, std_outfile=None): """ Reduce a single exposure Args: frames (:obj:`list`): List of 0-indexed rows in :attr:`fitstbl` with the frames to reduce. bg_frames (:obj:`list`, optional): List of frame indices for the background. std_outfile (:obj:`str`, optional): File with a previously reduced standard spectrum from PypeIt. Returns: dict: The dictionary containing the primary outputs of extraction. """ # if show is set, clear the ginga channels at the start of each new sci_ID if self.show: # TODO: Put this in a try/except block? display.clear_all(allow_new=True) has_bg = True if bg_frames is not None and len( bg_frames) > 0 else False # Is this an b/g subtraction reduction? if has_bg: self.bkg_redux = True # The default is to find_negative objects if the bg_frames are classified as "science", and to not find_negative # objects if the bg_frames are classified as "sky". This can be explicitly overridden if # par['reduce']['findobj']['find_negative'] is set to something other than the default of None. self.find_negative = (('science' in self.fitstbl['frametype'][bg_frames[0]]) | ('standard' in self.fitstbl['frametype'][bg_frames[0]]))\ if self.par['reduce']['findobj']['find_negative'] is None else self.par['reduce']['findobj']['find_negative'] else: self.bkg_redux = False self.find_negative = False # Container for all the Spec2DObj all_spec2d = spec2dobj.AllSpec2DObj() all_spec2d['meta']['bkg_redux'] = self.bkg_redux all_spec2d['meta']['find_negative'] = self.find_negative # TODO -- Should we reset/regenerate self.slits.mask for a new exposure # container for specobjs during first loop (objfind) all_specobjs_objfind = specobjs.SpecObjs() # container for specobjs during second loop (extraction) all_specobjs_extract = specobjs.SpecObjs() # list of global_sky obtained during objfind and used in extraction initial_sky_list = [] # list of sciImg sciImg_list = [] # List of detectors with successful calibration calibrated_det = [] # list of successful MasterSlits calibrations to be used in the extraction loop calib_slits = [] # List of objFind objects objFind_list = [] # Print status message msgs_string = 'Reducing target {:s}'.format( self.fitstbl['target'][frames[0]]) + msgs.newline() # TODO: Print these when the frames are actually combined, # backgrounds are used, etc? msgs_string += 'Combining frames:' + msgs.newline() for iframe in frames: msgs_string += '{0:s}'.format( self.fitstbl['filename'][iframe]) + msgs.newline() msgs.info(msgs_string) if has_bg: bg_msgs_string = '' for iframe in bg_frames: bg_msgs_string += '{0:s}'.format( self.fitstbl['filename'][iframe]) + msgs.newline() bg_msgs_string = msgs.newline( ) + 'Using background from frames:' + msgs.newline( ) + bg_msgs_string msgs.info(bg_msgs_string) # Find the detectors to reduce subset = self.par['rdx']['slitspatnum'] if self.par['rdx']['slitspatnum'] is not None \ else self.par['rdx']['detnum'] detectors = self.spectrograph.select_detectors(subset=subset) msgs.info(f'Detectors to work on: {detectors}') # Loop on Detectors # TODO: Attempt to put in a multiprocessing call here? # objfind for self.det in detectors: msgs.info("Working on detector {0}".format(self.det)) # run calibration self.caliBrate = self.calib_one(frames, self.det) if not self.caliBrate.success: msgs.warn( f'Calibrations for detector {self.det} were unsuccessful! The step ' f'that failed was {self.caliBrate.failed_step}. Continuing by ' f'skipping this detector.') continue # we save only the detectors that had a successful calibration, # and we use only those in the extract loop below calibrated_det.append(self.det) # we also save the successful MasterSlits calibrations because they are used and modified # in the slitmask stuff in between the two loops calib_slits.append(self.caliBrate.slits) # global_sky, skymask and sciImg are needed in the extract loop initial_sky, sobjs_obj, sciImg, objFind = self.objfind_one( frames, self.det, bg_frames, std_outfile=std_outfile) if len(sobjs_obj) > 0: all_specobjs_objfind.add_sobj(sobjs_obj) initial_sky_list.append(initial_sky) sciImg_list.append(sciImg) objFind_list.append(objFind) # slitmask stuff if self.par['reduce']['slitmask']['assign_obj']: # get object positions from slitmask design and slitmask offsets for all the detectors spat_flexure = np.array([ss.spat_flexure for ss in sciImg_list]) # Grab platescale with binning bin_spec, bin_spat = parse.parse_binning(self.binning) platescale = np.array( [ss.detector.platescale * bin_spat for ss in sciImg_list]) # get the dither offset if available if self.par['reduce']['slitmask']['use_dither_offset']: dither = self.spectrograph.parse_dither_pattern( [self.fitstbl.frame_paths(frames[0])]) dither_off = dither[2][0] if dither is not None else None else: dither_off = None calib_slits = slittrace.get_maskdef_objpos_offset_alldets( all_specobjs_objfind, calib_slits, spat_flexure, platescale, self.par['calibrations']['slitedges']['det_buffer'], self.par['reduce']['slitmask'], dither_off=dither_off) # determine if slitmask offsets exist and compute an average offsets over all the detectors calib_slits = slittrace.average_maskdef_offset( calib_slits, platescale[0], self.spectrograph.list_detectors( mosaic='MSC' in calib_slits[0].detname)) # slitmask design matching and add undetected objects all_specobjs_objfind = slittrace.assign_addobjs_alldets( all_specobjs_objfind, calib_slits, spat_flexure, platescale, self.par['reduce']['slitmask'], self.par['reduce']['findobj']['find_fwhm']) # Extract for i, self.det in enumerate(calibrated_det): # re-run (i.e., load) calibrations self.caliBrate = self.calib_one(frames, self.det) self.caliBrate.slits = calib_slits[i] detname = sciImg_list[i].detector.name # TODO: pass back the background frame, pass in background # files as an argument. extract one takes a file list as an # argument and instantiates science within if all_specobjs_objfind.nobj > 0: all_specobjs_on_det = all_specobjs_objfind[ all_specobjs_objfind.DET == detname] else: all_specobjs_on_det = all_specobjs_objfind # Extract all_spec2d[detname], tmp_sobjs \ = self.extract_one(frames, self.det, sciImg_list[i], objFind_list[i], initial_sky_list[i], all_specobjs_on_det) # Hold em if tmp_sobjs.nobj > 0: all_specobjs_extract.add_sobj(tmp_sobjs) # JFH TODO write out the background frame? # TODO -- Save here? Seems like we should. Would probably need to use update_det=True # Return return all_spec2d, all_specobjs_extract
def reduce_IR(A_files, B_files, caliBrate, spectrograph, det, parset, show=False, std_trace=None): """ Peform 2d extraction for a set of files at the same unique A-B offset location. Parameters ---------- A_files (list of strings): Files at A position for this offset B_files (list of strings) Files at B position for this offeset caliBrate (object): CaliBrate object spectrograph (object): spectrograph object det (int): Detector number parset (parsect object) Parset show (bool, optional): Show 2d reduction outputs. Default=False std_trace (string, optional) Trace for standard star. Default=None Returns ------- spec2DObj_A, spec2DObj_B spec2DObj_A (object, Spec2D): Spec2d Object for extraction at A position spec2DObj_B (object, Spec2D) Spec2d Object for extraction at B position """ # Build Science image sciImg = buildimage.buildimage_fromlist(spectrograph, det, parset['scienceframe'], list(A_files), bpm=caliBrate.msbpm, slits=caliBrate.slits, ignore_saturation=False) # Background Image? sciImg = sciImg.sub( buildimage.buildimage_fromlist(spectrograph, det, parset['scienceframe'], list(B_files), bpm=caliBrate.msbpm, slits=caliBrate.slits, ignore_saturation=False), parset['scienceframe']['process']) # Instantiate FindObjects object # Required for pypeline specific object # At instantiaton, the fullmask in self.sciImg is modified # DP: Should find_negative be True here? JFH: For quicklook yes! objFind = find_objects.FindObjects.get_instance(sciImg, spectrograph, parset, caliBrate, 'science', bkg_redux=True, find_negative=True, show=show) global_sky, sobjs_obj = objFind.run(std_trace=std_trace, show_peaks=show) # Instantiate Extract object extract = extraction.Extract.get_instance(sciImg, sobjs_obj, spectrograph, parset, caliBrate, 'science', bkg_redux=True, return_negative=True, show=show) skymodel, objmodel, ivarmodel, \ outmask, sobjs, waveimg, tilts = extract.run(global_sky, sobjs_obj) scaleimg = np.array([1.0], dtype=np.float) # np.array([1]) applies no scale # TODO -- Do this upstream # Tack on detector for sobj in sobjs: sobj.DETECTOR = sciImg.detector # Construct table of spectral flexure spec_flex_table = Table() spec_flex_table['spat_id'] = caliBrate.slits.spat_id spec_flex_table['sci_spec_flexure'] = extract.slitshift # Construct the Spec2DObj with the positive image spec2DObj_A = spec2dobj.Spec2DObj( sciimg=sciImg.image, ivarraw=sciImg.ivar, skymodel=skymodel, objmodel=objmodel, ivarmodel=ivarmodel, scaleimg=scaleimg, waveimg=waveimg, bpmmask=outmask, detector=sciImg.detector, sci_spat_flexure=sciImg.spat_flexure, sci_spec_flexure=spec_flex_table, vel_corr=None, vel_type=parset['calibrations']['wavelengths']['refframe'], tilts=tilts, slits=copy.deepcopy(caliBrate.slits), maskdef_designtab=None) spec2DObj_A.process_steps = sciImg.process_steps all_spec2d = spec2dobj.AllSpec2DObj() all_spec2d['meta']['bkg_redux'] = True all_spec2d[spec2DObj_A.detname] = spec2DObj_A # Construct the Spec2DObj with the negative image spec2DObj_B = spec2dobj.Spec2DObj( sciimg=-sciImg.image, ivarraw=sciImg.ivar, skymodel=-skymodel, objmodel=-objmodel, ivarmodel=ivarmodel, scaleimg=scaleimg, waveimg=waveimg, bpmmask=outmask, detector=sciImg.detector, sci_spat_flexure=sciImg.spat_flexure, sci_spec_flexure=spec_flex_table, vel_corr=None, vel_type=parset['calibrations']['wavelengths']['refframe'], tilts=tilts, slits=copy.deepcopy(caliBrate.slits), maskdef_designtab=None) return spec2DObj_A, spec2DObj_B
def reduce_exposure(self, frames, bg_frames=None, std_outfile=None): """ Reduce a single exposure Args: frame (:obj:`int`): 0-indexed row in :attr:`fitstbl` with the frame to reduce. bg_frames (:obj:`list`, optional): List of frame indices for the background. std_outfile (:obj:`str`, optional): File with a previously reduced standard spectrum from PypeIt. Returns: dict: The dictionary containing the primary outputs of extraction. """ # TODO: # - change doc string to reflect that more than one frame can be # provided # if show is set, clear the ginga channels at the start of each new sci_ID if self.show: # TODO: Put this in a try/except block? display.clear_all() has_bg = True if bg_frames is not None and len( bg_frames) > 0 else False # Is this an IR reduction? # TODO: Why specific to IR? self.ir_redux = True if has_bg else False # Container for all the Spec2DObj all_spec2d = spec2dobj.AllSpec2DObj() all_spec2d['meta']['ir_redux'] = self.ir_redux # TODO -- Should we reset/regenerate self.slits.mask for a new exposure all_specobjs = specobjs.SpecObjs() # Print status message msgs_string = 'Reducing target {:s}'.format( self.fitstbl['target'][frames[0]]) + msgs.newline() # TODO: Print these when the frames are actually combined, # backgrounds are used, etc? msgs_string += 'Combining frames:' + msgs.newline() for iframe in frames: msgs_string += '{0:s}'.format( self.fitstbl['filename'][iframe]) + msgs.newline() msgs.info(msgs_string) if has_bg: bg_msgs_string = '' for iframe in bg_frames: bg_msgs_string += '{0:s}'.format( self.fitstbl['filename'][iframe]) + msgs.newline() bg_msgs_string = msgs.newline( ) + 'Using background from frames:' + msgs.newline( ) + bg_msgs_string msgs.info(bg_msgs_string) # Find the detectors to reduce detectors = PypeIt.select_detectors( detnum=self.par['rdx']['detnum'], slitspatnum=self.par['rdx']['slitspatnum'], ndet=self.spectrograph.ndet) if len(detectors) != self.spectrograph.ndet: msgs.warn('Not reducing detectors: {0}'.format(' '.join([ str(d) for d in set(np.arange(self.spectrograph.ndet)) - set(detectors) ]))) # Loop on Detectors # TODO: Attempt to put in a multiprocessing call here? for self.det in detectors: msgs.info("Working on detector {0}".format(self.det)) # Instantiate Calibrations class self.caliBrate = calibrations.Calibrations.get_instance( self.fitstbl, self.par['calibrations'], self.spectrograph, self.calibrations_path, qadir=self.qa_path, reuse_masters=self.reuse_masters, show=self.show, slitspat_num=self.par['rdx']['slitspatnum']) # These need to be separate to accomodate COADD2D self.caliBrate.set_config(frames[0], self.det, self.par['calibrations']) self.caliBrate.run_the_steps() # Extract # TODO: pass back the background frame, pass in background # files as an argument. extract one takes a file list as an # argument and instantiates science within all_spec2d[self.det], tmp_sobjs \ = self.reduce_one(frames, self.det, bg_frames, std_outfile=std_outfile) # Hold em if tmp_sobjs.nobj > 0: all_specobjs.add_sobj(tmp_sobjs) # JFH TODO write out the background frame? # TODO -- Save here? Seems like we should. Would probably need to use update_det=True # Return return all_spec2d, all_specobjs