def test_deimos(): # Raw DEIMOS directory raw_dir = os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'keck_deimos') # Get the list of setup directories setups = glob.glob(os.path.join(raw_dir, '*')) # Set the output path and *remove if* if it already exists output_path = os.path.join(os.getcwd(), 'output') if os.path.isdir(output_path): shutil.rmtree(output_path) # Iterate through the setups for setup in setups: # Find the relevant pypeit file constructed by hand. by_hand_pypeit = os.path.join( os.getenv('PYPEIT_DEV'), 'pypeit_files', 'keck_deimos_{0}.pypeit'.format(os.path.split(setup)[1].lower())) if not os.path.isfile(by_hand_pypeit): # It doesn't exist, so assume there is no by-hand pypeit # file to compare to continue # Run pypeit_setup ps = PypeItSetup.from_file_root(setup, 'keck_deimos', output_path=output_path) ps.run(setup_only=True) # Write the automatically generated pypeit data pypeit_files = ps.fitstbl.write_pypeit(output_path, cfg_lines=ps.user_cfg) # Read the frame types from both the by-hand and automated # pypeit files _, _, by_hand_frametypes, _, _ = parse_pypeit_file(by_hand_pypeit, file_check=False) _, _, auto_frametypes, _, _ = parse_pypeit_file(pypeit_files[0], file_check=False) # For each file in the by-hand list, check that the frame types # in the automatically generated pypeit file are identical for f in by_hand_frametypes.keys(): type_list = np.sort(by_hand_frametypes[f].split(',')) if 'science' in type_list or 'standard' in type_list: # Only ensuring that calibrations are correctly typed continue assert f in auto_frametypes.keys(), \ 'Frame {0} not automatically parsed for setup {1}.'.format(f, setup) assert np.array_equal(type_list, np.sort(auto_frametypes[f].split(','))), \ 'Frame types differ for file {0} in setup {1}\n'.format(f, setup) \ + ' By-hand types: {0}'.format(by_hand_frametypes[f]) \ + ' Automated types: {0}'.format(auto_frametypes[f]) # Clean up after every setup shutil.rmtree(output_path)
def test_lris_blue_pypeit_overwrite(): f = os.path.join(os.environ['PYPEIT_DEV'], 'pypeit_files/keck_lris_blue_long_400_3400_d560.pypeit') assert os.path.isfile(f), 'Could not find pypeit file.' cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file( f, file_check=False) # Change the dev path for i in range(len(data_files)): path_list = data_files[i].split('/') for j, p in enumerate(path_list): if p == 'RAW_DATA': break data_files[i] = os.path.join(os.environ['PYPEIT_DEV'], '/'.join(path_list[j:])) # Read the fits table with and without the user data spectrograph = load_spectrograph('keck_lris_blue') par = spectrograph.default_pypeit_par() fitstbl = PypeItMetaData(spectrograph, par, files=data_files) fitstbl_usr = PypeItMetaData(spectrograph, par, files=data_files, usrdata=usrdata) assert fitstbl['target'][ 0] == 'unknown', 'Grating name changed in file header' assert fitstbl_usr['target'][ 0] == 'test', 'Grating name changed in pypeit file' assert fitstbl['target'][0] != fitstbl_usr['target'][0], \ 'Fits header value and input pypeit file value expected to be different.'
def load_pypeit(self, pypeit_file=None): # Load the pypeit file if pypeit_file is None: pypeit_file = self.pypeit_file msgs.info("Loading the PypeIt file: {}".format(pypeit_file)) if self.pypeit_file is not None: self.cfg_lines, self.data_files, self.frametype, self.usrdata, self.setups, self.sdict =\ parse_pypeit_file(pypeit_file, runtime=False)
def main(args): # Load fits file cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(args.file, runtime=False) # Get the raw fits file name sciIdx = get_science_frame(usrdata) fname = data_files[sciIdx] # Load the spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] spectrograph = load_spectrograph(spectrograph_name, ifile=data_files[sciIdx]) msgs.info('Loaded spectrograph {0}'.format(spectrograph.spectrograph)) spectrograph_cfg_lines = spectrograph.config_specific_par(fname).to_config() par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) # Get the master key file_base = os.path.basename(fname) ftdict = dict({file_base: 'science'}) fitstbl = PypeItMetaData(spectrograph, par, files=[data_files[sciIdx]], usrdata=Table(usrdata[sciIdx]), strict=True) fitstbl.finalize_usr_build(ftdict, setups[0]) mkey = fitstbl.master_key(0, det=args.det) # Load the frame data rawimage, _, _, datasec, _ = spectrograph.get_rawimage(fname, args.det) rawimage = procimg.trim_frame(rawimage, datasec < 1) frame = spectrograph.orient_image(rawimage, args.det) # Set paths if par['calibrations']['caldir'] == 'default': mdir = os.path.join(par['rdx']['redux_path'], 'Masters') else: mdir = par['calibrations']['caldir'] if not os.path.exists(mdir): mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir)) msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base)) mdir = mdir_base # Load the tslits_dict trc_file = os.path.join(mdir, MasterFrame.construct_file_name('Edges', mkey)) + '.gz' tslits_dict = edgetrace.EdgeTraceSet.from_file(trc_file).convert_to_tslits_dict() # Derive an appropriate output filename prefix = os.path.splitext(file_base) if prefix[1] == ".gz": prefix = os.path.splitext(prefix[0])[0] else: prefix = prefix[0] outname = "{0:s}_skyregions.fits".format(prefix) # Finally, initialise the GUI skyreg = SkySubGUI.initialize(args.det, frame, tslits_dict, outname=outname, runtime=False, printout=True) # Get the results skyreg.get_result()
def __init__(self, pypeit_file, det=0): self.pypeit_file = pypeit_file self.det = det self.iFile = None # if a single frame is being used, set the index, otherwise None self.spectrograph = None self.par = None # Load the pypeit file self.cfg_lines, self.data_files, self.frametype, self.usrdata, self.setups =\ parse_pypeit_file(pypeit_file, runtime=False)
def test_setup_keck_deimos_multiconfig(): root = os.path.join(os.environ['PYPEIT_DEV'], 'RAW_DATA', 'keck_deimos') files = glob.glob(os.path.join(root, '830G_L_8100', '*fits*')) files += glob.glob(os.path.join(root, '830G_L_8400', '*fits*')) output_path = os.path.join(os.getcwd(), 'output') if os.path.isdir(output_path): shutil.rmtree(output_path) os.makedirs(output_path) ps = pypeitsetup.PypeItSetup(files, spectrograph_name='keck_deimos') ps.run(setup_only=True, sort_dir=output_path) # Write the automatically generated pypeit data pypeit_files = ps.fitstbl.write_pypeit(output_path, cfg_lines=ps.user_cfg, write_bkg_pairs=True) assert len(pypeit_files) == 2, 'Should have created two pypeit files' # Test the pypeit files for the correct configuration and # calibration group results for f, s, c in zip(pypeit_files, ['A', 'B'], ['0', '1']): # TODO: All of this front-end stuff, pulled from pypeit.py, should # be put into a function. # Read the pypeit file cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file( f, runtime=True) # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph = load_spectrograph(cfg['rdx']['spectrograph']) # Configuration-specific parameters for idx, row in enumerate(usrdata): if 'science' in row['frametype'] or 'standard' in row['frametype']: break spectrograph_cfg_lines = spectrograph.config_specific_par( data_files[idx]).to_config() # PypeIt parameters par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) # Metadata fitstbl = PypeItMetaData(spectrograph, par, files=data_files, usrdata=usrdata, strict=True) fitstbl.finalize_usr_build(frametype, setups[0]) assert np.all(fitstbl['setup'] == s), 'Setup is wrong' assert np.all(fitstbl['calib'] == c), 'Calibration group is wrong' # Clean-up shutil.rmtree(output_path)
def test_setup_made_pypeit_file(): """ Test the .pypeit file(s) made by pypeit_setup This test depends on the one above """ pypeit_file = data_path('shane_kast_blue_A/shane_kast_blue_A.pypeit') cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file( pypeit_file) # Test assert len(data_files) == 2 assert frametype['b1.fits.gz'] == 'arc' assert setups[0] == 'A'
def test_setup_made_pypeit_file(): """ Test the .pypeit file(s) made by pypeit_setup This test depends on the one above """ pypeit_file = data_path('shane_kast_blue_A/shane_kast_blue_A.pypeit') cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(pypeit_file) # Test assert len(data_files) == 2 assert sorted(frametype['b1.fits.gz'].split(',')) == ['arc', 'tilt'] assert setups[0] == 'A' # Cleanup shutil.rmtree(data_path('shane_kast_blue_A'))
def test_lris_blue_pypeit_overwrite(): # JFH The RAW_DATA is at PYPEIT_DEV, but the github DEV suite where # the pypeit files are at a different path. To fix this, I've just # made pypeit_files directory in Cooked and copied this file over. # KBW: I'd prefer that you put symlinks in the github dev suite: # e.g.: # cd $PYPEIT_DEV # ln -s /Volumes/GoogleDrive/Team\ Drives/PHYS-GP-Hennawi/PypeIt/PypeIt-development-suite/RAW_DATA RAW_DATA # Read the dev suite pypeit file f = os.path.join( os.environ['PYPEIT_DEV'], 'Cooked/pypeit_files/keck_lris_blue_long_400_3400_d560.pypeit') if not os.path.isfile(f): f = os.path.join( os.environ['PYPEIT_DEV'], 'pypeit_files/keck_lris_blue_long_400_3400_d560.pypeit') assert os.path.isfile(f), 'Could not find pypeit file.' cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file( f, file_check=False) # Change the dev path for i in range(len(data_files)): path_list = data_files[i].split('/') for j, p in enumerate(path_list): if p == 'RAW_DATA': break data_files[i] = os.path.join(os.environ['PYPEIT_DEV'], '/'.join(path_list[j:])) # Read the fits table with and without the user data spectrograph = load_spectrograph('keck_lris_blue') par = spectrograph.default_pypeit_par() fitstbl = PypeItMetaData(spectrograph, par, file_list=data_files) fitstbl_usr = PypeItMetaData(spectrograph, par, file_list=data_files, usrdata=usrdata) assert fitstbl['target'][ 0] == 'unknown', 'Grating name changed in file header' assert fitstbl_usr['target'][ 0] == 'test', 'Grating name changed in pypeit file' assert fitstbl['target'][0] != fitstbl_usr['target'][0], \ 'Fits header value and input pypeit file value expected to be different.'
def from_pypeit_file(cls, filename): """ Instantiate the :class:`PypeitSetup` object using a pypeit file. Args: filename (str): Name of the pypeit file to read. Pypit files have a specific set of valid formats. A description can be found :ref:`pypeit_file`. Returns: :class:`PypeitSetup`: The instance of the class. """ cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(filename) return cls(data_files, frametype=frametype, usrdata=usrdata, setups=setups, cfg_lines=cfg_lines, pypeit_file=filename)
def test_read_combid(): # ------------------------------------------------------------------ # In case of failed tests setup_dir = data_path('setup_files') if os.path.isdir(setup_dir): shutil.rmtree(setup_dir) config_dir = data_path('shane_kast_blue_A') if os.path.isdir(config_dir): shutil.rmtree(config_dir) # ------------------------------------------------------------------ # Generate the pypeit file with the comb_id droot = data_path('b') pargs = Setup.parse_args([ '-r', droot, '-s', 'shane_kast_blue', '-c=all', '-b', '--extension=fits.gz', '--output_path={:s}'.format(data_path('')) ]) Setup.main(pargs) shutil.rmtree(setup_dir) pypeit_file = os.path.join(config_dir, 'shane_kast_blue_A.pypeit') cfg_lines, data_files, frametype, usrdata, setups, _ = parse_pypeit_file( pypeit_file) # Get the spectrograph spectrograph = None for l in cfg_lines: if 'spectrograph' in l: spectrograph = load_spectrograph(l.split(' ')[-1]) break assert spectrograph is not None, 'Did not appropriately read spectrograph' # Set the metadata pmd = PypeItMetaData(spectrograph, spectrograph.default_pypeit_par(), files=data_files, usrdata=usrdata, strict=False) indx = pmd['filename'] == 'b27.fits.gz' assert pmd['comb_id'][indx] == [1], 'Incorrect combination group ID' assert pmd['comb_id'][np.where(~indx) [0]][0] == -1, 'Incorrect combination group ID' shutil.rmtree(config_dir)
def test_pypeit_file(): # Read the PypeIt file cfg, data, frametype, usrdata, setups, _ \ = util.parse_pypeit_file(data_path('example_pypeit_file.pypeit'), file_check=False) # Long-winded way of getting the spectrograph name name = pypeitpar.PypeItPar.from_cfg_lines( merge_with=cfg)['rdx']['spectrograph'] # Instantiate the spectrograph spectrograph = load_spectrograph(name) # Get the spectrograph specific configuration spec_cfg = spectrograph.default_pypeit_par().to_config() # Initialize the PypeIt parameters merge in the user config _p = pypeitpar.PypeItPar.from_cfg_lines(cfg_lines=spec_cfg, merge_with=cfg) # Test everything was merged correctly # This is a PypeItPar default that's not changed #assert p['calibrations']['pinholeframe']['number'] == 0 # These are spectrograph specific defaults assert _p['fluxcalib'] is not None # These are user-level changes assert _p['calibrations']['traceframe']['process']['combine'] == 'median' assert _p['scienceframe']['process']['n_lohi'] == [8, 8]
def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None, show=False, redux_path=None): # Load cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(pypeit_file, runtime=True) self.pypeit_file = pypeit_file # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] self.spectrograph = load_spectrograph(spectrograph_name) # Par # Defaults spectrograph_def_par = self.spectrograph.default_pypeit_par() # Grab a science file for configuration specific parameters sci_file = None for idx, row in enumerate(usrdata): if 'science' in row['frametype']: sci_file = data_files[idx] break # Set spectrograph_cfg_lines = self.spectrograph.config_specific_par(spectrograph_def_par, sci_file).to_config() self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) # Fitstbl self.fitstbl = PypeItMetaData(self.spectrograph, self.par, file_list=data_files, usrdata=usrdata, strict=True) # The following could be put in a prepare_to_run() method in PypeItMetaData if 'setup' not in self.fitstbl.keys(): self.fitstbl['setup'] = setups[0] self.fitstbl.get_frame_types(user=frametype) # This sets them using the user inputs self.fitstbl.set_defaults() # Only does something if values not set in PypeIt file self.fitstbl._set_calib_group_bits() self.fitstbl._check_calib_groups() # Write .calib file (For QA naming amongst other things) calib_file = pypeit_file.replace('.pypeit', '.calib') self.fitstbl.write_calib(calib_file) # Other Internals self.logname = logname self.overwrite = overwrite # Currently the runtime argument determines the behavior for reuse_masters. There is also a reuse_masters # parameter in the parset but it is currently ignored. self.reuse_masters=reuse_masters self.show = show # Make the output directories self.par['rdx']['redux_path'] = os.getcwd() if redux_path is None else redux_path msgs.info("Setting reduction path to {:s}".format(self.par['rdx']['redux_path'])) paths.make_dirs(self.spectrograph.spectrograph, self.par['calibrations']['caldir'], self.par['rdx']['scidir'], self.par['rdx']['qadir'], overwrite=self.overwrite, redux_path=self.par['rdx']['redux_path']) # Instantiate Calibrations class self.caliBrate \ = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], self.spectrograph, redux_path=self.par['rdx']['redux_path'], reuse_masters=self.reuse_masters, save_masters=True, write_qa=True, show=self.show) # Init self.verbosity = verbosity # TODO: I don't think this ever used self.frame = None self.det = None self.tstart = None self.basename = None self.sciI = None self.obstime = None
def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None, show=False, redux_path=None): # Load cfg_lines, data_files, frametype, usrdata, setups \ = parse_pypeit_file(pypeit_file, runtime=True) self.pypeit_file = pypeit_file # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] self.spectrograph = load_spectrograph(spectrograph_name, ifile=data_files[0]) msgs.info('Loaded spectrograph {0}'.format( self.spectrograph.spectrograph)) # -------------------------------------------------------------- # Get the full set of PypeIt parameters # - Grab a science or standard file for configuration specific parameters scistd_file = None for idx, row in enumerate(usrdata): if ('science' in row['frametype']) or ('standard' in row['frametype']): scistd_file = data_files[idx] break # - Configuration specific parameters for the spectrograph if scistd_file is not None: msgs.info( 'Setting configuration-specific parameters using {0}'.format( os.path.split(scistd_file)[1])) spectrograph_cfg_lines = self.spectrograph.config_specific_par( scistd_file).to_config() # - Build the full set, merging with any user-provided # parameters self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) msgs.info('Built full PypeIt parameter set.') # Check the output paths are ready if redux_path is not None: self.par['rdx']['redux_path'] = redux_path # TODO: Write the full parameter set here? # -------------------------------------------------------------- # -------------------------------------------------------------- # Build the meta data # - Re-initilize based on the file data msgs.info('Compiling metadata') self.fitstbl = PypeItMetaData(self.spectrograph, self.par, files=data_files, usrdata=usrdata, strict=True) # - Interpret automated or user-provided data from the PypeIt # file self.fitstbl.finalize_usr_build(frametype, setups[0]) # -------------------------------------------------------------- # - Write .calib file (For QA naming amongst other things) calib_file = pypeit_file.replace('.pypeit', '.calib') self.fitstbl.write_calib(calib_file) # Other Internals self.logname = logname self.overwrite = overwrite # Currently the runtime argument determines the behavior for # reuse_masters. self.reuse_masters = reuse_masters self.show = show # Set paths if self.par['calibrations']['caldir'] == 'default': self.calibrations_path = os.path.join( self.par['rdx']['redux_path'], 'Masters') else: self.calibrations_path = self.par['calibrations']['caldir'] # Report paths msgs.info('Setting reduction path to {0}'.format( self.par['rdx']['redux_path'])) msgs.info('Master calibration data output to: {0}'.format( self.calibrations_path)) msgs.info('Science data output to: {0}'.format(self.science_path)) msgs.info('Quality assessment plots output to: {0}'.format( self.qa_path)) # TODO: Is anything written to the qa dir or only to qa/PNGs? # Should we have separate calibration and science QA # directories? # Instantiate Calibrations class self.caliBrate \ = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], self.spectrograph, caldir=self.calibrations_path, qadir=self.qa_path, reuse_masters=self.reuse_masters, show=self.show) # Init self.verbosity = verbosity # TODO: I don't think this ever used self.frame = None self.det = None self.tstart = None self.basename = None self.sciI = None self.obstime = None
def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None, show=False, redux_path=None, calib_only=False): # Load cfg_lines, data_files, frametype, usrdata, setups \ = parse_pypeit_file(pypeit_file, runtime=True) self.pypeit_file = pypeit_file self.calib_only = calib_only # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] self.spectrograph = load_spectrograph(spectrograph_name) msgs.info('Loaded spectrograph {0}'.format( self.spectrograph.spectrograph)) # -------------------------------------------------------------- # Get the full set of PypeIt parameters # - Grab a science or standard file for configuration specific parameters config_specific_file = None for idx, row in enumerate(usrdata): if ('science' in row['frametype']) or ('standard' in row['frametype']): config_specific_file = data_files[idx] # search for arcs, trace if no scistd was there if config_specific_file is None: for idx, row in enumerate(usrdata): if ('arc' in row['frametype']) or ('trace' in row['frametype']): config_specific_file = data_files[idx] if config_specific_file is not None: msgs.info( 'Setting configuration-specific parameters using {0}'.format( os.path.split(config_specific_file)[1])) spectrograph_cfg_lines = self.spectrograph.config_specific_par( config_specific_file).to_config() # - Build the full set, merging with any user-provided # parameters self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) msgs.info('Built full PypeIt parameter set.') # Check the output paths are ready if redux_path is not None: self.par['rdx']['redux_path'] = redux_path # TODO: Write the full parameter set here? # -------------------------------------------------------------- # -------------------------------------------------------------- # Build the meta data # - Re-initilize based on the file data msgs.info('Compiling metadata') self.fitstbl = PypeItMetaData(self.spectrograph, self.par, files=data_files, usrdata=usrdata, strict=True) # - Interpret automated or user-provided data from the PypeIt # file self.fitstbl.finalize_usr_build(frametype, setups[0]) # -------------------------------------------------------------- # - Write .calib file (For QA naming amongst other things) calib_file = pypeit_file.replace('.pypeit', '.calib') self.fitstbl.write_calib(calib_file) # Other Internals self.logname = logname self.overwrite = overwrite # Currently the runtime argument determines the behavior for # reuse_masters. self.reuse_masters = reuse_masters self.show = show # Set paths self.calibrations_path = os.path.join( self.par['rdx']['redux_path'], self.par['calibrations']['master_dir']) # Check for calibrations if not self.calib_only: calibrations.check_for_calibs( self.par, self.fitstbl, raise_error=self.par['calibrations']['raise_chk_error']) # Report paths msgs.info('Setting reduction path to {0}'.format( self.par['rdx']['redux_path'])) msgs.info('Master calibration data output to: {0}'.format( self.calibrations_path)) msgs.info('Science data output to: {0}'.format(self.science_path)) msgs.info('Quality assessment plots output to: {0}'.format( self.qa_path)) # TODO: Is anything written to the qa dir or only to qa/PNGs? # Should we have separate calibration and science QA # directories? # An html file wrapping them all too # # Instantiate Calibrations class # if self.spectrograph.pypeline in ['MultiSlit', 'Echelle']: # self.caliBrate \ # = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], # self.spectrograph, self.calibrations_path, # qadir=self.qa_path, # reuse_masters=self.reuse_masters, # show=self.show, # slitspat_num=self.par['rdx']['slitspatnum']) # elif self.spectrograph.pypeline in ['IFU']: # self.caliBrate \ # = calibrations.IFUCalibrations(self.fitstbl, self.par['calibrations'], # self.spectrograph, # self.calibrations_path, # qadir=self.qa_path, # reuse_masters=self.reuse_masters, # show=self.show) # else: # msgs.error("No calibration available to support pypeline: {0:s}".format(self.spectrograph.pypeline)) # Init self.verbosity = verbosity # TODO: I don't think this ever used self.det = None self.tstart = None self.basename = None self.sciI = None self.obstime = None