def __init__(self, file_list, path=None, frametype=None, usrdata=None, setups=None, cfg_lines=None, spectrograph_name=None, pypeit_file=None, setup_dict=None): # The provided list of files cannot be None if file_list is None or len(file_list) == 0: msgs.error('Must provide a list of files to be reduced!') # Save input self.file_list = file_list # TODO: It seems like path is never used... self.path = os.getcwd() if path is None else path self.frametype = frametype self.usrdata = usrdata self.setups = setups self.pypeit_file = pypeit_file self.user_cfg = cfg_lines self.setup_dict = setup_dict # Determine the spectrograph name _spectrograph_name = spectrograph_name if cfg_lines is None \ else PypeItPar.from_cfg_lines(merge_with=cfg_lines)['rdx']['spectrograph'] # Cannot proceed without spectrograph name if _spectrograph_name is None: msgs.error( 'Must provide spectrograph name directly or using configuration lines.' ) # Instantiate the spectrograph self.spectrograph = load_spectrograph( _spectrograph_name) #, ifile=file_list[0]) # Get the spectrograph specific configuration to be merged with # the user modifications. spectrograph_cfg_lines = self.spectrograph.default_pypeit_par( ).to_config() # Instantiate the pypeit parameters. The user input # configuration (cfg_lines) can be None. self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) # Prepare internals for execution self.fitstbl = None self.steps = []
def main(args): # Load fits file cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(args.file, runtime=False) # Get the raw fits file name sciIdx = get_science_frame(usrdata) fname = data_files[sciIdx] # Load the spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] spectrograph = load_spectrograph(spectrograph_name, ifile=data_files[sciIdx]) msgs.info('Loaded spectrograph {0}'.format(spectrograph.spectrograph)) spectrograph_cfg_lines = spectrograph.config_specific_par(fname).to_config() par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) # Get the master key file_base = os.path.basename(fname) ftdict = dict({file_base: 'science'}) fitstbl = PypeItMetaData(spectrograph, par, files=[data_files[sciIdx]], usrdata=Table(usrdata[sciIdx]), strict=True) fitstbl.finalize_usr_build(ftdict, setups[0]) mkey = fitstbl.master_key(0, det=args.det) # Load the frame data rawimage, _, _, datasec, _ = spectrograph.get_rawimage(fname, args.det) rawimage = procimg.trim_frame(rawimage, datasec < 1) frame = spectrograph.orient_image(rawimage, args.det) # Set paths if par['calibrations']['caldir'] == 'default': mdir = os.path.join(par['rdx']['redux_path'], 'Masters') else: mdir = par['calibrations']['caldir'] if not os.path.exists(mdir): mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir)) msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base)) mdir = mdir_base # Load the tslits_dict trc_file = os.path.join(mdir, MasterFrame.construct_file_name('Edges', mkey)) + '.gz' tslits_dict = edgetrace.EdgeTraceSet.from_file(trc_file).convert_to_tslits_dict() # Derive an appropriate output filename prefix = os.path.splitext(file_base) if prefix[1] == ".gz": prefix = os.path.splitext(prefix[0])[0] else: prefix = prefix[0] outname = "{0:s}_skyregions.fits".format(prefix) # Finally, initialise the GUI skyreg = SkySubGUI.initialize(args.det, frame, tslits_dict, outname=outname, runtime=False, printout=True) # Get the results skyreg.get_result()
def test_setup_keck_deimos_multiconfig(): root = os.path.join(os.environ['PYPEIT_DEV'], 'RAW_DATA', 'keck_deimos') files = glob.glob(os.path.join(root, '830G_L_8100', '*fits*')) files += glob.glob(os.path.join(root, '830G_L_8400', '*fits*')) output_path = os.path.join(os.getcwd(), 'output') if os.path.isdir(output_path): shutil.rmtree(output_path) os.makedirs(output_path) ps = pypeitsetup.PypeItSetup(files, spectrograph_name='keck_deimos') ps.run(setup_only=True, sort_dir=output_path) # Write the automatically generated pypeit data pypeit_files = ps.fitstbl.write_pypeit(output_path, cfg_lines=ps.user_cfg, write_bkg_pairs=True) assert len(pypeit_files) == 2, 'Should have created two pypeit files' # Test the pypeit files for the correct configuration and # calibration group results for f, s, c in zip(pypeit_files, ['A', 'B'], ['0', '1']): # TODO: All of this front-end stuff, pulled from pypeit.py, should # be put into a function. # Read the pypeit file cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file( f, runtime=True) # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph = load_spectrograph(cfg['rdx']['spectrograph']) # Configuration-specific parameters for idx, row in enumerate(usrdata): if 'science' in row['frametype'] or 'standard' in row['frametype']: break spectrograph_cfg_lines = spectrograph.config_specific_par( data_files[idx]).to_config() # PypeIt parameters par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) # Metadata fitstbl = PypeItMetaData(spectrograph, par, files=data_files, usrdata=usrdata, strict=True) fitstbl.finalize_usr_build(frametype, setups[0]) assert np.all(fitstbl['setup'] == s), 'Setup is wrong' assert np.all(fitstbl['calib'] == c), 'Calibration group is wrong' # Clean-up shutil.rmtree(output_path)
def load_spectrograph_parset(self, iFile=None): # Do some checks idx = self.check_index(iFile) # Grab the filename fname = self.data_files[idx] # Setup the configuration cfg = ConfigObj(self.cfg_lines) # Load the spectrograph spectrograph_name = cfg['rdx']['spectrograph'] self.spectrograph = load_spectrograph(spectrograph_name) msgs.info('Loaded spectrograph {0}'.format(self.spectrograph.name)) spectrograph_cfg_lines = self.spectrograph.config_specific_par(fname).to_config() # Load the parset self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=self.cfg_lines) return
def main(args): """ Args: args: Returns: astropy.table.Table: """ import os from IPython import embed import numpy as np from astropy import table from pypeit.pypeitsetup import PypeItSetup from pypeit import calibrations from pypeit import msgs from pypeit.par import PypeItPar import shutil # Check that the spectrograph is provided if using a file root if args.root is not None: if args.spectrograph is None: raise ValueError( 'Must provide spectrograph identifier with file root.') # Check that input spectrograph is supported if args.spectrograph not in available_spectrographs: raise ValueError( 'Instrument \'{0}\' unknown to PypeIt.\n'.format( args.spectrograph) + '\tOptions are: {0}\n'.format( ', '.join(available_spectrographs)) + '\tSelect an available instrument or consult the documentation ' + 'on how to add a new instrument.') # Initialize PypeItSetup based on the arguments ps = PypeItSetup.from_file_root(args.root, args.spectrograph, extension=args.extension) # Run the setup ps.run(setup_only=True) #, write_bkg_pairs=args.background) is_science = ps.fitstbl.find_frames('science') msgs.info('Loaded spectrograph {0}'.format(ps.spectrograph.name)) # Unique configurations uniq_cfg = ps.fitstbl.unique_configurations(copy=True) # Setup the table. Need to use object type for strings so that # they're not truncated. answers = table.Table() answers['setups'] = list(uniq_cfg.keys()) # Add the configuration columns for setup, setup_dict in uniq_cfg.items(): for key, value in setup_dict.items(): answers[key] = np.empty(len(answers), dtype=object) if isinstance(value, str) \ else type(value)(0) break answers['pass'] = False answers['scifiles'] = np.empty(len(answers), dtype=object) for i, setup in enumerate(uniq_cfg.keys()): for setup_key, setup_value in uniq_cfg[setup].items(): answers[setup_key] = setup_value if setup == 'None': print("There is a setup without science frames. Skipping...") answers['pass'][i] = False answers['scifiles'][i] = None continue msgs.info( '=======================================================================' ) msgs.info('Working on setup: {}'.format(setup)) msgs.info(str(uniq_cfg[setup])) msgs.info( '=======================================================================' ) # TODO: Make the snippet below, which is also in the init of # PypeIt a method somewhere in_cfg = ps.fitstbl['setup'] == setup config_specific_file = None # Grab a science/standard frame data_files = [ os.path.join(row['directory'], row['filename']) for row in ps.fitstbl[in_cfg] ] for idx, row in enumerate(ps.fitstbl[in_cfg]): if 'science' in row['frametype'] or 'standard' in row['frametype']: config_specific_file = data_files[idx] if config_specific_file is not None: msgs.info( 'Setting configuration-specific parameters using {0}'.format( os.path.split(config_specific_file)[1])) else: msgs.warn('No science or standard frame. Punting..') answers['pass'][i] = False answers['scifiles'][i] = None continue # spectrograph_cfg_lines = ps.spectrograph.config_specific_par( config_specific_file).to_config() # - Build the full set, merging with any user-provided # parameters par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines) # Print science frames if np.any(in_cfg & is_science): msgs.info('Your science frames are: {0}'.format( ps.fitstbl['filename'][in_cfg & is_science].tolist())) answers['scifiles'][i] = ', '.join( ps.fitstbl['filename'][in_cfg & is_science].tolist()) else: msgs.warn("This setup has no science frames!") answers['scifiles'][i] = '' # Check! answers['pass'][i] = calibrations.check_for_calibs(par, ps.fitstbl, raise_error=False, cut_cfg=in_cfg) if not answers['pass'][i]: msgs.warn( "Setup {} did not pass the calibration check!".format(setup)) print('= RESULTS ============================================') # Print answers.pprint_all() print('======================================================') # Remove setup_files if not args.save_setups: shutil.rmtree('setup_files') # Return objects used by unit tests return answers, ps
def main(args): """ Args: args: Returns: astropy.table.Table: """ import os import numpy as np from astropy import table from pypeit.pypeitsetup import PypeItSetup from pypeit import calibrations from pypeit import msgs from pypeit.par import PypeItPar # Check that the spectrograph is provided if using a file root if args.root is not None: if args.spectrograph is None: raise ValueError( 'Must provide spectrograph identifier with file root.') # Check that input spectrograph is supported instruments_served = defs.pypeit_spectrographs if args.spectrograph not in instruments_served: raise ValueError( 'Instrument \'{0}\' unknown to PypeIt.\n'.format( args.spectrograph) + '\tOptions are: {0}\n'.format(', '.join(instruments_served)) + '\tSelect an available instrument or consult the documentation ' + 'on how to add a new instrument.') # Initialize PypeItSetup based on the arguments ps = PypeItSetup.from_file_root(args.root, args.spectrograph, extension=args.extension) # Run the setup ps.run(setup_only=True) #, write_bkg_pairs=args.background) is_science = ps.fitstbl.find_frames('science') msgs.info('Loaded spectrograph {0}'.format(ps.spectrograph.spectrograph)) # Unique configurations setups, indx = ps.fitstbl.get_configuration_names(return_index=True) answers = table.Table() answers['setups'] = setups passes, scifiles, cfgs = [], [], [] for setup, i in zip(setups, indx): # Get the setup lines cfg = ps.fitstbl.get_setup(i, config_only=False) cfgs.append(cfg) if setup == 'None': print("There is a setup without science frames. Skipping...") passes.append(False) scifiles.append(None) continue in_cfg = ps.fitstbl['setup'] == setup # TODO -- Make the snippet below, which is also in the init of PypeIt a method somewhere config_specific_file = None msgs.info( '=======================================================================' ) msgs.info('Working on setup: {}'.format(setup)) msgs.info(str(cfg)) msgs.info( '=======================================================================' ) # Grab a science/standard frame data_files = [ os.path.join(row['directory'], row['filename']) for row in ps.fitstbl[in_cfg] ] for idx, row in enumerate(ps.fitstbl[in_cfg]): if ('science' in row['frametype']) or ('standard' in row['frametype']): config_specific_file = data_files[idx] if config_specific_file is not None: msgs.info( 'Setting configuration-specific parameters using {0}'.format( os.path.split(config_specific_file)[1])) else: msgs.warn('No science or standard frame. Punting..') passes.append(False) scifiles.append(None) continue # spectrograph_cfg_lines = ps.spectrograph.config_specific_par( config_specific_file).to_config() # - Build the full set, merging with any user-provided # parameters par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines) # Print science frames if np.any(in_cfg & is_science): msgs.info("Your science frames are: {}".format( ps.fitstbl['filename'][in_cfg & is_science])) scifiles.append(','.join(ps.fitstbl['filename'][in_cfg & is_science])) else: msgs.warn("This setup has no science frames!") scifiles.append('') # Check! passed = calibrations.check_for_calibs(par, ps.fitstbl, raise_error=False, cut_cfg=in_cfg) if not passed: msgs.warn( "Setup {} did not pass the calibration check!".format(setup)) # passes.append(passed) print('= RESULTS ============================================') # Pass/fail answers['pass'] = passes # Parse the configs pcfg = dict(disperser=[], angle=[], dichroic=[], decker=[], slitwid=[], binning=[]) for cfg in cfgs: # None? if len(cfg) == 0: for key in pcfg.keys(): pcfg[key].append(None) continue # Load it up key0 = list(cfg.keys())[0] subd = cfg[key0]['--'] # for convenience pcfg['disperser'].append(subd['disperser']['name']) pcfg['angle'].append(subd['disperser']['angle']) pcfg['dichroic'].append(subd['dichroic']) pcfg['decker'].append(subd['slit']['decker']) pcfg['slitwid'].append(subd['slit']['slitwid']) pcfg['binning'].append(subd['binning']) # Add for key in pcfg.keys(): answers[key] = pcfg[key] # Sci files [put this last as it can get large] answers['scifiles'] = scifiles # Print answers.pprint_all() print( '= ===================================================================================' ) # Return return answers, ps
def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None, show=False, redux_path=None): # Load cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(pypeit_file, runtime=True) self.pypeit_file = pypeit_file # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] self.spectrograph = load_spectrograph(spectrograph_name) # Par # Defaults spectrograph_def_par = self.spectrograph.default_pypeit_par() # Grab a science file for configuration specific parameters sci_file = None for idx, row in enumerate(usrdata): if 'science' in row['frametype']: sci_file = data_files[idx] break # Set spectrograph_cfg_lines = self.spectrograph.config_specific_par(spectrograph_def_par, sci_file).to_config() self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) # Fitstbl self.fitstbl = PypeItMetaData(self.spectrograph, self.par, file_list=data_files, usrdata=usrdata, strict=True) # The following could be put in a prepare_to_run() method in PypeItMetaData if 'setup' not in self.fitstbl.keys(): self.fitstbl['setup'] = setups[0] self.fitstbl.get_frame_types(user=frametype) # This sets them using the user inputs self.fitstbl.set_defaults() # Only does something if values not set in PypeIt file self.fitstbl._set_calib_group_bits() self.fitstbl._check_calib_groups() # Write .calib file (For QA naming amongst other things) calib_file = pypeit_file.replace('.pypeit', '.calib') self.fitstbl.write_calib(calib_file) # Other Internals self.logname = logname self.overwrite = overwrite # Currently the runtime argument determines the behavior for reuse_masters. There is also a reuse_masters # parameter in the parset but it is currently ignored. self.reuse_masters=reuse_masters self.show = show # Make the output directories self.par['rdx']['redux_path'] = os.getcwd() if redux_path is None else redux_path msgs.info("Setting reduction path to {:s}".format(self.par['rdx']['redux_path'])) paths.make_dirs(self.spectrograph.spectrograph, self.par['calibrations']['caldir'], self.par['rdx']['scidir'], self.par['rdx']['qadir'], overwrite=self.overwrite, redux_path=self.par['rdx']['redux_path']) # Instantiate Calibrations class self.caliBrate \ = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], self.spectrograph, redux_path=self.par['rdx']['redux_path'], reuse_masters=self.reuse_masters, save_masters=True, write_qa=True, show=self.show) # Init self.verbosity = verbosity # TODO: I don't think this ever used self.frame = None self.det = None self.tstart = None self.basename = None self.sciI = None self.obstime = None
def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None, show=False, redux_path=None): # Load cfg_lines, data_files, frametype, usrdata, setups \ = parse_pypeit_file(pypeit_file, runtime=True) self.pypeit_file = pypeit_file # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] self.spectrograph = load_spectrograph(spectrograph_name, ifile=data_files[0]) msgs.info('Loaded spectrograph {0}'.format( self.spectrograph.spectrograph)) # -------------------------------------------------------------- # Get the full set of PypeIt parameters # - Grab a science or standard file for configuration specific parameters scistd_file = None for idx, row in enumerate(usrdata): if ('science' in row['frametype']) or ('standard' in row['frametype']): scistd_file = data_files[idx] break # - Configuration specific parameters for the spectrograph if scistd_file is not None: msgs.info( 'Setting configuration-specific parameters using {0}'.format( os.path.split(scistd_file)[1])) spectrograph_cfg_lines = self.spectrograph.config_specific_par( scistd_file).to_config() # - Build the full set, merging with any user-provided # parameters self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) msgs.info('Built full PypeIt parameter set.') # Check the output paths are ready if redux_path is not None: self.par['rdx']['redux_path'] = redux_path # TODO: Write the full parameter set here? # -------------------------------------------------------------- # -------------------------------------------------------------- # Build the meta data # - Re-initilize based on the file data msgs.info('Compiling metadata') self.fitstbl = PypeItMetaData(self.spectrograph, self.par, files=data_files, usrdata=usrdata, strict=True) # - Interpret automated or user-provided data from the PypeIt # file self.fitstbl.finalize_usr_build(frametype, setups[0]) # -------------------------------------------------------------- # - Write .calib file (For QA naming amongst other things) calib_file = pypeit_file.replace('.pypeit', '.calib') self.fitstbl.write_calib(calib_file) # Other Internals self.logname = logname self.overwrite = overwrite # Currently the runtime argument determines the behavior for # reuse_masters. self.reuse_masters = reuse_masters self.show = show # Set paths if self.par['calibrations']['caldir'] == 'default': self.calibrations_path = os.path.join( self.par['rdx']['redux_path'], 'Masters') else: self.calibrations_path = self.par['calibrations']['caldir'] # Report paths msgs.info('Setting reduction path to {0}'.format( self.par['rdx']['redux_path'])) msgs.info('Master calibration data output to: {0}'.format( self.calibrations_path)) msgs.info('Science data output to: {0}'.format(self.science_path)) msgs.info('Quality assessment plots output to: {0}'.format( self.qa_path)) # TODO: Is anything written to the qa dir or only to qa/PNGs? # Should we have separate calibration and science QA # directories? # Instantiate Calibrations class self.caliBrate \ = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], self.spectrograph, caldir=self.calibrations_path, qadir=self.qa_path, reuse_masters=self.reuse_masters, show=self.show) # Init self.verbosity = verbosity # TODO: I don't think this ever used self.frame = None self.det = None self.tstart = None self.basename = None self.sciI = None self.obstime = None
def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None, show=False, redux_path=None, calib_only=False): # Load cfg_lines, data_files, frametype, usrdata, setups \ = parse_pypeit_file(pypeit_file, runtime=True) self.pypeit_file = pypeit_file self.calib_only = calib_only # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] self.spectrograph = load_spectrograph(spectrograph_name) msgs.info('Loaded spectrograph {0}'.format( self.spectrograph.spectrograph)) # -------------------------------------------------------------- # Get the full set of PypeIt parameters # - Grab a science or standard file for configuration specific parameters config_specific_file = None for idx, row in enumerate(usrdata): if ('science' in row['frametype']) or ('standard' in row['frametype']): config_specific_file = data_files[idx] # search for arcs, trace if no scistd was there if config_specific_file is None: for idx, row in enumerate(usrdata): if ('arc' in row['frametype']) or ('trace' in row['frametype']): config_specific_file = data_files[idx] if config_specific_file is not None: msgs.info( 'Setting configuration-specific parameters using {0}'.format( os.path.split(config_specific_file)[1])) spectrograph_cfg_lines = self.spectrograph.config_specific_par( config_specific_file).to_config() # - Build the full set, merging with any user-provided # parameters self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) msgs.info('Built full PypeIt parameter set.') # Check the output paths are ready if redux_path is not None: self.par['rdx']['redux_path'] = redux_path # TODO: Write the full parameter set here? # -------------------------------------------------------------- # -------------------------------------------------------------- # Build the meta data # - Re-initilize based on the file data msgs.info('Compiling metadata') self.fitstbl = PypeItMetaData(self.spectrograph, self.par, files=data_files, usrdata=usrdata, strict=True) # - Interpret automated or user-provided data from the PypeIt # file self.fitstbl.finalize_usr_build(frametype, setups[0]) # -------------------------------------------------------------- # - Write .calib file (For QA naming amongst other things) calib_file = pypeit_file.replace('.pypeit', '.calib') self.fitstbl.write_calib(calib_file) # Other Internals self.logname = logname self.overwrite = overwrite # Currently the runtime argument determines the behavior for # reuse_masters. self.reuse_masters = reuse_masters self.show = show # Set paths self.calibrations_path = os.path.join( self.par['rdx']['redux_path'], self.par['calibrations']['master_dir']) # Check for calibrations if not self.calib_only: calibrations.check_for_calibs( self.par, self.fitstbl, raise_error=self.par['calibrations']['raise_chk_error']) # Report paths msgs.info('Setting reduction path to {0}'.format( self.par['rdx']['redux_path'])) msgs.info('Master calibration data output to: {0}'.format( self.calibrations_path)) msgs.info('Science data output to: {0}'.format(self.science_path)) msgs.info('Quality assessment plots output to: {0}'.format( self.qa_path)) # TODO: Is anything written to the qa dir or only to qa/PNGs? # Should we have separate calibration and science QA # directories? # An html file wrapping them all too # # Instantiate Calibrations class # if self.spectrograph.pypeline in ['MultiSlit', 'Echelle']: # self.caliBrate \ # = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], # self.spectrograph, self.calibrations_path, # qadir=self.qa_path, # reuse_masters=self.reuse_masters, # show=self.show, # slitspat_num=self.par['rdx']['slitspatnum']) # elif self.spectrograph.pypeline in ['IFU']: # self.caliBrate \ # = calibrations.IFUCalibrations(self.fitstbl, self.par['calibrations'], # self.spectrograph, # self.calibrations_path, # qadir=self.qa_path, # reuse_masters=self.reuse_masters, # show=self.show) # else: # msgs.error("No calibration available to support pypeline: {0:s}".format(self.spectrograph.pypeline)) # Init self.verbosity = verbosity # TODO: I don't think this ever used self.det = None self.tstart = None self.basename = None self.sciI = None self.obstime = None