def test_lris_blue_pypeit_overwrite(): f = os.path.join(os.environ['PYPEIT_DEV'], 'pypeit_files/keck_lris_blue_long_400_3400_d560.pypeit') assert os.path.isfile(f), 'Could not find pypeit file.' cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file( f, file_check=False) # Change the dev path for i in range(len(data_files)): path_list = data_files[i].split('/') for j, p in enumerate(path_list): if p == 'RAW_DATA': break data_files[i] = os.path.join(os.environ['PYPEIT_DEV'], '/'.join(path_list[j:])) # Read the fits table with and without the user data spectrograph = load_spectrograph('keck_lris_blue') par = spectrograph.default_pypeit_par() fitstbl = PypeItMetaData(spectrograph, par, files=data_files) fitstbl_usr = PypeItMetaData(spectrograph, par, files=data_files, usrdata=usrdata) assert fitstbl['target'][ 0] == 'unknown', 'Grating name changed in file header' assert fitstbl_usr['target'][ 0] == 'test', 'Grating name changed in pypeit file' assert fitstbl['target'][0] != fitstbl_usr['target'][0], \ 'Fits header value and input pypeit file value expected to be different.'
def test_lris_blue_pypeit_overwrite(): # JFH The RAW_DATA is at PYPEIT_DEV, but the github DEV suite where # the pypeit files are at a different path. To fix this, I've just # made pypeit_files directory in Cooked and copied this file over. # KBW: I'd prefer that you put symlinks in the github dev suite: # e.g.: # cd $PYPEIT_DEV # ln -s /Volumes/GoogleDrive/Team\ Drives/PHYS-GP-Hennawi/PypeIt/PypeIt-development-suite/RAW_DATA RAW_DATA # Read the dev suite pypeit file f = os.path.join( os.environ['PYPEIT_DEV'], 'Cooked/pypeit_files/keck_lris_blue_long_400_3400_d560.pypeit') if not os.path.isfile(f): f = os.path.join( os.environ['PYPEIT_DEV'], 'pypeit_files/keck_lris_blue_long_400_3400_d560.pypeit') assert os.path.isfile(f), 'Could not find pypeit file.' cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file( f, file_check=False) # Change the dev path for i in range(len(data_files)): path_list = data_files[i].split('/') for j, p in enumerate(path_list): if p == 'RAW_DATA': break data_files[i] = os.path.join(os.environ['PYPEIT_DEV'], '/'.join(path_list[j:])) # Read the fits table with and without the user data spectrograph = load_spectrograph('keck_lris_blue') par = spectrograph.default_pypeit_par() fitstbl = PypeItMetaData(spectrograph, par, file_list=data_files) fitstbl_usr = PypeItMetaData(spectrograph, par, file_list=data_files, usrdata=usrdata) assert fitstbl['target'][ 0] == 'unknown', 'Grating name changed in file header' assert fitstbl_usr['target'][ 0] == 'test', 'Grating name changed in pypeit file' assert fitstbl['target'][0] != fitstbl_usr['target'][0], \ 'Fits header value and input pypeit file value expected to be different.'
def build_fitstbl(self, strict=True): """ Construct the table with metadata for the frames to reduce. Largely a wrapper for :func:`pypeit.core.load.create_fitstbl`. Args: strict (:obj:`bool`, optional): Function will fault if :func:`fits.getheader` fails to read the headers of any of the files in :attr:`file_list`. Set to False to only report a warning and continue. Returns: :obj:`astropy.table.Table`: Table with the metadata for each fits file to reduce. Note this is different from :attr:`fitstbl` which is a :obj:`PypeItMetaData` object """ # Build and sort the table self.fitstbl = PypeItMetaData(self.spectrograph, par=self.par, files=self.file_list, usrdata=self.usrdata, strict=strict) # Sort by the time if 'time' in self.fitstbl.keys(): self.fitstbl.sort('time') # Add this to the completed steps self.steps.append(inspect.stack()[0][3]) # Return the table return self.fitstbl.table
def get_master_dirkey(self, iFile=None): # Do some checks if self.spectrograph is None or self.par is None: msgs.error() idx = self.check_index(iFile) # Get the master dir if self.par['calibrations']['master_dir'] == 'default': mdir = os.path.join(self.par['rdx']['redux_path'], 'Masters') else: mdir = self.par['calibrations']['master_dir'] if not os.path.exists(mdir): mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir)) msgs.warn('Master file dir: {0} does not exist. Using {1}'.format( mdir, mdir_base)) mdir = mdir_base # Get the base name file_base = self.get_basename(iFile=iFile) ftdict = dict({file_base: 'science'}) fitstbl = PypeItMetaData(self.spectrograph, self.par, files=[self.data_files[idx]], usrdata=Table(self.usrdata[idx]), strict=True) fitstbl.finalize_usr_build(ftdict, self.setups[0]) mkey = fitstbl.master_key(0, det=self.det) # Return the result return mdir, mkey
def read_old_fitstbl(spectrograph, f): fitstbl = PypeItMetaData(spectrograph, data=Table.read(f)) type_bits = np.zeros(len(fitstbl), dtype=fitstbl.bitmask.minimum_dtype()) for bit in fitstbl.bitmask.keys(): type_bits[fitstbl[bit]] = fitstbl.bitmask.turn_on( type_bits[fitstbl[bit]], flag=bit) fitstbl.set_frame_types(type_bits) return fitstbl
def main(args): # Load fits file cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(args.file, runtime=False) # Get the raw fits file name sciIdx = get_science_frame(usrdata) fname = data_files[sciIdx] # Load the spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] spectrograph = load_spectrograph(spectrograph_name, ifile=data_files[sciIdx]) msgs.info('Loaded spectrograph {0}'.format(spectrograph.spectrograph)) spectrograph_cfg_lines = spectrograph.config_specific_par(fname).to_config() par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) # Get the master key file_base = os.path.basename(fname) ftdict = dict({file_base: 'science'}) fitstbl = PypeItMetaData(spectrograph, par, files=[data_files[sciIdx]], usrdata=Table(usrdata[sciIdx]), strict=True) fitstbl.finalize_usr_build(ftdict, setups[0]) mkey = fitstbl.master_key(0, det=args.det) # Load the frame data rawimage, _, _, datasec, _ = spectrograph.get_rawimage(fname, args.det) rawimage = procimg.trim_frame(rawimage, datasec < 1) frame = spectrograph.orient_image(rawimage, args.det) # Set paths if par['calibrations']['caldir'] == 'default': mdir = os.path.join(par['rdx']['redux_path'], 'Masters') else: mdir = par['calibrations']['caldir'] if not os.path.exists(mdir): mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir)) msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base)) mdir = mdir_base # Load the tslits_dict trc_file = os.path.join(mdir, MasterFrame.construct_file_name('Edges', mkey)) + '.gz' tslits_dict = edgetrace.EdgeTraceSet.from_file(trc_file).convert_to_tslits_dict() # Derive an appropriate output filename prefix = os.path.splitext(file_base) if prefix[1] == ".gz": prefix = os.path.splitext(prefix[0])[0] else: prefix = prefix[0] outname = "{0:s}_skyregions.fits".format(prefix) # Finally, initialise the GUI skyreg = SkySubGUI.initialize(args.det, frame, tslits_dict, outname=outname, runtime=False, printout=True) # Get the results skyreg.get_result()
def test_add_reduce(): """Test adding history entries for reducing an exposure""" # Create a PypeItMetadata object to simulate # what would be created in PypeIt::reduce_all # This will contain two calibration groups, one # that combines two frames and has no calibration # frames, and another that combines and subtracts # frames and has calibration frames spectrograph = load_spectrograph('keck_deimos') spectrograph_def_par = spectrograph.default_pypeit_par() ftype_bitmask = FrameTypeBitMask() science_value = ftype_bitmask.turn_on(np.int_(0),'science') standard_value = ftype_bitmask.turn_on(np.int_(0),'standard') arc_value = ftype_bitmask.turn_on(np.int_(0),'arc') tilt_arc_science_value = ftype_bitmask.turn_on(np.int_(0), 'tilt') | arc_value | science_value t = Table(names=['filename', 'frametype', 'framebit', 'target', 'calib', 'comb_id', 'bkg_id'], rows=[['file1.fits','science', science_value, 'targ1', '1', '-1', '-1'], ['file2.fits','standard', standard_value, 'targ2', '1', '-1', '-1'], ['file3.fits','tilt/arc/science',tilt_arc_science_value, 'targ3', '2', '2', '3'], ['file4.fits','tilt/arc/science',tilt_arc_science_value, 'targ4', '2', '3', '2'], ['file5.fits','tilt/arc/science',tilt_arc_science_value, 'targ3', '2', '2', '3'], ['file6.fits','tilt/arc/science',tilt_arc_science_value, 'targ4', '2', '3', '2'], ['file7.fits','arc', arc_value, 'targ5', '2', '-1', '-1']]) metadata = PypeItMetaData(spectrograph, spectrograph_def_par, data=t) metadata.set_calibration_groups() #Create history entries for both calibration groups history = History() history.add_reduce(1, metadata, [0], []) history.add_reduce(2, metadata, [2, 4], [3, 5]) # Verify we get the expected entries expected_history = [' PypeIt Reducing target targ1', 'Combining frames:', '"file1.fits"', ' PypeIt Reducing target targ3', 'Combining frames:', '"file3.fits"', '"file5.fits"', 'Subtracted background from frames:', '"file4.fits"', '"file6.fits"', 'Callibration frames:', 'tilt/arc/science "file3.fits"', 'tilt/arc/science "file4.fits"', 'tilt/arc/science "file5.fits"', 'tilt/arc/science "file6.fits"', 'arc "file7.fits"'] assert verify_history(history, expected_history)
def test_setup_keck_deimos_multiconfig(): root = os.path.join(os.environ['PYPEIT_DEV'], 'RAW_DATA', 'keck_deimos') files = glob.glob(os.path.join(root, '830G_L_8100', '*fits*')) files += glob.glob(os.path.join(root, '830G_L_8400', '*fits*')) output_path = os.path.join(os.getcwd(), 'output') if os.path.isdir(output_path): shutil.rmtree(output_path) os.makedirs(output_path) ps = pypeitsetup.PypeItSetup(files, spectrograph_name='keck_deimos') ps.run(setup_only=True, sort_dir=output_path) # Write the automatically generated pypeit data pypeit_files = ps.fitstbl.write_pypeit(output_path, cfg_lines=ps.user_cfg, write_bkg_pairs=True) assert len(pypeit_files) == 2, 'Should have created two pypeit files' # Test the pypeit files for the correct configuration and # calibration group results for f, s, c in zip(pypeit_files, ['A', 'B'], ['0', '1']): # TODO: All of this front-end stuff, pulled from pypeit.py, should # be put into a function. # Read the pypeit file cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file( f, runtime=True) # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph = load_spectrograph(cfg['rdx']['spectrograph']) # Configuration-specific parameters for idx, row in enumerate(usrdata): if 'science' in row['frametype'] or 'standard' in row['frametype']: break spectrograph_cfg_lines = spectrograph.config_specific_par( data_files[idx]).to_config() # PypeIt parameters par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) # Metadata fitstbl = PypeItMetaData(spectrograph, par, files=data_files, usrdata=usrdata, strict=True) fitstbl.finalize_usr_build(frametype, setups[0]) assert np.all(fitstbl['setup'] == s), 'Setup is wrong' assert np.all(fitstbl['calib'] == c), 'Calibration group is wrong' # Clean-up shutil.rmtree(output_path)
def load_metadata(self, fits_file): """ Load the fitstbl from disk (a binary FITS table) Args: fits_file (str): Name of PypeItMetaData file Returns: obj:`PypeItMetaData`: The so-called fitstbl """ self.fitstbl = PypeItMetaData(self.spectrograph, self.par, data=Table.read(fits_file)) msgs.info("Loaded fitstbl from {:s}".format(fits_file)) return self.fitstbl.table
def test_read_combid(): # ------------------------------------------------------------------ # In case of failed tests setup_dir = data_path('setup_files') if os.path.isdir(setup_dir): shutil.rmtree(setup_dir) config_dir = data_path('shane_kast_blue_A') if os.path.isdir(config_dir): shutil.rmtree(config_dir) # ------------------------------------------------------------------ # Generate the pypeit file with the comb_id droot = data_path('b') pargs = Setup.parse_args([ '-r', droot, '-s', 'shane_kast_blue', '-c=all', '-b', '--extension=fits.gz', '--output_path={:s}'.format(data_path('')) ]) Setup.main(pargs) shutil.rmtree(setup_dir) pypeit_file = os.path.join(config_dir, 'shane_kast_blue_A.pypeit') cfg_lines, data_files, frametype, usrdata, setups, _ = parse_pypeit_file( pypeit_file) # Get the spectrograph spectrograph = None for l in cfg_lines: if 'spectrograph' in l: spectrograph = load_spectrograph(l.split(' ')[-1]) break assert spectrograph is not None, 'Did not appropriately read spectrograph' # Set the metadata pmd = PypeItMetaData(spectrograph, spectrograph.default_pypeit_par(), files=data_files, usrdata=usrdata, strict=False) indx = pmd['filename'] == 'b27.fits.gz' assert pmd['comb_id'][indx] == [1], 'Incorrect combination group ID' assert pmd['comb_id'][np.where(~indx) [0]][0] == -1, 'Incorrect combination group ID' shutil.rmtree(config_dir)
def dummy_fitstbl(nfile=10, spectro_name='shane_kast_blue', directory='', notype=False): """ Generate a dummy fitstbl for testing Parameters ---------- nfile : int, optional Number of files to mimic spectro_name : str, optional Name of spectrograph to mimic notype : bool (optional) If True, do not add image type info to the fitstbl Returns ------- fitstbl : PypeItMetaData """ fitsdict = {} fitsdict['index'] = np.arange(nfile) fitsdict['directory'] = [directory] * nfile fitsdict['filename'] = ['b{:03d}.fits.gz'.format(i) for i in range(nfile)] # TODO: The below will fail at 60 dates = ['2015-01-23T00:{:02d}:11.04'.format(i) for i in range(nfile)] ttime = time.Time(dates, format='isot') fitsdict['mjd'] = ttime.mjd fitsdict['target'] = ['Dummy'] * nfile fitsdict['ra'] = ['00:00:00'] * nfile fitsdict['dec'] = ['+00:00:00'] * nfile fitsdict['exptime'] = [300.] * nfile fitsdict['dispname'] = ['600/4310'] * nfile fitsdict['dichroic'] = ['560'] * nfile fitsdict["binning"] = ['1,1'] * nfile fitsdict["airmass"] = [1.0] * nfile if spectro_name == 'shane_kast_blue': fitsdict['numamplifiers'] = [1] * nfile # Lamps for i in range(1, 17): fitsdict['lampstat{:02d}'.format(i)] = ['off'] * nfile fitsdict['exptime'][0] = 0 # Bias fitsdict['lampstat06'][1] = 'on' # Arc fitsdict['exptime'][1] = 30 # Arc fitsdict['lampstat01'][2] = 'on' # Trace, pixel, slit flat fitsdict['lampstat01'][3] = 'on' # Trace, pixel, slit flat fitsdict['exptime'][2] = 30 # flat fitsdict['exptime'][3] = 30 # flat fitsdict['ra'][4] = '05:06:36.6' # Standard fitsdict['dec'][4] = '52:52:01.0' fitsdict['airmass'][4] = 1.2 fitsdict['ra'][5] = '07:06:23.45' # Random object fitsdict['dec'][5] = '+30:20:50.5' fitsdict['decker'] = ['0.5 arcsec'] * nfile # arrays for k in fitsdict.keys(): fitsdict[k] = np.array(fitsdict[k]) spectrograph = load_spectrograph(spectro_name) fitstbl = PypeItMetaData(spectrograph, spectrograph.default_pypeit_par(), data=fitsdict) fitstbl['instrume'] = spectro_name type_bits = np.zeros(len(fitstbl), dtype=fitstbl.type_bitmask.minimum_dtype()) # Image typing if not notype: if spectro_name == 'shane_kast_blue': #fitstbl['sci_ID'] = 1 # This links all the files to the science object type_bits[0] = fitstbl.type_bitmask.turn_on(type_bits[0], flag='bias') type_bits[1] = fitstbl.type_bitmask.turn_on(type_bits[1], flag='arc') type_bits[1] = fitstbl.type_bitmask.turn_on(type_bits[1], flag='tilt') type_bits[2:4] = fitstbl.type_bitmask.turn_on( type_bits[2:4], flag=['pixelflat', 'trace', 'illumflat']) type_bits[4] = fitstbl.type_bitmask.turn_on(type_bits[4], flag='standard') type_bits[5:] = fitstbl.type_bitmask.turn_on(type_bits[5:], flag='science') fitstbl.set_frame_types(type_bits) # Calibration groups cfgs = fitstbl.unique_configurations( ignore_frames=['bias', 'dark']) fitstbl.set_configurations(cfgs) fitstbl.set_calibration_groups(global_frames=['bias', 'dark']) return fitstbl
def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None, show=False, redux_path=None): # Load cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(pypeit_file, runtime=True) self.pypeit_file = pypeit_file # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] self.spectrograph = load_spectrograph(spectrograph_name) # Par # Defaults spectrograph_def_par = self.spectrograph.default_pypeit_par() # Grab a science file for configuration specific parameters sci_file = None for idx, row in enumerate(usrdata): if 'science' in row['frametype']: sci_file = data_files[idx] break # Set spectrograph_cfg_lines = self.spectrograph.config_specific_par(spectrograph_def_par, sci_file).to_config() self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) # Fitstbl self.fitstbl = PypeItMetaData(self.spectrograph, self.par, file_list=data_files, usrdata=usrdata, strict=True) # The following could be put in a prepare_to_run() method in PypeItMetaData if 'setup' not in self.fitstbl.keys(): self.fitstbl['setup'] = setups[0] self.fitstbl.get_frame_types(user=frametype) # This sets them using the user inputs self.fitstbl.set_defaults() # Only does something if values not set in PypeIt file self.fitstbl._set_calib_group_bits() self.fitstbl._check_calib_groups() # Write .calib file (For QA naming amongst other things) calib_file = pypeit_file.replace('.pypeit', '.calib') self.fitstbl.write_calib(calib_file) # Other Internals self.logname = logname self.overwrite = overwrite # Currently the runtime argument determines the behavior for reuse_masters. There is also a reuse_masters # parameter in the parset but it is currently ignored. self.reuse_masters=reuse_masters self.show = show # Make the output directories self.par['rdx']['redux_path'] = os.getcwd() if redux_path is None else redux_path msgs.info("Setting reduction path to {:s}".format(self.par['rdx']['redux_path'])) paths.make_dirs(self.spectrograph.spectrograph, self.par['calibrations']['caldir'], self.par['rdx']['scidir'], self.par['rdx']['qadir'], overwrite=self.overwrite, redux_path=self.par['rdx']['redux_path']) # Instantiate Calibrations class self.caliBrate \ = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], self.spectrograph, redux_path=self.par['rdx']['redux_path'], reuse_masters=self.reuse_masters, save_masters=True, write_qa=True, show=self.show) # Init self.verbosity = verbosity # TODO: I don't think this ever used self.frame = None self.det = None self.tstart = None self.basename = None self.sciI = None self.obstime = None
def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None, show=False, redux_path=None): # Load cfg_lines, data_files, frametype, usrdata, setups \ = parse_pypeit_file(pypeit_file, runtime=True) self.pypeit_file = pypeit_file # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] self.spectrograph = load_spectrograph(spectrograph_name, ifile=data_files[0]) msgs.info('Loaded spectrograph {0}'.format( self.spectrograph.spectrograph)) # -------------------------------------------------------------- # Get the full set of PypeIt parameters # - Grab a science or standard file for configuration specific parameters scistd_file = None for idx, row in enumerate(usrdata): if ('science' in row['frametype']) or ('standard' in row['frametype']): scistd_file = data_files[idx] break # - Configuration specific parameters for the spectrograph if scistd_file is not None: msgs.info( 'Setting configuration-specific parameters using {0}'.format( os.path.split(scistd_file)[1])) spectrograph_cfg_lines = self.spectrograph.config_specific_par( scistd_file).to_config() # - Build the full set, merging with any user-provided # parameters self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) msgs.info('Built full PypeIt parameter set.') # Check the output paths are ready if redux_path is not None: self.par['rdx']['redux_path'] = redux_path # TODO: Write the full parameter set here? # -------------------------------------------------------------- # -------------------------------------------------------------- # Build the meta data # - Re-initilize based on the file data msgs.info('Compiling metadata') self.fitstbl = PypeItMetaData(self.spectrograph, self.par, files=data_files, usrdata=usrdata, strict=True) # - Interpret automated or user-provided data from the PypeIt # file self.fitstbl.finalize_usr_build(frametype, setups[0]) # -------------------------------------------------------------- # - Write .calib file (For QA naming amongst other things) calib_file = pypeit_file.replace('.pypeit', '.calib') self.fitstbl.write_calib(calib_file) # Other Internals self.logname = logname self.overwrite = overwrite # Currently the runtime argument determines the behavior for # reuse_masters. self.reuse_masters = reuse_masters self.show = show # Set paths if self.par['calibrations']['caldir'] == 'default': self.calibrations_path = os.path.join( self.par['rdx']['redux_path'], 'Masters') else: self.calibrations_path = self.par['calibrations']['caldir'] # Report paths msgs.info('Setting reduction path to {0}'.format( self.par['rdx']['redux_path'])) msgs.info('Master calibration data output to: {0}'.format( self.calibrations_path)) msgs.info('Science data output to: {0}'.format(self.science_path)) msgs.info('Quality assessment plots output to: {0}'.format( self.qa_path)) # TODO: Is anything written to the qa dir or only to qa/PNGs? # Should we have separate calibration and science QA # directories? # Instantiate Calibrations class self.caliBrate \ = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], self.spectrograph, caldir=self.calibrations_path, qadir=self.qa_path, reuse_masters=self.reuse_masters, show=self.show) # Init self.verbosity = verbosity # TODO: I don't think this ever used self.frame = None self.det = None self.tstart = None self.basename = None self.sciI = None self.obstime = None
def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None, show=False, redux_path=None, calib_only=False): # Load cfg_lines, data_files, frametype, usrdata, setups \ = parse_pypeit_file(pypeit_file, runtime=True) self.pypeit_file = pypeit_file self.calib_only = calib_only # Spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] self.spectrograph = load_spectrograph(spectrograph_name) msgs.info('Loaded spectrograph {0}'.format( self.spectrograph.spectrograph)) # -------------------------------------------------------------- # Get the full set of PypeIt parameters # - Grab a science or standard file for configuration specific parameters config_specific_file = None for idx, row in enumerate(usrdata): if ('science' in row['frametype']) or ('standard' in row['frametype']): config_specific_file = data_files[idx] # search for arcs, trace if no scistd was there if config_specific_file is None: for idx, row in enumerate(usrdata): if ('arc' in row['frametype']) or ('trace' in row['frametype']): config_specific_file = data_files[idx] if config_specific_file is not None: msgs.info( 'Setting configuration-specific parameters using {0}'.format( os.path.split(config_specific_file)[1])) spectrograph_cfg_lines = self.spectrograph.config_specific_par( config_specific_file).to_config() # - Build the full set, merging with any user-provided # parameters self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) msgs.info('Built full PypeIt parameter set.') # Check the output paths are ready if redux_path is not None: self.par['rdx']['redux_path'] = redux_path # TODO: Write the full parameter set here? # -------------------------------------------------------------- # -------------------------------------------------------------- # Build the meta data # - Re-initilize based on the file data msgs.info('Compiling metadata') self.fitstbl = PypeItMetaData(self.spectrograph, self.par, files=data_files, usrdata=usrdata, strict=True) # - Interpret automated or user-provided data from the PypeIt # file self.fitstbl.finalize_usr_build(frametype, setups[0]) # -------------------------------------------------------------- # - Write .calib file (For QA naming amongst other things) calib_file = pypeit_file.replace('.pypeit', '.calib') self.fitstbl.write_calib(calib_file) # Other Internals self.logname = logname self.overwrite = overwrite # Currently the runtime argument determines the behavior for # reuse_masters. self.reuse_masters = reuse_masters self.show = show # Set paths self.calibrations_path = os.path.join( self.par['rdx']['redux_path'], self.par['calibrations']['master_dir']) # Check for calibrations if not self.calib_only: calibrations.check_for_calibs( self.par, self.fitstbl, raise_error=self.par['calibrations']['raise_chk_error']) # Report paths msgs.info('Setting reduction path to {0}'.format( self.par['rdx']['redux_path'])) msgs.info('Master calibration data output to: {0}'.format( self.calibrations_path)) msgs.info('Science data output to: {0}'.format(self.science_path)) msgs.info('Quality assessment plots output to: {0}'.format( self.qa_path)) # TODO: Is anything written to the qa dir or only to qa/PNGs? # Should we have separate calibration and science QA # directories? # An html file wrapping them all too # # Instantiate Calibrations class # if self.spectrograph.pypeline in ['MultiSlit', 'Echelle']: # self.caliBrate \ # = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], # self.spectrograph, self.calibrations_path, # qadir=self.qa_path, # reuse_masters=self.reuse_masters, # show=self.show, # slitspat_num=self.par['rdx']['slitspatnum']) # elif self.spectrograph.pypeline in ['IFU']: # self.caliBrate \ # = calibrations.IFUCalibrations(self.fitstbl, self.par['calibrations'], # self.spectrograph, # self.calibrations_path, # qadir=self.qa_path, # reuse_masters=self.reuse_masters, # show=self.show) # else: # msgs.error("No calibration available to support pypeline: {0:s}".format(self.spectrograph.pypeline)) # Init self.verbosity = verbosity # TODO: I don't think this ever used self.det = None self.tstart = None self.basename = None self.sciI = None self.obstime = None