Exemple #1
0
    def __init__(self,
                 file_list,
                 path=None,
                 frametype=None,
                 usrdata=None,
                 setups=None,
                 cfg_lines=None,
                 spectrograph_name=None,
                 pypeit_file=None,
                 setup_dict=None):

        # The provided list of files cannot be None
        if file_list is None or len(file_list) == 0:
            msgs.error('Must provide a list of files to be reduced!')

        # Save input
        self.file_list = file_list
        # TODO: It seems like path is never used...
        self.path = os.getcwd() if path is None else path
        self.frametype = frametype
        self.usrdata = usrdata
        self.setups = setups
        self.pypeit_file = pypeit_file
        self.user_cfg = cfg_lines
        self.setup_dict = setup_dict

        # Determine the spectrograph name
        _spectrograph_name = spectrograph_name if cfg_lines is None \
                    else PypeItPar.from_cfg_lines(merge_with=cfg_lines)['rdx']['spectrograph']

        # Cannot proceed without spectrograph name
        if _spectrograph_name is None:
            msgs.error(
                'Must provide spectrograph name directly or using configuration lines.'
            )

        # Instantiate the spectrograph
        self.spectrograph = load_spectrograph(
            _spectrograph_name)  #, ifile=file_list[0])

        # Get the spectrograph specific configuration to be merged with
        # the user modifications.
        spectrograph_cfg_lines = self.spectrograph.default_pypeit_par(
        ).to_config()

        # Instantiate the pypeit parameters.  The user input
        # configuration (cfg_lines) can be None.
        self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines,
                                            merge_with=cfg_lines)

        # Prepare internals for execution
        self.fitstbl = None
        self.steps = []
Exemple #2
0
def main(args):

    # Load fits file
    cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(args.file, runtime=False)

    # Get the raw fits file name
    sciIdx = get_science_frame(usrdata)
    fname = data_files[sciIdx]

    # Load the spectrograph
    cfg = ConfigObj(cfg_lines)
    spectrograph_name = cfg['rdx']['spectrograph']
    spectrograph = load_spectrograph(spectrograph_name, ifile=data_files[sciIdx])
    msgs.info('Loaded spectrograph {0}'.format(spectrograph.spectrograph))
    spectrograph_cfg_lines = spectrograph.config_specific_par(fname).to_config()
    par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines)

    # Get the master key
    file_base = os.path.basename(fname)
    ftdict = dict({file_base: 'science'})
    fitstbl = PypeItMetaData(spectrograph, par, files=[data_files[sciIdx]], usrdata=Table(usrdata[sciIdx]), strict=True)
    fitstbl.finalize_usr_build(ftdict, setups[0])
    mkey = fitstbl.master_key(0, det=args.det)

    # Load the frame data
    rawimage, _, _, datasec, _ = spectrograph.get_rawimage(fname, args.det)
    rawimage = procimg.trim_frame(rawimage, datasec < 1)
    frame = spectrograph.orient_image(rawimage, args.det)

    # Set paths
    if par['calibrations']['caldir'] == 'default':
        mdir = os.path.join(par['rdx']['redux_path'], 'Masters')
    else:
        mdir = par['calibrations']['caldir']

    if not os.path.exists(mdir):
        mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir))
        msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base))
        mdir = mdir_base

    # Load the tslits_dict
    trc_file = os.path.join(mdir, MasterFrame.construct_file_name('Edges', mkey)) + '.gz'
    tslits_dict = edgetrace.EdgeTraceSet.from_file(trc_file).convert_to_tslits_dict()

    # Derive an appropriate output filename
    prefix = os.path.splitext(file_base)
    if prefix[1] == ".gz":
        prefix = os.path.splitext(prefix[0])[0]
    else:
        prefix = prefix[0]
    outname = "{0:s}_skyregions.fits".format(prefix)

    # Finally, initialise the GUI
    skyreg = SkySubGUI.initialize(args.det, frame, tslits_dict, outname=outname, runtime=False, printout=True)

    # Get the results
    skyreg.get_result()
Exemple #3
0
def test_setup_keck_deimos_multiconfig():

    root = os.path.join(os.environ['PYPEIT_DEV'], 'RAW_DATA', 'keck_deimos')
    files = glob.glob(os.path.join(root, '830G_L_8100', '*fits*'))
    files += glob.glob(os.path.join(root, '830G_L_8400', '*fits*'))

    output_path = os.path.join(os.getcwd(), 'output')
    if os.path.isdir(output_path):
        shutil.rmtree(output_path)
    os.makedirs(output_path)

    ps = pypeitsetup.PypeItSetup(files, spectrograph_name='keck_deimos')
    ps.run(setup_only=True, sort_dir=output_path)
    # Write the automatically generated pypeit data
    pypeit_files = ps.fitstbl.write_pypeit(output_path,
                                           cfg_lines=ps.user_cfg,
                                           write_bkg_pairs=True)

    assert len(pypeit_files) == 2, 'Should have created two pypeit files'

    # Test the pypeit files for the correct configuration and
    # calibration group results
    for f, s, c in zip(pypeit_files, ['A', 'B'], ['0', '1']):

        # TODO: All of this front-end stuff, pulled from pypeit.py, should
        # be put into a function.

        # Read the pypeit file
        cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(
            f, runtime=True)
        # Spectrograph
        cfg = ConfigObj(cfg_lines)
        spectrograph = load_spectrograph(cfg['rdx']['spectrograph'])
        # Configuration-specific parameters
        for idx, row in enumerate(usrdata):
            if 'science' in row['frametype'] or 'standard' in row['frametype']:
                break
        spectrograph_cfg_lines = spectrograph.config_specific_par(
            data_files[idx]).to_config()
        #  PypeIt parameters
        par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines,
                                       merge_with=cfg_lines)
        #  Metadata
        fitstbl = PypeItMetaData(spectrograph,
                                 par,
                                 files=data_files,
                                 usrdata=usrdata,
                                 strict=True)
        fitstbl.finalize_usr_build(frametype, setups[0])

        assert np.all(fitstbl['setup'] == s), 'Setup is wrong'
        assert np.all(fitstbl['calib'] == c), 'Calibration group is wrong'

    # Clean-up
    shutil.rmtree(output_path)
Exemple #4
0
 def load_spectrograph_parset(self, iFile=None):
     # Do some checks
     idx = self.check_index(iFile)
     # Grab the filename
     fname = self.data_files[idx]
     # Setup the configuration
     cfg = ConfigObj(self.cfg_lines)
     # Load the spectrograph
     spectrograph_name = cfg['rdx']['spectrograph']
     self.spectrograph = load_spectrograph(spectrograph_name)
     msgs.info('Loaded spectrograph {0}'.format(self.spectrograph.name))
     spectrograph_cfg_lines = self.spectrograph.config_specific_par(fname).to_config()
     # Load the parset
     self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=self.cfg_lines)
     return
Exemple #5
0
def main(args):
    """

    Args:
        args:

    Returns:
        astropy.table.Table:

    """

    import os

    from IPython import embed

    import numpy as np

    from astropy import table

    from pypeit.pypeitsetup import PypeItSetup
    from pypeit import calibrations
    from pypeit import msgs
    from pypeit.par import PypeItPar

    import shutil

    # Check that the spectrograph is provided if using a file root
    if args.root is not None:
        if args.spectrograph is None:
            raise ValueError(
                'Must provide spectrograph identifier with file root.')
        # Check that input spectrograph is supported
        if args.spectrograph not in available_spectrographs:
            raise ValueError(
                'Instrument \'{0}\' unknown to PypeIt.\n'.format(
                    args.spectrograph) + '\tOptions are: {0}\n'.format(
                        ', '.join(available_spectrographs)) +
                '\tSelect an available instrument or consult the documentation '
                + 'on how to add a new instrument.')

    # Initialize PypeItSetup based on the arguments
    ps = PypeItSetup.from_file_root(args.root,
                                    args.spectrograph,
                                    extension=args.extension)

    # Run the setup
    ps.run(setup_only=True)  #, write_bkg_pairs=args.background)
    is_science = ps.fitstbl.find_frames('science')

    msgs.info('Loaded spectrograph {0}'.format(ps.spectrograph.name))

    # Unique configurations
    uniq_cfg = ps.fitstbl.unique_configurations(copy=True)

    # Setup the table. Need to use object type for strings so that
    # they're not truncated.
    answers = table.Table()
    answers['setups'] = list(uniq_cfg.keys())
    # Add the configuration columns
    for setup, setup_dict in uniq_cfg.items():
        for key, value in setup_dict.items():
            answers[key] = np.empty(len(answers), dtype=object) if isinstance(value, str) \
                                else type(value)(0)
        break
    answers['pass'] = False
    answers['scifiles'] = np.empty(len(answers), dtype=object)

    for i, setup in enumerate(uniq_cfg.keys()):
        for setup_key, setup_value in uniq_cfg[setup].items():
            answers[setup_key] = setup_value
        if setup == 'None':
            print("There is a setup without science frames.  Skipping...")
            answers['pass'][i] = False
            answers['scifiles'][i] = None
            continue

        msgs.info(
            '======================================================================='
        )
        msgs.info('Working on setup: {}'.format(setup))
        msgs.info(str(uniq_cfg[setup]))
        msgs.info(
            '======================================================================='
        )

        # TODO: Make the snippet below, which is also in the init of
        # PypeIt a method somewhere
        in_cfg = ps.fitstbl['setup'] == setup
        config_specific_file = None

        # Grab a science/standard frame
        data_files = [
            os.path.join(row['directory'], row['filename'])
            for row in ps.fitstbl[in_cfg]
        ]
        for idx, row in enumerate(ps.fitstbl[in_cfg]):
            if 'science' in row['frametype'] or 'standard' in row['frametype']:
                config_specific_file = data_files[idx]
        if config_specific_file is not None:
            msgs.info(
                'Setting configuration-specific parameters using {0}'.format(
                    os.path.split(config_specific_file)[1]))
        else:
            msgs.warn('No science or standard frame.  Punting..')
            answers['pass'][i] = False
            answers['scifiles'][i] = None
            continue
        #
        spectrograph_cfg_lines = ps.spectrograph.config_specific_par(
            config_specific_file).to_config()

        #   - Build the full set, merging with any user-provided
        #     parameters
        par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines)
        # Print science frames
        if np.any(in_cfg & is_science):
            msgs.info('Your science frames are: {0}'.format(
                ps.fitstbl['filename'][in_cfg & is_science].tolist()))
            answers['scifiles'][i] = ', '.join(
                ps.fitstbl['filename'][in_cfg & is_science].tolist())
        else:
            msgs.warn("This setup has no science frames!")
            answers['scifiles'][i] = ''

        # Check!
        answers['pass'][i] = calibrations.check_for_calibs(par,
                                                           ps.fitstbl,
                                                           raise_error=False,
                                                           cut_cfg=in_cfg)
        if not answers['pass'][i]:
            msgs.warn(
                "Setup {} did not pass the calibration check!".format(setup))

    print('= RESULTS ============================================')
    # Print
    answers.pprint_all()
    print('======================================================')
    # Remove setup_files
    if not args.save_setups:
        shutil.rmtree('setup_files')
    # Return objects used by unit tests
    return answers, ps
Exemple #6
0
def main(args):
    """

    Args:
        args:

    Returns:
        astropy.table.Table:

    """

    import os
    import numpy as np

    from astropy import table

    from pypeit.pypeitsetup import PypeItSetup
    from pypeit import calibrations
    from pypeit import msgs
    from pypeit.par import PypeItPar

    # Check that the spectrograph is provided if using a file root
    if args.root is not None:
        if args.spectrograph is None:
            raise ValueError(
                'Must provide spectrograph identifier with file root.')
        # Check that input spectrograph is supported
        instruments_served = defs.pypeit_spectrographs
        if args.spectrograph not in instruments_served:
            raise ValueError(
                'Instrument \'{0}\' unknown to PypeIt.\n'.format(
                    args.spectrograph) +
                '\tOptions are: {0}\n'.format(', '.join(instruments_served)) +
                '\tSelect an available instrument or consult the documentation '
                + 'on how to add a new instrument.')

    # Initialize PypeItSetup based on the arguments
    ps = PypeItSetup.from_file_root(args.root,
                                    args.spectrograph,
                                    extension=args.extension)

    # Run the setup
    ps.run(setup_only=True)  #, write_bkg_pairs=args.background)
    is_science = ps.fitstbl.find_frames('science')

    msgs.info('Loaded spectrograph {0}'.format(ps.spectrograph.spectrograph))

    # Unique configurations
    setups, indx = ps.fitstbl.get_configuration_names(return_index=True)

    answers = table.Table()
    answers['setups'] = setups
    passes, scifiles, cfgs = [], [], []

    for setup, i in zip(setups, indx):
        # Get the setup lines
        cfg = ps.fitstbl.get_setup(i, config_only=False)
        cfgs.append(cfg)
        if setup == 'None':
            print("There is a setup without science frames.  Skipping...")
            passes.append(False)
            scifiles.append(None)
            continue
        in_cfg = ps.fitstbl['setup'] == setup
        # TODO -- Make the snippet below, which is also in the init of PypeIt a method somewhere
        config_specific_file = None

        msgs.info(
            '======================================================================='
        )
        msgs.info('Working on setup: {}'.format(setup))
        msgs.info(str(cfg))
        msgs.info(
            '======================================================================='
        )

        # Grab a science/standard frame
        data_files = [
            os.path.join(row['directory'], row['filename'])
            for row in ps.fitstbl[in_cfg]
        ]
        for idx, row in enumerate(ps.fitstbl[in_cfg]):
            if ('science' in row['frametype']) or ('standard'
                                                   in row['frametype']):
                config_specific_file = data_files[idx]
        if config_specific_file is not None:
            msgs.info(
                'Setting configuration-specific parameters using {0}'.format(
                    os.path.split(config_specific_file)[1]))
        else:
            msgs.warn('No science or standard frame.  Punting..')
            passes.append(False)
            scifiles.append(None)
            continue
        #
        spectrograph_cfg_lines = ps.spectrograph.config_specific_par(
            config_specific_file).to_config()

        #   - Build the full set, merging with any user-provided
        #     parameters
        par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines)
        # Print science frames
        if np.any(in_cfg & is_science):
            msgs.info("Your science frames are: {}".format(
                ps.fitstbl['filename'][in_cfg & is_science]))
            scifiles.append(','.join(ps.fitstbl['filename'][in_cfg
                                                            & is_science]))
        else:
            msgs.warn("This setup has no science frames!")
            scifiles.append('')
        # Check!
        passed = calibrations.check_for_calibs(par,
                                               ps.fitstbl,
                                               raise_error=False,
                                               cut_cfg=in_cfg)
        if not passed:
            msgs.warn(
                "Setup {} did not pass the calibration check!".format(setup))
        #
        passes.append(passed)

    print('= RESULTS ============================================')

    # Pass/fail
    answers['pass'] = passes

    # Parse the configs
    pcfg = dict(disperser=[],
                angle=[],
                dichroic=[],
                decker=[],
                slitwid=[],
                binning=[])
    for cfg in cfgs:
        # None?
        if len(cfg) == 0:
            for key in pcfg.keys():
                pcfg[key].append(None)
            continue
        # Load it up
        key0 = list(cfg.keys())[0]
        subd = cfg[key0]['--']  # for convenience
        pcfg['disperser'].append(subd['disperser']['name'])
        pcfg['angle'].append(subd['disperser']['angle'])
        pcfg['dichroic'].append(subd['dichroic'])
        pcfg['decker'].append(subd['slit']['decker'])
        pcfg['slitwid'].append(subd['slit']['slitwid'])
        pcfg['binning'].append(subd['binning'])

    # Add
    for key in pcfg.keys():
        answers[key] = pcfg[key]

    # Sci files [put this last as it can get large]
    answers['scifiles'] = scifiles

    # Print
    answers.pprint_all()
    print(
        '= ==================================================================================='
    )
    # Return
    return answers, ps
Exemple #7
0
    def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None,
                 show=False, redux_path=None):

        # Load
        cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(pypeit_file, runtime=True)
        self.pypeit_file = pypeit_file

        # Spectrograph
        cfg = ConfigObj(cfg_lines)
        spectrograph_name = cfg['rdx']['spectrograph']
        self.spectrograph = load_spectrograph(spectrograph_name)

        # Par
        # Defaults
        spectrograph_def_par = self.spectrograph.default_pypeit_par()
        # Grab a science file for configuration specific parameters
        sci_file = None
        for idx, row in enumerate(usrdata):
            if 'science' in row['frametype']:
                sci_file = data_files[idx]
                break

        # Set
        spectrograph_cfg_lines = self.spectrograph.config_specific_par(spectrograph_def_par, sci_file).to_config()
        self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines)

        # Fitstbl
        self.fitstbl = PypeItMetaData(self.spectrograph, self.par, file_list=data_files,
                                      usrdata=usrdata, strict=True)

        # The following could be put in a prepare_to_run() method in PypeItMetaData
        if 'setup' not in self.fitstbl.keys():
            self.fitstbl['setup'] = setups[0]
        self.fitstbl.get_frame_types(user=frametype)  # This sets them using the user inputs
        self.fitstbl.set_defaults()  # Only does something if values not set in PypeIt file
        self.fitstbl._set_calib_group_bits()
        self.fitstbl._check_calib_groups()
        # Write .calib file (For QA naming amongst other things)
        calib_file = pypeit_file.replace('.pypeit', '.calib')
        self.fitstbl.write_calib(calib_file)

        # Other Internals
        self.logname = logname
        self.overwrite = overwrite
        # Currently the runtime argument determines the behavior for reuse_masters. There is also a reuse_masters
        # parameter in the parset but it is currently ignored.
        self.reuse_masters=reuse_masters
        self.show = show

        # Make the output directories
        self.par['rdx']['redux_path'] = os.getcwd() if redux_path is None else redux_path
        msgs.info("Setting reduction path to {:s}".format(self.par['rdx']['redux_path']))
        paths.make_dirs(self.spectrograph.spectrograph, self.par['calibrations']['caldir'],
                        self.par['rdx']['scidir'], self.par['rdx']['qadir'],
                        overwrite=self.overwrite, redux_path=self.par['rdx']['redux_path'])

        # Instantiate Calibrations class
        self.caliBrate \
            = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], self.spectrograph,
                                                 redux_path=self.par['rdx']['redux_path'],
                                                 reuse_masters=self.reuse_masters,
                                                 save_masters=True, write_qa=True,
                                                 show=self.show)
        # Init
        self.verbosity = verbosity
        # TODO: I don't think this ever used

        self.frame = None
        self.det = None

        self.tstart = None
        self.basename = None
        self.sciI = None
        self.obstime = None
Exemple #8
0
    def __init__(self,
                 pypeit_file,
                 verbosity=2,
                 overwrite=True,
                 reuse_masters=False,
                 logname=None,
                 show=False,
                 redux_path=None):

        # Load
        cfg_lines, data_files, frametype, usrdata, setups \
                = parse_pypeit_file(pypeit_file, runtime=True)
        self.pypeit_file = pypeit_file

        # Spectrograph
        cfg = ConfigObj(cfg_lines)
        spectrograph_name = cfg['rdx']['spectrograph']
        self.spectrograph = load_spectrograph(spectrograph_name,
                                              ifile=data_files[0])
        msgs.info('Loaded spectrograph {0}'.format(
            self.spectrograph.spectrograph))

        # --------------------------------------------------------------
        # Get the full set of PypeIt parameters
        #   - Grab a science or standard file for configuration specific parameters
        scistd_file = None
        for idx, row in enumerate(usrdata):
            if ('science' in row['frametype']) or ('standard'
                                                   in row['frametype']):
                scistd_file = data_files[idx]
                break
        #   - Configuration specific parameters for the spectrograph
        if scistd_file is not None:
            msgs.info(
                'Setting configuration-specific parameters using {0}'.format(
                    os.path.split(scistd_file)[1]))
        spectrograph_cfg_lines = self.spectrograph.config_specific_par(
            scistd_file).to_config()
        #   - Build the full set, merging with any user-provided
        #     parameters
        self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines,
                                            merge_with=cfg_lines)
        msgs.info('Built full PypeIt parameter set.')

        # Check the output paths are ready
        if redux_path is not None:
            self.par['rdx']['redux_path'] = redux_path

        # TODO: Write the full parameter set here?
        # --------------------------------------------------------------

        # --------------------------------------------------------------
        # Build the meta data
        #   - Re-initilize based on the file data
        msgs.info('Compiling metadata')
        self.fitstbl = PypeItMetaData(self.spectrograph,
                                      self.par,
                                      files=data_files,
                                      usrdata=usrdata,
                                      strict=True)
        #   - Interpret automated or user-provided data from the PypeIt
        #   file
        self.fitstbl.finalize_usr_build(frametype, setups[0])
        # --------------------------------------------------------------
        #   - Write .calib file (For QA naming amongst other things)
        calib_file = pypeit_file.replace('.pypeit', '.calib')
        self.fitstbl.write_calib(calib_file)

        # Other Internals
        self.logname = logname
        self.overwrite = overwrite

        # Currently the runtime argument determines the behavior for
        # reuse_masters.
        self.reuse_masters = reuse_masters
        self.show = show

        # Set paths
        if self.par['calibrations']['caldir'] == 'default':
            self.calibrations_path = os.path.join(
                self.par['rdx']['redux_path'], 'Masters')
        else:
            self.calibrations_path = self.par['calibrations']['caldir']

        # Report paths
        msgs.info('Setting reduction path to {0}'.format(
            self.par['rdx']['redux_path']))
        msgs.info('Master calibration data output to: {0}'.format(
            self.calibrations_path))
        msgs.info('Science data output to: {0}'.format(self.science_path))
        msgs.info('Quality assessment plots output to: {0}'.format(
            self.qa_path))
        # TODO: Is anything written to the qa dir or only to qa/PNGs?
        # Should we have separate calibration and science QA
        # directories?

        # Instantiate Calibrations class
        self.caliBrate \
            = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'],
                                                 self.spectrograph,
                                                 caldir=self.calibrations_path,
                                                 qadir=self.qa_path,
                                                 reuse_masters=self.reuse_masters,
                                                 show=self.show)
        # Init
        self.verbosity = verbosity
        # TODO: I don't think this ever used

        self.frame = None
        self.det = None

        self.tstart = None
        self.basename = None
        self.sciI = None
        self.obstime = None
Exemple #9
0
    def __init__(self,
                 pypeit_file,
                 verbosity=2,
                 overwrite=True,
                 reuse_masters=False,
                 logname=None,
                 show=False,
                 redux_path=None,
                 calib_only=False):

        # Load
        cfg_lines, data_files, frametype, usrdata, setups \
                = parse_pypeit_file(pypeit_file, runtime=True)
        self.pypeit_file = pypeit_file
        self.calib_only = calib_only

        # Spectrograph
        cfg = ConfigObj(cfg_lines)
        spectrograph_name = cfg['rdx']['spectrograph']
        self.spectrograph = load_spectrograph(spectrograph_name)
        msgs.info('Loaded spectrograph {0}'.format(
            self.spectrograph.spectrograph))

        # --------------------------------------------------------------
        # Get the full set of PypeIt parameters
        #   - Grab a science or standard file for configuration specific parameters

        config_specific_file = None
        for idx, row in enumerate(usrdata):
            if ('science' in row['frametype']) or ('standard'
                                                   in row['frametype']):
                config_specific_file = data_files[idx]
        # search for arcs, trace if no scistd was there
        if config_specific_file is None:
            for idx, row in enumerate(usrdata):
                if ('arc' in row['frametype']) or ('trace'
                                                   in row['frametype']):
                    config_specific_file = data_files[idx]
        if config_specific_file is not None:
            msgs.info(
                'Setting configuration-specific parameters using {0}'.format(
                    os.path.split(config_specific_file)[1]))
        spectrograph_cfg_lines = self.spectrograph.config_specific_par(
            config_specific_file).to_config()

        #   - Build the full set, merging with any user-provided
        #     parameters
        self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines,
                                            merge_with=cfg_lines)
        msgs.info('Built full PypeIt parameter set.')

        # Check the output paths are ready
        if redux_path is not None:
            self.par['rdx']['redux_path'] = redux_path

        # TODO: Write the full parameter set here?
        # --------------------------------------------------------------

        # --------------------------------------------------------------
        # Build the meta data
        #   - Re-initilize based on the file data
        msgs.info('Compiling metadata')
        self.fitstbl = PypeItMetaData(self.spectrograph,
                                      self.par,
                                      files=data_files,
                                      usrdata=usrdata,
                                      strict=True)
        #   - Interpret automated or user-provided data from the PypeIt
        #   file
        self.fitstbl.finalize_usr_build(frametype, setups[0])
        # --------------------------------------------------------------
        #   - Write .calib file (For QA naming amongst other things)
        calib_file = pypeit_file.replace('.pypeit', '.calib')
        self.fitstbl.write_calib(calib_file)

        # Other Internals
        self.logname = logname
        self.overwrite = overwrite

        # Currently the runtime argument determines the behavior for
        # reuse_masters.
        self.reuse_masters = reuse_masters
        self.show = show

        # Set paths
        self.calibrations_path = os.path.join(
            self.par['rdx']['redux_path'],
            self.par['calibrations']['master_dir'])

        # Check for calibrations
        if not self.calib_only:
            calibrations.check_for_calibs(
                self.par,
                self.fitstbl,
                raise_error=self.par['calibrations']['raise_chk_error'])

        # Report paths
        msgs.info('Setting reduction path to {0}'.format(
            self.par['rdx']['redux_path']))
        msgs.info('Master calibration data output to: {0}'.format(
            self.calibrations_path))
        msgs.info('Science data output to: {0}'.format(self.science_path))
        msgs.info('Quality assessment plots output to: {0}'.format(
            self.qa_path))
        # TODO: Is anything written to the qa dir or only to qa/PNGs?
        # Should we have separate calibration and science QA
        # directories?
        # An html file wrapping them all too

        #        # Instantiate Calibrations class
        #        if self.spectrograph.pypeline in ['MultiSlit', 'Echelle']:
        #            self.caliBrate \
        #                = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'],
        #                                                     self.spectrograph, self.calibrations_path,
        #                                                     qadir=self.qa_path,
        #                                                     reuse_masters=self.reuse_masters,
        #                                                     show=self.show,
        #                                                     slitspat_num=self.par['rdx']['slitspatnum'])
        #        elif self.spectrograph.pypeline in ['IFU']:
        #            self.caliBrate \
        #                = calibrations.IFUCalibrations(self.fitstbl, self.par['calibrations'],
        #                                               self.spectrograph,
        #                                               self.calibrations_path,
        #                                               qadir=self.qa_path,
        #                                               reuse_masters=self.reuse_masters,
        #                                               show=self.show)
        #        else:
        #            msgs.error("No calibration available to support pypeline: {0:s}".format(self.spectrograph.pypeline))

        # Init
        self.verbosity = verbosity
        # TODO: I don't think this ever used

        self.det = None

        self.tstart = None
        self.basename = None
        self.sciI = None
        self.obstime = None