Ejemplo n.º 1
0
def test_deimos():
    # Raw DEIMOS directory
    raw_dir = os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'keck_deimos')

    # Get the list of setup directories
    setups = glob.glob(os.path.join(raw_dir, '*'))

    # Set the output path and *remove if* if it already exists
    output_path = os.path.join(os.getcwd(), 'output')
    if os.path.isdir(output_path):
        shutil.rmtree(output_path)

    # Iterate through the setups
    for setup in setups:

        # Find the relevant pypeit file constructed by hand.
        by_hand_pypeit = os.path.join(
            os.getenv('PYPEIT_DEV'), 'pypeit_files',
            'keck_deimos_{0}.pypeit'.format(os.path.split(setup)[1].lower()))

        if not os.path.isfile(by_hand_pypeit):
            # It doesn't exist, so assume there is no by-hand pypeit
            # file to compare to
            continue

        # Run pypeit_setup
        ps = PypeItSetup.from_file_root(setup,
                                        'keck_deimos',
                                        output_path=output_path)
        ps.run(setup_only=True)
        # Write the automatically generated pypeit data
        pypeit_files = ps.fitstbl.write_pypeit(output_path,
                                               cfg_lines=ps.user_cfg)

        # Read the frame types from both the by-hand and automated
        # pypeit files
        _, _, by_hand_frametypes, _, _ = parse_pypeit_file(by_hand_pypeit,
                                                           file_check=False)
        _, _, auto_frametypes, _, _ = parse_pypeit_file(pypeit_files[0],
                                                        file_check=False)

        # For each file in the by-hand list, check that the frame types
        # in the automatically generated pypeit file are identical
        for f in by_hand_frametypes.keys():
            type_list = np.sort(by_hand_frametypes[f].split(','))
            if 'science' in type_list or 'standard' in type_list:
                # Only ensuring that calibrations are correctly typed
                continue
            assert f in auto_frametypes.keys(), \
                'Frame {0} not automatically parsed for setup {1}.'.format(f, setup)
            assert np.array_equal(type_list, np.sort(auto_frametypes[f].split(','))), \
                'Frame types differ for file {0} in setup {1}\n'.format(f, setup) \
                 + '    By-hand types: {0}'.format(by_hand_frametypes[f]) \
                 + '    Automated types: {0}'.format(auto_frametypes[f])

        # Clean up after every setup
        shutil.rmtree(output_path)
Ejemplo n.º 2
0
def test_lris_blue_pypeit_overwrite():
    f = os.path.join(os.environ['PYPEIT_DEV'],
                     'pypeit_files/keck_lris_blue_long_400_3400_d560.pypeit')
    assert os.path.isfile(f), 'Could not find pypeit file.'

    cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(
        f, file_check=False)

    # Change the dev path
    for i in range(len(data_files)):
        path_list = data_files[i].split('/')
        for j, p in enumerate(path_list):
            if p == 'RAW_DATA':
                break
        data_files[i] = os.path.join(os.environ['PYPEIT_DEV'],
                                     '/'.join(path_list[j:]))

    # Read the fits table with and without the user data
    spectrograph = load_spectrograph('keck_lris_blue')
    par = spectrograph.default_pypeit_par()
    fitstbl = PypeItMetaData(spectrograph, par, files=data_files)
    fitstbl_usr = PypeItMetaData(spectrograph,
                                 par,
                                 files=data_files,
                                 usrdata=usrdata)

    assert fitstbl['target'][
        0] == 'unknown', 'Grating name changed in file header'
    assert fitstbl_usr['target'][
        0] == 'test', 'Grating name changed in pypeit file'
    assert fitstbl['target'][0] != fitstbl_usr['target'][0], \
            'Fits header value and input pypeit file value expected to be different.'
Ejemplo n.º 3
0
 def load_pypeit(self, pypeit_file=None):
     # Load the pypeit file
     if pypeit_file is None:
         pypeit_file = self.pypeit_file
     msgs.info("Loading the PypeIt file: {}".format(pypeit_file))
     if self.pypeit_file is not None:
         self.cfg_lines, self.data_files, self.frametype, self.usrdata, self.setups, self.sdict =\
             parse_pypeit_file(pypeit_file, runtime=False)
Ejemplo n.º 4
0
def main(args):

    # Load fits file
    cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(args.file, runtime=False)

    # Get the raw fits file name
    sciIdx = get_science_frame(usrdata)
    fname = data_files[sciIdx]

    # Load the spectrograph
    cfg = ConfigObj(cfg_lines)
    spectrograph_name = cfg['rdx']['spectrograph']
    spectrograph = load_spectrograph(spectrograph_name, ifile=data_files[sciIdx])
    msgs.info('Loaded spectrograph {0}'.format(spectrograph.spectrograph))
    spectrograph_cfg_lines = spectrograph.config_specific_par(fname).to_config()
    par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines)

    # Get the master key
    file_base = os.path.basename(fname)
    ftdict = dict({file_base: 'science'})
    fitstbl = PypeItMetaData(spectrograph, par, files=[data_files[sciIdx]], usrdata=Table(usrdata[sciIdx]), strict=True)
    fitstbl.finalize_usr_build(ftdict, setups[0])
    mkey = fitstbl.master_key(0, det=args.det)

    # Load the frame data
    rawimage, _, _, datasec, _ = spectrograph.get_rawimage(fname, args.det)
    rawimage = procimg.trim_frame(rawimage, datasec < 1)
    frame = spectrograph.orient_image(rawimage, args.det)

    # Set paths
    if par['calibrations']['caldir'] == 'default':
        mdir = os.path.join(par['rdx']['redux_path'], 'Masters')
    else:
        mdir = par['calibrations']['caldir']

    if not os.path.exists(mdir):
        mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir))
        msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base))
        mdir = mdir_base

    # Load the tslits_dict
    trc_file = os.path.join(mdir, MasterFrame.construct_file_name('Edges', mkey)) + '.gz'
    tslits_dict = edgetrace.EdgeTraceSet.from_file(trc_file).convert_to_tslits_dict()

    # Derive an appropriate output filename
    prefix = os.path.splitext(file_base)
    if prefix[1] == ".gz":
        prefix = os.path.splitext(prefix[0])[0]
    else:
        prefix = prefix[0]
    outname = "{0:s}_skyregions.fits".format(prefix)

    # Finally, initialise the GUI
    skyreg = SkySubGUI.initialize(args.det, frame, tslits_dict, outname=outname, runtime=False, printout=True)

    # Get the results
    skyreg.get_result()
Ejemplo n.º 5
0
    def __init__(self, pypeit_file, det=0):
        self.pypeit_file = pypeit_file
        self.det = det
        self.iFile = None  # if a single frame is being used, set the index, otherwise None
        self.spectrograph = None
        self.par = None

        # Load the pypeit file
        self.cfg_lines, self.data_files, self.frametype, self.usrdata, self.setups =\
            parse_pypeit_file(pypeit_file, runtime=False)
Ejemplo n.º 6
0
def test_setup_keck_deimos_multiconfig():

    root = os.path.join(os.environ['PYPEIT_DEV'], 'RAW_DATA', 'keck_deimos')
    files = glob.glob(os.path.join(root, '830G_L_8100', '*fits*'))
    files += glob.glob(os.path.join(root, '830G_L_8400', '*fits*'))

    output_path = os.path.join(os.getcwd(), 'output')
    if os.path.isdir(output_path):
        shutil.rmtree(output_path)
    os.makedirs(output_path)

    ps = pypeitsetup.PypeItSetup(files, spectrograph_name='keck_deimos')
    ps.run(setup_only=True, sort_dir=output_path)
    # Write the automatically generated pypeit data
    pypeit_files = ps.fitstbl.write_pypeit(output_path,
                                           cfg_lines=ps.user_cfg,
                                           write_bkg_pairs=True)

    assert len(pypeit_files) == 2, 'Should have created two pypeit files'

    # Test the pypeit files for the correct configuration and
    # calibration group results
    for f, s, c in zip(pypeit_files, ['A', 'B'], ['0', '1']):

        # TODO: All of this front-end stuff, pulled from pypeit.py, should
        # be put into a function.

        # Read the pypeit file
        cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(
            f, runtime=True)
        # Spectrograph
        cfg = ConfigObj(cfg_lines)
        spectrograph = load_spectrograph(cfg['rdx']['spectrograph'])
        # Configuration-specific parameters
        for idx, row in enumerate(usrdata):
            if 'science' in row['frametype'] or 'standard' in row['frametype']:
                break
        spectrograph_cfg_lines = spectrograph.config_specific_par(
            data_files[idx]).to_config()
        #  PypeIt parameters
        par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines,
                                       merge_with=cfg_lines)
        #  Metadata
        fitstbl = PypeItMetaData(spectrograph,
                                 par,
                                 files=data_files,
                                 usrdata=usrdata,
                                 strict=True)
        fitstbl.finalize_usr_build(frametype, setups[0])

        assert np.all(fitstbl['setup'] == s), 'Setup is wrong'
        assert np.all(fitstbl['calib'] == c), 'Calibration group is wrong'

    # Clean-up
    shutil.rmtree(output_path)
Ejemplo n.º 7
0
def test_setup_made_pypeit_file():
    """ Test the .pypeit file(s) made by pypeit_setup
    This test depends on the one above
    """
    pypeit_file = data_path('shane_kast_blue_A/shane_kast_blue_A.pypeit')
    cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(
        pypeit_file)
    # Test
    assert len(data_files) == 2
    assert frametype['b1.fits.gz'] == 'arc'
    assert setups[0] == 'A'
Ejemplo n.º 8
0
def test_setup_made_pypeit_file():
    """ Test the .pypeit file(s) made by pypeit_setup
    This test depends on the one above
    """
    pypeit_file = data_path('shane_kast_blue_A/shane_kast_blue_A.pypeit')
    cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(pypeit_file)
    # Test
    assert len(data_files) == 2
    assert sorted(frametype['b1.fits.gz'].split(',')) == ['arc', 'tilt']
    assert setups[0] == 'A'

    # Cleanup
    shutil.rmtree(data_path('shane_kast_blue_A'))
Ejemplo n.º 9
0
def test_lris_blue_pypeit_overwrite():

    # JFH The RAW_DATA is at PYPEIT_DEV, but the github DEV suite where
    # the pypeit files are at a different path. To fix this, I've just
    # made pypeit_files directory in Cooked and copied this file over.

    # KBW: I'd prefer that you put symlinks in the github dev suite:
    # e.g.:
    #   cd $PYPEIT_DEV
    #   ln -s /Volumes/GoogleDrive/Team\ Drives/PHYS-GP-Hennawi/PypeIt/PypeIt-development-suite/RAW_DATA  RAW_DATA

    # Read the dev suite pypeit file
    f = os.path.join(
        os.environ['PYPEIT_DEV'],
        'Cooked/pypeit_files/keck_lris_blue_long_400_3400_d560.pypeit')
    if not os.path.isfile(f):
        f = os.path.join(
            os.environ['PYPEIT_DEV'],
            'pypeit_files/keck_lris_blue_long_400_3400_d560.pypeit')
    assert os.path.isfile(f), 'Could not find pypeit file.'

    cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(
        f, file_check=False)

    # Change the dev path
    for i in range(len(data_files)):
        path_list = data_files[i].split('/')
        for j, p in enumerate(path_list):
            if p == 'RAW_DATA':
                break
        data_files[i] = os.path.join(os.environ['PYPEIT_DEV'],
                                     '/'.join(path_list[j:]))

    # Read the fits table with and without the user data
    spectrograph = load_spectrograph('keck_lris_blue')
    par = spectrograph.default_pypeit_par()
    fitstbl = PypeItMetaData(spectrograph, par, file_list=data_files)
    fitstbl_usr = PypeItMetaData(spectrograph,
                                 par,
                                 file_list=data_files,
                                 usrdata=usrdata)

    assert fitstbl['target'][
        0] == 'unknown', 'Grating name changed in file header'
    assert fitstbl_usr['target'][
        0] == 'test', 'Grating name changed in pypeit file'
    assert fitstbl['target'][0] != fitstbl_usr['target'][0], \
            'Fits header value and input pypeit file value expected to be different.'
Ejemplo n.º 10
0
    def from_pypeit_file(cls, filename):
        """
        Instantiate the :class:`PypeitSetup` object using a pypeit file.

        Args:
            filename (str):
                Name of the pypeit file to read.  Pypit files have a
                specific set of valid formats. A description can be
                found :ref:`pypeit_file`.
        
        Returns:
            :class:`PypeitSetup`: The instance of the class.
        """
        cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(filename)
        return cls(data_files, frametype=frametype, usrdata=usrdata, setups=setups,
                   cfg_lines=cfg_lines, pypeit_file=filename)
Ejemplo n.º 11
0
def test_read_combid():

    # ------------------------------------------------------------------
    # In case of failed tests
    setup_dir = data_path('setup_files')
    if os.path.isdir(setup_dir):
        shutil.rmtree(setup_dir)
    config_dir = data_path('shane_kast_blue_A')
    if os.path.isdir(config_dir):
        shutil.rmtree(config_dir)
    # ------------------------------------------------------------------

    # Generate the pypeit file with the comb_id
    droot = data_path('b')
    pargs = Setup.parse_args([
        '-r', droot, '-s', 'shane_kast_blue', '-c=all', '-b',
        '--extension=fits.gz', '--output_path={:s}'.format(data_path(''))
    ])
    Setup.main(pargs)
    shutil.rmtree(setup_dir)

    pypeit_file = os.path.join(config_dir, 'shane_kast_blue_A.pypeit')
    cfg_lines, data_files, frametype, usrdata, setups, _ = parse_pypeit_file(
        pypeit_file)

    # Get the spectrograph
    spectrograph = None
    for l in cfg_lines:
        if 'spectrograph' in l:
            spectrograph = load_spectrograph(l.split(' ')[-1])
            break
    assert spectrograph is not None, 'Did not appropriately read spectrograph'

    # Set the metadata
    pmd = PypeItMetaData(spectrograph,
                         spectrograph.default_pypeit_par(),
                         files=data_files,
                         usrdata=usrdata,
                         strict=False)

    indx = pmd['filename'] == 'b27.fits.gz'
    assert pmd['comb_id'][indx] == [1], 'Incorrect combination group ID'
    assert pmd['comb_id'][np.where(~indx)
                          [0]][0] == -1, 'Incorrect combination group ID'

    shutil.rmtree(config_dir)
Ejemplo n.º 12
0
def test_pypeit_file():
    # Read the PypeIt file
    cfg, data, frametype, usrdata, setups, _ \
            = util.parse_pypeit_file(data_path('example_pypeit_file.pypeit'), file_check=False)
    # Long-winded way of getting the spectrograph name
    name = pypeitpar.PypeItPar.from_cfg_lines(
        merge_with=cfg)['rdx']['spectrograph']
    # Instantiate the spectrograph
    spectrograph = load_spectrograph(name)
    # Get the spectrograph specific configuration
    spec_cfg = spectrograph.default_pypeit_par().to_config()
    # Initialize the PypeIt parameters merge in the user config
    _p = pypeitpar.PypeItPar.from_cfg_lines(cfg_lines=spec_cfg, merge_with=cfg)
    # Test everything was merged correctly
    # This is a PypeItPar default that's not changed
    #assert p['calibrations']['pinholeframe']['number'] == 0
    # These are spectrograph specific defaults
    assert _p['fluxcalib'] is not None
    # These are user-level changes
    assert _p['calibrations']['traceframe']['process']['combine'] == 'median'
    assert _p['scienceframe']['process']['n_lohi'] == [8, 8]
Ejemplo n.º 13
0
    def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_masters=False, logname=None,
                 show=False, redux_path=None):

        # Load
        cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(pypeit_file, runtime=True)
        self.pypeit_file = pypeit_file

        # Spectrograph
        cfg = ConfigObj(cfg_lines)
        spectrograph_name = cfg['rdx']['spectrograph']
        self.spectrograph = load_spectrograph(spectrograph_name)

        # Par
        # Defaults
        spectrograph_def_par = self.spectrograph.default_pypeit_par()
        # Grab a science file for configuration specific parameters
        sci_file = None
        for idx, row in enumerate(usrdata):
            if 'science' in row['frametype']:
                sci_file = data_files[idx]
                break

        # Set
        spectrograph_cfg_lines = self.spectrograph.config_specific_par(spectrograph_def_par, sci_file).to_config()
        self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines)

        # Fitstbl
        self.fitstbl = PypeItMetaData(self.spectrograph, self.par, file_list=data_files,
                                      usrdata=usrdata, strict=True)

        # The following could be put in a prepare_to_run() method in PypeItMetaData
        if 'setup' not in self.fitstbl.keys():
            self.fitstbl['setup'] = setups[0]
        self.fitstbl.get_frame_types(user=frametype)  # This sets them using the user inputs
        self.fitstbl.set_defaults()  # Only does something if values not set in PypeIt file
        self.fitstbl._set_calib_group_bits()
        self.fitstbl._check_calib_groups()
        # Write .calib file (For QA naming amongst other things)
        calib_file = pypeit_file.replace('.pypeit', '.calib')
        self.fitstbl.write_calib(calib_file)

        # Other Internals
        self.logname = logname
        self.overwrite = overwrite
        # Currently the runtime argument determines the behavior for reuse_masters. There is also a reuse_masters
        # parameter in the parset but it is currently ignored.
        self.reuse_masters=reuse_masters
        self.show = show

        # Make the output directories
        self.par['rdx']['redux_path'] = os.getcwd() if redux_path is None else redux_path
        msgs.info("Setting reduction path to {:s}".format(self.par['rdx']['redux_path']))
        paths.make_dirs(self.spectrograph.spectrograph, self.par['calibrations']['caldir'],
                        self.par['rdx']['scidir'], self.par['rdx']['qadir'],
                        overwrite=self.overwrite, redux_path=self.par['rdx']['redux_path'])

        # Instantiate Calibrations class
        self.caliBrate \
            = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'], self.spectrograph,
                                                 redux_path=self.par['rdx']['redux_path'],
                                                 reuse_masters=self.reuse_masters,
                                                 save_masters=True, write_qa=True,
                                                 show=self.show)
        # Init
        self.verbosity = verbosity
        # TODO: I don't think this ever used

        self.frame = None
        self.det = None

        self.tstart = None
        self.basename = None
        self.sciI = None
        self.obstime = None
Ejemplo n.º 14
0
    def __init__(self,
                 pypeit_file,
                 verbosity=2,
                 overwrite=True,
                 reuse_masters=False,
                 logname=None,
                 show=False,
                 redux_path=None):

        # Load
        cfg_lines, data_files, frametype, usrdata, setups \
                = parse_pypeit_file(pypeit_file, runtime=True)
        self.pypeit_file = pypeit_file

        # Spectrograph
        cfg = ConfigObj(cfg_lines)
        spectrograph_name = cfg['rdx']['spectrograph']
        self.spectrograph = load_spectrograph(spectrograph_name,
                                              ifile=data_files[0])
        msgs.info('Loaded spectrograph {0}'.format(
            self.spectrograph.spectrograph))

        # --------------------------------------------------------------
        # Get the full set of PypeIt parameters
        #   - Grab a science or standard file for configuration specific parameters
        scistd_file = None
        for idx, row in enumerate(usrdata):
            if ('science' in row['frametype']) or ('standard'
                                                   in row['frametype']):
                scistd_file = data_files[idx]
                break
        #   - Configuration specific parameters for the spectrograph
        if scistd_file is not None:
            msgs.info(
                'Setting configuration-specific parameters using {0}'.format(
                    os.path.split(scistd_file)[1]))
        spectrograph_cfg_lines = self.spectrograph.config_specific_par(
            scistd_file).to_config()
        #   - Build the full set, merging with any user-provided
        #     parameters
        self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines,
                                            merge_with=cfg_lines)
        msgs.info('Built full PypeIt parameter set.')

        # Check the output paths are ready
        if redux_path is not None:
            self.par['rdx']['redux_path'] = redux_path

        # TODO: Write the full parameter set here?
        # --------------------------------------------------------------

        # --------------------------------------------------------------
        # Build the meta data
        #   - Re-initilize based on the file data
        msgs.info('Compiling metadata')
        self.fitstbl = PypeItMetaData(self.spectrograph,
                                      self.par,
                                      files=data_files,
                                      usrdata=usrdata,
                                      strict=True)
        #   - Interpret automated or user-provided data from the PypeIt
        #   file
        self.fitstbl.finalize_usr_build(frametype, setups[0])
        # --------------------------------------------------------------
        #   - Write .calib file (For QA naming amongst other things)
        calib_file = pypeit_file.replace('.pypeit', '.calib')
        self.fitstbl.write_calib(calib_file)

        # Other Internals
        self.logname = logname
        self.overwrite = overwrite

        # Currently the runtime argument determines the behavior for
        # reuse_masters.
        self.reuse_masters = reuse_masters
        self.show = show

        # Set paths
        if self.par['calibrations']['caldir'] == 'default':
            self.calibrations_path = os.path.join(
                self.par['rdx']['redux_path'], 'Masters')
        else:
            self.calibrations_path = self.par['calibrations']['caldir']

        # Report paths
        msgs.info('Setting reduction path to {0}'.format(
            self.par['rdx']['redux_path']))
        msgs.info('Master calibration data output to: {0}'.format(
            self.calibrations_path))
        msgs.info('Science data output to: {0}'.format(self.science_path))
        msgs.info('Quality assessment plots output to: {0}'.format(
            self.qa_path))
        # TODO: Is anything written to the qa dir or only to qa/PNGs?
        # Should we have separate calibration and science QA
        # directories?

        # Instantiate Calibrations class
        self.caliBrate \
            = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'],
                                                 self.spectrograph,
                                                 caldir=self.calibrations_path,
                                                 qadir=self.qa_path,
                                                 reuse_masters=self.reuse_masters,
                                                 show=self.show)
        # Init
        self.verbosity = verbosity
        # TODO: I don't think this ever used

        self.frame = None
        self.det = None

        self.tstart = None
        self.basename = None
        self.sciI = None
        self.obstime = None
Ejemplo n.º 15
0
    def __init__(self,
                 pypeit_file,
                 verbosity=2,
                 overwrite=True,
                 reuse_masters=False,
                 logname=None,
                 show=False,
                 redux_path=None,
                 calib_only=False):

        # Load
        cfg_lines, data_files, frametype, usrdata, setups \
                = parse_pypeit_file(pypeit_file, runtime=True)
        self.pypeit_file = pypeit_file
        self.calib_only = calib_only

        # Spectrograph
        cfg = ConfigObj(cfg_lines)
        spectrograph_name = cfg['rdx']['spectrograph']
        self.spectrograph = load_spectrograph(spectrograph_name)
        msgs.info('Loaded spectrograph {0}'.format(
            self.spectrograph.spectrograph))

        # --------------------------------------------------------------
        # Get the full set of PypeIt parameters
        #   - Grab a science or standard file for configuration specific parameters

        config_specific_file = None
        for idx, row in enumerate(usrdata):
            if ('science' in row['frametype']) or ('standard'
                                                   in row['frametype']):
                config_specific_file = data_files[idx]
        # search for arcs, trace if no scistd was there
        if config_specific_file is None:
            for idx, row in enumerate(usrdata):
                if ('arc' in row['frametype']) or ('trace'
                                                   in row['frametype']):
                    config_specific_file = data_files[idx]
        if config_specific_file is not None:
            msgs.info(
                'Setting configuration-specific parameters using {0}'.format(
                    os.path.split(config_specific_file)[1]))
        spectrograph_cfg_lines = self.spectrograph.config_specific_par(
            config_specific_file).to_config()

        #   - Build the full set, merging with any user-provided
        #     parameters
        self.par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines,
                                            merge_with=cfg_lines)
        msgs.info('Built full PypeIt parameter set.')

        # Check the output paths are ready
        if redux_path is not None:
            self.par['rdx']['redux_path'] = redux_path

        # TODO: Write the full parameter set here?
        # --------------------------------------------------------------

        # --------------------------------------------------------------
        # Build the meta data
        #   - Re-initilize based on the file data
        msgs.info('Compiling metadata')
        self.fitstbl = PypeItMetaData(self.spectrograph,
                                      self.par,
                                      files=data_files,
                                      usrdata=usrdata,
                                      strict=True)
        #   - Interpret automated or user-provided data from the PypeIt
        #   file
        self.fitstbl.finalize_usr_build(frametype, setups[0])
        # --------------------------------------------------------------
        #   - Write .calib file (For QA naming amongst other things)
        calib_file = pypeit_file.replace('.pypeit', '.calib')
        self.fitstbl.write_calib(calib_file)

        # Other Internals
        self.logname = logname
        self.overwrite = overwrite

        # Currently the runtime argument determines the behavior for
        # reuse_masters.
        self.reuse_masters = reuse_masters
        self.show = show

        # Set paths
        self.calibrations_path = os.path.join(
            self.par['rdx']['redux_path'],
            self.par['calibrations']['master_dir'])

        # Check for calibrations
        if not self.calib_only:
            calibrations.check_for_calibs(
                self.par,
                self.fitstbl,
                raise_error=self.par['calibrations']['raise_chk_error'])

        # Report paths
        msgs.info('Setting reduction path to {0}'.format(
            self.par['rdx']['redux_path']))
        msgs.info('Master calibration data output to: {0}'.format(
            self.calibrations_path))
        msgs.info('Science data output to: {0}'.format(self.science_path))
        msgs.info('Quality assessment plots output to: {0}'.format(
            self.qa_path))
        # TODO: Is anything written to the qa dir or only to qa/PNGs?
        # Should we have separate calibration and science QA
        # directories?
        # An html file wrapping them all too

        #        # Instantiate Calibrations class
        #        if self.spectrograph.pypeline in ['MultiSlit', 'Echelle']:
        #            self.caliBrate \
        #                = calibrations.MultiSlitCalibrations(self.fitstbl, self.par['calibrations'],
        #                                                     self.spectrograph, self.calibrations_path,
        #                                                     qadir=self.qa_path,
        #                                                     reuse_masters=self.reuse_masters,
        #                                                     show=self.show,
        #                                                     slitspat_num=self.par['rdx']['slitspatnum'])
        #        elif self.spectrograph.pypeline in ['IFU']:
        #            self.caliBrate \
        #                = calibrations.IFUCalibrations(self.fitstbl, self.par['calibrations'],
        #                                               self.spectrograph,
        #                                               self.calibrations_path,
        #                                               qadir=self.qa_path,
        #                                               reuse_masters=self.reuse_masters,
        #                                               show=self.show)
        #        else:
        #            msgs.error("No calibration available to support pypeline: {0:s}".format(self.spectrograph.pypeline))

        # Init
        self.verbosity = verbosity
        # TODO: I don't think this ever used

        self.det = None

        self.tstart = None
        self.basename = None
        self.sciI = None
        self.obstime = None