def reduce(self):
if self.archival == True:
print("Using archival data")
grism = self.get_correct_grism()
print(grism)
self.create_archival_data_symlinks(grism)
if self.masters == True:
self.copy_archival_masters(grism)
if self.args.root is not None:
ps = PypeItSetup.from_file_root(
self.args.root,
"vlt_fors2",
extension=".fits",
output_path=self.working_dir,
)
else:
raise IOError("Need to set -r!")
ps.run(setup_only=True, sort_dir=self.working_dir, write_bkg_pairs=False)
self.read_pypeit_sorted()
self.change_flat_type()
self.choose_detector_chip()
self.use_bias_frames()
print(self.sorted)
self.write_virt_pypeit_file()
pipeline = pypeit.PypeIt(
self.tmp.name,
verbosity=self.args.verbosity,
reuse_masters=self.masters,
overwrite=self.args.overwrite,
)
pipeline.reduce_all()
self.flux_spectra()
def run_setup(self, root, extension=None, **kwargs):
"""
Generate a PypeItSetup instance and run it
Args:
root (str):
extension (str, optional):
**kwargs:
Passed to ps.run()
Returns:
tuple: PypeItSetup, setups, indx
"""
# Initialize PypeItSetup based on the arguments
ps = PypeItSetup.from_file_root(root,
self.spectrograph.name,
extension=extension)
# Run the setup
ps.run(setup_only=True, **kwargs) # , write_bkg_pairs=args.background)
# Unique configurations
setups, indx = ps.fitstbl.get_configuration_names(return_index=True)
# Return
return ps, setups, indx
def main(args):
from pypeit.pypeitsetup import PypeItSetup
# if args.root is None:
# raise IOError('root is a required argument. Use the -r, --root command-line option.')
if args.spectrograph is None:
raise IOError('spectrograph is a required argument. Use the -s, --spectrograph '
'command-line option.')
# Check that input spectrograph is supported
if args.spectrograph not in available_spectrographs:
raise ValueError('Instrument \'{0}\' unknown to PypeIt.\n'.format(args.spectrograph)
+ '\tOptions are: {0}\n'.format(', '.join(available_spectrographs))
+ '\tSelect an available instrument or consult the documentation '
+ 'on how to add a new instrument.')
# Get the output directory
sort_dir = os.path.join(args.output_path, 'setup_files')
# Initialize PypeItSetup based on the arguments
ps = PypeItSetup.from_file_root(args.root, args.spectrograph, extension=args.extension,
output_path=sort_dir)
# Run the setup
ps.run(setup_only=True, sort_dir=sort_dir, write_bkg_pairs=args.background, obslog=True)
# Use PypeItMetaData to write the complete PypeIt file
# TODO: Set cfg_split to 'all' by default?
if args.cfg_split is not None:
ps.fitstbl.write_pypeit(args.output_path, cfg_lines=ps.user_cfg,
write_bkg_pairs=args.background,
configs=[item.strip() for item in args.cfg_split.split(',')])
def test_deimos():
# Raw DEIMOS directory
raw_dir = os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'keck_deimos')
# Get the list of setup directories
setups = glob.glob(os.path.join(raw_dir, '*'))
# Set the output path and *remove if* if it already exists
output_path = os.path.join(os.getcwd(), 'output')
if os.path.isdir(output_path):
shutil.rmtree(output_path)
# Iterate through the setups
for setup in setups:
# Find the relevant pypeit file constructed by hand.
by_hand_pypeit = os.path.join(
os.getenv('PYPEIT_DEV'), 'pypeit_files',
'keck_deimos_{0}.pypeit'.format(os.path.split(setup)[1].lower()))
if not os.path.isfile(by_hand_pypeit):
# It doesn't exist, so assume there is no by-hand pypeit
# file to compare to
continue
# Run pypeit_setup
ps = PypeItSetup.from_file_root(setup,
'keck_deimos',
output_path=output_path)
ps.run(setup_only=True)
# Write the automatically generated pypeit data
pypeit_files = ps.fitstbl.write_pypeit(output_path,
cfg_lines=ps.user_cfg)
# Read the frame types from both the by-hand and automated
# pypeit files
_, _, by_hand_frametypes, _, _ = parse_pypeit_file(by_hand_pypeit,
file_check=False)
_, _, auto_frametypes, _, _ = parse_pypeit_file(pypeit_files[0],
file_check=False)
# For each file in the by-hand list, check that the frame types
# in the automatically generated pypeit file are identical
for f in by_hand_frametypes.keys():
type_list = np.sort(by_hand_frametypes[f].split(','))
if 'science' in type_list or 'standard' in type_list:
# Only ensuring that calibrations are correctly typed
continue
assert f in auto_frametypes.keys(), \
'Frame {0} not automatically parsed for setup {1}.'.format(f, setup)
assert np.array_equal(type_list, np.sort(auto_frametypes[f].split(','))), \
'Frame types differ for file {0} in setup {1}\n'.format(f, setup) \
+ ' By-hand types: {0}'.format(by_hand_frametypes[f]) \
+ ' Automated types: {0}'.format(auto_frametypes[f])
# Clean up after every setup
shutil.rmtree(output_path)
def main(args):
import os
import pdb as debugger
from pypeit import msgs
from pypeit.pypeitsetup import PypeItSetup
# Check that the spectrograph is provided if using a file root
if args.root is not None:
if args.spectrograph is None:
raise ValueError(
'Must provide spectrograph identifier with file root.')
# Check that input spectrograph is supported
instruments_served = valid_spectrographs()
if args.spectrograph not in instruments_served:
raise ValueError(
'Instrument \'{0}\' unknown to PypeIt.\n'.format(
args.spectrograph) +
'\tOptions are: {0}\n'.format(', '.join(instruments_served)) +
'\tSelect an available instrument or consult the documentation '
+ 'on how to add a new instrument.')
# Get the output directory
output_path = os.getcwd() if args.output_path is None else args.output_path
sort_dir = os.path.join(output_path, 'setup_files')
# Initialize PypeItSetup based on the arguments
if args.root is not None:
ps = PypeItSetup.from_file_root(args.root,
args.spectrograph,
extension=args.extension,
output_path=sort_dir)
else:
# Should never reach here
raise IOError('Need to set -r !!')
# Run the setup
ps.run(setup_only=True, sort_dir=sort_dir, write_bkg_pairs=args.background)
# Use PypeItMetaData to write the complete PypeIt file
if args.cfg_split is not None:
pypeit_file = os.path.join(output_path,
'{0}.pypeit'.format(args.spectrograph))
config_list = [item.strip() for item in args.cfg_split.split(',')]
ps.fitstbl.write_pypeit(pypeit_file,
cfg_lines=ps.user_cfg,
write_bkg_pairs=args.background,
configs=config_list)
return 0
def test_trace_add_rm():
# Define the output directories (HARDCODED!!)
setupdir = os.path.join(os.getcwd(), 'setup_files')
outdir = os.path.join(os.getcwd(), 'shane_kast_blue_A')
masterdir = os.path.join(os.getcwd(), 'shane_kast_blue_A', 'Masters')
# Remove them if they already exist
if os.path.isdir(setupdir):
shutil.rmtree(setupdir)
if os.path.isdir(outdir):
shutil.rmtree(outdir)
droot = os.path.join(os.environ['PYPEIT_DEV'],
'RAW_DATA/shane_kast_blue/600_4310_d55')
# Run the setup
ps = PypeItSetup.from_file_root(droot,
'shane_kast_blue',
output_path=setupdir)
ps.run(setup_only=True, sort_dir=setupdir)
# Add lines to remove and add slits. This removes the one slit that
# is found and adds another.
ps.user_cfg += [
'[calibrations]', '[[slitedges]]', 'rm_slits = 1:1028:170',
'add_slits = 1:1028:30:300', 'add_predict = straight'
]
# Use PypeItMetaData to write the complete PypeIt file
pypeit_file = ps.fitstbl.write_pypeit(output_path=os.getcwd(),
cfg_lines=ps.user_cfg,
configs=['all'])[0]
# Run the tracing
scripts.trace_edges.TraceEdges.main(
scripts.trace_edges.TraceEdges.parse_args(['-f', pypeit_file]))
# Define the edges master file (HARDCODED!!)
trace_file = os.path.join(outdir, 'Masters',
'MasterEdges_A_1_DET01.fits.gz')
# Check that the correct number of traces were found
edges = edgetrace.EdgeTraceSet.from_file(trace_file)
assert edges.ntrace == 2, 'Did not find the expected number of traces.'
# Clean up
shutil.rmtree(setupdir)
shutil.rmtree(outdir)
def main(args):
from pypeit.spectrographs.util import load_spectrograph
from pypeit.pypeitsetup import PypeItSetup
# Check that input spectrograph is supported
if args.spec not in available_spectrographs:
raise ValueError(
'Instrument \'{0}\' unknown to PypeIt.\n'.format(args.spec) +
'\tOptions are: {0}\n'.format(', '.join(available_spectrographs)) +
'\tSelect an available instrument or consult the documentation ' +
'on how to add a new instrument.')
if args.keys:
# Only print the metadata to header card mapping
load_spectrograph(args.spec).meta_key_map()
return
if args.bad_types not in ['keep', 'rm', 'only']:
raise ValueError(
f'{args.bad_types} is not a valid keyword for the --bad_types argument.'
)
# Generate the metadata table
ps = PypeItSetup.from_file_root(args.root,
args.spec,
extension=args.extension)
ps.run(setup_only=True,
write_files=False,
groupings=args.groupings,
clean_config=args.bad_frames)
# Check the file can be written (this is here because the spectrograph
# needs to be defined first)
_file = args.file
if _file == 'default':
_file = f'{ps.spectrograph.name}.obslog'
if _file is not None:
_odir, _file = os.path.split(_file)
_file = os.path.join(args.output_path, _file)
if not os.path.isdir(args.output_path):
os.makedirs(args.output_path)
if not args.interact and os.path.isfile(_file) and not args.overwrite:
raise FileExistsError(
f'{_file} already exists. Use -o to overwrite.')
# Write/Print the data
header = [
'Auto-generated PypeIt Observing Log',
'{0}'.format(time.strftime("%a %d %b %Y %H:%M:%S", time.localtime())),
f'Root file string: {args.root}'
]
if args.bad_types == 'keep':
nrows = len(ps.fitstbl)
indx = np.ones(nrows, dtype=bool)
elif args.bad_types == 'rm':
indx = ps.fitstbl['frametype'] != 'None'
elif args.bad_types == 'only':
indx = ps.fitstbl['frametype'] == 'None'
else:
raise ValueError('CODING ERROR: Should never get here.')
fitstbl = ps.fitstbl.write(output='table' if args.interact else _file,
rows=indx,
columns=args.columns,
sort_col=args.sort,
overwrite=args.overwrite,
header=header)
if args.interact:
embed()
def main(args):
"""
Args:
args:
Returns:
astropy.table.Table:
"""
import os
from IPython import embed
import numpy as np
from astropy import table
from pypeit.pypeitsetup import PypeItSetup
from pypeit import calibrations
from pypeit import msgs
from pypeit.par import PypeItPar
import shutil
# Check that the spectrograph is provided if using a file root
if args.root is not None:
if args.spectrograph is None:
raise ValueError(
'Must provide spectrograph identifier with file root.')
# Check that input spectrograph is supported
if args.spectrograph not in available_spectrographs:
raise ValueError(
'Instrument \'{0}\' unknown to PypeIt.\n'.format(
args.spectrograph) + '\tOptions are: {0}\n'.format(
', '.join(available_spectrographs)) +
'\tSelect an available instrument or consult the documentation '
+ 'on how to add a new instrument.')
# Initialize PypeItSetup based on the arguments
ps = PypeItSetup.from_file_root(args.root,
args.spectrograph,
extension=args.extension)
# Run the setup
ps.run(setup_only=True) #, write_bkg_pairs=args.background)
is_science = ps.fitstbl.find_frames('science')
msgs.info('Loaded spectrograph {0}'.format(ps.spectrograph.name))
# Unique configurations
uniq_cfg = ps.fitstbl.unique_configurations(copy=True)
# Setup the table. Need to use object type for strings so that
# they're not truncated.
answers = table.Table()
answers['setups'] = list(uniq_cfg.keys())
# Add the configuration columns
for setup, setup_dict in uniq_cfg.items():
for key, value in setup_dict.items():
answers[key] = np.empty(len(answers), dtype=object) if isinstance(value, str) \
else type(value)(0)
break
answers['pass'] = False
answers['scifiles'] = np.empty(len(answers), dtype=object)
for i, setup in enumerate(uniq_cfg.keys()):
for setup_key, setup_value in uniq_cfg[setup].items():
answers[setup_key] = setup_value
if setup == 'None':
print("There is a setup without science frames. Skipping...")
answers['pass'][i] = False
answers['scifiles'][i] = None
continue
msgs.info(
'======================================================================='
)
msgs.info('Working on setup: {}'.format(setup))
msgs.info(str(uniq_cfg[setup]))
msgs.info(
'======================================================================='
)
# TODO: Make the snippet below, which is also in the init of
# PypeIt a method somewhere
in_cfg = ps.fitstbl['setup'] == setup
config_specific_file = None
# Grab a science/standard frame
data_files = [
os.path.join(row['directory'], row['filename'])
for row in ps.fitstbl[in_cfg]
]
for idx, row in enumerate(ps.fitstbl[in_cfg]):
if 'science' in row['frametype'] or 'standard' in row['frametype']:
config_specific_file = data_files[idx]
if config_specific_file is not None:
msgs.info(
'Setting configuration-specific parameters using {0}'.format(
os.path.split(config_specific_file)[1]))
else:
msgs.warn('No science or standard frame. Punting..')
answers['pass'][i] = False
answers['scifiles'][i] = None
continue
#
spectrograph_cfg_lines = ps.spectrograph.config_specific_par(
config_specific_file).to_config()
# - Build the full set, merging with any user-provided
# parameters
par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines)
# Print science frames
if np.any(in_cfg & is_science):
msgs.info('Your science frames are: {0}'.format(
ps.fitstbl['filename'][in_cfg & is_science].tolist()))
answers['scifiles'][i] = ', '.join(
ps.fitstbl['filename'][in_cfg & is_science].tolist())
else:
msgs.warn("This setup has no science frames!")
answers['scifiles'][i] = ''
# Check!
answers['pass'][i] = calibrations.check_for_calibs(par,
ps.fitstbl,
raise_error=False,
cut_cfg=in_cfg)
if not answers['pass'][i]:
msgs.warn(
"Setup {} did not pass the calibration check!".format(setup))
print('= RESULTS ============================================')
# Print
answers.pprint_all()
print('======================================================')
# Remove setup_files
if not args.save_setups:
shutil.rmtree('setup_files')
# Return objects used by unit tests
return answers, ps
def main(args):
"""
Args:
args:
Returns:
astropy.table.Table:
"""
import os
import numpy as np
from astropy import table
from pypeit.pypeitsetup import PypeItSetup
from pypeit import calibrations
from pypeit import msgs
from pypeit.par import PypeItPar
# Check that the spectrograph is provided if using a file root
if args.root is not None:
if args.spectrograph is None:
raise ValueError(
'Must provide spectrograph identifier with file root.')
# Check that input spectrograph is supported
instruments_served = defs.pypeit_spectrographs
if args.spectrograph not in instruments_served:
raise ValueError(
'Instrument \'{0}\' unknown to PypeIt.\n'.format(
args.spectrograph) +
'\tOptions are: {0}\n'.format(', '.join(instruments_served)) +
'\tSelect an available instrument or consult the documentation '
+ 'on how to add a new instrument.')
# Initialize PypeItSetup based on the arguments
ps = PypeItSetup.from_file_root(args.root,
args.spectrograph,
extension=args.extension)
# Run the setup
ps.run(setup_only=True) #, write_bkg_pairs=args.background)
is_science = ps.fitstbl.find_frames('science')
msgs.info('Loaded spectrograph {0}'.format(ps.spectrograph.spectrograph))
# Unique configurations
setups, indx = ps.fitstbl.get_configuration_names(return_index=True)
answers = table.Table()
answers['setups'] = setups
passes, scifiles, cfgs = [], [], []
for setup, i in zip(setups, indx):
# Get the setup lines
cfg = ps.fitstbl.get_setup(i, config_only=False)
cfgs.append(cfg)
if setup == 'None':
print("There is a setup without science frames. Skipping...")
passes.append(False)
scifiles.append(None)
continue
in_cfg = ps.fitstbl['setup'] == setup
# TODO -- Make the snippet below, which is also in the init of PypeIt a method somewhere
config_specific_file = None
msgs.info(
'======================================================================='
)
msgs.info('Working on setup: {}'.format(setup))
msgs.info(str(cfg))
msgs.info(
'======================================================================='
)
# Grab a science/standard frame
data_files = [
os.path.join(row['directory'], row['filename'])
for row in ps.fitstbl[in_cfg]
]
for idx, row in enumerate(ps.fitstbl[in_cfg]):
if ('science' in row['frametype']) or ('standard'
in row['frametype']):
config_specific_file = data_files[idx]
if config_specific_file is not None:
msgs.info(
'Setting configuration-specific parameters using {0}'.format(
os.path.split(config_specific_file)[1]))
else:
msgs.warn('No science or standard frame. Punting..')
passes.append(False)
scifiles.append(None)
continue
#
spectrograph_cfg_lines = ps.spectrograph.config_specific_par(
config_specific_file).to_config()
# - Build the full set, merging with any user-provided
# parameters
par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines)
# Print science frames
if np.any(in_cfg & is_science):
msgs.info("Your science frames are: {}".format(
ps.fitstbl['filename'][in_cfg & is_science]))
scifiles.append(','.join(ps.fitstbl['filename'][in_cfg
& is_science]))
else:
msgs.warn("This setup has no science frames!")
scifiles.append('')
# Check!
passed = calibrations.check_for_calibs(par,
ps.fitstbl,
raise_error=False,
cut_cfg=in_cfg)
if not passed:
msgs.warn(
"Setup {} did not pass the calibration check!".format(setup))
#
passes.append(passed)
print('= RESULTS ============================================')
# Pass/fail
answers['pass'] = passes
# Parse the configs
pcfg = dict(disperser=[],
angle=[],
dichroic=[],
decker=[],
slitwid=[],
binning=[])
for cfg in cfgs:
# None?
if len(cfg) == 0:
for key in pcfg.keys():
pcfg[key].append(None)
continue
# Load it up
key0 = list(cfg.keys())[0]
subd = cfg[key0]['--'] # for convenience
pcfg['disperser'].append(subd['disperser']['name'])
pcfg['angle'].append(subd['disperser']['angle'])
pcfg['dichroic'].append(subd['dichroic'])
pcfg['decker'].append(subd['slit']['decker'])
pcfg['slitwid'].append(subd['slit']['slitwid'])
pcfg['binning'].append(subd['binning'])
# Add
for key in pcfg.keys():
answers[key] = pcfg[key]
# Sci files [put this last as it can get large]
answers['scifiles'] = scifiles
# Print
answers.pprint_all()
print(
'= ==================================================================================='
)
# Return
return answers, ps
