def argf_diff_and_dup():
""" Compares default argf values against those in the ARMS and AMRED files
Returns
-------
"""
# Load default argf file
baseargf = arparse.get_argflag_class(('BaseArgFlag', '.tmp'))
base_lines = baseargf.load_file()
baseargf.set_paramlist(base_lines)
# ARMS
msgs.info("===============================================")
msgs.info("Working on ARMS vs. Base")
arms = arparse.get_argflag_class(('ARMS', '.tmp'))
arms_lines = arms.load_file()
arms.set_paramlist(arms_lines)
# Look for duplicates and diffs
for key in baseargf._argflag.keys():
if key in arms._argflag.keys():
compare_dicts(key, baseargf._argflag[key], arms._argflag[key])
# ARMED
msgs.info("===============================================")
msgs.info("Working on ARMED vs. Base")
armed = arparse.get_argflag_class(('ARMED', '.tmp'))
armed_lines = armed.load_file()
armed.set_paramlist(armed_lines)
# Look for duplicates and diffs
for key in baseargf._argflag.keys():
if key in armed._argflag.keys():
compare_dicts(key, baseargf._argflag[key], armed._argflag[key])
def dummy_settings(pypitdir=None,
nfile=10,
spectrograph='shane_kast_blue',
set_idx=True):
""" Generate settings classes
Parameters
----------
pypitdir
nfile
spectrograph
set_idx : bool, optional
Set dummy index values for science and calibs
Returns
-------
"""
from pypit import arparse
# Dummy argflag
if spectrograph != 'shane_kast_blue':
msgs.error(
"Only setup for Kast Blue") # You will need to fuss with scidx
argf = arparse.get_argflag_class(("ARMLSD", spectrograph))
argf.init_param()
if pypitdir is None:
pypitdir = __file__[0:__file__.rfind('/')]
# Run specific
argf.set_param('run pypitdir {0:s}'.format(pypitdir))
argf.set_param('run spectrograph {:s}'.format(spectrograph))
argf.set_param('run directory science ./')
# Dummy spect
spect = arparse.get_spect_class(("ARMLSD", spectrograph, "dummy"))
lines = spect.load_file(base=True) # Base spectrograph settings
spect.set_paramlist(lines)
lines = spect.load_file()
spect.set_paramlist(lines)
if set_idx:
for jj, key in enumerate(spect._spect.keys()):
if key in ['det']:
continue
if 'index' in spect._spect[key].keys():
if spectrograph == 'shane_kast_blue': # Science frames from idx = 5 to 9
assert nfile == 10
for kk in [5, 6, 7, 8, 9]:
if key == 'science':
spect._spect[key]['index'] += [np.array([kk])]
elif key == 'arc':
spect._spect[key]['index'] += [np.array([1])]
elif key == 'standard':
spect._spect[key]['index'] += [np.array([4])]
elif key == 'bias':
spect._spect[key]['index'] += [np.array([0])]
elif key == 'trace':
spect._spect[key]['index'] += [np.array([2, 3])]
elif key == 'pixelflat':
spect._spect[key]['index'] += [np.array([2, 3])]
arparse.init(argf, spect)
return
def test_load_armlsd():
""" Test loading of ARMLSD
"""
argf = arparse.get_argflag_class(('ARMLSD', ''))
argf.init_param()
argf.set_param('run pypitdir {0:s}'.format('pypit'))
argf.set_param('run redname {0:s}'.format('shane_kast_blue_setup_01'))
# Test
assert argf._argflag['run']['redname'] == 'shane_kast_blue_setup_01'
assert argf._argflag['reduce']['trim'] == True
assert argf._argflag['reduce']['skysub']['perform'] == True
# Save
argf.save()
def PYPIT(redname,
debug=None,
progname=__file__,
quick=False,
ncpus=1,
verbosity=1,
use_masters=False,
devtest=False,
logname=None):
""" Main driver of the PYPIT code. Default settings and
user-specified changes are made, and passed to the
appropriate code for data reduction.
Parameters
----------
redname : string
Input reduction script
debug : dict, optional
Debug dict
progname : string
Name of the program
quick : bool
If True, a quick reduction (but possibly less
accurate) will be performed. This flag is most
useful for observing at a telescope, but not
for publication quality results.
ncpus : int
Number of CPUs to use for multiprocessing the
data reduction (sometimes not used)
verbosity : int (0,1,2)
Level of verbosity:
0 = No output
1 = Minimal output (default - suitable for the average user)
2 = All output
use_masters : bool, optional
Load calibration files from MasterFrames directory, if they exist
devtest : bool, optional
Running PYPIT-Development suite; will turn instrument-specific options
logname : str or None
The name of an ascii log file which is used to
save the output details of the reduction
debug : dict
A PYPIT debug dict (from ardebug.init)
version : str
last_updated : str
---------------------------------------------------
"""
from pypit import ardebug
# Init logger
if debug is None:
debug = ardebug.init()
msgs = armsgs.get_logger((logname, debug, verbosity))
msgs.pypit_file = redname
# This needs to be loaded after msgs
from pypit import arparse
# version checking
try:
archeck.version_check()
except archeck.VersionError as err:
msgs.error(err.message)
# First send all signals to messages to be dealt with (i.e. someone hits ctrl+c)
sigsignal(SIGINT, msgs.signal_handler)
# Ignore all warnings given by python
resetwarnings()
simplefilter("ignore")
# Record the starting time
tstart = time()
# Load the input file
pyp_dict = load_input(redname, msgs)
parlines, datlines, spclines = [
pyp_dict[ii] for ii in ['par', 'dat', 'spc']
]
# Initialize the arguments and flags
# argflag = arload.argflag_init()
# settings.argflag['run']['ncpus'] = ncpus
# settings.argflag['output']['verbosity'] = verbosity
# Determine the name of the spectrograph
specname = None
for i in range(len(parlines)):
parspl = parlines[i].split()
if len(parspl) < 3:
msgs.error(
"There appears to be a missing argument on the following input line"
+ msgs.newline() + parlines[i])
if (parspl[0] == 'run') and (parspl[1] == 'spectrograph'):
specname = parspl[2]
break
if specname is None:
msgs.error(
"Please specify the spectrograph settings to be used with the command"
+ msgs.newline() + "run spectrograph <name>")
msgs.info("Reducing data from the {0:s} spectrograph".format(specname))
# Determine the type of reduction used for this spectrograph
redtype = None
# Get the software path
prgn_spl = progname.split('/')
tfname = "/".join(prgn_spl[:-1]) + "/"
# Settings file
fname = tfname + 'data/settings/settings.' + specname
try:
spl = open(fname, 'r').readlines()
except IOError:
msgs.error(
"The following instrument settings file cannot be found:" +
msgs.newline() + fname + msgs.newline() +
"Please check the settings file exists, and that the instrument name is spelt correctly."
)
for i in range(len(spl)):
parspl = spl[i].split()
if len(parspl) < 3:
continue
if (parspl[0] == 'mosaic') and (parspl[1] == 'reduction'):
redtype = parspl[2]
break
if redtype is None:
msgs.bug(
"The {0:s} instrument settings file must contain the reduction type"
.format(specname))
msgs.error("Please specify the reduction type with the command" +
msgs.newline() + "mosaic reduction <type>")
# Load default reduction arguments/flags, and set any command line arguments
argf = arparse.get_argflag_class(
(redtype.upper(), ".".join(redname.split(".")[:-1])))
argf.init_param()
# Run specific
argf.set_param('run pypitdir {0:s}'.format(tfname))
argf.set_param('run progname {0:s}'.format(progname))
argf.set_param('run redname {0:s}'.format(redname))
# Load user changes to the arguments/flags
plines = argf.load_lines(parlines)
argf.set_paramlist(plines)
# If the user wishes to load a settings file, do that now
if argf.__dict__['_argflag']['run']['load']['settings'] is not None:
lines = argf.load_file(
argf.__dict__['_argflag']['run']['load']['settings'])
argf.set_paramlist(lines)
# Load default spectrograph settings
spect = arparse.get_spect_class(
(redtype.upper(), specname, ".".join(redname.split(".")[:-1])))
lines = spect.load_file(base=True) # Base spectrograph settings
spect.set_paramlist(lines)
lines = spect.load_file() # Instrument specific
spect.set_paramlist(lines)
# Load frametype numbers, as relevant
if len(pyp_dict['ftype']) > 0:
ftlines = spect.load_ftype(pyp_dict['ftype'])
plines = spect.load_lines(ftlines)
spect.set_paramlist(plines)
# Load user changes to the arguments/flags
plines = spect.load_lines(spclines)
spect.set_paramlist(plines)
if argf.__dict__['_argflag']['run']['load']['spect'] is not None:
lines = spect.load_file(
argf.__dict__['_argflag']['run']['load']['spect'])
spect.set_paramlist(lines)
# If the instrument settings file sets some argflag settings, implement those changes now
if len(spect.__dict__['_settings']) != 0:
argf.set_paramlist(spect.__dict__['_settings'])
# Load command line changes
argf.set_param('run ncpus {0:d}'.format(ncpus))
argf.set_param('output verbosity {0:d}'.format(verbosity))
if use_masters:
argf.set_param('reduce masters reuse True')
msgs.work("Make appropriate changes to quick reduction")
# Load Development suite changes
if devtest:
msgs.info(
"Loading instrument specific argurment for Development Suite tests"
)
from pypit import ardevtest
ardevtest.set_param(argf, specname)
if quick:
# If a quick reduction has been requested, make sure the requested pipeline
# is the quick implementation (if it exists), otherwise run the standard pipeline.
msgs.work("QUICK REDUCTION TO STILL BE DONE")
# Setup from PYPIT file?
if len(pyp_dict['setup']['name']) == 1:
argf.set_param('setup name {:s}'.format(pyp_dict['setup']['name'][0]))
# Finally, save the arguments/flags and spectrograph settings used for this reduction
argf.save()
spect.save()
# Now that all of the relevant settings are loaded, globalize the settings
arparse.init(argf, spect)
'''
# Test that a maximum of one .setup files is present
from pypit import arsort
setup_file, nexist = arsort.get_setup_file()
if nexist == 1:
msgs.info("Found setup_file: {:s}".format(setup_file))
msgs.info("Will use this to guide the data reduction.")
'''
# Load the important information from the fits headers
from pypit.arload import load_headers
fitsdict, updates = load_headers(datlines)
# If some settings were updated because of the fits headers, globalize the settings again
if len(updates) != 0:
spect.set_paramlist(updates)
arparse.init(argf, spect)
# If the dispersion direction is 1, flip the axes
if arparse.argflag['trace']['dispersion']['direction'] == 1:
# Update the keywords of all fits files
for ff in range(len(fitsdict['naxis0'])):
temp = fitsdict['naxis0'][ff]
fitsdict['naxis0'][ff] = fitsdict['naxis1'][ff]
fitsdict['naxis1'][ff] = temp
# Update the spectrograph settings for all detectors in the mosaic
for dd in range(arparse.spect['mosaic']['ndet']):
ddnum = arparse.get_dnum(dd + 1)
# Change the user-specified (x,y) pixel sizes
tmp = arparse.spect[ddnum]['xgap']
arparse.spect[ddnum]['xgap'] = arparse.spect[ddnum]['ygap']
arparse.spect[ddnum]['ygap'] = tmp
arparse.spect[ddnum]['ysize'] = 1.0 / arparse.spect[ddnum]['ysize']
# Update the amplifier/data/overscan sections
for i in range(arparse.spect[ddnum]['numamplifiers']):
# Flip the order of the sections
arparse.spect[ddnum]['datasec{0:02d}'.format(
i + 1)] = arparse.spect[ddnum]['datasec{0:02d}'.format(
i + 1)][::-1]
arparse.spect[ddnum]['oscansec{0:02d}'.format(
i + 1)] = arparse.spect[ddnum]['oscansec{0:02d}'.format(
i + 1)][::-1]
# Reduce the data!
status = 0
# Send the data away to be reduced
if spect.__dict__['_spect']['mosaic']['reduction'] == 'ARMLSD':
msgs.info("Data reduction will be performed using PYPIT-ARMLSD")
from pypit import armlsd
status = armlsd.ARMLSD(fitsdict)
elif spect.__dict__['_spect']['mosaic']['reduction'] == 'ARMED':
msgs.info("Data reduction will be performed using PYPIT-ARMED")
from pypit import armed
status = armed.ARMED(fitsdict)
# Check for successful reduction
if status == 0:
from pypit import arqa
msgs.info("Data reduction complete")
# QA HTML
msgs.info("Generating QA HTML")
arqa.gen_mf_html(redname)
arqa.gen_exp_html()
elif status == 1:
msgs.info("Setup complete")
elif status == 2:
msgs.info("Calcheck complete")
else:
msgs.error("Data reduction failed with status ID {0:d}".format(status))
# Capture the end time and print it to user
tend = time()
codetime = tend - tstart
if codetime < 60.0:
msgs.info("Data reduction execution time: {0:.2f}s".format(codetime))
elif codetime / 60.0 < 60.0:
mns = int(codetime / 60.0)
scs = codetime - 60.0 * mns
msgs.info("Data reduction execution time: {0:d}m {1:.2f}s".format(
mns, scs))
else:
hrs = int(codetime / 3600.0)
mns = int(60.0 * (codetime / 3600.0 - hrs))
scs = codetime - 60.0 * mns - 3600.0 * hrs
msgs.info(
"Data reduction execution time: {0:d}h {1:d}m {2:.2f}s".format(
hrs, mns, scs))
return