def main(args):
import sys, os
from pypit import pypit
from pypit.scripts import qa_html
import traceback
# Import PYPIT routines
# Initiate logging for bugs and command line help
# These messages will not be saved to a log file
# Set the default variables
qck = False
cpu = 1
#vrb = 2
# Load options from command line
debug['develop'] = debug['develop'] or args.develop
debug['arc'] = debug['arc'] or args.debug_arc
splitnm = os.path.splitext(args.pypit_file)
if splitnm[1] != '.pypit':
initmsgs.error("Bad extension for PYPIT reduction file."+initmsgs.newline()+".pypit is required")
logname = splitnm[0] + ".log"
# Execute the reduction, and catch any bugs for printout
if debug['develop']:
pypit.PYPIT(args.pypit_file, progname=pypit.__file__, quick=qck, ncpus=cpu, verbosity=args.verbosity,
use_masters=args.use_masters, devtest=args.devtest, logname=logname, debug=debug)
else:
try:
pypit.PYPIT(args.pypit_file, progname=pypit.__file__, quick=qck, ncpus=cpu, verbosity=args.verbosity,
use_masters=args.use_masters, devtest=args.devtest, logname=logname, debug=debug)
except:
# There is a bug in the code, print the file and line number of the error.
et, ev, tb = sys.exc_info()
filename, line_no = "<filename>", "<line_no>"
while tb:
co = tb.tb_frame.f_code
filename = str(co.co_filename)
try:
line_no = str(traceback.tb_lineno(tb))
except AttributeError: # Python 3
line_no = 'UNDEFINED'
tb = tb.tb_next
filename = filename.split('/')[-1]
if str(ev) != "":
initmsgs.bug("There appears to be a bug on Line " + line_no + " of " + filename + " with error:" +
initmsgs.newline() + str(ev) + initmsgs.newline() +
"---> please contact the authors")
# Get armsgs instance to terminate
from pypit.armsgs import get_logger
get_logger().close()
return 1
return 0
from astropy.table import Table as tTable, Column
from astropy import units as u
from linetools import utils as ltu
try:
from xastropy.xutils import xdebug as debugger
except:
import pdb as debugger
try:
basestring
except NameError:
basestring = str
# Logging
msgs = armsgs.get_logger()
def sort_data(argflag, spect, fitsdict):
"""
Create an exposure class for every science frame
Parameters
----------
argflag : dict
Arguments and flags used for reduction
spect : dict
Properties of the spectrograph.
fitsdict : dict
Contains relevant information from fits header files
from __future__ import (print_function, absolute_import, division,
unicode_literals)
import numpy as np
from astropy.table import Table, Column, vstack
import glob, copy
import yaml
from pypit import armsgs
from pypit import arparse as settings
from pypit import ardebug as debugger
# Logging
msgs = armsgs.get_logger()
def parse_nist(slf, ion):
"""Parse a NIST ASCII table. Note that the long ---- should have
been commented out and also the few lines at the start.
Parameters
----------
ion : str
Name of ion
"""
# Root (for development only)
if slf is None:
from pypit import arutils as arut
msgs.warn("Using arutils.dummy_self. Better know what you are doing.")
slf = arut.dummy_self()
def main(args):
import sys, os
from pypit import pypit
import traceback
# Import PYPIT routines
# Initiate logging for bugs and command line help
# These messages will not be saved to a log file
# Set the default variables
qck = False
cpu = 1
# vrb = 2
# Load options from command line
debug["develop"] = debug["develop"] or args.develop
debug["arc"] = debug["arc"] or args.debug_arc
splitnm = os.path.splitext(args.pypit_file)
if splitnm[1] != ".pypit":
initmsgs.error("Bad extension for PYPIT reduction file." + initmsgs.newline() + ".pypit is required")
logname = splitnm[0] + ".log"
# Execute the reduction, and catch any bugs for printout
if debug["develop"]:
pypit.PYPIT(
args.pypit_file,
progname=pypit.__file__,
quick=qck,
ncpus=cpu,
verbose=args.verbose,
use_masters=args.use_masters,
logname=logname,
debug=debug,
)
else:
try:
pypit.PYPIT(
args.pypit_file,
progname=pypit.__file__,
quick=qck,
ncpus=cpu,
verbose=args.verbose,
use_masters=args.use_masters,
logname=logname,
debug=debug,
)
except:
# There is a bug in the code, print the file and line number of the error.
et, ev, tb = sys.exc_info()
filename, line_no = "<filename>", "<line_no>"
while tb:
co = tb.tb_frame.f_code
filename = str(co.co_filename)
line_no = str(traceback.tb_lineno(tb))
tb = tb.tb_next
filename = filename.split("/")[-1]
if str(ev) != "":
initmsgs.bug(
"There appears to be a bug on Line "
+ line_no
+ " of "
+ filename
+ " with error:"
+ initmsgs.newline()
+ str(ev)
+ initmsgs.newline()
+ "---> please contact the authors"
)
# Get armsgs instance to terminate
from pypit.armsgs import get_logger
get_logger().close()
def PYPIT(redname,
debug=None,
progname=__file__,
quick=False,
ncpus=1,
verbosity=1,
use_masters=False,
devtest=False,
logname=None):
""" Main driver of the PYPIT code. Default settings and
user-specified changes are made, and passed to the
appropriate code for data reduction.
Parameters
----------
redname : string
Input reduction script
debug : dict, optional
Debug dict
progname : string
Name of the program
quick : bool
If True, a quick reduction (but possibly less
accurate) will be performed. This flag is most
useful for observing at a telescope, but not
for publication quality results.
ncpus : int
Number of CPUs to use for multiprocessing the
data reduction (sometimes not used)
verbosity : int (0,1,2)
Level of verbosity:
0 = No output
1 = Minimal output (default - suitable for the average user)
2 = All output
use_masters : bool, optional
Load calibration files from MasterFrames directory, if they exist
devtest : bool, optional
Running PYPIT-Development suite; will turn instrument-specific options
logname : str or None
The name of an ascii log file which is used to
save the output details of the reduction
debug : dict
A PYPIT debug dict (from ardebug.init)
version : str
last_updated : str
---------------------------------------------------
"""
from pypit import ardebug
# Init logger
if debug is None:
debug = ardebug.init()
msgs = armsgs.get_logger((logname, debug, verbosity))
msgs.pypit_file = redname
# This needs to be loaded after msgs
from pypit import arparse
# version checking
try:
archeck.version_check()
except archeck.VersionError as err:
msgs.error(err.message)
# First send all signals to messages to be dealt with (i.e. someone hits ctrl+c)
sigsignal(SIGINT, msgs.signal_handler)
# Ignore all warnings given by python
resetwarnings()
simplefilter("ignore")
# Record the starting time
tstart = time()
# Load the input file
pyp_dict = load_input(redname, msgs)
parlines, datlines, spclines = [
pyp_dict[ii] for ii in ['par', 'dat', 'spc']
]
# Initialize the arguments and flags
# argflag = arload.argflag_init()
# settings.argflag['run']['ncpus'] = ncpus
# settings.argflag['output']['verbosity'] = verbosity
# Determine the name of the spectrograph
specname = None
for i in range(len(parlines)):
parspl = parlines[i].split()
if len(parspl) < 3:
msgs.error(
"There appears to be a missing argument on the following input line"
+ msgs.newline() + parlines[i])
if (parspl[0] == 'run') and (parspl[1] == 'spectrograph'):
specname = parspl[2]
break
if specname is None:
msgs.error(
"Please specify the spectrograph settings to be used with the command"
+ msgs.newline() + "run spectrograph <name>")
msgs.info("Reducing data from the {0:s} spectrograph".format(specname))
# Determine the type of reduction used for this spectrograph
redtype = None
# Get the software path
prgn_spl = progname.split('/')
tfname = "/".join(prgn_spl[:-1]) + "/"
# Settings file
fname = tfname + 'data/settings/settings.' + specname
try:
spl = open(fname, 'r').readlines()
except IOError:
msgs.error(
"The following instrument settings file cannot be found:" +
msgs.newline() + fname + msgs.newline() +
"Please check the settings file exists, and that the instrument name is spelt correctly."
)
for i in range(len(spl)):
parspl = spl[i].split()
if len(parspl) < 3:
continue
if (parspl[0] == 'mosaic') and (parspl[1] == 'reduction'):
redtype = parspl[2]
break
if redtype is None:
msgs.bug(
"The {0:s} instrument settings file must contain the reduction type"
.format(specname))
msgs.error("Please specify the reduction type with the command" +
msgs.newline() + "mosaic reduction <type>")
# Load default reduction arguments/flags, and set any command line arguments
argf = arparse.get_argflag_class(
(redtype.upper(), ".".join(redname.split(".")[:-1])))
argf.init_param()
# Run specific
argf.set_param('run pypitdir {0:s}'.format(tfname))
argf.set_param('run progname {0:s}'.format(progname))
argf.set_param('run redname {0:s}'.format(redname))
# Load user changes to the arguments/flags
plines = argf.load_lines(parlines)
argf.set_paramlist(plines)
# If the user wishes to load a settings file, do that now
if argf.__dict__['_argflag']['run']['load']['settings'] is not None:
lines = argf.load_file(
argf.__dict__['_argflag']['run']['load']['settings'])
argf.set_paramlist(lines)
# Load default spectrograph settings
spect = arparse.get_spect_class(
(redtype.upper(), specname, ".".join(redname.split(".")[:-1])))
lines = spect.load_file(base=True) # Base spectrograph settings
spect.set_paramlist(lines)
lines = spect.load_file() # Instrument specific
spect.set_paramlist(lines)
# Load frametype numbers, as relevant
if len(pyp_dict['ftype']) > 0:
ftlines = spect.load_ftype(pyp_dict['ftype'])
plines = spect.load_lines(ftlines)
spect.set_paramlist(plines)
# Load user changes to the arguments/flags
plines = spect.load_lines(spclines)
spect.set_paramlist(plines)
if argf.__dict__['_argflag']['run']['load']['spect'] is not None:
lines = spect.load_file(
argf.__dict__['_argflag']['run']['load']['spect'])
spect.set_paramlist(lines)
# If the instrument settings file sets some argflag settings, implement those changes now
if len(spect.__dict__['_settings']) != 0:
argf.set_paramlist(spect.__dict__['_settings'])
# Load command line changes
argf.set_param('run ncpus {0:d}'.format(ncpus))
argf.set_param('output verbosity {0:d}'.format(verbosity))
if use_masters:
argf.set_param('reduce masters reuse True')
msgs.work("Make appropriate changes to quick reduction")
# Load Development suite changes
if devtest:
msgs.info(
"Loading instrument specific argurment for Development Suite tests"
)
from pypit import ardevtest
ardevtest.set_param(argf, specname)
if quick:
# If a quick reduction has been requested, make sure the requested pipeline
# is the quick implementation (if it exists), otherwise run the standard pipeline.
msgs.work("QUICK REDUCTION TO STILL BE DONE")
# Setup from PYPIT file?
if len(pyp_dict['setup']['name']) == 1:
argf.set_param('setup name {:s}'.format(pyp_dict['setup']['name'][0]))
# Finally, save the arguments/flags and spectrograph settings used for this reduction
argf.save()
spect.save()
# Now that all of the relevant settings are loaded, globalize the settings
arparse.init(argf, spect)
'''
# Test that a maximum of one .setup files is present
from pypit import arsort
setup_file, nexist = arsort.get_setup_file()
if nexist == 1:
msgs.info("Found setup_file: {:s}".format(setup_file))
msgs.info("Will use this to guide the data reduction.")
'''
# Load the important information from the fits headers
from pypit.arload import load_headers
fitsdict, updates = load_headers(datlines)
# If some settings were updated because of the fits headers, globalize the settings again
if len(updates) != 0:
spect.set_paramlist(updates)
arparse.init(argf, spect)
# If the dispersion direction is 1, flip the axes
if arparse.argflag['trace']['dispersion']['direction'] == 1:
# Update the keywords of all fits files
for ff in range(len(fitsdict['naxis0'])):
temp = fitsdict['naxis0'][ff]
fitsdict['naxis0'][ff] = fitsdict['naxis1'][ff]
fitsdict['naxis1'][ff] = temp
# Update the spectrograph settings for all detectors in the mosaic
for dd in range(arparse.spect['mosaic']['ndet']):
ddnum = arparse.get_dnum(dd + 1)
# Change the user-specified (x,y) pixel sizes
tmp = arparse.spect[ddnum]['xgap']
arparse.spect[ddnum]['xgap'] = arparse.spect[ddnum]['ygap']
arparse.spect[ddnum]['ygap'] = tmp
arparse.spect[ddnum]['ysize'] = 1.0 / arparse.spect[ddnum]['ysize']
# Update the amplifier/data/overscan sections
for i in range(arparse.spect[ddnum]['numamplifiers']):
# Flip the order of the sections
arparse.spect[ddnum]['datasec{0:02d}'.format(
i + 1)] = arparse.spect[ddnum]['datasec{0:02d}'.format(
i + 1)][::-1]
arparse.spect[ddnum]['oscansec{0:02d}'.format(
i + 1)] = arparse.spect[ddnum]['oscansec{0:02d}'.format(
i + 1)][::-1]
# Reduce the data!
status = 0
# Send the data away to be reduced
if spect.__dict__['_spect']['mosaic']['reduction'] == 'ARMLSD':
msgs.info("Data reduction will be performed using PYPIT-ARMLSD")
from pypit import armlsd
status = armlsd.ARMLSD(fitsdict)
elif spect.__dict__['_spect']['mosaic']['reduction'] == 'ARMED':
msgs.info("Data reduction will be performed using PYPIT-ARMED")
from pypit import armed
status = armed.ARMED(fitsdict)
# Check for successful reduction
if status == 0:
from pypit import arqa
msgs.info("Data reduction complete")
# QA HTML
msgs.info("Generating QA HTML")
arqa.gen_mf_html(redname)
arqa.gen_exp_html()
elif status == 1:
msgs.info("Setup complete")
elif status == 2:
msgs.info("Calcheck complete")
else:
msgs.error("Data reduction failed with status ID {0:d}".format(status))
# Capture the end time and print it to user
tend = time()
codetime = tend - tstart
if codetime < 60.0:
msgs.info("Data reduction execution time: {0:.2f}s".format(codetime))
elif codetime / 60.0 < 60.0:
mns = int(codetime / 60.0)
scs = codetime - 60.0 * mns
msgs.info("Data reduction execution time: {0:d}m {1:.2f}s".format(
mns, scs))
else:
hrs = int(codetime / 3600.0)
mns = int(60.0 * (codetime / 3600.0 - hrs))
scs = codetime - 60.0 * mns - 3600.0 * hrs
msgs.info(
"Data reduction execution time: {0:d}h {1:d}m {2:.2f}s".format(
hrs, mns, scs))
return
def PYPIT(redname, debug=None, progname=__file__, quick=False, ncpus=1, verbose=1,
use_masters=False, logname=None):
"""
Main driver of the PYPIT code. Default settings and
user-specified changes are made, and passed to the
appropriate code for data reduction.
Parameters
----------
redname : string
Input reduction script
debug : dict, optional
Debug dict
progname : string
Name of the program
quick : bool
If True, a quick reduction (but possibly less
accurate) will be performed. This flag is most
useful for observing at a telescope, but not
for publication quality results.
ncpus : int
Number of CPUs to use for multiprocessing the
data reduction (sometimes not used)
verbose : int (0,1,2)
Level of verbosity:
0 = No output
1 = Minimal output (default - suitable for the average user)
2 = All output
use_masters : bool, optional
Load calibration files from MasterFrames directory, if they exist
logname : str or None
The name of an ascii log file which is used to
save the output details of the reduction
debug : dict
A PYPIT debug dict (from ardebug.init)
version : str
last_updated : str
---------------------------------------------------
"""
from pypit import ardebug
# Init logger
if debug is None:
debug=ardebug.init()
msgs = armsgs.get_logger((logname, debug, verbose))
from pypit import arload # This needs to be after msgs is defined!
# First send all signals to messages to be dealt with (i.e. someone hits ctrl+c)
sigsignal(SIGINT, msgs.signal_handler)
# Ignore all warnings given by python
resetwarnings()
simplefilter("ignore")
# Record the starting time
tstart = time()
# Load the input file
parlines, datlines, spclines = arload.load_input(redname)
# Initialize the arguments and flags
argflag = arload.argflag_init()
argflag['run']['ncpus'] = ncpus
argflag['out']['verbose'] = verbose
# Determine the name of the spectrograph
specname = None
for i in range(len(parlines)):
parspl = parlines[i].split()
if len(parspl) < 3:
msgs.error("There appears to be a missing argument on the following input line" + msgs.newline() +
parlines[i])
if (parspl[0] == 'run') and (parspl[1] == 'spectrograph'):
specname = parspl[2]
if specname is None:
msgs.error("Please specify the spectrograph settings to be used with the command" + msgs.newline() +
"run spectrograph <name>")
msgs.info("Reducing data from the {0:s} spectrograph".format(specname))
# Load the Spectrograph settings
spect = arload.load_spect(progname, specname)
# Load default reduction arguments/flags, and set any command line arguments
#argflag = arload.optarg(argflag, cmdlnarg, spect['mosaic']['reduction'].lower())
# Load the default settings
prgn_spl = progname.split('/')
tfname = ""
for i in range(0,len(prgn_spl)-1): tfname += prgn_spl[i]+"/"
#fname = tfname + prgn_spl[-2] + '/settings.' + spect['mosaic']['reduction'].lower()
fname = tfname + '/settings.' + spect['mosaic']['reduction'].lower()
argflag = arload.load_settings(fname, argflag)
argflag['run']['prognm'] = progname
argflag['run']['pypitdir'] = tfname
# Now update the settings based on the user input file
argflag = arload.set_params(parlines, argflag, setstr="Input ")
# Check the input file
arload.check_argflag(argflag)
# Load any changes to the spectrograph settings based on the user input file
spect = arload.load_spect(progname, specname, spect=spect, lines=spclines)
# Command line arguments
if use_masters:
argflag['masters']['use'] = True
# If a quick reduction has been requested, make sure the requested pipeline
# is the quick implementation (if it exists), otherwise run the standard pipeline.
if quick:
# Change to a "quick" settings file
msgs.work("QUICK REDUCTION TO STILL BE DONE")
# Load the important information from the fits headers
fitsdict = arload.load_headers(argflag, spect, datlines)
# Reduce the data!
status = 0
msgs.work("Make appropriate changes to quick reduction")
if quick:
msgs.work("define what is needed here for quick reduction")
# Send the data away to be reduced
if spect['mosaic']['reduction'] == 'ARMLSD':
msgs.info("Data reduction will be performed using PYPIT-ARMLSD")
from pypit import armlsd
status = armlsd.ARMLSD(argflag, spect, fitsdict)
elif spect['mosaic']['reduction'] == 'ARMED':
msgs.info("Data reduction will be performed using PYPIT-ARMED")
from pypit import armed
status = armed.ARMED(argflag, spect, fitsdict)
# Check for successful reduction
if status == 0:
msgs.info("Data reduction complete")
else:
msgs.error("Data reduction failed with status ID {0:d}".format(status))
# Capture the end time and print it to user
tend = time()
codetime = tend-tstart
if codetime < 60.0:
msgs.info("Data reduction execution time: {0:.2f}s".format(codetime))
elif codetime/60.0 < 60.0:
mns = int(codetime/60.0)
scs = codetime - 60.0*mns
msgs.info("Data reduction execution time: {0:d}m {1:.2f}s".format(mns, scs))
else:
hrs = int(codetime/3600.0)
mns = int(60.0*(codetime/3600.0 - hrs))
scs = codetime - 60.0*mns - 3600.0*hrs
msgs.info("Data reduction execution time: {0:d}h {1:d}m {2:.2f}s".format(hrs, mns, scs))
return
import numpy as np
from pypit import armsgs
from pypit import pyputils
# CANNOT LOAD DEBUGGER AS THIS MODULE IS CALLED BY ARDEBUG
#from pypit import ardebug as debugger
import pdb as debugger
try:
basestring
except NameError:
basestring = str
# Logging
msgs = armsgs.get_logger() # THESE MAY NOT WORK..
'''
if msgs is None: # For usage outside of PYPIT
msgs = pyputils.get_dummy_logger()
armsgs.pypit_logger = msgs
'''
def connect_to_ginga(host='localhost', port=9000):
""" Connect to an active RC Ginga
Parameters
----------
host : str, optional
port : int, optional
Returns