def test_instr_setup(fitsdict):
""" Test instrument setup naming convention
"""
from pypit import arsciexp
arutils.dummy_settings(spectrograph='shane_kast_blue', set_idx=False)
# Load
settings.argflag['run']['setup'] = True # Over-ride default numbers
filesort = arsort.sort_data(fitsdict)
# Match and test
arsort.match_science(fitsdict, filesort)
# Make a science frame
sciexp = arsciexp.ScienceExposure(0, fitsdict, do_qa=False)
# Get an ID
setup_dict = {}
setupID = arsort.instr_setup(sciexp, 1, fitsdict, setup_dict)
assert setupID == 'A_01_aa'
# Should get same thing
setupID = arsort.instr_setup(sciexp, 1, fitsdict, setup_dict)
assert setupID == 'A_01_aa'
# New det (fake out kast_blue)
settings.spect['det02'] = dict(numamplifiers=1)
setupID2 = arsort.instr_setup(sciexp, 2, fitsdict, setup_dict)
assert setupID2 == 'A_02_aa'
# New calib set
settings.spect['arc']['index'][1] = np.array(
[9]) # Not really an arc, but ok
sciexp1 = arsciexp.ScienceExposure(1, fitsdict, do_qa=False)
setupID3 = arsort.instr_setup(sciexp1, 1, fitsdict, setup_dict)
assert setupID3 == 'A_01_ab'
assert setup_dict['A']['ab']['arcs'][0] == 'b009.fits'
def test_user_frametype(fitsdict):
""" Test setting frametype manually
"""
arutils.dummy_settings(spectrograph='shane_kast_blue', set_idx=False)
# Modify settings -- WARNING: THIS IS GLOBAL!
settings.spect['set'] = {}
settings.spect['set']['standard'] = ['b009.fits']
filesort = arsort.sort_data(fitsdict)
assert 9 in filesort['standard']
settings.spect['set'] = {}
def test_sort_data(fitsdict):
""" Test sort_data
"""
arutils.dummy_settings(spectrograph='shane_kast_blue', set_idx=False)
# Sort
filesort = arsort.sort_data(fitsdict)
assert filesort['bias'][0] == 0
assert filesort['arc'][0] == 1
assert filesort['trace'][0] == 2
assert filesort['standard'][0] == 4
assert len(filesort['science']) == 5
def test_neg_match_science(fitsdict):
""" Test using negative number for calibs
"""
arutils.dummy_settings(spectrograph='shane_kast_blue', set_idx=False)
# Load
filesort = arsort.sort_data(fitsdict)
# Use negative number
for ftype in ['arc', 'pixelflat', 'trace', 'bias']:
settings.spect[ftype]['number'] = 1
settings.spect['trace']['number'] = -1
arsort.match_science(fitsdict, filesort)
assert len(settings.spect['trace']['index'][1]) == 2
def test_match_science(fitsdict):
""" Test match_science routine
"""
arutils.dummy_settings(spectrograph='shane_kast_blue', set_idx=False)
# Load
settings.argflag['run']['setup'] = True # Over-ride default numbers
filesort = arsort.sort_data(fitsdict)
# Match and test
arsort.match_science(fitsdict, filesort)
assert settings.spect['arc']['index'][1][0] == 1
assert settings.spect['standard']['index'][1][0] == 4
assert len(settings.spect['trace']['index'][0]) == 2
def Setup(self):
# Sort the data
msgs.bug("Files and folders should not be deleted -- there should be an option to overwrite files automatically if they already exist, or choose to rename them if necessary")
self._filesort = arsort.sort_data(self)
# Write out the details of the sorted files
if self._argflag['out']['sorted'] is not None: arsort.sort_write(self)
# Match Science frames to calibration frames
arsort.match_science(self)
# If the user is only debugging, then exit now
if self._argflag['run']['calcheck']:
msgs.info("Calibration check complete. Change the 'calcheck' flag to continue with data reduction")
sys.exit()
# Make directory structure for different objects
self._sci_targs = arsort.make_dirs(self)
return
def SetupScience(argflag, spect, fitsdict):
""" Create an exposure class for every science frame
Also links to standard star frames
Parameters
----------
argflag : dict
Arguments and flags used for reduction
spect : dict
Properties of the spectrograph.
fitsdict : dict
Contains relevant information from fits header files
Returns
-------
sciexp : list
A list containing all science exposure classes
"""
# Sort the data
msgs.bug(
"Files and folders should not be deleted -- there should be an option to overwrite files automatically if they already exist, or choose to rename them if necessary"
)
filesort = arsort.sort_data(argflag, spect, fitsdict)
# Write out the details of the sorted files
if argflag['out']['sorted'] is not None:
arsort.sort_write(argflag['out']['sorted'], spect, fitsdict, filesort)
# Match calibration frames to science frames
spect = arsort.match_science(argflag, spect, fitsdict, filesort)
# If the user is only debugging, then exit now
if argflag['run']['calcheck']:
msgs.info(
"Calibration check complete. Change the 'calcheck' flag to continue with data reduction"
)
sys.exit()
# Make directory structure for different objects
sci_targs = arsort.make_dirs(argflag, fitsdict, filesort)
# Create the list of science exposures
numsci = np.size(filesort['science'])
sciexp = []
for i in xrange(numsci):
sciexp.append(arsciexp.ScienceExposure(i, argflag, spect, fitsdict))
return sciexp
def SetupScience(argflag, spect, fitsdict):
""" Create an exposure class for every science frame
Also links to standard star frames
Parameters
----------
argflag : dict
Arguments and flags used for reduction
spect : dict
Properties of the spectrograph.
fitsdict : dict
Contains relevant information from fits header files
Returns
-------
sciexp : list
A list containing all science exposure classes
"""
# Sort the data
msgs.bug("Files and folders should not be deleted -- there should be an option to overwrite files automatically if they already exist, or choose to rename them if necessary")
filesort = arsort.sort_data(argflag, spect, fitsdict)
# Write out the details of the sorted files
if argflag['out']['sorted'] is not None:
arsort.sort_write(argflag['out']['sorted'], spect, fitsdict, filesort)
# Match calibration frames to science frames
spect = arsort.match_science(argflag, spect, fitsdict, filesort)
# If the user is only debugging, then exit now
if argflag['run']['calcheck']:
msgs.info("Calibration check complete. Change the 'calcheck' flag to continue with data reduction")
sys.exit()
# Make directory structure for different objects
sci_targs = arsort.make_dirs(argflag, fitsdict, filesort)
# Create the list of science exposures
numsci = np.size(filesort['science'])
sciexp = []
for i in range(numsci):
sciexp.append(arsciexp.ScienceExposure(i, argflag, spect, fitsdict))
return sciexp
def SetupScience(fitsdict):
""" Create an exposure class for every science frame
Also links to standard star frames and calibrations
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
Returns
-------
sciexp : list
A list containing all science exposure classes
"""
# Init
if settings.argflag['run']['calcheck'] or settings.argflag['run']['setup']:
do_qa = False
bad_to_unknown = True
else:
do_qa = True
bad_to_unknown = False
if settings.argflag['run']['setup']:
skip_cset = True
else:
skip_cset = False
# Sort the data
msgs.bug(
"Files and folders should not be deleted -- there should be an option to overwrite files automatically if they already exist, or choose to rename them if necessary"
)
filesort = arsort.sort_data(fitsdict, flag_unknown=bad_to_unknown)
# Write out the details of the sorted files into a .lst file
if settings.argflag['output']['sorted'] is not None:
srt_tbl = arsort.sort_write(fitsdict, filesort)
# Match calibration frames to science frames
arsort.match_science(fitsdict, filesort)
# Make directory structure for different objects
if do_qa:
sci_targs = arsort.make_dirs(fitsdict, filesort)
# Create the list of science exposures
numsci = np.size(settings.spect['science']['index'])
sciexp = []
for i in range(numsci):
sciexp.append(arsciexp.ScienceExposure(i, fitsdict, do_qa=do_qa))
# Generate setup and group dicts
setup_dict = {}
if settings.argflag['reduce']['masters']['force']:
# Check that setup was input
if len(settings.argflag['reduce']['masters']['setup']) == 0:
msgs.error(
"Need to specify reduce masters setup in your PYPIT file!")
# setup_dict
setup = settings.argflag['reduce']['masters']['setup']
setup_dict = {}
setup_dict[setup[0]] = {}
for ii in range(1, 20): # Dummy detectors
setup_dict[setup[0]][arparse.get_dnum(ii)] = dict(binning='1x1')
setup_dict[setup[0]][setup[-2:]] = {}
iSCI = filesort['science']
setup_dict[setup[0]][setup[-2:]]['sci'] = [
fitsdict['filename'][i] for i in iSCI
]
# Write
calib_file = arsort.write_calib(setup_dict)
return sciexp, setup_dict
# Run through the setups to fill setup_dict
setupIDs = []
for sc in range(numsci):
for kk in range(settings.spect['mosaic']['ndet']):
# Use user-supplied setup name (useful for some book-keeping)?
try:
cname = settings.argflag['setup']['name']
except KeyError:
cname = None
setupID = arsort.instr_setup(sciexp[sc],
kk + 1,
fitsdict,
setup_dict,
skip_cset=skip_cset,
config_name=cname)
if kk == 0: # Only save the first detector for run setup
setupIDs.append(setupID)
# Calib IDs
group_dict = {}
if settings.argflag['run']['setup']: # Collate all matching files
for sc, setupID in enumerate(setupIDs):
scidx = sciexp[sc]._idx_sci[0]
# Set group_key
config_key = setupID[0]
# Plan init
if config_key not in group_dict.keys():
group_dict[config_key] = {}
for key in filesort.keys():
if key not in ['unknown', 'dark']:
group_dict[config_key][key] = []
group_dict[config_key]['sciobj'] = []
group_dict[config_key]['stdobj'] = []
# Fill group_dict too
for key in filesort.keys():
if key in ['unknown', 'dark', 'failures']:
continue
for idx in settings.spect[key]['index'][sc]:
# Only add if new
if fitsdict['filename'][idx] not in group_dict[config_key][
key]:
group_dict[config_key][key].append(
fitsdict['filename'][idx])
if key == 'standard': # Add target name
group_dict[config_key]['stdobj'].append(
fitsdict['target'][idx])
if key == 'science': # Add target name
group_dict[config_key]['sciobj'].append(
fitsdict['target'][scidx])
#debugger.set_trace()
# Write .sorted file
if len(group_dict) > 0:
arsort.write_sorted(srt_tbl, group_dict, setup_dict)
else:
msgs.warn("No group dict entries and therefore no .sorted file")
# Write setup -- only if not present
setup_file, nexist = arsort.get_setup_file()
# Write calib file (if not in setup mode)
if not settings.argflag['run']['setup']:
calib_file = arsort.write_calib(setup_dict)
else:
arsort.write_setup(setup_dict)
# Finish calcheck or setup
if settings.argflag['run']['calcheck'] or settings.argflag['run']['setup']:
if settings.argflag['run']['calcheck']:
msgs.info("Inspect the .calib file: {:s}".format(calib_file))
msgs.info(
"*********************************************************")
msgs.info("Calibration check complete and successful!")
msgs.info(
"Set 'run calcheck False' to continue with data reduction")
msgs.info(
"*********************************************************")
# Instrument specific (might push into a separate file)
if settings.argflag['run']['spectrograph'] in ['keck_lris_blue']:
if settings.argflag['reduce']['flatfield']['useframe'] in [
'pixelflat'
]:
msgs.warn(
"We recommend a slitless flat for your instrument.")
return 'calcheck', None
elif settings.argflag['run']['setup']:
for idx in filesort['failures']:
msgs.warn(
"No Arc found: Skipping object {:s} with file {:s}".format(
fitsdict['target'][idx], fitsdict['filename'][idx]))
msgs.info("Setup is complete.")
msgs.info("Inspect the .setups file: {:s}".format(setup_file))
return 'setup', None
else:
msgs.error("Should not get here")
return sciexp, setup_dict