def test_find_standard():
# G191b2b
std_ra = '05:06:36.6'
std_dec = '52:52:01.0'
# Grab
std_dict = arflx.find_standard_file((std_ra, std_dec))
# Test
assert std_dict['name'] == 'G191B2B'
assert std_dict['file'] == '/data/standards/calspec/g191b2b_mod_005.fits'
assert std_dict['fmt'] == 1
# Fail to find
# near G191b2b
std_ra = '05:06:36.6'
std_dec = '52:22:01.0'
std_dict = arflx.find_standard_file((std_ra, std_dec))
assert std_dict is None
def test_find_standard():
from pypit import arutils as arut
from pypit import arflux as arflx
# Dummy self
slf = arut.dummy_self()
# G191b2b
std_ra = '05:06:36.6'
std_dec = '52:52:01.0'
# Grab
std_dict = arflx.find_standard_file(slf._argflag, (std_ra, std_dec))
# Test
assert std_dict['name'] == 'G191B2B'
assert std_dict['file'] == '/data/standards/calspec/g191b2b_mod_005.fits'
assert std_dict['fmt'] == 1
# Fail to find
# near G191b2b
std_ra = '05:06:36.6'
std_dec = '52:22:01.0'
std_dict = arflx.find_standard_file(slf._argflag, (std_ra, std_dec))
assert std_dict is None
def test_find_standard():
from pypit import arutils as arut
from pypit import arflux as arflx
# Dummy self
slf = arut.dummy_self()
# G191b2b
std_ra = "05:06:36.6"
std_dec = "52:52:01.0"
# Grab
std_dict = arflx.find_standard_file(slf._argflag, (std_ra, std_dec))
# Test
assert std_dict["name"] == "G191B2B"
assert std_dict["file"] == "/data/standards/calspec/g191b2b_mod_005.fits"
assert std_dict["fmt"] == 1
# Fail to find
# near G191b2b
std_ra = "05:06:36.6"
std_dec = "52:22:01.0"
std_dict = arflx.find_standard_file(slf._argflag, (std_ra, std_dec))
assert std_dict is None
def sort_data(argflag, spect, fitsdict):
"""
Create an exposure class for every science frame
Parameters
----------
argflag : dict
Arguments and flags used for reduction
spect : dict
Properties of the spectrograph.
fitsdict : dict
Contains relevant information from fits header files
Returns
-------
ftag : dict
A dictionary of filetypes
"""
msgs.bug("There appears to be a bug with the assignment of arc frames when only one science frame is supplied")
msgs.info("Sorting files")
numfiles = fitsdict['filename'].size
# Set the filetype dictionary
ftag = dict({'science': np.array([], dtype=np.int),
'standard': np.array([], dtype=np.int),
'bias': np.array([], dtype=np.int),
'pixflat': np.array([], dtype=np.int),
'blzflat': np.array([], dtype=np.int),
'trace': np.array([], dtype=np.int),
'arc': np.array([], dtype=np.int)})
fkey = np.array(ftag.keys())
# Create an array where 1 means it is a certain type of frame and 0 means it isn't.
filarr = np.zeros((len(fkey), numfiles), dtype=np.int)
setarr = np.zeros((len(fkey), numfiles), dtype=np.int)
# Identify the frames:
for i in range(len(fkey)):
# Self identification
if argflag['run']['use_idname']:
w = np.where(fitsdict['idname'] == spect[fkey[i]]['idname'])[0]
msgs.info("Sorting files")
else:
w = np.arange(numfiles)
n = np.arange(numfiles)
n = np.intersect1d(n, w)
# Perform additional checks in order to make sure this identification is true
chkk = spect[fkey[i]]['check'].keys()
for ch in chkk:
if ch[0:9] == 'condition':
# Deal with a conditional argument
conds = re.split("(\||\&)", spect[fkey[i]]['check'][ch])
ntmp = chk_condition(fitsdict, conds[0])
# And more
for cn in range((len(conds)-1)//2):
if conds[2*cn+1] == "|":
ntmp = ntmp | chk_condition(fitsdict, conds[2*cn+2])
elif conds[2*cn+1] == "&":
ntmp = ntmp & chk_condition(fitsdict, conds[2*cn+2])
w = np.where(ntmp)[0]
else:
if fitsdict[ch].dtype.char == 'S': # Numpy string array
# Strip numpy string array of all whitespace
w = np.where(np.char.strip(fitsdict[ch]) == spect[fkey[i]]['check'][ch])[0]
else:
w = np.where(fitsdict[ch] == spect[fkey[i]]['check'][ch])[0]
n = np.intersect1d(n, w)
# Assign these filetypes
filarr[i,:][n] = 1
# Check if these files can also be another type
if spect[fkey[i]]['canbe'] is not None:
for cb in spect[fkey[i]]['canbe']:
# Assign these filetypes
fa = np.where(fkey == cb)[0]
if np.size(fa) == 1: filarr[fa[0],:][n] = 1
else: msgs.error("Unknown type for argument 'canbe': {0:s}".format(cb))
# # Check for filetype clashes
# bdf=np.where(np.sum(filarr,axis=0)[n] != 0)[0]
# if np.size(bdf) != 0:
# msgs.info("Clash with file identifications when assigning frames as type {0:s}:".format(fkey[i]))
# # Check if this clash is allowed:
# clashfound=False
# for b in range(np.size(bdf)):
# # For each file with a clash, get all frames that have been assigned to it
# tarr = np.where(filarr[:,n[bdf[b]]]==1)[0]
# for a in tarr:
# if spect[fkey[i]]['canbe'] is None:
# print "{0:s} can only be of type: {1:s}, and is marked as {2:s}".format(fitsdict['filename'][n[bdf[b]]],fkey[i],fkey[a])
# clashfound=True
# elif fkey[a] not in spect[fkey[i]]['canbe']:
# print "{0:s} current filetype: {1:s}".format(fitsdict['filename'][n[bdf[b]]],fkey[a])
# clashfound=True
# if clashfound: msgs.error("Check these files and your settings.{0:s} file before continuing.".format(argflag['run']['spectrograph'])+msgs.newline()+"You can use the 'canbe' option to allow one frame to have multiple types.")
# else: msgs.info("Clash permitted")
# Identify the standard stars
# Find the nearest standard star to each science frame
wscistd = np.where(filarr[np.where(fkey == 'standard')[0], :].flatten() == 1)[0]
for i in range(wscistd.size):
# set_trace()
radec = (fitsdict['ra'][wscistd[i]], fitsdict['dec'][wscistd[i]])
# If an object exists within 20 arcmins of a listed standard, then it is probably a standard star
foundstd = find_standard_file(argflag, radec, toler=20.*u.arcmin, check=True)
if foundstd:
filarr[np.where(fkey == 'science')[0], wscistd[i]] = 0
else:
filarr[np.where(fkey == 'standard')[0], wscistd[i]] = 0
# Make any forced changes
msgs.info("Making forced file identification changes")
skeys = spect['set'].keys()
for sk in skeys:
for j in spect['set'][sk]:
w = np.where(fitsdict['filename']==j)[0]
filarr[:,w]=0
setarr[np.where(fkey==sk)[0],w]=1
del w
filarr = filarr + setarr
# Check that all files have an identification
badfiles = np.where(np.sum(filarr, axis=0) == 0)[0]
if np.size(badfiles) != 0:
msgs.info("Couldn't identify the following files:")
for i in range(np.size(badfiles)):
msgs.info(fitsdict['filename'][badfiles[i]])
msgs.error("Check these files and your settings.{0:s} file before continuing".format(argflag['run']['spectrograph']))
# Now identify the dark frames
wdark = np.where((filarr[np.where(fkey == 'bias')[0],:] == 1).flatten() &
(fitsdict['exptime'].astype(np.float64) > spect['mosaic']['minexp']))[0]
ftag['dark'] = wdark
# Store the frames in the ftag array
for i in range(len(fkey)):
ftag[fkey[i]] = np.where(filarr[i,:] == 1)[0]
# Finally check there are no duplicates (the arrays will automatically sort with np.unique)
msgs.info("Finalising frame sorting, and removing duplicates")
for key in ftag.keys():
ftag[key] = np.unique(ftag[key])
if np.size(ftag[key]) == 1:
msgs.info("Found {0:d} {1:s} frame".format(np.size(ftag[key]),key))
else:
msgs.info("Found {0:d} {1:s} frames".format(np.size(ftag[key]),key))
# Return ftag!
msgs.info("Sorting completed successfully")
return ftag
def sort_data(argflag, spect, fitsdict):
"""
Create an exposure class for every science frame
Parameters
----------
argflag : dict
Arguments and flags used for reduction
spect : dict
Properties of the spectrograph.
fitsdict : dict
Contains relevant information from fits header files
Returns
-------
ftag : dict
A dictionary of filetypes
"""
msgs.bug("There appears to be a bug with the assignment of arc frames when only one science frame is supplied")
msgs.info("Sorting files")
numfiles = fitsdict['filename'].size
# Set the filetype dictionary
ftag = dict({'science': np.array([], dtype=np.int),
'standard': np.array([], dtype=np.int),
'bias': np.array([], dtype=np.int),
'pixflat': np.array([], dtype=np.int),
'blzflat': np.array([], dtype=np.int),
'trace': np.array([], dtype=np.int),
'arc': np.array([], dtype=np.int)})
fkey = np.array(ftag.keys())
# Create an array where 1 means it is a certain type of frame and 0 means it isn't.
filarr = np.zeros((len(fkey), numfiles), dtype=np.int)
setarr = np.zeros((len(fkey), numfiles), dtype=np.int)
# Identify the frames:
for i in xrange(len(fkey)):
# Self identification
if argflag['run']['use_idname']:
w = np.where(fitsdict['idname'] == spect[fkey[i]]['idname'])[0]
msgs.info("Sorting files")
else:
w = np.arange(numfiles)
n = np.arange(numfiles)
n = np.intersect1d(n, w)
# Perform additional checks in order to make sure this identification is true
chkk = spect[fkey[i]]['check'].keys()
for ch in chkk:
if ch[0:9] == 'condition':
# Deal with a conditional argument
conds = re.split("(\||\&)", spect[fkey[i]]['check'][ch])
ntmp = chk_condition(fitsdict, conds[0])
# And more
for cn in xrange((len(conds)-1)//2):
if conds[2*cn+1] == "|":
ntmp = ntmp | chk_condition(fitsdict, conds[2*cn+2])
elif conds[2*cn+1] == "&":
ntmp = ntmp & chk_condition(fitsdict, conds[2*cn+2])
w = np.where(ntmp)[0]
else:
if fitsdict[ch].dtype.char == 'S': # Numpy string array
# Strip numpy string array of all whitespace
w = np.where(np.char.strip(fitsdict[ch]) == spect[fkey[i]]['check'][ch])[0]
else:
w = np.where(fitsdict[ch] == spect[fkey[i]]['check'][ch])[0]
n = np.intersect1d(n, w)
# Assign these filetypes
filarr[i,:][n] = 1
# Check if these files can also be another type
if spect[fkey[i]]['canbe'] is not None:
for cb in spect[fkey[i]]['canbe']:
# Assign these filetypes
fa = np.where(fkey == cb)[0]
if np.size(fa) == 1: filarr[fa[0],:][n] = 1
else: msgs.error("Unknown type for argument 'canbe': {0:s}".format(cb))
# # Check for filetype clashes
# bdf=np.where(np.sum(filarr,axis=0)[n] != 0)[0]
# if np.size(bdf) != 0:
# msgs.info("Clash with file identifications when assigning frames as type {0:s}:".format(fkey[i]))
# # Check if this clash is allowed:
# clashfound=False
# for b in xrange(np.size(bdf)):
# # For each file with a clash, get all frames that have been assigned to it
# tarr = np.where(filarr[:,n[bdf[b]]]==1)[0]
# for a in tarr:
# if spect[fkey[i]]['canbe'] is None:
# print "{0:s} can only be of type: {1:s}, and is marked as {2:s}".format(fitsdict['filename'][n[bdf[b]]],fkey[i],fkey[a])
# clashfound=True
# elif fkey[a] not in spect[fkey[i]]['canbe']:
# print "{0:s} current filetype: {1:s}".format(fitsdict['filename'][n[bdf[b]]],fkey[a])
# clashfound=True
# if clashfound: msgs.error("Check these files and your settings.{0:s} file before continuing.".format(argflag['run']['spectrograph'])+msgs.newline()+"You can use the 'canbe' option to allow one frame to have multiple types.")
# else: msgs.info("Clash permitted")
# Identify the standard stars
# Find the nearest standard star to each science frame
wscistd = np.where(filarr[np.where(fkey == 'standard')[0], :].flatten() == 1)[0]
for i in xrange(wscistd.size):
# set_trace()
radec = (fitsdict['ra'][wscistd[i]], fitsdict['dec'][wscistd[i]])
# If an object exists within 20 arcmins of a listed standard, then it is probably a standard star
foundstd = find_standard_file(argflag, radec, toler=20.*u.arcmin, check=True)
if foundstd:
filarr[np.where(fkey == 'science')[0], wscistd[i]] = 0
else:
filarr[np.where(fkey == 'standard')[0], wscistd[i]] = 0
# Make any forced changes
msgs.info("Making forced file identification changes")
skeys = spect['set'].keys()
for sk in skeys:
for j in spect['set'][sk]:
w = np.where(fitsdict['filename']==j)[0]
filarr[:,w]=0
setarr[np.where(fkey==sk)[0],w]=1
del w
filarr = filarr + setarr
# Check that all files have an identification
badfiles = np.where(np.sum(filarr, axis=0) == 0)[0]
if np.size(badfiles) != 0:
msgs.info("Couldn't identify the following files:")
for i in xrange(np.size(badfiles)):
msgs.info(fitsdict['filename'][badfiles[i]])
msgs.error("Check these files and your settings.{0:s} file before continuing".format(argflag['run']['spectrograph']))
# Now identify the dark frames
wdark = np.where((filarr[np.where(fkey == 'bias')[0],:] == 1).flatten() &
(fitsdict['exptime'].astype(np.float64) > spect['mosaic']['minexp']))[0]
ftag['dark'] = wdark
# Store the frames in the ftag array
for i in xrange(len(fkey)):
ftag[fkey[i]] = np.where(filarr[i,:] == 1)[0]
# Finally check there are no duplicates (the arrays will automatically sort with np.unique)
msgs.info("Finalising frame sorting, and removing duplicates")
for key in ftag.keys():
ftag[key] = np.unique(ftag[key])
if np.size(ftag[key]) == 1:
msgs.info("Found {0:d} {1:s} frame".format(np.size(ftag[key]),key))
else:
msgs.info("Found {0:d} {1:s} frames".format(np.size(ftag[key]),key))
# Return ftag!
msgs.info("Sorting completed successfully")
return ftag
