def test_find_standard(): # G191b2b std_ra = '05:06:36.6' std_dec = '52:52:01.0' # Grab std_dict = arflx.find_standard_file((std_ra, std_dec)) # Test assert std_dict['name'] == 'G191B2B' assert std_dict['file'] == '/data/standards/calspec/g191b2b_mod_005.fits' assert std_dict['fmt'] == 1 # Fail to find # near G191b2b std_ra = '05:06:36.6' std_dec = '52:22:01.0' std_dict = arflx.find_standard_file((std_ra, std_dec)) assert std_dict is None
def test_find_standard(): from pypit import arutils as arut from pypit import arflux as arflx # Dummy self slf = arut.dummy_self() # G191b2b std_ra = '05:06:36.6' std_dec = '52:52:01.0' # Grab std_dict = arflx.find_standard_file(slf._argflag, (std_ra, std_dec)) # Test assert std_dict['name'] == 'G191B2B' assert std_dict['file'] == '/data/standards/calspec/g191b2b_mod_005.fits' assert std_dict['fmt'] == 1 # Fail to find # near G191b2b std_ra = '05:06:36.6' std_dec = '52:22:01.0' std_dict = arflx.find_standard_file(slf._argflag, (std_ra, std_dec)) assert std_dict is None
def test_find_standard(): from pypit import arutils as arut from pypit import arflux as arflx # Dummy self slf = arut.dummy_self() # G191b2b std_ra = "05:06:36.6" std_dec = "52:52:01.0" # Grab std_dict = arflx.find_standard_file(slf._argflag, (std_ra, std_dec)) # Test assert std_dict["name"] == "G191B2B" assert std_dict["file"] == "/data/standards/calspec/g191b2b_mod_005.fits" assert std_dict["fmt"] == 1 # Fail to find # near G191b2b std_ra = "05:06:36.6" std_dec = "52:22:01.0" std_dict = arflx.find_standard_file(slf._argflag, (std_ra, std_dec)) assert std_dict is None
def sort_data(argflag, spect, fitsdict): """ Create an exposure class for every science frame Parameters ---------- argflag : dict Arguments and flags used for reduction spect : dict Properties of the spectrograph. fitsdict : dict Contains relevant information from fits header files Returns ------- ftag : dict A dictionary of filetypes """ msgs.bug("There appears to be a bug with the assignment of arc frames when only one science frame is supplied") msgs.info("Sorting files") numfiles = fitsdict['filename'].size # Set the filetype dictionary ftag = dict({'science': np.array([], dtype=np.int), 'standard': np.array([], dtype=np.int), 'bias': np.array([], dtype=np.int), 'pixflat': np.array([], dtype=np.int), 'blzflat': np.array([], dtype=np.int), 'trace': np.array([], dtype=np.int), 'arc': np.array([], dtype=np.int)}) fkey = np.array(ftag.keys()) # Create an array where 1 means it is a certain type of frame and 0 means it isn't. filarr = np.zeros((len(fkey), numfiles), dtype=np.int) setarr = np.zeros((len(fkey), numfiles), dtype=np.int) # Identify the frames: for i in range(len(fkey)): # Self identification if argflag['run']['use_idname']: w = np.where(fitsdict['idname'] == spect[fkey[i]]['idname'])[0] msgs.info("Sorting files") else: w = np.arange(numfiles) n = np.arange(numfiles) n = np.intersect1d(n, w) # Perform additional checks in order to make sure this identification is true chkk = spect[fkey[i]]['check'].keys() for ch in chkk: if ch[0:9] == 'condition': # Deal with a conditional argument conds = re.split("(\||\&)", spect[fkey[i]]['check'][ch]) ntmp = chk_condition(fitsdict, conds[0]) # And more for cn in range((len(conds)-1)//2): if conds[2*cn+1] == "|": ntmp = ntmp | chk_condition(fitsdict, conds[2*cn+2]) elif conds[2*cn+1] == "&": ntmp = ntmp & chk_condition(fitsdict, conds[2*cn+2]) w = np.where(ntmp)[0] else: if fitsdict[ch].dtype.char == 'S': # Numpy string array # Strip numpy string array of all whitespace w = np.where(np.char.strip(fitsdict[ch]) == spect[fkey[i]]['check'][ch])[0] else: w = np.where(fitsdict[ch] == spect[fkey[i]]['check'][ch])[0] n = np.intersect1d(n, w) # Assign these filetypes filarr[i,:][n] = 1 # Check if these files can also be another type if spect[fkey[i]]['canbe'] is not None: for cb in spect[fkey[i]]['canbe']: # Assign these filetypes fa = np.where(fkey == cb)[0] if np.size(fa) == 1: filarr[fa[0],:][n] = 1 else: msgs.error("Unknown type for argument 'canbe': {0:s}".format(cb)) # # Check for filetype clashes # bdf=np.where(np.sum(filarr,axis=0)[n] != 0)[0] # if np.size(bdf) != 0: # msgs.info("Clash with file identifications when assigning frames as type {0:s}:".format(fkey[i])) # # Check if this clash is allowed: # clashfound=False # for b in range(np.size(bdf)): # # For each file with a clash, get all frames that have been assigned to it # tarr = np.where(filarr[:,n[bdf[b]]]==1)[0] # for a in tarr: # if spect[fkey[i]]['canbe'] is None: # print "{0:s} can only be of type: {1:s}, and is marked as {2:s}".format(fitsdict['filename'][n[bdf[b]]],fkey[i],fkey[a]) # clashfound=True # elif fkey[a] not in spect[fkey[i]]['canbe']: # print "{0:s} current filetype: {1:s}".format(fitsdict['filename'][n[bdf[b]]],fkey[a]) # clashfound=True # if clashfound: msgs.error("Check these files and your settings.{0:s} file before continuing.".format(argflag['run']['spectrograph'])+msgs.newline()+"You can use the 'canbe' option to allow one frame to have multiple types.") # else: msgs.info("Clash permitted") # Identify the standard stars # Find the nearest standard star to each science frame wscistd = np.where(filarr[np.where(fkey == 'standard')[0], :].flatten() == 1)[0] for i in range(wscistd.size): # set_trace() radec = (fitsdict['ra'][wscistd[i]], fitsdict['dec'][wscistd[i]]) # If an object exists within 20 arcmins of a listed standard, then it is probably a standard star foundstd = find_standard_file(argflag, radec, toler=20.*u.arcmin, check=True) if foundstd: filarr[np.where(fkey == 'science')[0], wscistd[i]] = 0 else: filarr[np.where(fkey == 'standard')[0], wscistd[i]] = 0 # Make any forced changes msgs.info("Making forced file identification changes") skeys = spect['set'].keys() for sk in skeys: for j in spect['set'][sk]: w = np.where(fitsdict['filename']==j)[0] filarr[:,w]=0 setarr[np.where(fkey==sk)[0],w]=1 del w filarr = filarr + setarr # Check that all files have an identification badfiles = np.where(np.sum(filarr, axis=0) == 0)[0] if np.size(badfiles) != 0: msgs.info("Couldn't identify the following files:") for i in range(np.size(badfiles)): msgs.info(fitsdict['filename'][badfiles[i]]) msgs.error("Check these files and your settings.{0:s} file before continuing".format(argflag['run']['spectrograph'])) # Now identify the dark frames wdark = np.where((filarr[np.where(fkey == 'bias')[0],:] == 1).flatten() & (fitsdict['exptime'].astype(np.float64) > spect['mosaic']['minexp']))[0] ftag['dark'] = wdark # Store the frames in the ftag array for i in range(len(fkey)): ftag[fkey[i]] = np.where(filarr[i,:] == 1)[0] # Finally check there are no duplicates (the arrays will automatically sort with np.unique) msgs.info("Finalising frame sorting, and removing duplicates") for key in ftag.keys(): ftag[key] = np.unique(ftag[key]) if np.size(ftag[key]) == 1: msgs.info("Found {0:d} {1:s} frame".format(np.size(ftag[key]),key)) else: msgs.info("Found {0:d} {1:s} frames".format(np.size(ftag[key]),key)) # Return ftag! msgs.info("Sorting completed successfully") return ftag
def sort_data(argflag, spect, fitsdict): """ Create an exposure class for every science frame Parameters ---------- argflag : dict Arguments and flags used for reduction spect : dict Properties of the spectrograph. fitsdict : dict Contains relevant information from fits header files Returns ------- ftag : dict A dictionary of filetypes """ msgs.bug("There appears to be a bug with the assignment of arc frames when only one science frame is supplied") msgs.info("Sorting files") numfiles = fitsdict['filename'].size # Set the filetype dictionary ftag = dict({'science': np.array([], dtype=np.int), 'standard': np.array([], dtype=np.int), 'bias': np.array([], dtype=np.int), 'pixflat': np.array([], dtype=np.int), 'blzflat': np.array([], dtype=np.int), 'trace': np.array([], dtype=np.int), 'arc': np.array([], dtype=np.int)}) fkey = np.array(ftag.keys()) # Create an array where 1 means it is a certain type of frame and 0 means it isn't. filarr = np.zeros((len(fkey), numfiles), dtype=np.int) setarr = np.zeros((len(fkey), numfiles), dtype=np.int) # Identify the frames: for i in xrange(len(fkey)): # Self identification if argflag['run']['use_idname']: w = np.where(fitsdict['idname'] == spect[fkey[i]]['idname'])[0] msgs.info("Sorting files") else: w = np.arange(numfiles) n = np.arange(numfiles) n = np.intersect1d(n, w) # Perform additional checks in order to make sure this identification is true chkk = spect[fkey[i]]['check'].keys() for ch in chkk: if ch[0:9] == 'condition': # Deal with a conditional argument conds = re.split("(\||\&)", spect[fkey[i]]['check'][ch]) ntmp = chk_condition(fitsdict, conds[0]) # And more for cn in xrange((len(conds)-1)//2): if conds[2*cn+1] == "|": ntmp = ntmp | chk_condition(fitsdict, conds[2*cn+2]) elif conds[2*cn+1] == "&": ntmp = ntmp & chk_condition(fitsdict, conds[2*cn+2]) w = np.where(ntmp)[0] else: if fitsdict[ch].dtype.char == 'S': # Numpy string array # Strip numpy string array of all whitespace w = np.where(np.char.strip(fitsdict[ch]) == spect[fkey[i]]['check'][ch])[0] else: w = np.where(fitsdict[ch] == spect[fkey[i]]['check'][ch])[0] n = np.intersect1d(n, w) # Assign these filetypes filarr[i,:][n] = 1 # Check if these files can also be another type if spect[fkey[i]]['canbe'] is not None: for cb in spect[fkey[i]]['canbe']: # Assign these filetypes fa = np.where(fkey == cb)[0] if np.size(fa) == 1: filarr[fa[0],:][n] = 1 else: msgs.error("Unknown type for argument 'canbe': {0:s}".format(cb)) # # Check for filetype clashes # bdf=np.where(np.sum(filarr,axis=0)[n] != 0)[0] # if np.size(bdf) != 0: # msgs.info("Clash with file identifications when assigning frames as type {0:s}:".format(fkey[i])) # # Check if this clash is allowed: # clashfound=False # for b in xrange(np.size(bdf)): # # For each file with a clash, get all frames that have been assigned to it # tarr = np.where(filarr[:,n[bdf[b]]]==1)[0] # for a in tarr: # if spect[fkey[i]]['canbe'] is None: # print "{0:s} can only be of type: {1:s}, and is marked as {2:s}".format(fitsdict['filename'][n[bdf[b]]],fkey[i],fkey[a]) # clashfound=True # elif fkey[a] not in spect[fkey[i]]['canbe']: # print "{0:s} current filetype: {1:s}".format(fitsdict['filename'][n[bdf[b]]],fkey[a]) # clashfound=True # if clashfound: msgs.error("Check these files and your settings.{0:s} file before continuing.".format(argflag['run']['spectrograph'])+msgs.newline()+"You can use the 'canbe' option to allow one frame to have multiple types.") # else: msgs.info("Clash permitted") # Identify the standard stars # Find the nearest standard star to each science frame wscistd = np.where(filarr[np.where(fkey == 'standard')[0], :].flatten() == 1)[0] for i in xrange(wscistd.size): # set_trace() radec = (fitsdict['ra'][wscistd[i]], fitsdict['dec'][wscistd[i]]) # If an object exists within 20 arcmins of a listed standard, then it is probably a standard star foundstd = find_standard_file(argflag, radec, toler=20.*u.arcmin, check=True) if foundstd: filarr[np.where(fkey == 'science')[0], wscistd[i]] = 0 else: filarr[np.where(fkey == 'standard')[0], wscistd[i]] = 0 # Make any forced changes msgs.info("Making forced file identification changes") skeys = spect['set'].keys() for sk in skeys: for j in spect['set'][sk]: w = np.where(fitsdict['filename']==j)[0] filarr[:,w]=0 setarr[np.where(fkey==sk)[0],w]=1 del w filarr = filarr + setarr # Check that all files have an identification badfiles = np.where(np.sum(filarr, axis=0) == 0)[0] if np.size(badfiles) != 0: msgs.info("Couldn't identify the following files:") for i in xrange(np.size(badfiles)): msgs.info(fitsdict['filename'][badfiles[i]]) msgs.error("Check these files and your settings.{0:s} file before continuing".format(argflag['run']['spectrograph'])) # Now identify the dark frames wdark = np.where((filarr[np.where(fkey == 'bias')[0],:] == 1).flatten() & (fitsdict['exptime'].astype(np.float64) > spect['mosaic']['minexp']))[0] ftag['dark'] = wdark # Store the frames in the ftag array for i in xrange(len(fkey)): ftag[fkey[i]] = np.where(filarr[i,:] == 1)[0] # Finally check there are no duplicates (the arrays will automatically sort with np.unique) msgs.info("Finalising frame sorting, and removing duplicates") for key in ftag.keys(): ftag[key] = np.unique(ftag[key]) if np.size(ftag[key]) == 1: msgs.info("Found {0:d} {1:s} frame".format(np.size(ftag[key]),key)) else: msgs.info("Found {0:d} {1:s} frames".format(np.size(ftag[key]),key)) # Return ftag! msgs.info("Sorting completed successfully") return ftag