def check(filename, head=False):
paths = u.find(filename,path+'Spectra')
try:
for f in paths:
print f
wav, flx, err = u.unc(a.read_spec(f, errors=True, atomicron=True, negtonan=True, verbose=False)[0])
plt.plot(wav,flx), plt.fill_between(wav, flx-err, flx+err), plt.title(filename)
if head: return pf.getheader(f)
except KeyError:
print filename
def main(spInput, grav=''):
# 1. LOAD RELEVANT MODULES ---------------------------------------------------------
import astrotools as at
import asciidata
import pyfits
import matplotlib.pyplot as plt
import numpy
import sys
import pdb
# 2. SET UP VARIABLES --------------------------------------------------------------
FOLDER_ROOT = '/Users/alejo/KCData/' # Location of NIR and OPT folders
FOLDER_OUT = 'Output/NOCN/'
OPTNIR_KEYS = ['OPT', 'NIR']
BAND_NAME = ['NIR']
data = ''
dataRaw = ''
specFiles = ''
spectraRaw = ''
spectra = ''
# For TXT objects file (updatable here directly)
FILE_IN = 'nir_spex_prism_with_optical_12aug15.txt' # ASCII file w/ data
HDR_FILE_IN = ('Ref','Designation','J','H','K','SpType','SpType_T','NIRFobs',\
'NIRFtel','NIRfile','OPTobs','OPTtel','OPTinst','OPTfile',\
'Young?','Dusty?','Blue?','Multiple?','Pec?')
colNameRef = HDR_FILE_IN[0]
colNameDesig = HDR_FILE_IN[1]
colNameJ = HDR_FILE_IN[2]
colNameK = HDR_FILE_IN[4]
colNameJK = 'J-K'
colNameType = HDR_FILE_IN[6]
colNameYng = HDR_FILE_IN[14]
colNameDust = HDR_FILE_IN[15]
colNameBlue = HDR_FILE_IN[16]
colNamePec = HDR_FILE_IN[18]
# For TXT exclude-objects file
EXCL_FILE = 'Exclude_Objs.txt' # ASCII file w/ U#s of objects to exclude
# 3. READ DATA FROM INPUT FILES ----------------------------------------------------
NULL_CHAR = '' # Null character
DELL_CHAR = '\t' # Delimiter character
COMM_CHAR = '#' # Comment character
# File with objects (query in Access)
dataRaw = asciidata.open(FOLDER_ROOT + FILE_IN, NULL_CHAR, DELL_CHAR, COMM_CHAR)
# Store data in a dictionary-type object
data = {}.fromkeys(HDR_FILE_IN)
for colIdx,colData in enumerate(dataRaw):
data[HDR_FILE_IN[colIdx]] = colData.tonumpy()
# 4. FORMAT SOME ASCII COLUMNS -----------------------------------------------------
# 4.1 Convert into unicode the Spectral Type-Text column
uniSpType = [None] * len(data[colNameType])
for sIdx,sType in enumerate(data[colNameType]):
uniSpType[sIdx] = sType.decode('utf-8')
data[colNameType] = numpy.array(uniSpType)
# 4.2 Calculate J-K Color And Add J-K Column
data[colNameJK] = data[colNameJ] - data[colNameK]
# 4.3 Format Designation Number from Designation Column
for desigIdx,desig in enumerate(data[colNameDesig]):
desig = ''.join(desig.split())
signType = '+'
signPos = desig.find(signType)
if signPos == -1:
signType = '-'
signPos = desig.find(signType)
desigProper = desig[:4] + signType + desig[signPos+1:signPos+5]
data[colNameDesig][desigIdx] = desigProper
# 5. FILTER DATA BY USER INPUT IN spInput ------------------------------------------
# Find all spectra of same spectral type
specIdx = []
for spIdx,spType in enumerate(data[colNameType]):
if spType.upper().startswith(spInput.upper()):
specIdx.append(spIdx)
if not specIdx:
print 'No target found for given input.'
return
spTypeInput = spInput.upper()
# Sort relevant objects by JKmag value
specIdx = numpy.array(specIdx)
specSortIdx = data[colNameJK][specIdx].argsort()
# 6. READ SPECTRAL DATA FROM SPECTRAL FILES ----------------------------------------
spectraRaw = {}.fromkeys(OPTNIR_KEYS) # Used to store the raw data from fits files
specFilesDict = {}.fromkeys(OPTNIR_KEYS) # Used for reference purposes
for key in OPTNIR_KEYS:
specFiles = [None] * len(specSortIdx)
for sortIdx,specSort in enumerate(specSortIdx):
tmpFullName = FOLDER_ROOT + key + '/' + data[key + 'file'][specIdx[specSort]]
specFiles[sortIdx] = tmpFullName
specFilesDict[key] = specFiles
spectraRaw[key] = at.read_spec(specFiles, atomicron=True, negtonan=True, \
errors=True, verbose=False)
# Clear out spectral data for objects missing either OPT or NIR data
allNone = True
for spIdx in range(0,len(spectraRaw['OPT'])):
if spectraRaw['OPT'][spIdx] is None:
spectraRaw['NIR'][spIdx] = None
elif spectraRaw['NIR'][spIdx] is None:
spectraRaw['OPT'][spIdx] = None
else:
allNone = False
if allNone:
print 'No spectral data found for objects of the given spectral type.'
return
# Convert spectraRaw contents into lists if only one spectral data
for key in spectraRaw.keys():
if spectraRaw[key][0] is not None:
if len(spectraRaw[key][0]) > 3:
spectraRaw[key] = [spectraRaw[key],]
# 7. GATHER OBJECTS' NAMES----------------------------------------------------------
# Filtered objects
refs = [None] * len(specSortIdx)
for idx,spIdx in enumerate(specSortIdx):
tmpRef = data[colNameRef][specIdx[spIdx]]
refs[idx] = str(int(tmpRef))
#8. SMOOTH SPECTRA -----------------------------------------------------------------
# Smooth the flux data to a reasonable resolution
spectraS = at.smooth_spec(spectraRaw['NIR'], specFile=specFilesDict['NIR'], \
winWidth=0)
# 9. SET LIMITS FOR BAND AND NORMALIZING SECTION------------------------------------
# Initialize dictionary to store limits
BAND_LIMS = {}.fromkeys(BAND_NAME)
for bandKey in BAND_NAME:
BAND_LIMS[bandKey] = dict(lim = [None] * 2, limN = [None] * 2)
# Set wl limits for band
# Limits are in microns
BAND_LIMS['NIR']['lim'][0] = 0.8
BAND_LIMS['NIR']['lim'][1] = 2.4
# Set wl limits for normalizing sections; this is the peak of the J band
# Limits are in microns
BAND_LIMS['NIR']['limN'][0] = 1.28
BAND_LIMS['NIR']['limN'][1] = 1.32
# 10. SELECT SPECTRAL DATA FOR NIR BAND---------------------------------------------
# Initialize variables
spectraN = {}.fromkeys(BAND_NAME)
# Gather reference numbers of objects
objRef = data[colNameRef][specIdx[specSortIdx]]
# Select band
spectra = at.sel_band(spectraS, BAND_LIMS['NIR']['lim'], objRef)
# Normalize band
spectraN['NIR'] = at.norm_spec(spectra, BAND_LIMS['NIR']['limN'])
# 11. CHARACTERIZE TARGETS (i.e. identify young, blue, to exclude...)---------------
# Determine which targets to exclude using the "Exclude_Objs" file
toExclude = [False] * len(refs)
dataExcl = asciidata.open(FOLDER_ROOT + EXCL_FILE, NULL_CHAR, DELL_CHAR, COMM_CHAR)
if len(dataExcl[0]) > 0:
# Extract data from "Exclude_Objs" file
excludeObjs = [None] * len(dataExcl[0])
for rowIdx, rowData in enumerate(dataExcl[0]):
excludeObjs[rowIdx] = str(rowData)
# Find intersection of exclude-obj list and filtered targets list
setExclude = set(excludeObjs).intersection(set(refs))
# Create list with intersection targets
if len(setExclude) != 0:
for exclIdx in setExclude:
tmpExclIdx = numpy.where(numpy.array(refs) == exclIdx)
toExclude[tmpExclIdx[0]] = True
# Determine which targets are blue
blueObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBlue][spIdx].upper() == 'YES':
blueObjs[idx] = True
# Determine which targets are dusty
dustyObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameDust][spIdx].upper() == 'YES':
dustyObjs[idx] = True
# Determine which targets are peculiar
pecObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNamePec][spIdx].upper() == 'YES':
pecObjs[idx] = True
# Determine which plots are young objects
youngObjs = [False] * len(refs)
for idx,spIdx in enumerate(specSortIdx):
if data[colNameYng][specIdx[spIdx]].upper() == 'YES':
youngObjs[idx] = True
# Determine which targets are GAMMA
gammaObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb3"
if utcA == '\xce' and utcB == '\xb3':
gammaObjs[idx] = True
# Determine which targets are BETA
betaObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb2"
if utcA == '\xce' and utcB == '\xb2':
betaObjs[idx] = True
# Determine which targets to include in plots (based on user input)
# Consolidate plotting instructions
grav = grav.upper()
plotInstructions = ['exclude'] * len(refs)
if grav == 'Y': # If plot request is Young, include gamma, beta & young targets
for plotIdx in range(len(refs)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx] or betaObjs[plotIdx] or youngObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'G': # If plot request is Gamma, include only gamma targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'B': # If plot request is Beta, include only beta targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'F': # If plot request is Field, include Field & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx] or gammaObjs[plotIdx] or youngObjs[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'field'
else: # Otherwise, print Field, gamma, beta, young & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
if youngObjs[plotIdx]:
plotInstructions[plotIdx] = 'young'
else:
plotInstructions[plotIdx] = 'field'
# If all plot instructions are "exclude", then stop procedure
allExcl = True
for instr in plotInstructions:
if instr != 'exclude':
allExcl = False
if allExcl:
if not uniqueSpec:
print 'No spectral data to plot based on your request.'
return
# 12. PLOT DATA --------------------------------------------------------------------
# Gather info on each object (for legend purposes)
objInfo = [None] * len(refs)
for posIdx,spIdx in enumerate(specIdx[specSortIdx]):
tmpDesig = data[colNameDesig][spIdx]
tmpJK = data[colNameJK][spIdx]
tmpSPtype = data[colNameType][spIdx]
tmpSPtype = tmpSPtype + ' ' * (5 - len(tmpSPtype)) # For alignment purposes
objInfo[posIdx] = (tmpDesig + ' ' + tmpSPtype + ' ' + '%.2f' %tmpJK)
# Create Figure with Subplots
figObj = plotspec(spectraN, BAND_NAME, BAND_LIMS, objInfo, spTypeInput, grav, \
plotInstructions)
figObj.savefig(FOLDER_ROOT + FOLDER_OUT + spTypeInput + grav + '_fan.pdf', \
dpi=800)
spTypes = []
spectra = {}.fromkeys(BANDS)
for band in BANDS:
spectra[band] = []
for idxTp, spTp in enumerate(SPTYPES):
if not(spTp in SPSTDNM): continue
# Fetch and normalize special standard (L2, L5 and L7 only)
if spTp in SPSTDNM:
if spTp == 'L2':
isp = 0
elif spTp == 'L5':
isp = 1
else:
isp = 2
tmpspec = at.read_spec('../more data/NIR/' + SPSTD[isp], errors=False, \
atomicron=True, negtonan=True)
for band in BANDS:
tmpband = at.sel_band(tmpspec, BAND_LIMS[band]['lim'])
if idxTp in [1,2,3] and band == 'H':
norm_lims = SPECIAL_H_NORM_LIM
else:
norm_lims = NORM_LIMS[band]['lim']
stdToPlot = at.norm_spec(tmpband, norm_lims)[0]
spectra[band].append(stdToPlot)
spTypes.append(spTp)
# # Fetch standard
# tmpStd = nocs.main(spTp, GRAV, plot=False, std=True, normalize=False)
# # Normalize standard
# for bdIdx, band in enumerate(BANDS):
# if idxTp in [1,2,3] and band == 'H':
def main(spInput,
grav='',
plot=True,
templ=False,
std=False,
lbl=False,
normalize=True):
# 1. LOAD RELEVANT MODULES ------------------------------------------------
#import asciidata
import astrotools as at
import numpy as np
import sys
import pdb
import matplotlib.pyplot as plt
from astropy.io import ascii
# 2. SET UP VARIABLES -----------------------------------------------------
# Customizable variables <><><><><><><><><><><><><><><><><><><><><><><><><><><>
FOLDER_ROOT = '/Users/alejo/Dropbox/Project_0/more data/' # Location of NIR and OPT folders
FOLDER_IN = '/Users/alejo/Dropbox/Project_0/data/' # Location of input files
FOLDER_OUT = '/Users/alejo/Dropbox/Project_0/plots/' # Location to save output figures
FILE_IN = 'nir_spex_prism_with_optical.txt' # ASCII file w/ data
# <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
# For TXT objects list file
HDR_FILE_IN = ('Ref','Designation`','J','H','K','SpType','SpType_T','NIRFobs',\
'NIRFtel','NIRfile','OPTobs','OPTtel','OPTinst','OPTfile',\
'Young?','Dusty?','Blue?','Binary?','Pec?')
# For TXT standards file
FILE_IN_STD = 'NIR_Standards.txt' # ASCII file w/ standards
HDR_FILE_IN_STD = ('Ref', 'Designation', 'NIR SpType', 'OPT SpType')
colNameNIRS = HDR_FILE_IN_STD[2]
colNameOPTS = HDR_FILE_IN_STD[3]
# For TXT exclude-objects file
EXCL_FILE = 'Exclude_Objs.txt' # ASCII file w/ unums of objects to exclude
OPTNIR_KEYS = ['OPT', 'NIR']
BANDS_NAMES = ['K', 'H', 'J', 'OPT']
data = ''
dataRaw = ''
specFiles = ''
spectraRaw = ''
spectra = ''
colNameRef = HDR_FILE_IN[0]
colNameDesig = HDR_FILE_IN[1]
colNameJ = HDR_FILE_IN[2]
colNameK = HDR_FILE_IN[4]
colNameJK = 'J-K'
colNameType = HDR_FILE_IN[6]
colNameNIRfile = HDR_FILE_IN[9]
colNameYng = HDR_FILE_IN[14]
colNameDust = HDR_FILE_IN[15]
colNameBlue = HDR_FILE_IN[16]
colNameBin = HDR_FILE_IN[17]
colNamePec = HDR_FILE_IN[18]
# Initialize dictionary to store NIR bands limits and normalizing sections
BAND_LIMS = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
BAND_LIMS[bandKey] = dict(lim=[None] * 2, limN=[None] * 2)
# Set wavelength limits for bands
# Limits are in microns
BAND_LIMS['OPT']['lim'][0] = 0.65
BAND_LIMS['OPT']['lim'][1] = 0.90
BAND_LIMS['J']['lim'][0] = 0.8
BAND_LIMS['J']['lim'][1] = 1.4
BAND_LIMS['H']['lim'][0] = 1.4
BAND_LIMS['H']['lim'][1] = 1.9
BAND_LIMS['K']['lim'][0] = 1.9
BAND_LIMS['K']['lim'][1] = 2.4
# Set wl limits for normalizing sections
# Limits are in microns
BAND_LIMS['OPT']['limN'][0] = 0.66
BAND_LIMS['OPT']['limN'][1] = 0.89
BAND_LIMS['J']['limN'][0] = 0.87
BAND_LIMS['J']['limN'][1] = 1.39
BAND_LIMS['H']['limN'][0] = 1.41
BAND_LIMS['H']['limN'][1] = 1.89
BAND_LIMS['K']['limN'][0] = 1.91
BAND_LIMS['K']['limN'][1] = 2.39
# 3. READ DATA FROM INPUT FILES -------------------------------------------
DELL_CHAR = '\t' # Delimiter character
COMM_CHAR = '#' # Comment character
# File with objects (source: query in Access)
dataRaw = ascii.read(FOLDER_IN + FILE_IN, format='no_header', \
delimiter=DELL_CHAR, comment=COMM_CHAR, data_start=1)
# Store data in a dictionary-type object
data = {}.fromkeys(HDR_FILE_IN)
for colIdx, colname in enumerate(dataRaw.colnames):
data[HDR_FILE_IN[colIdx]] = np.array(dataRaw[colname])
# File with standards (source: manually generated)
dataRawS = ascii.read(FOLDER_IN + FILE_IN_STD, data_start=0)
# Store standard data in a dictionary-type object
dataS = {}.fromkeys(HDR_FILE_IN_STD)
for colIdx, colname in enumerate(dataRawS.colnames):
dataS[HDR_FILE_IN_STD[colIdx]] = np.array(dataRawS[colname])
# 4. FORMAT SOME ASCII COLUMNS --------------------------------------------
# 4.1 Convert into unicode the Spectral Type-Text column
uniSpType = [None] * len(data[colNameType])
for sIdx, sType in enumerate(data[colNameType]):
uniSpType[sIdx] = sType #.decode('utf-8')
data[colNameType] = np.array(uniSpType)
# 4.2 Calculate J-K Color
data[colNameJK] = data[colNameJ] - data[colNameK]
# 4.3 Format Designation Number in Designation Column
# (From "XX XX XX.X +XX XX XX.X" to "XXXX+XXXX")
for desigIdx, desig in enumerate(data[colNameDesig]):
desig = ''.join(desig.split())
signType = '+'
signPos = desig.find(signType)
if signPos == -1:
signType = '-'
signPos = desig.find(signType)
desigProper = desig[:4] + signType + desig[signPos + 1:signPos + 5]
data[colNameDesig][desigIdx] = desigProper
# 5. FILTER DATA BY USER INPUT IN spInput ---------------------------------
uniqueSpec = False
specIdx = []
if spInput.upper().startswith('L'):
# If input is a spectral type, then find all spectra of same spectral type
for spIdx, spType in enumerate(data[colNameType]):
if spType.upper().startswith(spInput.upper()):
specIdx.append(spIdx)
if not specIdx:
print('No targets found for given input.')
if std is False:
return
spTypeInput = spInput.upper()
else:
# If input is one single spectrum, then find it
for spIdx, spType in enumerate(data[colNameRef]):
if str(spType) == spInput.upper():
specIdx.append(spIdx)
if not specIdx:
print('Requested target not found.')
if std is False:
return
else:
spTypeInput = data[colNameType][specIdx[0]][0:2]
uniqueSpec = True
# Find NIR standard target that matches user's spectral type
stdIdx = []
for spIdx, spType in enumerate(dataS[colNameNIRS]):
if spType.upper().startswith(spTypeInput):
stdIdx.append(spIdx)
# Add NIR standard target to list of filtered objects if not there already
# (It may not be included in first filter because OPT SpT != NIR SpT)
if not uniqueSpec:
if dataS[colNameNIRS][stdIdx] != dataS[colNameOPTS][stdIdx]:
for spIdx, spRef in enumerate(data[colNameRef]):
if spRef == dataS[colNameRef][stdIdx][0]:
if spIdx not in specIdx:
specIdx.append(spIdx)
# Sort relevant objects by JKmag value
specIdx = np.array(specIdx)
specSortIdx = data[colNameJK][specIdx].argsort()
# 6. READ SPECTRAL DATA FROM SPECTRAL FILES -------------------------------
spectraRaw = {}.fromkeys(
OPTNIR_KEYS) # Used to store the raw data from fits files
specFilesDict = {}.fromkeys(OPTNIR_KEYS) # Used for reference purposes
for key in OPTNIR_KEYS:
specFiles = [None] * len(specSortIdx)
for sortIdx, specSort in enumerate(specSortIdx):
if data[key + 'file'][specIdx[specSort]][-4:] == '.dat': continue
if data[key + 'file'][specIdx[specSort]] == 'include': continue
tmpFullName = FOLDER_ROOT + key + '/' + data[key \
+ 'file'][specIdx[specSort]]
specFiles[sortIdx] = tmpFullName
specFilesDict[key] = specFiles
spectraRaw[key] = at.read_spec(specFiles, atomicron=True, negtonan=True, \
errors=True, verbose=False)
# Clear out spectral data for objects missing either OPT or NIR data
allNone = True
for spIdx in range(0, len(spectraRaw['OPT'])):
if spectraRaw['OPT'][spIdx] is None:
spectraRaw['NIR'][spIdx] = None
elif spectraRaw['NIR'][spIdx] is None:
spectraRaw['OPT'][spIdx] = None
else:
allNone = False
if allNone:
print('No spectral data found for objects of the given spectral type.')
if std is False:
return
# Convert spectraRaw contents into lists if only one spectral data
# (This reduces the dimensions of the object holding the data)
for key in spectraRaw.keys():
if spectraRaw[key][0] is not None:
if len(spectraRaw[key][0]) > 3:
spectraRaw[key] = [
spectraRaw[key],
]
# 7. GATHER OBJECTS' NAMES ------------------------------------------------
# Filtered objects
refs = [None] * len(specSortIdx)
for idx, spIdx in enumerate(specSortIdx):
tmpRef = data[colNameRef][specIdx[spIdx]]
refs[idx] = str(tmpRef)
# Standard objects
refsStd = [None] * len(dataS[colNameRef])
for idx, spIdx in enumerate(dataS[colNameRef]):
tmpRef = dataS[colNameRef][idx]
refsStd[idx] = str(tmpRef)
# Gather reference numbers of objects
objRef = data[colNameRef][specIdx[specSortIdx]]
#8. SMOOTH SPECTRA --------------------------------------------------------
# Smooth the flux data to a reasonable resolution
spectraS = {}.fromkeys(OPTNIR_KEYS)
tmpSpOPT = at.smooth_spec(spectraRaw['OPT'], specFile=specFilesDict['OPT'], \
winWidth=10)
tmpSpNIR = at.smooth_spec(spectraRaw['NIR'], specFile=specFilesDict['NIR'], \
winWidth=0)
spectraS['OPT'] = tmpSpOPT
spectraS['NIR'] = tmpSpNIR
# 9. SELECT SPECTRAL DATA FOR THE DIFFERENT BANDS -------------------------
# Initialize variables
spectra = {}.fromkeys(BANDS_NAMES)
spectraN = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
if bandKey == 'OPT':
optNIR = 'OPT'
else:
optNIR = 'NIR'
# Select band
spectra[bandKey] = at.sel_band(spectraS[optNIR], BAND_LIMS[bandKey]['lim'], \
objRef)
if spectra[bandKey] is None:
break
# Normalize band
spectraN[bandKey], flagN = at.norm_spec(spectra[bandKey], \
BAND_LIMS[bandKey]['limN'], flag=True)
if flagN:
print('LIMITS for normalization changed!')
if spectraN[bandKey] is None:
break
# 10. CHARACTERIZE TARGETS (i.e. identify young, blue, to exclude...) -----
# Determine which targets to exclude
# (source: file manually generated)
toExclude = [False] * len(refs) # FORCE TO INCLUDE ALL TARGETS
# dataExcl = ascii.read(FOLDER_IN + EXCL_FILE, data_start=0, delimiter=DELL_CHAR, \
# comment=COMM_CHAR, names=['ID'])
# if len(dataExcl['ID']) > 0:
# # Extract data from "Exclude_Objs" file
# excludeObjs = np.array(dataExcl['ID'], dtype='string')
#
# # Find intersection of exclude-obj list and filtered targets list
# setExclude = set(excludeObjs).intersection(set(refs))
#
# # Create list with intersection targets
# if len(setExclude) != 0:
# for exclIdx in setExclude:
# tmpExclIdx = np.where(np.array(refs) == exclIdx)
# toExclude[tmpExclIdx[0]] = True
# Determine which target is the NIR Standard object
O_standard = [None] * 3 # Holds standard for output
stdObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameRef][spIdx] == dataS[colNameRef][stdIdx]:
stdObjs[idx] = True
if normalize:
O_standard[0] = spectraN['J'][idx]
O_standard[1] = spectraN['H'][idx]
O_standard[2] = spectraN['K'][idx]
else:
O_standard[0] = spectra['J'][idx]
O_standard[1] = spectra['H'][idx]
O_standard[2] = spectra['K'][idx]
# Determine which targets are blue
blueObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBlue][spIdx].upper() == 'YES':
blueObjs[idx] = True
# Determine which targets are dusty
dustyObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameDust][spIdx].upper() == 'YES':
dustyObjs[idx] = True
# Determine which targets are binary
binObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBin][spIdx].upper() == 'YES':
binObjs[idx] = True
# Determine which targets are peculiar
pecObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNamePec][spIdx].upper() == 'YES':
pecObjs[idx] = True
# Determine which targets are young
youngObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameYng][spIdx].upper() == 'YES':
youngObjs[idx] = True
# Determine which targets are GAMMA
gammaObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb3"
if utcA == '\xce' and utcB == '\xb3':
gammaObjs[idx] = True
# Determine which targets are BETA
betaObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb2"
if utcA == '\xce' and utcB == '\xb2':
betaObjs[idx] = True
# Determine which targets to include in plots (based on user input)
# Consolidate plotting & template-flux instructions
grav = grav.upper()
plotInstructions = ['exclude'] * len(refs)
templInstructions = [False] * len(refs)
if grav == 'Y': # If plot request is Young, include gamma, beta & young targets
for plotIdx in range(len(refs)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx] or betaObjs[plotIdx] or youngObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] \
or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
templInstructions[plotIdx] = True
elif grav == 'G': # If plot request is Gamma, include only gamma targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] \
or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
templInstructions[plotIdx] = True
elif grav == 'B': # If plot request is Beta, include only beta targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] \
or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
templInstructions[plotIdx] = True
elif grav == 'F': # If plot request is Field, include Field & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx] or gammaObjs[plotIdx] or youngObjs[plotIdx]:
continue
#if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] \
# or binObjs[plotIdx]:
# continue
if stdObjs[plotIdx]:
plotInstructions[plotIdx] = 'standard'
else:
plotInstructions[plotIdx] = 'field'
templInstructions[plotIdx] = True
else: # Otherwise, print Field, gamma, beta, young & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] \
or binObjs[plotIdx]:
continue
if youngObjs[plotIdx]:
plotInstructions[plotIdx] = 'young'
elif stdObjs[plotIdx]:
plotInstructions[plotIdx] = 'standard'
else:
plotInstructions[plotIdx] = 'field'
templInstructions[plotIdx] = True
# If all plot instructions are "exclude", then stop procedure (for spectral types)
allExcl = True
for instr in plotInstructions:
if instr != 'exclude':
allExcl = False
if allExcl:
if std:
return O_standard
if not uniqueSpec:
print('No spectral data to plot based on your request.')
return
# 11. CALCULATE TEMPLATE SPECTRA FOR SELECTED SET OF SPECTRA -----------------------
# Gather spectra to use to calculate template spectrum
# if not allExcl:
# O_template = [None] * 3 # Holds calculated template for output
# templCalculated = False
# for bandIdx, bandKey in enumerate(BANDS_NAMES):
# if bandKey == 'OPT':
# continue
#
# templSpecs = []
# for spIdx, spex in enumerate(spectraN[bandKey]):
# if templInstructions[spIdx]:
# # Check that spectrum exists
# if spex is None:
# templInstructions[spIdx] = False
# continue
#
# if bandKey == 'OPT':
# templSpecs.append(spex)
# else:
# # Check that spectrum comes with error values (NIR bands only)
# notNansBool = np.isfinite(spex[2])
# notNans = np.any(notNansBool)
# if notNans:
# templSpecs.append(spex)
# else:
# print(str(objRef[spIdx]) + ' excluded from template')
#
# # Calculate template spectrum
# if len(templSpecs) > 1:
# template = at.mean_comb(templSpecs)
# templCalculated = True
#
# # Append template to list of spectra to plot in the next step
# spectraN[bandKey].append(template)
# # Append template to output object
# if bandIdx == 0:
# tempIdx = 2
# elif bandIdx == 2:
# tempIdx = 0
# else:
# tempIdx = 1
# O_template[tempIdx] = template
#
# if templCalculated:
# refs.append('template')
# plotInstructions.append('template')
# else:
# O_template = None
# 12. PLOT DATA -----------------------------------------------------------
if lbl or plot:
# Gather info on each target
objInfo = [None] * len(refs)
for posIdx, spIdx in enumerate(specIdx[specSortIdx]):
tmpDesig = data[colNameDesig][spIdx]
tmpJK = data[colNameJK][spIdx]
tmpSPtype = data[colNameType][spIdx]
tmpSPtype = tmpSPtype + ' ' * (5 - len(tmpSPtype)
) # For alignment purposes
objInfo[posIdx] = (tmpDesig + ' ' + tmpSPtype + ' ' +
'%.2f' % tmpJK)
if objInfo[-1] is None:
objInfo[-1] = 'template'
if plot:
# Create Figure with Subplots and Annotations
tmpspectraN = {key: spectraN[key] for key in ['J', 'H', 'K']}
tmpBANDS_NAMES = BANDS_NAMES[:-1]
tmpBAND_LIMS = {key: BAND_LIMS[key] for key in ['J', 'H', 'K']}
figObj = plotspec(tmpspectraN, tmpBANDS_NAMES, tmpBAND_LIMS, objInfo, \
spTypeInput, grav, plotInstructions)
figObj.savefig(FOLDER_OUT + spTypeInput + grav + '.pdf', dpi=600)
# 13. DETERMINE OUTPUT ----------------------------------------------------
if templ:
if std:
return O_template, O_standard
else:
return O_template
elif std:
return O_standard
else:
if lbl:
return spectraN, objInfo
else:
return spectraN
def add_fits(self, fitsPath, source_id, unc_fitsPath='', wavelength_units='', flux_units='', publication_id='', obs_date='', wavelength_order='', regime='', instrument_id='', telescope_id='', mode_id='', airmass=0, comment='', wlog=False, SDSS=False):
'''
Checks the header of the **fitsFile** and inserts the data with **source_id**.
'''
filename, header = os.path.basename(fitsPath), pf.getheader(fitsPath)
# x- and y-units
if not wavelength_units:
try:
wavelength_units = header['XUNITS']
if 'microns' in wavelength_units or 'Microns' in wavelength_units or 'um' in wavelength_units: wavelength_units = 'um'
except KeyError:
try:
if header['BUNIT']: wavelength_units = 'um'
except KeyError: wavelength_units = ''
if not flux_units:
try: flux_units = header['YUNITS'].replace(' ','')
except KeyError:
try: flux_units = header['BUNIT'].replace(' ','')
except KeyError: flux_units = ''
if 'erg' in flux_units and 'A' in flux_units: flux_units = 'ergs-1cm-2A-1' if 'erg' in flux_units and 'A' in flux_units else 'ergs-1cm-2um-1' if 'erg' in flux_units and 'um' in flux_units else 'Wm-2um-1' if 'W' in flux_units and 'um' in flux_units else 'Wm-2A-1' if 'W' in flux_units and 'A' in flux_units else ''
# Date, object name, telescope and instrument
if not obs_date:
try: obs_date = header['DATE_OBS']
except KeyError:
try: obs_date = header['DATE-OBS']
except KeyError:
try: obs_date = header['DATE']
except KeyError: obs_date = ''
if not telescope_id:
try:
n = header['TELESCOP'].lower() if isinstance(header['TELESCOP'],str) else ''
telescope_id = 5 if 'hst' in n else 6 if 'spitzer' in n else 7 if 'irtf' in n else 9 if 'keck' in n and 'ii' in n else 8 if 'keck' in n and 'i' in n else 10 if 'kp' in n and '4' in n else 11 if 'kp' in n and '2' in n else 12 if 'bok' in n else 13 if 'mmt' in n else 14 if 'ctio' in n and '1' in n else 15 if 'ctio' in n and '4' in n else 16 if 'gemini' in n and 'north' in n else 17 if 'gemini' in n and 'south' in n else 18 if 'vlt' in n else 19 if '3.5m' in n else 20 if 'subaru' in n else 21 if ('mag' in n and 'ii' in n) or ('clay' in n) else 22 if ('mag' in n and 'i' in n) or ('baade' in n) else None
except KeyError: telescope_id = ''
if not instrument_id:
try:
i = header['INSTRUME'].lower()
instrument_id = 1 if 'r-c spec' in i or 'test' in i or 'nod' in i else 2 if 'gmos-n' in i else 3 if 'gmos-s' in i else 4 if 'fors' in i else 5 if 'lris' in i else 6 if 'spex' in i else 7 if 'ldss3' in i else 8 if 'focas' in i else 9 if 'nirspec' in i else 0
except KeyError: instrument_id = ''
try: airmass = header['AIRMASS']
except: airmass = 0
try:
if SDSS:
data = pf.open(fitsPath, memmap=True)[1].data
flx, wav, err = map(np.array,zip(*data)[:3])
flx, wav, err, wavelength_units, flux_units = flx*10**-17, 10**wav, np.sqrt(1/err)*10**-17, 'A', 'ergs-1cm-2A-1'
else:
data = a.read_spec(fitsPath, errors=True, atomicron=True, negtonan=True, verbose=False, wlog=wlog)[0]
wav, flx = data[:2]
try: err = a.read_spec(unc_fitsPath, errors=True, atomicron=True, negtonan=True, verbose=False, wlog=wlog)[0][1] if unc_fitsPath else data[2]
except: err = ''
try: snr = flx/err if any(flx) and any(err) else None
except (TypeError,IndexError): snr = None
if not regime:
if wav[0]<500 or wavelength_units=='um': regime = 'OPT' if wav[0]<0.8 and wav[-1]<1.2 else 'NIR' if wav[0]<1.2 and wav[-1]>2 else 'MIR' if wav[-1]>2.5 else None
else: regime = 'OPT' if wav[0]<8000 and wav[-1]<12000 else 'NIR' if wav[0]<12000 and wav[-1]>20000 else 'MIR' if wav[-1]>25000 else None
spec_id = sorted(list(set(range(1,self.query.execute("SELECT max(id) FROM spectra").fetchone()[0]+2))-set(zip(*self.query.execute("SELECT id FROM spectra").fetchall())[0])))[0]
try: self.query.execute("INSERT INTO spectra VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (spec_id, source_id, wav, wavelength_units, flx, flux_units, err, snr, wavelength_order, regime, publication_id, obs_date, instrument_id, telescope_id, mode_id, airmass, filename, comment, header)), self.modify.commit()
except: self.query.execute("INSERT INTO spectra VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (spec_id, source_id, wav, wavelength_units, flx, flux_units, err, snr, wavelength_order, regime, publication_id, obs_date, instrument_id, telescope_id, mode_id, airmass, filename, comment, None)), self.modify.commit()
u.printer(['spec_id','source_id','wavelength_unit','flux_unit','regime','publication_id','obs_date', 'instrument_id', 'telescope_id', 'mode_id', 'airmass', 'filename', 'comment'],[[spec_id,source_id, wavelength_units, flux_units, regime, publication_id, obs_date, instrument_id, telescope_id, mode_id, airmass, filename, comment]], empties=True)
# self.clean_up('spectra')
except KeyError: print "Couldn't add fits file {}".format(fitsPath); print [filename, source_id, wavelength_units, flux_units, obs_date, instrument_id, telescope_id, mode_id, airmass, comment]
def add_data(self, overwrite=False):
"""
Add spectral and/or photometry data for a target. If target does not exist in database, it will create a Target instance automatically using the *add_target* method. The data to upload is read form the *upload.txt* file located in the same folder as the database file in your computer. Read *upload.txt* header for more information.
*overwrite*
Boolean, whether to overwrite existing data or not.
"""
# 1. Initialize variables ---------------------------------------------
DB_FILE = 'BDNYCData.txt'
UP_FILE = 'upload.txt' # ascii file with upload data
UP_HEADERS = ('unum','name','ra', 'dec', 'sptype', 'standard', \
'rng', 'res', 'instr', 'date', 'ord_filt', 'fitsname', \
'survey', 'band_1', 'val_1', 'err_1', \
'band_2', 'val_2', 'err_2', \
'band_3', 'val_3', 'err_3')
NULL = '' # Null character in ascii file
DEL = '\t' # Delimiter character in ascii file
COMM = '#' # Comment character in ascii file
NUM_BANDS = 3
colNmUnum = UP_HEADERS[0]
colNmName = UP_HEADERS[1]
colNmRa = UP_HEADERS[2]
colNmDec = UP_HEADERS[3]
colNmSptype = UP_HEADERS[4]
colNmStd = UP_HEADERS[5]
colNmRng = UP_HEADERS[6]
colNmRes = UP_HEADERS[7]
colNmInstr = UP_HEADERS[8]
colNmDate = UP_HEADERS[9]
colNmOrdfilt = UP_HEADERS[10]
colNmFits = UP_HEADERS[11]
colNmSurvey = UP_HEADERS[12]
colNmBand1 = UP_HEADERS[13]
colNmVal1 = UP_HEADERS[14]
colNmErr1 = UP_HEADERS[15]
colNmBand2 = UP_HEADERS[16]
colNmVal2 = UP_HEADERS[17]
colNmErr2 = UP_HEADERS[18]
colNmBand3 = UP_HEADERS[19]
colNmVal3 = UP_HEADERS[20]
colNmErr3 = UP_HEADERS[21]
# 2. Load ascii file --------------------------------------------------
dataRaw = ad.open(UP_FILE, null=NULL, delimiter=DEL, comment_char=COMM)
# Store ascii data in a dictionary-type object
data = {}.fromkeys(UP_HEADERS)
for colIdx, colData in enumerate(dataRaw):
data[UP_HEADERS[colIdx]] = colData.tonumpy()
if data[colNmUnum] is None:
print 'Upload file empty.'
return
# 3. Upload data to database ------------------------------------------
somethingAdded = False
for row in range(len(data[colNmUnum])):
# 3.1 Check if target already exists in database
unum = data[colNmUnum][row].upper()
try:
unum + 0
print 'U-number invalid.'
continue
except TypeError:
if len(unum) != 6:
print 'U-number invalid.'
continue
if unum[0] != 'U':
print 'U-number invalid.'
continue
dbIdx = self.match_unum(unum)
if dbIdx is None:
newTgt = True
else:
newTgt = False
# 3.2 Get target attributes
if newTgt:
name = data[colNmName][row]
if name == '':
name = None
sptype = data[colNmSptype][row].capitalize()
if sptype == '':
sptype = 'XX'
ra = data[colNmRa][row]
if len(ra) < 8 or ra == '':
print unum + ' Right ascension invalid.'
continue
dec = data[colNmDec][row]
if len(dec) < 9:
print unum + ' Declination invalid.'
continue
std = data[colNmStd][row].capitalize()
if not (std == 'Yes' or std == 'No'):
print unum + ' Standard column must be Yes or No.'
continue
# 3.3 Get range of data
rng = data[colNmRng][row].lower()
if not (rng == 'opt' or rng == 'nir' or rng == 'mir'):
print unum + ' Range invalid.'
continue
# 3.4 Get spectrum data
# 3.4.1 Open & read fits file
specAdd = False
fitsName = data[colNmFits][row]
if fitsName != '':
fitsRaw = at.read_spec(fitsName, errors=True)
if fitsRaw[0] is not None:
wl = fitsRaw[0][0]
flux = fitsRaw[0][1]
# 3.4.2 Determine if 3rd dimension is uncertainty or snr
errNm = None
if len(fitsRaw[0]) == 3:
errVals = None
med = np.median(flux / fitsRaw[0][2])
if med < 10**3 and med > 10**-3:
errVals = fitsRaw[0][2]
errNm = 'uncertainty'
else:
errVals = fitsRaw[0][2]
errNm = 'snr'
# 3.4.3 Get spectrum attributes
attsOK = True
res = data[colNmRes][row].lower()
if not (res == 'high' or res == 'med' or res == 'low'):
print unum + ' Resolution invalid.'
attsOK = False
instr = data[colNmInstr][row]
if instr == '':
print unum + ' Must provide Instrument for spectrum.'
attsOK = False
date = data[colNmDate][row].lower()
if len(date) != 9:
print unum + ' Date invalid.'
attsOK = False
if date[-1] == '\r':
date = date[:-1]
ord_filt = data[colNmOrdfilt][row]
if ord_filt == '':
print unum + ' Must provide Order/Filter for spectrum.'
attsOK = False
# 3.4.4 Create dictionary structure with spectrum data
if attsOK:
specAdd = True
specDict = {instr:{date:{ord_filt:{'wl':wl, \
'flux':flux, errNm:errVals}}}}
# 3.5 Get target photometry attributes
# 3.5.1 Check if photometry data was provided
photAdd = False
try:
survey = data[colNmSurvey][row]
except TypeError:
survey = ''
# 3.5.2 Get data for bands
if survey != '':
bands = []
vals = []
errs = []
photOK = True
for bndNum in range(1, NUM_BANDS + 1):
bndNm = 'band_' + str(bndNum)
valNm = 'val_' + str(bndNum)
errNm = 'err_' + str(bndNum)
try:
# Get band name
bands.append(data[bndNm][row])
if bands[bndNum - 1] == '':
photOK = False
else:
try:
bands[bndNum - 1][0]
except IndexError:
photOK = False
# Get band photometry value
try:
vals.append(data[valNm][row])
try:
vals[bndNum - 1] + 0
except TypeError:
if vals[bndNum - 1][-1] == '\r':
vals[bndNum - 1] = vals[bndNum - 1][:-1]
try:
vals[bndNum - 1] + 0
except TypeError:
photOK = False
else:
photOK = False
except TypeError:
photOK = False
# Get band photometry value error
try:
errs.append(data[errNm][row])
if errs[bndNum - 1] == '':
errs[bndNum - 1] = None
else:
try:
errs[bndNum - 1] + 0
except TypeError:
if errs[bndNum - 1][0] == '\r':
errs[bndNum - 1] = None
else:
try:
errs[bndNum - 1] = float(
errs[bndNum - 1])
except ValueError:
photOK = False
except TypeError:
errs.append(None)
except TypeError:
if bndNum == 1:
photOK = False
# 3.5.3 Create dictionary structure with photometry data
if photOK:
photDict = {survey: {}}
for bdIdx, band in enumerate(bands):
if band != '':
photAdd = True
photDict[survey][band] = {'val': vals[bdIdx], \
'err': errs[bdIdx]}
else:
print unum + ' Photometry data invalid.'
# 3.6 Create range-level dictionary with all data
if photAdd and specAdd:
rngDict = {res: specDict, 'phot': photDict}
elif photAdd and not specAdd:
rngDict = {'phot': photDict}
elif not photAdd and specAdd:
rngDict = {res: specDict}
else:
print 'No data to add for ' + unum
continue
# 3.7 Add new target to database if necessary
if newTgt:
# 3.7.1 Create Target instance
if rng == 'opt':
target = Target(name, unum, ra, dec, sptype, \
rngDict, {}, {}, std)
elif rng == 'nir':
target = Target(name, unum, ra, dec, sptype, \
{}, rngDict, {}, std)
elif rng == 'mir':
target = Target(name, unum, ra, dec, sptype, \
{}, {}, rngDict, std)
# 3.7.2 Add to database
self.add_target(target, verbose=False)
somethingAdded = True
print unum + ' new target added to database: Index # ' + \
str(len(self.targets) - 1)
# 3.8 Add new data to database if target already exists
else:
addedPhot = False
addedSpec = False
if rng == 'opt':
currentTgt = self.targets[dbIdx].opt
elif rng == 'nir':
currentTgt = self.targets[dbIdx].nir
elif rng == 'mir':
currentTgt = self.targets[dbIdx].mir
# 3.8.1 Check dictionary level where to add data
if currentTgt != {}:
# For photometry data
if photAdd:
try:
currentTgt['phot'][survey]
for bnd in rngDict['phot'][survey].keys():
try:
currentTgt['phot'][survey][bnd]
# Overwrite existing data if requested
if overwrite:
addedPhot = True
currentTgt['phot'][survey][bnd] = \
rngDict['phot'][survey][bnd]
else:
print unum + ' ' + bnd + \
' photometry already exists' + \
' in database.'
except KeyError:
addedPhot = True
currentTgt['phot'][survey][bnd] = \
rngDict['phot'][survey][bnd]
except KeyError:
addedPhot = True
currentTgt['phot'][survey] = \
rngDict['phot'][survey]
# For spectrum data
if specAdd:
try:
currentTgt[res][instr]
try:
currentTgt[res][instr][date]
try:
currentTgt[res][instr][date][ord_filt]
# Overwrite existing data if requested
if overwrite:
addedSpec = True
currentTgt[res][instr][date][ord_filt] \
= rngDict[res][instr][date][ord_filt]
else:
print unum + ', ' + \
data[colNmFits][row] + \
' spectrum already exists in database.'
except KeyError:
addedSpec = True
currentTgt[res][instr][date][ord_filt] = \
rngDict[res][instr][date][ord_filt]
except KeyError:
addedSpec = True
currentTgt[res][instr][date] = \
rngDict[res][instr][date]
except KeyError:
addedSpec = True
currentTgt[res][instr] = rngDict[res][instr]
else:
addedSpec = True
currentTgt = rngDict
if addedSpec or addedPhot:
somethingAdded = True
print unum + ' new data added to target in database.'
# 4. Commit additions to database file --------------------------------
if somethingAdded:
# Check that database txt file exists in current folder
try:
f = open(DB_FILE, 'rb')
except IOError:
print DB_FILE + ' could not be loaded. Check that it is ' + \
'in the current folder. Process stopped.'
return
f.close()
f = open(DB_FILE, 'wb')
print 'Updating ' + DB_FILE + '...'
pickle.dump(self, f)
f.close()
print 'Remember to push updated ' + DB_FILE + ' to github.'
return
def add_data(self, overwrite=False):
"""
Add spectral and/or photometry data for a target. If target does not exist in database, it will create a Target instance automatically using the *add_target* method. The data to upload is read form the *upload.txt* file located in the same folder as the database file in your computer. Read *upload.txt* header for more information.
*overwrite*
Boolean, whether to overwrite existing data or not.
"""
# 1. Initialize variables ---------------------------------------------
DB_FILE = 'BDNYCData.txt'
UP_FILE = 'upload.txt' # ascii file with upload data
UP_HEADERS = ('unum','name','ra', 'dec', 'sptype', 'standard', \
'rng', 'res', 'instr', 'date', 'ord_filt', 'fitsname', \
'survey', 'band_1', 'val_1', 'err_1', \
'band_2', 'val_2', 'err_2', \
'band_3', 'val_3', 'err_3')
NULL = '' # Null character in ascii file
DEL = '\t' # Delimiter character in ascii file
COMM = '#' # Comment character in ascii file
NUM_BANDS = 3
colNmUnum = UP_HEADERS[0]
colNmName = UP_HEADERS[1]
colNmRa = UP_HEADERS[2]
colNmDec = UP_HEADERS[3]
colNmSptype = UP_HEADERS[4]
colNmStd = UP_HEADERS[5]
colNmRng = UP_HEADERS[6]
colNmRes = UP_HEADERS[7]
colNmInstr = UP_HEADERS[8]
colNmDate = UP_HEADERS[9]
colNmOrdfilt = UP_HEADERS[10]
colNmFits = UP_HEADERS[11]
colNmSurvey = UP_HEADERS[12]
colNmBand1 = UP_HEADERS[13]
colNmVal1 = UP_HEADERS[14]
colNmErr1 = UP_HEADERS[15]
colNmBand2 = UP_HEADERS[16]
colNmVal2 = UP_HEADERS[17]
colNmErr2 = UP_HEADERS[18]
colNmBand3 = UP_HEADERS[19]
colNmVal3 = UP_HEADERS[20]
colNmErr3 = UP_HEADERS[21]
# 2. Load ascii file --------------------------------------------------
dataRaw = ad.open(UP_FILE, null=NULL, delimiter=DEL, comment_char=COMM)
# Store ascii data in a dictionary-type object
data = {}.fromkeys(UP_HEADERS)
for colIdx, colData in enumerate(dataRaw):
data[UP_HEADERS[colIdx]] = colData.tonumpy()
if data[colNmUnum] is None:
print 'Upload file empty.'
return
# 3. Upload data to database ------------------------------------------
somethingAdded = False
for row in range(len(data[colNmUnum])):
# 3.1 Check if target already exists in database
unum = data[colNmUnum][row].upper()
try:
unum + 0
print 'U-number invalid.'
continue
except TypeError:
if len(unum) != 6:
print 'U-number invalid.'
continue
if unum[0] != 'U':
print 'U-number invalid.'
continue
dbIdx = self.match_unum(unum)
if dbIdx is None:
newTgt = True
else:
newTgt = False
# 3.2 Get target attributes
if newTgt:
name = data[colNmName][row]
if name == '':
name = None
sptype = data[colNmSptype][row].capitalize()
if sptype == '':
sptype = 'XX'
ra = data[colNmRa][row]
if len(ra) < 8 or ra == '':
print unum + ' Right ascension invalid.'
continue
dec = data[colNmDec][row]
if len(dec) < 9:
print unum + ' Declination invalid.'
continue
std = data[colNmStd][row].capitalize()
if not (std == 'Yes' or std == 'No'):
print unum + ' Standard column must be Yes or No.'
continue
# 3.3 Get range of data
rng = data[colNmRng][row].lower()
if not (rng == 'opt' or rng == 'nir' or rng == 'mir'):
print unum + ' Range invalid.'
continue
# 3.4 Get spectrum data
# 3.4.1 Open & read fits file
specAdd = False
fitsName = data[colNmFits][row]
if fitsName != '':
fitsRaw = at.read_spec(fitsName, errors=True)
if fitsRaw[0] is not None:
wl = fitsRaw[0][0]
flux = fitsRaw[0][1]
# 3.4.2 Determine if 3rd dimension is uncertainty or snr
errNm = None
if len(fitsRaw[0]) == 3:
errVals = None
med = np.median(flux/fitsRaw[0][2])
if med < 10**3 and med > 10**-3:
errVals = fitsRaw[0][2]
errNm = 'uncertainty'
else:
errVals = fitsRaw[0][2]
errNm = 'snr'
# 3.4.3 Get spectrum attributes
attsOK = True
res = data[colNmRes][row].lower()
if not (res == 'high' or res == 'med' or res == 'low'):
print unum + ' Resolution invalid.'
attsOK = False
instr = data[colNmInstr][row]
if instr == '':
print unum + ' Must provide Instrument for spectrum.'
attsOK = False
date = data[colNmDate][row].lower()
if len(date) != 9:
print unum + ' Date invalid.'
attsOK = False
if date[-1] == '\r':
date = date[:-1]
ord_filt = data[colNmOrdfilt][row]
if ord_filt == '':
print unum + ' Must provide Order/Filter for spectrum.'
attsOK = False
# 3.4.4 Create dictionary structure with spectrum data
if attsOK:
specAdd = True
specDict = {instr:{date:{ord_filt:{'wl':wl, \
'flux':flux, errNm:errVals}}}}
# 3.5 Get target photometry attributes
# 3.5.1 Check if photometry data was provided
photAdd = False
try:
survey = data[colNmSurvey][row]
except TypeError:
survey = ''
# 3.5.2 Get data for bands
if survey != '':
bands = []
vals = []
errs = []
photOK = True
for bndNum in range(1,NUM_BANDS + 1):
bndNm = 'band_' + str(bndNum)
valNm = 'val_' + str(bndNum)
errNm = 'err_' + str(bndNum)
try:
# Get band name
bands.append(data[bndNm][row])
if bands[bndNum - 1] == '':
photOK = False
else:
try:
bands[bndNum - 1][0]
except IndexError:
photOK = False
# Get band photometry value
try:
vals.append(data[valNm][row])
try:
vals[bndNum - 1] + 0
except TypeError:
if vals[bndNum - 1][-1] == '\r':
vals[bndNum - 1] = vals[bndNum - 1][:-1]
try:
vals[bndNum - 1] + 0
except TypeError:
photOK = False
else:
photOK = False
except TypeError:
photOK = False
# Get band photometry value error
try:
errs.append(data[errNm][row])
if errs[bndNum - 1] == '':
errs[bndNum - 1] = None
else:
try:
errs[bndNum - 1] + 0
except TypeError:
if errs[bndNum - 1][0] == '\r':
errs[bndNum - 1] = None
else:
try:
errs[bndNum - 1] = float(errs[bndNum - 1])
except ValueError:
photOK = False
except TypeError:
errs.append(None)
except TypeError:
if bndNum == 1:
photOK = False
# 3.5.3 Create dictionary structure with photometry data
if photOK:
photDict = {survey:{}}
for bdIdx, band in enumerate(bands):
if band != '':
photAdd = True
photDict[survey][band] = {'val': vals[bdIdx], \
'err': errs[bdIdx]}
else:
print unum + ' Photometry data invalid.'
# 3.6 Create range-level dictionary with all data
if photAdd and specAdd:
rngDict = {res:specDict, 'phot':photDict}
elif photAdd and not specAdd:
rngDict = {'phot':photDict}
elif not photAdd and specAdd:
rngDict = {res:specDict}
else:
print 'No data to add for ' + unum
continue
# 3.7 Add new target to database if necessary
if newTgt:
# 3.7.1 Create Target instance
if rng == 'opt':
target = Target(name, unum, ra, dec, sptype, \
rngDict, {}, {}, std)
elif rng == 'nir':
target = Target(name, unum, ra, dec, sptype, \
{}, rngDict, {}, std)
elif rng == 'mir':
target = Target(name, unum, ra, dec, sptype, \
{}, {}, rngDict, std)
# 3.7.2 Add to database
self.add_target(target, verbose=False)
somethingAdded = True
print unum + ' new target added to database: Index # ' + \
str(len(self.targets) - 1)
# 3.8 Add new data to database if target already exists
else:
addedPhot = False
addedSpec = False
if rng == 'opt':
currentTgt = self.targets[dbIdx].opt
elif rng == 'nir':
currentTgt = self.targets[dbIdx].nir
elif rng == 'mir':
currentTgt = self.targets[dbIdx].mir
# 3.8.1 Check dictionary level where to add data
if currentTgt != {}:
# For photometry data
if photAdd:
try:
currentTgt['phot'][survey]
for bnd in rngDict['phot'][survey].keys():
try:
currentTgt['phot'][survey][bnd]
# Overwrite existing data if requested
if overwrite:
addedPhot = True
currentTgt['phot'][survey][bnd] = \
rngDict['phot'][survey][bnd]
else:
print unum + ' ' + bnd + \
' photometry already exists' + \
' in database.'
except KeyError:
addedPhot = True
currentTgt['phot'][survey][bnd] = \
rngDict['phot'][survey][bnd]
except KeyError:
addedPhot = True
currentTgt['phot'][survey] = \
rngDict['phot'][survey]
# For spectrum data
if specAdd:
try:
currentTgt[res][instr]
try:
currentTgt[res][instr][date]
try:
currentTgt[res][instr][date][ord_filt]
# Overwrite existing data if requested
if overwrite:
addedSpec = True
currentTgt[res][instr][date][ord_filt] \
= rngDict[res][instr][date][ord_filt]
else:
print unum + ', ' + \
data[colNmFits][row] + \
' spectrum already exists in database.'
except KeyError:
addedSpec = True
currentTgt[res][instr][date][ord_filt] = \
rngDict[res][instr][date][ord_filt]
except KeyError:
addedSpec = True
currentTgt[res][instr][date] = \
rngDict[res][instr][date]
except KeyError:
addedSpec = True
currentTgt[res][instr] = rngDict[res][instr]
else:
addedSpec = True
currentTgt = rngDict
if addedSpec or addedPhot:
somethingAdded = True
print unum + ' new data added to target in database.'
# 4. Commit additions to database file --------------------------------
if somethingAdded:
# Check that database txt file exists in current folder
try:
f = open(DB_FILE,'rb')
except IOError:
print DB_FILE + ' could not be loaded. Check that it is ' + \
'in the current folder. Process stopped.'
return
f.close()
f = open(DB_FILE,'wb')
print 'Updating ' + DB_FILE + '...'
pickle.dump(self, f)
f.close()
print 'Remember to push updated ' + DB_FILE + ' to github.'
return
def main(spInput, grav='', plot=True, templ=False, std=False, excluded=False, normalize=True):
# 1. LOAD RELEVANT MODULES ------------------------------------------------
from astropy.io import ascii
import astrotools as at
import numpy as np
import sys
import os
import pdb
import matplotlib.pyplot as plt
# 2. SET UP VARIABLES -----------------------------------------------------
# Customizable variables <><><><><><><><><><><><><><><><><><><><><><><><><><><>
FOLDER_ROOT = '/Users/alejo/Dropbox/Project_0/more data/' # Location of NIR and OPT folders
FOLDER_IN = '/Users/alejo/Dropbox/Project_0/data/' # Location of input files
FOLDER_OUT = '/Users/alejo/Dropbox/Project_0/plots/' # Location to save output figures
FILE_IN = 'nir_spex_prism_with_optical.txt' # ASCII file w/ data
# <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
# For TXT objects file (updatable here directly)
HDR_FILE_IN = ('Ref','Designation`','J','H','K','SpType','SpType_T','NIRFobs',\
'NIRFtel','NIRfile','OPTobs','OPTtel','OPTinst','OPTfile',\
'Young?','Dusty?','Blue?','Binary?','Pec?')
# For TXT standards file
FILE_IN_STD = 'NIR_Standards_K10.txt' # ASCII file w/ standards
HDR_FILE_IN_STD = ('Ref','Designation','NIR SpType','OPT SpType')
colNameNIRS = HDR_FILE_IN_STD[2]
colNameOPTS = HDR_FILE_IN_STD[3]
OPTNIR_KEYS = ['OPT','NIR']
BANDS_NAMES = ['K','H','J','OPT']
data = ''
dataRaw = ''
specFiles = ''
spectraRaw = ''
spectra = ''
colNameRef = HDR_FILE_IN[0]
colNameDesig = HDR_FILE_IN[1]
colNameJ = HDR_FILE_IN[2]
colNameK = HDR_FILE_IN[4]
colNameJK = 'J-K'
colNameType = HDR_FILE_IN[6]
colNameNIRfile = HDR_FILE_IN[9]
colNameYng = HDR_FILE_IN[14]
colNameDust = HDR_FILE_IN[15]
colNameBlue = HDR_FILE_IN[16]
colNameBin = HDR_FILE_IN[17]
colNamePec = HDR_FILE_IN[18]
# Initialize dictionary to store NIR bands limits and normalizing sections
BAND_LIMS = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
BAND_LIMS[bandKey] = dict(lim = [None] * 2, limN = [None] * 2)
# Set wavelength limits for bands
# Limits are in microns
BAND_LIMS['OPT']['lim'][0] = 0.65
BAND_LIMS['OPT']['lim'][1] = 0.90
BAND_LIMS['J' ]['lim'][0] = 0.8
BAND_LIMS['J' ]['lim'][1] = 1.4
BAND_LIMS['H' ]['lim'][0] = 1.4
BAND_LIMS['H' ]['lim'][1] = 1.9
BAND_LIMS['K' ]['lim'][0] = 1.9
BAND_LIMS['K' ]['lim'][1] = 2.4
# Set wl limits for normalizing sections
# Limits are in microns
BAND_LIMS['OPT']['limN'][0] = 0.66
BAND_LIMS['OPT']['limN'][1] = 0.89
BAND_LIMS['J' ]['limN'][0] = 0.87
BAND_LIMS['J' ]['limN'][1] = 1.39
BAND_LIMS['H' ]['limN'][0] = 1.41
BAND_LIMS['H' ]['limN'][1] = 1.89
BAND_LIMS['K' ]['limN'][0] = 1.91
BAND_LIMS['K' ]['limN'][1] = 2.39
# 3. READ DATA FROM MAIN INPUT FILE ---------------------------------------
DELL_CHAR = '\t' # Delimiter character
COMM_CHAR = '#' # Comment character
# File with ALL objects (source: query in Access)
dataRaw = ascii.read(FOLDER_IN + FILE_IN, format='no_header', \
delimiter=DELL_CHAR, comment=COMM_CHAR, data_start=1)
# Store data in a dictionary-type object
data = {}.fromkeys(HDR_FILE_IN)
for colIdx,colname in enumerate(dataRaw.colnames):
data[HDR_FILE_IN[colIdx]] = np.array(dataRaw[colname])
# File with standards (source: manually generated)
dataRawS = ascii.read(FOLDER_IN + FILE_IN_STD, data_start=0)
# Store standard data in a dictionary-type object
dataS = {}.fromkeys(HDR_FILE_IN_STD)
for colIdx,colname in enumerate(dataRawS.colnames):
dataS[HDR_FILE_IN_STD[colIdx]] = np.array(dataRawS[colname])
# 4. FORMAT SOME ASCII COLUMNS --------------------------------------------
# 4.1 Convert into unicode the Spectral Type-Text column
uniSpType = [None] * len(data[colNameType])
for sIdx,sType in enumerate(data[colNameType]):
uniSpType[sIdx] = sType #.decode('utf-8')
data[colNameType] = np.array(uniSpType)
# 4.2 Calculate J-K Color
data[colNameJK] = data[colNameJ] - data[colNameK]
# 4.3 Format Designation Number from Designation Column
# (From "XX XX XX.X +XX XX XX.X" to "XXXX+XXXX")
for desigIdx,desig in enumerate(data[colNameDesig]):
desig = ''.join(desig.split())
signType = '+'
signPos = desig.find(signType)
if signPos == -1:
signType = '-'
signPos = desig.find(signType)
desigProper = desig[:4] + signType + desig[signPos+1:signPos+5]
data[colNameDesig][desigIdx] = desigProper
# 5. FILTER DATA BY USER INPUT IN spInput ---------------------------------
specIdx = []
# Find all spectra of same spectral type
for spIdx,spType in enumerate(data[colNameType]):
if spType.upper().startswith(spInput.upper()):
specIdx.append(spIdx)
if not specIdx:
print('No targets found for given input.')
if std is False:
return
spTypeInput = spInput.upper()
# Find NIR standard target that matches user's spectral type
stdIdx = []
for spIdx,spType in enumerate(dataS[colNameNIRS]):
if spType.upper().startswith(spTypeInput):
stdIdx.append(spIdx)
# Add NIR standard target to list of filtered objects if not there already
# (It may not be included in first filter because OPT SpT != NIR SpT)
if dataS[colNameNIRS][stdIdx] != dataS[colNameOPTS][stdIdx]:
for spIdx,spRef in enumerate(data[colNameRef]):
if spRef == int(dataS[colNameRef][stdIdx][0]):
if spIdx not in specIdx:
specIdx.append(spIdx)
# Sort relevant objects by JKmag value
specIdx = np.array(specIdx)
specSortIdx = data[colNameJK][specIdx].argsort()
# 6. READ SPECTRAL DATA FROM SPECTRAL FILES -------------------------------
spectraRaw = {}.fromkeys(OPTNIR_KEYS) # Used to store the raw data from fits files
specFilesDict = {}.fromkeys(OPTNIR_KEYS) # Used for reference purposes
for key in OPTNIR_KEYS:
specFiles = [None] * len(specSortIdx)
for sortIdx,specSort in enumerate(specSortIdx):
if data[key + 'file'][specIdx[specSort]][-4:] == '.dat': continue
if data[key + 'file'][specIdx[specSort]] == 'include': continue
tmpFullName = FOLDER_ROOT + key + '/' + data[key + 'file'][specIdx[specSort]]
specFiles[sortIdx] = tmpFullName
specFilesDict[key] = specFiles
spectraRaw[key] = at.read_spec(specFiles, atomicron=True, negtonan=True, \
errors=True, verbose=False)
# Clear out spectral data for objects missing either OPT or NIR data
allNone = True
for spIdx in range(0,len(spectraRaw['OPT'])):
if spectraRaw['OPT'][spIdx] is None:
spectraRaw['NIR'][spIdx] = None
elif spectraRaw['NIR'][spIdx] is None:
spectraRaw['OPT'][spIdx] = None
else:
allNone = False
if allNone:
print('No spectral data found for objects of the given spectral type.')
if std is False:
return
# Convert spectraRaw contents into lists if only one spectral data
# (This reduces the dimensions of the object holding the data)
for key in spectraRaw.keys():
if spectraRaw[key][0] is not None:
if len(spectraRaw[key][0]) > 3:
spectraRaw[key] = [spectraRaw[key],]
# 7. GATHER OBJECTS' NAMES ------------------------------------------------
# Filtered objects
refs = [None] * len(specSortIdx)
NIRfilenames = [None] * len(specSortIdx)
for idx,spIdx in enumerate(specSortIdx):
tmpRef = data[colNameRef][specIdx[spIdx]]
refs[idx] = str(int(tmpRef))
NIRfilenames[idx] = data[colNameNIRfile][specIdx[spIdx]]
# Standard objects
refsStd = [None] * len(dataS[colNameRef])
for idx,spIdx in enumerate(dataS[colNameRef]):
tmpRef = dataS[colNameRef][idx]
refsStd[idx] = str(int(tmpRef))
# Gather reference numbers of objects
objRef = data[colNameRef][specIdx[specSortIdx]]
# 8. SMOOTH SPECTRA -------------------------------------------------------
# Smooth the flux data to a reasonable resolution
spectraS = {}.fromkeys(OPTNIR_KEYS)
tmpSpOPT = at.smooth_spec(spectraRaw['OPT'], specFile=specFilesDict['OPT'], \
winWidth=10)
tmpSpNIR = at.smooth_spec(spectraRaw['NIR'], specFile=specFilesDict['NIR'], \
winWidth=0)
spectraS['OPT'] = tmpSpOPT
spectraS['NIR'] = tmpSpNIR
# 9. SELECT SPECTRAL DATA FOR THE DIFFERENT BANDS -------------------------
# Initialize variables
spectra = {}.fromkeys(BANDS_NAMES)
spectraN = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
if bandKey == 'OPT':
optNIR = 'OPT'
else:
optNIR = 'NIR'
# Select band
spectra[bandKey] = at.sel_band(spectraS[optNIR], BAND_LIMS[bandKey]['lim'], \
objRef)
if spectra[bandKey] is None:
break
# Normalize band
spectraN[bandKey], flagN = at.norm_spec(spectra[bandKey], \
BAND_LIMS[bandKey]['limN'], flag=True)
if flagN:
print(bandKey + ' LIMITS for normalization changed!')
if spectraN[bandKey] is None:
break
# 10. CHARACTERIZE TARGETS (i.e. identify young, field, and excluded) -----
grav = grav.lower()
toInclude = [False] * len(refs)
# toInclude_LG = [False] * len(refs)
toExclude = [False] * len(refs)
dataIncl = []
# dataIncl_LG = []
dataExcl = []
# 10.1 Extract NIR file names from "keepers" file
fileslist = os.listdir(FOLDER_IN)
inclFile = ''
for fl in fileslist:
if fl.find('keepers') == -1 or fl.find(spInput) == -1: continue
tmpfl = fl.split('_')
if len(tmpfl[1]) == 2 and grav == 'f':
inclFile = fl
break
elif len(tmpfl[1]) == 3:
if tmpfl[1][-1] == grav:
inclFile = fl
break
if inclFile != '':
dataIncl = ascii.read(FOLDER_IN + inclFile, format='no_header', \
delimiter=DELL_CHAR, comment=COMM_CHAR)
if len(dataIncl) > 0:
includeObjs = np.array(dataIncl['col1']).astype(object)
includeObjs = includeObjs + np.repeat('.fits', len(dataIncl))
# Find intersection of include-obj list and filtered targets list
setInclude = set(includeObjs).intersection(set(NIRfilenames))
# Create list with intersection targets
if len(setInclude) != 0:
for inclIdx in setInclude:
tmpInclIdx = np.where(np.array(NIRfilenames) == inclIdx)[0]
toInclude[tmpInclIdx] = True
# 10.2 Extract NIR file names from "rejects" file
exclFile = ''
for fl in fileslist:
if fl.find('rejects') == -1 or fl.find(spInput) == -1: continue
tmpfl = fl.split('_')
if len(tmpfl[1]) == 2 and grav == 'f':
exclFile = fl
break
elif len(tmpfl[1]) == 3:
if tmpfl[1][-1] == grav:
exclFile = fl
break
if exclFile != '':
try:
dataExcl = ascii.read(FOLDER_IN + exclFile, format='no_header', \
delimiter=DELL_CHAR, comment=COMM_CHAR)
except:
dataExcl = []
if len(dataExcl) == 0 and excluded:
print('No objects found in REJECTS file. Nothing to plot.')
return
elif len(dataExcl) > 0:
excludeObjs = np.array(dataExcl['col1']).astype(object)
excludeObjs = excludeObjs + np.repeat('.fits', len(dataExcl))
# Find intersection of exclude-obj list and filtered targets list
setExclude = set(excludeObjs).intersection(set(NIRfilenames))
# Create list with intersection targets
if len(setExclude) != 0:
for exclIdx in setExclude:
tmpExclIdx = np.where(np.array(NIRfilenames) == exclIdx)[0]
toExclude[tmpExclIdx] = True
# 10.3 Determine which target is the NIR Standard object
O_standard = [None] * 3 # Holds standard for output
stdObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameRef][spIdx] == dataS[colNameRef][stdIdx]:
stdObjs[idx] = True
if normalize:
O_standard[0] = spectraN['J'][idx]
O_standard[1] = spectraN['H'][idx]
O_standard[2] = spectraN['K'][idx]
else:
O_standard[0] = spectra['J'][idx]
O_standard[1] = spectra['H'][idx]
O_standard[2] = spectra['K'][idx]
# 10.4 Determine which targets to include in plots (based on user input)
# Consolidate plotting & template-flux instructions
plotInstructions = ['no'] * len(refs)
templInstructions = [False] * len(refs)
if grav == 'f':
plotinstlbl = 'field'
elif grav == 'lg':
plotinstlbl = 'low'
elif grav == 'g':
plotinstlbl = 'gamma'
elif grav == 'b':
plotinstlbl = 'beta'
else:
print('Wrong gravity input.')
return
for plotIdx in range(len(refs)):
if toInclude[plotIdx]:
plotInstructions[plotIdx] = plotinstlbl
templInstructions[plotIdx] = True
if toExclude[plotIdx] and excluded:
plotInstructions[plotIdx] = 'excluded'
# If all plot instructions are "no", then stop procedure (for spectral types)
allExcl = True
for instr in plotInstructions:
if instr != 'no':
allExcl = False
if allExcl:
print('No spectral data to plot based on your request.')
return
# 11. CALCULATE TEMPLATE SPECTRA FOR SELECTED SET OF SPECTRA ---------------------
# Gather spectra to use to calculate template spectrum
if not allExcl:
O_template = [None] * 3 # Holds calculated template for output
templCalculated = False
for bandIdx, bandKey in enumerate(BANDS_NAMES):
template = None
templSpecs = []
for spIdx, spex in enumerate(spectraN[bandKey]):
if templInstructions[spIdx]:
# Check that spectrum exists
if spex is None:
templInstructions[spIdx] = False
continue
if bandKey == 'OPT':
# Manually skip including OPT spectrum of some specific targets
# which use the same NIR fits file as both OPT and NIR spectrum
# so OPT spectrum is very bad
if refs[spIdx] == '50246':
continue
if refs[spIdx] == '50061':
continue
elif refs[spIdx] == '50188':
continue
templSpecs.append(spex)
else:
# Check that spectrum comes with error values (NIR bands only)
notNansBool = np.isfinite(spex[2])
notNans = np.any(notNansBool)
if notNans:
templSpecs.append(spex)
else:
print(str(objRef[spIdx]) + ' excluded from template')
templInstructions[spIdx] = False
# Calculate template spectrum using spec uncertainties as weights
if len(templSpecs) > 1:
if bandKey == 'OPT':
template = at.mean_comb(templSpecs, extremes=True)
else:
template_first, renormSpecs = at.mean_comb(templSpecs, renormalize=True)
# Re-calculate template using re-normalized spectra
template = at.mean_comb(renormSpecs, extremes=True)
# To calculate simple standard deviation, recalculate template without
# any weights, and just use the simple variance that comes out of that.
tmptempl = at.mean_comb(renormSpecs, forcesimple=True)
template[2] = tmptempl[2].copy()
templCalculated = True
# Append template to list of spectra to plot in the next step
if templCalculated:
spectraN[bandKey].append(template)
# Append template to output object
if bandIdx == 0:
tempIdx = 2
elif bandIdx == 2:
tempIdx = 0
elif bandIdx == 1:
tempIdx = 1
else:
tempIdx = None
if tempIdx is not None:
O_template[tempIdx] = template
if templCalculated:
refs.append('template')
plotInstructions.append('template')
else:
O_template = None
# 12. PLOT DATA -----------------------------------------------------------
if plot:
# Gather info on each target
objInfo = [None] * len(refs)
for posIdx,spIdx in enumerate(specIdx[specSortIdx]):
tmpDesig = data[colNameDesig][spIdx]
tmpJK = data[colNameJK][spIdx]
# Append description of special object to its spectral type when missing
spDesc = ''
try:
loc = data[colNameType][spIdx].index(spDesc)
except ValueError:
loc = None
if loc is None:
tmpSPtype = data[colNameType][spIdx] + spDesc
else:
tmpSPtype = data[colNameType][spIdx]
tmpSPtype = tmpSPtype + ' ' * (7 - len(tmpSPtype)) # For alignment purposes
if tmpDesig == '1126-5003':
objInfo[posIdx] = (tmpDesig + ' ' + tmpSPtype + '%.2f' %tmpJK)
else:
objInfo[posIdx] = (tmpDesig + ' ' + tmpSPtype + ' ' + '%.2f' %tmpJK)
if objInfo[-1] is None:
objInfo[-1] = 'template'
# Create Figure with Subplots and Annotations
figObj = plotspec(spectraN, BANDS_NAMES, BAND_LIMS, objInfo, spTypeInput, \
grav, plotInstructions, excluded)
if plot:
if excluded:
sptxt = '_excluded'
else:
sptxt = ''
figObj.savefig(FOLDER_OUT + spTypeInput + 'strip_' + \
grav + sptxt + '.pdf', dpi=300)
# 13. DETERMINE OUTPUT ----------------------------------------------------
if templ:
if std:
return O_template, O_standard
else:
return O_template
elif std:
return O_standard
else:
return spectraN
def main(spInput, grav=''):
# 1. LOAD RELEVANT MODULES ---------------------------------------------------------
import astrotools as at
from astropy.io import ascii
import matplotlib.pyplot as plt
import numpy as np
import sys
import pdb
# 2. SET UP VARIABLES --------------------------------------------------------------
# Customizable variables <><><><><><><><><><><><><><><><><><><><><><><><><><><>
FOLDER_ROOT = '/Users/alejo/Dropbox/Project_0/more data/' # Location of NIR and OPT folders
FOLDER_IN = '/Users/alejo/Dropbox/Project_0/data/' # Location of input files
FOLDER_OUT = '/Users/alejo/Dropbox/Project_0/plots/' # Location to save output figures
FILE_IN = 'nir_spex_prism_with_optical.txt' # ASCII file w/ data
# <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
# For TXT objects file (updatable here directly)
HDR_FILE_IN = ('Ref','Designation`','J','H','K','SpType','SpType_T','NIRFobs',\
'NIRFtel','NIRfile','OPTobs','OPTtel','OPTinst','OPTfile',\
'Young?','Dusty?','Blue?','Binary?','Pec?')
OPTNIR_KEYS = ['OPT', 'NIR']
BAND_NAME = ['NIR']
data = ''
dataRaw = ''
specFiles = ''
spectraRaw = ''
spectra = ''
colNameRef = HDR_FILE_IN[0]
colNameDesig = HDR_FILE_IN[1]
colNameJ = HDR_FILE_IN[2]
colNameK = HDR_FILE_IN[4]
colNameJK = 'J-K'
colNameType = HDR_FILE_IN[6]
colNameNIRfile = HDR_FILE_IN[9]
colNameYng = HDR_FILE_IN[14]
colNameDust = HDR_FILE_IN[15]
colNameBlue = HDR_FILE_IN[16]
colNameBin = HDR_FILE_IN[17]
colNamePec = HDR_FILE_IN[18]
# Initialize dictionary to store NIR bands limits and normalizing sections
BAND_LIMS = {}.fromkeys(BAND_NAME)
for bandKey in BAND_NAME:
BAND_LIMS[bandKey] = dict(lim = [None] * 2, limN = [None] * 2)
# Set wl limits for band
# Limits are in microns
BAND_LIMS['NIR']['lim'][0] = 0.8
BAND_LIMS['NIR']['lim'][1] = 2.4
# Set wl limits for normalizing sections; this is the peak of the J band
# Limits are in microns
BAND_LIMS['NIR']['limN'][0] = 1.28
BAND_LIMS['NIR']['limN'][1] = 1.32
# 3. READ DATA FROM INPUT FILES ----------------------------------------------------
DELL_CHAR = '\t' # Delimiter character
COMM_CHAR = '#' # Comment character
# File with objects (source: query in Access)
dataRaw = ascii.read(FOLDER_IN + FILE_IN, format='no_header', \
delimiter=DELL_CHAR, comment=COMM_CHAR, data_start=1)
# Store data in a dictionary-type object
data = {}.fromkeys(HDR_FILE_IN)
for colIdx,colname in enumerate(dataRaw.colnames):
data[HDR_FILE_IN[colIdx]] = np.array(dataRaw[colname])
# 4. FORMAT SOME ASCII COLUMNS -----------------------------------------------------
# 4.1 Convert into unicode the Spectral Type-Text column
uniSpType = [None] * len(data[colNameType])
for sIdx,sType in enumerate(data[colNameType]):
uniSpType[sIdx] = sType#.decode('utf-8')
data[colNameType] = np.array(uniSpType)
# 4.2 Calculate J-K Color
data[colNameJK] = data[colNameJ] - data[colNameK]
# 4.3 Format Designation Number from Designation Column
for desigIdx,desig in enumerate(data[colNameDesig]):
desig = ''.join(desig.split())
signType = '+'
signPos = desig.find(signType)
if signPos == -1:
signType = '-'
signPos = desig.find(signType)
desigProper = desig[:4] + signType + desig[signPos+1:signPos+5]
data[colNameDesig][desigIdx] = desigProper
# 5. FILTER DATA BY USER INPUT IN spInput ------------------------------------------
# Find all spectra of same spectral type
specIdx = []
for spIdx,spType in enumerate(data[colNameType]):
if spType.upper().startswith(spInput.upper()):
specIdx.append(spIdx)
if not specIdx:
print('No target found for given input.')
return
spTypeInput = spInput.upper()
# Sort relevant objects by JKmag value
specIdx = np.array(specIdx)
specSortIdx = data[colNameJK][specIdx].argsort()
# 6. READ SPECTRAL DATA FROM SPECTRAL FILES ----------------------------------------
spectraRaw = {}.fromkeys(OPTNIR_KEYS) # Used to store the raw data from fits files
specFilesDict = {}.fromkeys(OPTNIR_KEYS) # Used for reference purposes
for key in OPTNIR_KEYS:
specFiles = [None] * len(specSortIdx)
for sortIdx,specSort in enumerate(specSortIdx):
if data[key + 'file'][specIdx[specSort]][-4:] == '.dat': continue
if data[key + 'file'][specIdx[specSort]] == 'include': continue
tmpFullName = FOLDER_ROOT + key + '/' + data[key \
+ 'file'][specIdx[specSort]]
specFiles[sortIdx] = tmpFullName
specFilesDict[key] = specFiles
spectraRaw[key] = at.read_spec(specFiles, atomicron=True, negtonan=True, \
errors=True, verbose=False)
# Clear out spectral data for objects missing either OPT or NIR data
allNone = True
for spIdx in range(0,len(spectraRaw['OPT'])):
if spectraRaw['OPT'][spIdx] is None:
spectraRaw['NIR'][spIdx] = None
elif spectraRaw['NIR'][spIdx] is None:
spectraRaw['OPT'][spIdx] = None
else:
allNone = False
if allNone:
print('No spectral data found for objects of the given spectral type.')
return
# Convert spectraRaw contents into lists if only one spectral data
for key in spectraRaw.keys():
if spectraRaw[key][0] is not None:
if len(spectraRaw[key][0]) > 3:
spectraRaw[key] = [spectraRaw[key],]
# 7. GATHER OBJECTS' NAMES ---------------------------------------------------------
# Filtered objects
refs = [None] * len(specSortIdx)
for idx,spIdx in enumerate(specSortIdx):
tmpRef = data[colNameRef][specIdx[spIdx]]
refs[idx] = str(int(tmpRef))
#8. SMOOTH SPECTRA -----------------------------------------------------------------
# Smooth the flux data to a reasonable resolution
spectraS = at.smooth_spec(spectraRaw['NIR'], specFile=specFilesDict['NIR'], \
winWidth=0)
# 9. SELECT SPECTRAL DATA FOR NIR BAND --------------------------------------------
# Initialize variables
spectraN = {}.fromkeys(BAND_NAME)
# Gather reference numbers of objects
objRef = data[colNameRef][specIdx[specSortIdx]]
# Select band
spectra = at.sel_band(spectraS, BAND_LIMS['NIR']['lim'], objRef)
# Normalize band
spectraN['NIR'] = at.norm_spec(spectra, BAND_LIMS['NIR']['limN'])
# 11. CHARACTERIZE TARGETS (i.e. identify young, blue, to exclude...) --------------
# Force to include all targerts
toExclude = [False] * len(refs)
# Determine which targets are blue
blueObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBlue][spIdx].upper() == 'YES':
blueObjs[idx] = True
# Determine which targets are dusty
dustyObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameDust][spIdx].upper() == 'YES':
dustyObjs[idx] = True
# Determine which targets are peculiar
pecObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNamePec][spIdx].upper() == 'YES':
pecObjs[idx] = True
# Determine which plots are young objects
youngObjs = [False] * len(refs)
for idx,spIdx in enumerate(specSortIdx):
if data[colNameYng][specIdx[spIdx]].upper() == 'YES':
youngObjs[idx] = True
# Determine which targets are GAMMA
gammaObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb3"
if utcA == '\xce' and utcB == '\xb3':
gammaObjs[idx] = True
# Determine which targets are BETA
betaObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb2"
if utcA == '\xce' and utcB == '\xb2':
betaObjs[idx] = True
# Determine which targets to include in plots (based on user input)
# Consolidate plotting instructions
grav = grav.upper()
plotInstructions = ['exclude'] * len(refs)
if grav == 'Y': # If plot request is Young, include gamma, beta & young targets
for plotIdx in range(len(refs)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx] or betaObjs[plotIdx] or youngObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'G': # If plot request is Gamma, include only gamma targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'B': # If plot request is Beta, include only beta targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'F': # If plot request is Field, include Field & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx] or gammaObjs[plotIdx] or youngObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'field'
else: # Otherwise, print Field, gamma, beta, young & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
if youngObjs[plotIdx]:
plotInstructions[plotIdx] = 'young'
else:
plotInstructions[plotIdx] = 'field'
# If all plot instructions are "exclude", then stop procedure
allExcl = True
for instr in plotInstructions:
if instr != 'exclude':
allExcl = False
if allExcl:
if not uniqueSpec:
print('No spectral data to plot based on your request.')
return
# 11. PLOT DATA --------------------------------------------------------------------
# Gather info on each object (for legend purposes)
objInfo = [None] * len(refs)
for posIdx,spIdx in enumerate(specIdx[specSortIdx]):
tmpDesig = data[colNameDesig][spIdx]
tmpJK = data[colNameJK][spIdx]
tmpSPtype = data[colNameType][spIdx]
tmpSPtype = tmpSPtype + ' ' * (5 - len(tmpSPtype)) # For alignment purposes
objInfo[posIdx] = (tmpDesig + ' ' + tmpSPtype + ' ' + '%.2f' %tmpJK)
# Create Figure with Subplots
figObj = plotspec(spectraN, BAND_NAME, BAND_LIMS, objInfo, spTypeInput, grav, \
plotInstructions)
figObj.savefig(FOLDER_OUT + spTypeInput + grav + '_fan.pdf', \
dpi=800)
tgtNames = []
fitsNames = []
for tgt in dataOrg:
tgtNames.append(tgt[0])
fitsNames.append(FOLDER_NIR + tgt[1])
# Break into several plots if too many objects
numData = len(dataOrg)
if numData > 6:
numPlots = int(np.ceil(numData / 6.))
else:
numPlots = 1
# 3. GET SPECTRA --------------------------------------------------------------
spectra = at.read_spec(fitsNames, aToMicron=True, negToZero=True, errors=False, plot=False)
# 4. CLEAN AND NORMALIZE SPECTRA ----------------------------------------------
spectraC = at.sel_band(spectra, BAND_LIMS)
spectraN = at.norm_spec(spectraC, BAND_LIMS)
# 6. PLOT ALL SPECTRA ---------------------------------------------------------
# Parameter to space spectra more
if FILE_LBL == 'emln_galaxies':
broad = True
else:
broad = False
start = 0
stop = 6
for plIdx in range(numPlots):
if stop > numData:
def main(spInput, grav="", plot=True, templ=False, std=False, lbl=False):
# 1. LOAD RELEVANT MODULES ---------------------------------------------------------
import asciidata
import astrotools as at
import pyfits
import numpy
import sys
import pdb
import matplotlib.pyplot as plt
# 2. SET UP VARIABLES --------------------------------------------------------------
# General variables
FOLDER_ROOT = "/Users/alejo/KCData/" # Location of NIR and OPT folders
FOLDER_OUT = "Output/NOC/"
OPTNIR_KEYS = ["OPT", "NIR"]
BANDS_NAMES = ["K", "H", "J", "OPT"]
data = ""
dataRaw = ""
specFiles = ""
spectraRaw = ""
spectra = ""
# For TXT objects file (updatable here directly)
FILE_IN = "nir_spex_prism_with_optical_12aug15.txt" # ASCII file w/ data
HDR_FILE_IN = (
"Ref",
"Designation",
"J",
"H",
"K",
"SpType",
"SpType_T",
"NIRFobs",
"NIRFtel",
"NIRfile",
"OPTobs",
"OPTtel",
"OPTinst",
"OPTfile",
"Young?",
"Dusty?",
"Blue?",
"Multiple?",
"Pec?",
)
colNameRef = HDR_FILE_IN[0]
colNameDesig = HDR_FILE_IN[1]
colNameJ = HDR_FILE_IN[2]
colNameK = HDR_FILE_IN[4]
colNameJK = "J-K"
colNameType = HDR_FILE_IN[6]
colNameYng = HDR_FILE_IN[14]
colNameDust = HDR_FILE_IN[15]
colNameBlue = HDR_FILE_IN[16]
colNameBin = HDR_FILE_IN[17]
colNamePec = HDR_FILE_IN[18]
# For TXT standards file
FILE_IN_STD = "NIR_Standards.txt" # ASCII file w/ standards
HDR_FILE_IN_STD = ("Ref", "Designation", "NIR SpType", "OPT SpType")
colNameNIRS = HDR_FILE_IN_STD[2]
colNameOPTS = HDR_FILE_IN_STD[3]
# For TXT exclude-objects file
EXCL_FILE = "Exclude_Objs.txt" # ASCII file w/ unums of objects to exclude
# 3. READ DATA FROM INPUT FILES-----------------------------------------------------
NULL_CHAR = "" # Null character
DELL_CHAR = "\t" # Delimiter character
COMM_CHAR = "#" # Comment character
# File with objects (query in Access)
dataRaw = asciidata.open(FOLDER_ROOT + FILE_IN, NULL_CHAR, DELL_CHAR, COMM_CHAR)
# Store data in a dictionary-type object
data = {}.fromkeys(HDR_FILE_IN)
for colIdx, colData in enumerate(dataRaw):
data[HDR_FILE_IN[colIdx]] = colData.tonumpy()
# File with standards
dataRawS = asciidata.open(FOLDER_ROOT + FILE_IN_STD, NULL_CHAR, DELL_CHAR, COMM_CHAR)
# Store standard data in a dictionary-type object
dataS = {}.fromkeys(HDR_FILE_IN_STD)
for colIdx, colData in enumerate(dataRawS):
dataS[HDR_FILE_IN_STD[colIdx]] = colData.tonumpy()
# 4. FORMAT SOME ASCII COLUMNS -----------------------------------------------------
# 4.1 Convert into unicode the Spectral Type-Text column
uniSpType = [None] * len(data[colNameType])
for sIdx, sType in enumerate(data[colNameType]):
uniSpType[sIdx] = sType.decode("utf-8")
data[colNameType] = numpy.array(uniSpType)
# 4.2 Calculate J-K Color And Add J-K Column
data[colNameJK] = data[colNameJ] - data[colNameK]
# 4.3 Format Designation Number from Designation Column
# (From "XX XX XX.X +XX XX XX.X" to "XXXX+XXXX")
for desigIdx, desig in enumerate(data[colNameDesig]):
desig = "".join(desig.split())
signType = "+"
signPos = desig.find(signType)
if signPos == -1:
signType = "-"
signPos = desig.find(signType)
desigProper = desig[:4] + signType + desig[signPos + 1 : signPos + 5]
data[colNameDesig][desigIdx] = desigProper
# 5. FILTER DATA BY USER INPUT IN spInput -------------------------------------------
uniqueSpec = False
specIdx = []
if spInput.upper().startswith("L"):
# If input is a spectral type, then find all spectra of same spectral type
for spIdx, spType in enumerate(data[colNameType]):
if spType.upper().startswith(spInput.upper()):
specIdx.append(spIdx)
if not specIdx:
print "No targets found for given input."
if std is False:
return
spTypeInput = spInput.upper()
else:
# If input is one single spectrum, then find it
for spIdx, spType in enumerate(data[colNameRef]):
if str(spType) == spInput.upper():
specIdx.append(spIdx)
if not specIdx:
print "Requested target not found."
if std is False:
return
else:
spTypeInput = data[colNameType][specIdx[0]][0:2]
uniqueSpec = True
# Find NIR standard target that matches user's spectral type
stdIdx = []
for spIdx, spType in enumerate(dataS[colNameNIRS]):
if spType.upper().startswith(spTypeInput):
stdIdx.append(spIdx)
# Add NIR standard target to list of filtered objects if not there already
# (It may not be included in first filter because OPT SpT != NIR SpT)
if not uniqueSpec:
if dataS[colNameNIRS][stdIdx] != dataS[colNameOPTS][stdIdx]:
for spIdx, spRef in enumerate(data[colNameRef]):
if spRef == int(dataS[colNameRef][stdIdx][0]):
if spIdx not in specIdx:
specIdx.append(spIdx)
# Sort relevant objects by JKmag value
specIdx = numpy.array(specIdx)
specSortIdx = data[colNameJK][specIdx].argsort()
# 6. READ SPECTRAL DATA FROM SPECTRAL FILES ----------------------------------------
spectraRaw = {}.fromkeys(OPTNIR_KEYS) # Used to store the raw data from fits files
specFilesDict = {}.fromkeys(OPTNIR_KEYS) # Used for reference purposes
for key in OPTNIR_KEYS:
specFiles = [None] * len(specSortIdx)
for sortIdx, specSort in enumerate(specSortIdx):
tmpFullName = FOLDER_ROOT + key + "/" + data[key + "file"][specIdx[specSort]]
specFiles[sortIdx] = tmpFullName
specFilesDict[key] = specFiles
spectraRaw[key] = at.read_spec(specFiles, atomicron=True, negtonan=True, errors=True, verbose=False)
# Clear out spectral data for objects missing either OPT or NIR data
allNone = True
for spIdx in range(0, len(spectraRaw["OPT"])):
if spectraRaw["OPT"][spIdx] is None:
spectraRaw["NIR"][spIdx] = None
elif spectraRaw["NIR"][spIdx] is None:
spectraRaw["OPT"][spIdx] = None
else:
allNone = False
if allNone:
print "No spectral data found for objects of the given spectral type."
if std is False:
return
# Convert spectraRaw contents into lists if only one spectral data
# (This reduces the dimensions of the object holding the data)
for key in spectraRaw.keys():
if spectraRaw[key][0] is not None:
if len(spectraRaw[key][0]) > 3:
spectraRaw[key] = [spectraRaw[key]]
# 7. GATHER OBJECTS' NAMES----------------------------------------------------------
# Filtered objects
refs = [None] * len(specSortIdx)
for idx, spIdx in enumerate(specSortIdx):
tmpRef = data[colNameRef][specIdx[spIdx]]
refs[idx] = str(int(tmpRef))
# Standard objects
refsStd = [None] * len(dataS[colNameRef])
for idx, spIdx in enumerate(dataS[colNameRef]):
tmpRef = dataS[colNameRef][idx]
refsStd[idx] = str(int(tmpRef))
# Gather reference numbers of objects
objRef = data[colNameRef][specIdx[specSortIdx]]
# 8. SMOOTH SPECTRA -----------------------------------------------------------------
# Smooth the flux data to a reasonable resolution
spectraS = {}.fromkeys(OPTNIR_KEYS)
tmpSpOPT = at.smooth_spec(spectraRaw["OPT"], specFile=specFilesDict["OPT"], winWidth=10)
tmpSpNIR = at.smooth_spec(spectraRaw["NIR"], specFile=specFilesDict["NIR"], winWidth=0)
spectraS["OPT"] = tmpSpOPT
spectraS["NIR"] = tmpSpNIR
# 9. SET LIMITS FOR BANDS AND NORMALIZING SECTIONS----------------------------------
# Initialize dictionary to store limits
BAND_LIMS = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
BAND_LIMS[bandKey] = dict(lim=[None] * 2, limN=[None] * 2)
# Set wavelength limits for bands
# Limits are in microns
BAND_LIMS["OPT"]["lim"][0] = 0.65
BAND_LIMS["OPT"]["lim"][1] = 0.90
BAND_LIMS["J"]["lim"][0] = 0.8
BAND_LIMS["J"]["lim"][1] = 1.4
BAND_LIMS["H"]["lim"][0] = 1.4
BAND_LIMS["H"]["lim"][1] = 1.9
BAND_LIMS["K"]["lim"][0] = 1.9
BAND_LIMS["K"]["lim"][1] = 2.4
# Set wl limits for normalizing sections
# Limits are in microns
BAND_LIMS["OPT"]["limN"][0] = 0.66
BAND_LIMS["OPT"]["limN"][1] = 0.89
BAND_LIMS["J"]["limN"][0] = 0.87
BAND_LIMS["J"]["limN"][1] = 1.39
BAND_LIMS["H"]["limN"][0] = 1.41
BAND_LIMS["H"]["limN"][1] = 1.89
BAND_LIMS["K"]["limN"][0] = 1.91
BAND_LIMS["K"]["limN"][1] = 2.39
# 10. SELECT SPECTRAL DATA FOR OPTICAL, J-BAND, H-BAND, & K-BAND--------------------
# Initialize variables
spectra = {}.fromkeys(BANDS_NAMES)
spectraN = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
if bandKey == "OPT":
optNIR = "OPT"
else:
optNIR = "NIR"
# Select band
spectra[bandKey] = at.sel_band(spectraS[optNIR], BAND_LIMS[bandKey]["lim"], objRef)
if spectra[bandKey] is None:
break
# Normalize band
spectraN[bandKey], flagN = at.norm_spec(spectra[bandKey], BAND_LIMS[bandKey]["limN"], flag=True)
if flagN:
print "LIMITS for normalization changed!"
if spectraN[bandKey] is None:
break
# 11. CHARACTERIZE TARGETS (i.e. identify young, blue, to exclude...)---------------
# Determine which targets to exclude using the "Exclude_Objs" file
toExclude = [False] * len(refs)
dataExcl = asciidata.open(FOLDER_ROOT + EXCL_FILE, NULL_CHAR, DELL_CHAR, COMM_CHAR)
if len(dataExcl[0]) > 0:
# Extract data from "Exclude_Objs" file
excludeObjs = [None] * len(dataExcl[0])
for rowIdx, rowData in enumerate(dataExcl[0]):
excludeObjs[rowIdx] = str(rowData)
# Find intersection of exclude-obj list and filtered targets list
setExclude = set(excludeObjs).intersection(set(refs))
# Create list with intersection targets
if len(setExclude) != 0:
for exclIdx in setExclude:
tmpExclIdx = numpy.where(numpy.array(refs) == exclIdx)
toExclude[tmpExclIdx[0]] = True
# Determine which target is the NIR Standard object
O_standard = [None] * 3 # Holds standard for output
stdObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameRef][spIdx] == dataS[colNameRef][stdIdx]:
stdObjs[idx] = True
O_standard[0] = spectraN["J"][idx]
O_standard[1] = spectraN["H"][idx]
O_standard[2] = spectraN["K"][idx]
# Determine which targets are blue
blueObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBlue][spIdx].upper() == "YES":
blueObjs[idx] = True
# Determine which targets are dusty
dustyObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameDust][spIdx].upper() == "YES":
dustyObjs[idx] = True
# Determine which targets are binary
binObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBin][spIdx].upper() == "YES":
binObjs[idx] = True
# Determine which targets are peculiar
pecObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNamePec][spIdx].upper() == "YES":
pecObjs[idx] = True
# Determine which targets are young
youngObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameYng][spIdx].upper() == "YES":
youngObjs[idx] = True
# Determine which targets are GAMMA
gammaObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode("utf-8")
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb3"
if utcA == "\xce" and utcB == "\xb3":
gammaObjs[idx] = True
# Determine which targets are BETA
betaObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode("utf-8")
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb2"
if utcA == "\xce" and utcB == "\xb2":
betaObjs[idx] = True
# Determine which targets to include in plots (based on user input)
# Consolidate plotting & template-flux instructions
grav = grav.upper()
plotInstructions = ["exclude"] * len(refs)
templInstructions = [False] * len(refs)
if grav == "Y": # If plot request is Young, include gamma, beta & young targets
for plotIdx in range(len(refs)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx] or betaObjs[plotIdx] or youngObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = "young"
templInstructions[plotIdx] = True
elif grav == "G": # If plot request is Gamma, include only gamma targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = "young"
templInstructions[plotIdx] = True
elif grav == "B": # If plot request is Beta, include only beta targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = "young"
templInstructions[plotIdx] = True
elif grav == "F": # If plot request is Field, include Field & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx] or gammaObjs[plotIdx] or youngObjs[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] or binObjs[plotIdx]:
continue
if stdObjs[plotIdx]:
plotInstructions[plotIdx] = "standard"
else:
plotInstructions[plotIdx] = "field"
templInstructions[plotIdx] = True
else: # Otherwise, print Field, gamma, beta, young & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] or binObjs[plotIdx]:
continue
if youngObjs[plotIdx]:
plotInstructions[plotIdx] = "young"
elif stdObjs[plotIdx]:
plotInstructions[plotIdx] = "standard"
else:
plotInstructions[plotIdx] = "field"
templInstructions[plotIdx] = True
# If all plot instructions are "exclude", then stop procedure (for spectral types)
allExcl = True
for instr in plotInstructions:
if instr != "exclude":
allExcl = False
if allExcl:
if std:
return O_standard
if not uniqueSpec:
print "No spectral data to plot based on your request."
return
# 12. CALCULATE TEMPLATE SPECTRA FOR SELECTED SET OF SPECTRA -----------------------
# Gather spectra to use to calculate template spectrum
if not allExcl:
O_template = [None] * 3 # Holds calculated template for output
templCalculated = False
for bandIdx, bandKey in enumerate(BANDS_NAMES):
if bandKey == "OPT":
continue
templSpecs = []
for spIdx, spex in enumerate(spectraN[bandKey]):
if templInstructions[spIdx]:
# Check that spectrum exists
if spex is None:
templInstructions[spIdx] = False
continue
if bandKey == "OPT":
templSpecs.append(spex)
else:
# Check that spectrum comes with error values (NIR bands only)
notNansBool = numpy.isfinite(spex[2])
notNans = numpy.any(notNansBool)
if notNans:
templSpecs.append(spex)
else:
print str(objRef[spIdx]) + " excluded from template"
# Calculate template spectrum
if len(templSpecs) > 1:
template = at.mean_comb(templSpecs)
templCalculated = True
# Append template to list of spectra to plot in the next step
spectraN[bandKey].append(template)
# Append template to output object
if bandIdx == 0:
tempIdx = 2
elif bandIdx == 2:
tempIdx = 0
else:
tempIdx = 1
O_template[tempIdx] = template
if templCalculated:
refs.append("template")
plotInstructions.append("template")
else:
O_template = None
# 13. PLOT DATA --------------------------------------------------------------------
if lbl or plot:
# Gather info on each target
objInfo = [None] * len(refs)
for posIdx, spIdx in enumerate(specIdx[specSortIdx]):
tmpDesig = data[colNameDesig][spIdx]
tmpJK = data[colNameJK][spIdx]
tmpSPtype = data[colNameType][spIdx]
tmpSPtype = tmpSPtype + " " * (5 - len(tmpSPtype)) # For alignment purposes
objInfo[posIdx] = tmpDesig + " " + tmpSPtype + " " + "%.2f" % tmpJK
if objInfo[-1] is None:
objInfo[-1] = "template"
if plot:
# Create Figure with Subplots and Annotations
figObj = plotspec(spectraN, BANDS_NAMES, BAND_LIMS, objInfo, spTypeInput, grav, plotInstructions)
figObj.savefig(FOLDER_ROOT + FOLDER_OUT + spTypeInput + grav + ".pdf", dpi=600)
# 14. DETERMINE OUTPUT -------------------------------------------------------------
if templ:
if std:
return O_template, O_standard
else:
return O_template
elif std:
return O_standard
else:
if lbl:
return spectraN, objInfo
else:
return spectraN
spTypes = []
spectra = {}.fromkeys(BANDS)
for band in BANDS:
spectra[band] = []
for idxTp, spTp in enumerate(SPTYPES):
if not(spTp in SPSTDNM): continue
# Fetch and normalize special standard (L2, L5 and L7 only)
if spTp in SPSTDNM:
if spTp == 'L2':
isp = 0
elif spTp == 'L5':
isp = 1
else:
isp = 2
tmpspec = at.read_spec(FOLDER_DATASPEC + 'NIR/' + SPSTD[isp], errors=False, \
atomicron=True, negtonan=True)
for band in BANDS:
tmpband = at.sel_band(tmpspec, BAND_LIMS[band]['lim'])
if idxTp in [1,2,3] and band == 'H':
norm_lims = SPECIAL_H_NORM_LIM
else:
norm_lims = NORM_LIMS[band]['lim']
stdToPlot = at.norm_spec(tmpband, norm_lims)[0]
spectra[band].append(stdToPlot)
spTypes.append(spTp)
# # Fetch standard
# tmpStd = nocs.main(spTp, GRAV, plot=False, std=True, normalize=False)
# # Normalize standard
# for bdIdx, band in enumerate(BANDS):
# if idxTp in [1,2,3] and band == 'H':
BAND_NORMS[band] = [None] * 2
BAND_LIMS['J'][0] = 0.8
BAND_LIMS['J'][1] = 1.4
BAND_LIMS['H'][0] = 1.4
BAND_LIMS['H'][1] = 1.9
BAND_LIMS['K'][0] = 1.9
BAND_LIMS['K'][1] = 2.4
BAND_NORMS['J'][0] = 0.87
BAND_NORMS['J'][1] = 1.39
BAND_NORMS['H'][0] = 1.41
BAND_NORMS['H'][1] = 1.89
BAND_NORMS['K'][0] = 1.91
BAND_NORMS['K'][1] = 2.39
# 3. GET SPECTRUM OF TARGET ---------------------------------------------------
specRaw = at.read_spec(interestObject)[0]
specRaw = np.array(specRaw)
if specRaw is None:
print 'Could not get spectrum from file.'
sys.exit(0)
# Separate spectrum by bands
specSep = separate_bands(specRaw, BANDS, BAND_LIMS)
# Normalize spectrum
specN = [None] * 3
for bIdx, band in enumerate(BANDS):
specN[bIdx] = at.norm_spec(specSep[bIdx], BAND_NORMS[band])[0]
if specN[0] is None:
'Error normalizing spectra.'
