def get_spec(unum, separate=False, bandnames=None, bandlimits=None, bandnorms=None, retCoord=False, retST=False):
'''
This function finds and pulls spectrum from the BDNYC database. Specifically, it looks for low-res, NIR SpeX Prism spectra. It can split and normalize it by the NIR bands (J, H, and K).
*unum*
String with the U-number of the target (e.g. U20268).
*separate*
Boolean, whether to split and normalize the spectrum by NIR bands (J, H, and K)
*bandnames*
List with strings containing the band names used to separate the spectrum (when separate=True)
*bandlimits*
Dictionary with keys *bandnames*, each key containing a list with float numbers specifying the bands limits (e.g. {'J': [0.8,1.4], 'H': [1.4,1.9], 'K': [1.9,2.4]}
*bandnorms*
Same structure as *bandlimits*, this time the float numbers specify the wavelength ranges used to normalize the bands (when separate=True)
*retCoord*
Boolean, whether to pull from the database the target coordinates
*retST*
Boolean, whether to pull from the database the target spectral type
'''
import pickle
import BDNYC
import astrotools as at
# 1. Initialize variables ---------------------------------------
FOLDER_DB = '/Users/alejo/Dropbox/Python/Python_Database/'
FILE_DB = 'BDNYCData.txt'
# 2. Load database ----------------------------------------------
f = open(FOLDER_DB + FILE_DB,'rb')
bdnyc = pickle.load(f)
f.close()
# 3. Check data available for target ----------------------------
availData = bdnyc.show_data(unum, dump=True)
if availData is None:
return
# 4. Find Spex Prism data ---------------------------------------
spexFound = False
for row in availData:
# Check that row is data row
try:
row[0] + 1
except TypeError:
continue
# Check that row is nir row
try:
loc = row.index('nir')
except ValueError:
continue
# Check that row is low res row
try:
loc = row.index('low')
except ValueError:
continue
# Check that row is Spex Prism row
instr = row[3].lower()
loc = instr.find('spex')
if loc != -1:
spexFound = True
specIdx = row[0]
break
if not spexFound:
print 'Spectrum for target not found.'
return
# 5. Fetch target parameters if requested -----------------------
params = []
if retCoord:
ra = availData[4][1][0:5]
dec = availData[5][1][0:6]
coord = ra + dec
coord = coord.replace(' ','')
params.append(coord)
if retST:
st = availData[3][1]
params.append(st)
# 6. Get spectrum -----------------------------------------------
specRaw = bdnyc.get_data(unum, specIdx)
# 7. Separate spectrum by bands ---------------------------------
if separate:
spec = [None] * 3
for bIdx, band in enumerate(bandnames):
bLim = bandlimits[band][0]
bMax = bandlimits[band][1]
idx1 = np.where(specRaw[0,:] >= bLim)
idx2 = np.where(specRaw[0,:] <= bMax)
idx = np.intersect1d(idx1[0], idx2[0])
if len(idx) == 0:
print 'Error in spectrum range.'
return
spec[bIdx] = specRaw[:,idx]
# 8. Normalize spectrum -------------------------------------
specNorm = [None] * 3
for bIdx, band in enumerate(bandnames):
specNorm[bIdx] = at.norm_spec(spec[bIdx], bandnorms[band])[0]
if params != []:
return specNorm, params
else:
return specNorm
else:
if params != []:
return specRaw, params
else:
return specRaw
def main(spInput, grav=''):
# 1. LOAD RELEVANT MODULES ---------------------------------------------------------
import astrotools as at
import asciidata
import pyfits
import matplotlib.pyplot as plt
import numpy
import sys
import pdb
# 2. SET UP VARIABLES --------------------------------------------------------------
FOLDER_ROOT = '/Users/alejo/KCData/' # Location of NIR and OPT folders
FOLDER_OUT = 'Output/NOCN/'
OPTNIR_KEYS = ['OPT', 'NIR']
BAND_NAME = ['NIR']
data = ''
dataRaw = ''
specFiles = ''
spectraRaw = ''
spectra = ''
# For TXT objects file (updatable here directly)
FILE_IN = 'nir_spex_prism_with_optical_12aug15.txt' # ASCII file w/ data
HDR_FILE_IN = ('Ref','Designation','J','H','K','SpType','SpType_T','NIRFobs',\
'NIRFtel','NIRfile','OPTobs','OPTtel','OPTinst','OPTfile',\
'Young?','Dusty?','Blue?','Multiple?','Pec?')
colNameRef = HDR_FILE_IN[0]
colNameDesig = HDR_FILE_IN[1]
colNameJ = HDR_FILE_IN[2]
colNameK = HDR_FILE_IN[4]
colNameJK = 'J-K'
colNameType = HDR_FILE_IN[6]
colNameYng = HDR_FILE_IN[14]
colNameDust = HDR_FILE_IN[15]
colNameBlue = HDR_FILE_IN[16]
colNamePec = HDR_FILE_IN[18]
# For TXT exclude-objects file
EXCL_FILE = 'Exclude_Objs.txt' # ASCII file w/ U#s of objects to exclude
# 3. READ DATA FROM INPUT FILES ----------------------------------------------------
NULL_CHAR = '' # Null character
DELL_CHAR = '\t' # Delimiter character
COMM_CHAR = '#' # Comment character
# File with objects (query in Access)
dataRaw = asciidata.open(FOLDER_ROOT + FILE_IN, NULL_CHAR, DELL_CHAR, COMM_CHAR)
# Store data in a dictionary-type object
data = {}.fromkeys(HDR_FILE_IN)
for colIdx,colData in enumerate(dataRaw):
data[HDR_FILE_IN[colIdx]] = colData.tonumpy()
# 4. FORMAT SOME ASCII COLUMNS -----------------------------------------------------
# 4.1 Convert into unicode the Spectral Type-Text column
uniSpType = [None] * len(data[colNameType])
for sIdx,sType in enumerate(data[colNameType]):
uniSpType[sIdx] = sType.decode('utf-8')
data[colNameType] = numpy.array(uniSpType)
# 4.2 Calculate J-K Color And Add J-K Column
data[colNameJK] = data[colNameJ] - data[colNameK]
# 4.3 Format Designation Number from Designation Column
for desigIdx,desig in enumerate(data[colNameDesig]):
desig = ''.join(desig.split())
signType = '+'
signPos = desig.find(signType)
if signPos == -1:
signType = '-'
signPos = desig.find(signType)
desigProper = desig[:4] + signType + desig[signPos+1:signPos+5]
data[colNameDesig][desigIdx] = desigProper
# 5. FILTER DATA BY USER INPUT IN spInput ------------------------------------------
# Find all spectra of same spectral type
specIdx = []
for spIdx,spType in enumerate(data[colNameType]):
if spType.upper().startswith(spInput.upper()):
specIdx.append(spIdx)
if not specIdx:
print 'No target found for given input.'
return
spTypeInput = spInput.upper()
# Sort relevant objects by JKmag value
specIdx = numpy.array(specIdx)
specSortIdx = data[colNameJK][specIdx].argsort()
# 6. READ SPECTRAL DATA FROM SPECTRAL FILES ----------------------------------------
spectraRaw = {}.fromkeys(OPTNIR_KEYS) # Used to store the raw data from fits files
specFilesDict = {}.fromkeys(OPTNIR_KEYS) # Used for reference purposes
for key in OPTNIR_KEYS:
specFiles = [None] * len(specSortIdx)
for sortIdx,specSort in enumerate(specSortIdx):
tmpFullName = FOLDER_ROOT + key + '/' + data[key + 'file'][specIdx[specSort]]
specFiles[sortIdx] = tmpFullName
specFilesDict[key] = specFiles
spectraRaw[key] = at.read_spec(specFiles, atomicron=True, negtonan=True, \
errors=True, verbose=False)
# Clear out spectral data for objects missing either OPT or NIR data
allNone = True
for spIdx in range(0,len(spectraRaw['OPT'])):
if spectraRaw['OPT'][spIdx] is None:
spectraRaw['NIR'][spIdx] = None
elif spectraRaw['NIR'][spIdx] is None:
spectraRaw['OPT'][spIdx] = None
else:
allNone = False
if allNone:
print 'No spectral data found for objects of the given spectral type.'
return
# Convert spectraRaw contents into lists if only one spectral data
for key in spectraRaw.keys():
if spectraRaw[key][0] is not None:
if len(spectraRaw[key][0]) > 3:
spectraRaw[key] = [spectraRaw[key],]
# 7. GATHER OBJECTS' NAMES----------------------------------------------------------
# Filtered objects
refs = [None] * len(specSortIdx)
for idx,spIdx in enumerate(specSortIdx):
tmpRef = data[colNameRef][specIdx[spIdx]]
refs[idx] = str(int(tmpRef))
#8. SMOOTH SPECTRA -----------------------------------------------------------------
# Smooth the flux data to a reasonable resolution
spectraS = at.smooth_spec(spectraRaw['NIR'], specFile=specFilesDict['NIR'], \
winWidth=0)
# 9. SET LIMITS FOR BAND AND NORMALIZING SECTION------------------------------------
# Initialize dictionary to store limits
BAND_LIMS = {}.fromkeys(BAND_NAME)
for bandKey in BAND_NAME:
BAND_LIMS[bandKey] = dict(lim = [None] * 2, limN = [None] * 2)
# Set wl limits for band
# Limits are in microns
BAND_LIMS['NIR']['lim'][0] = 0.8
BAND_LIMS['NIR']['lim'][1] = 2.4
# Set wl limits for normalizing sections; this is the peak of the J band
# Limits are in microns
BAND_LIMS['NIR']['limN'][0] = 1.28
BAND_LIMS['NIR']['limN'][1] = 1.32
# 10. SELECT SPECTRAL DATA FOR NIR BAND---------------------------------------------
# Initialize variables
spectraN = {}.fromkeys(BAND_NAME)
# Gather reference numbers of objects
objRef = data[colNameRef][specIdx[specSortIdx]]
# Select band
spectra = at.sel_band(spectraS, BAND_LIMS['NIR']['lim'], objRef)
# Normalize band
spectraN['NIR'] = at.norm_spec(spectra, BAND_LIMS['NIR']['limN'])
# 11. CHARACTERIZE TARGETS (i.e. identify young, blue, to exclude...)---------------
# Determine which targets to exclude using the "Exclude_Objs" file
toExclude = [False] * len(refs)
dataExcl = asciidata.open(FOLDER_ROOT + EXCL_FILE, NULL_CHAR, DELL_CHAR, COMM_CHAR)
if len(dataExcl[0]) > 0:
# Extract data from "Exclude_Objs" file
excludeObjs = [None] * len(dataExcl[0])
for rowIdx, rowData in enumerate(dataExcl[0]):
excludeObjs[rowIdx] = str(rowData)
# Find intersection of exclude-obj list and filtered targets list
setExclude = set(excludeObjs).intersection(set(refs))
# Create list with intersection targets
if len(setExclude) != 0:
for exclIdx in setExclude:
tmpExclIdx = numpy.where(numpy.array(refs) == exclIdx)
toExclude[tmpExclIdx[0]] = True
# Determine which targets are blue
blueObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBlue][spIdx].upper() == 'YES':
blueObjs[idx] = True
# Determine which targets are dusty
dustyObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameDust][spIdx].upper() == 'YES':
dustyObjs[idx] = True
# Determine which targets are peculiar
pecObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNamePec][spIdx].upper() == 'YES':
pecObjs[idx] = True
# Determine which plots are young objects
youngObjs = [False] * len(refs)
for idx,spIdx in enumerate(specSortIdx):
if data[colNameYng][specIdx[spIdx]].upper() == 'YES':
youngObjs[idx] = True
# Determine which targets are GAMMA
gammaObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb3"
if utcA == '\xce' and utcB == '\xb3':
gammaObjs[idx] = True
# Determine which targets are BETA
betaObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb2"
if utcA == '\xce' and utcB == '\xb2':
betaObjs[idx] = True
# Determine which targets to include in plots (based on user input)
# Consolidate plotting instructions
grav = grav.upper()
plotInstructions = ['exclude'] * len(refs)
if grav == 'Y': # If plot request is Young, include gamma, beta & young targets
for plotIdx in range(len(refs)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx] or betaObjs[plotIdx] or youngObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'G': # If plot request is Gamma, include only gamma targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'B': # If plot request is Beta, include only beta targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'F': # If plot request is Field, include Field & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx] or gammaObjs[plotIdx] or youngObjs[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'field'
else: # Otherwise, print Field, gamma, beta, young & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
if youngObjs[plotIdx]:
plotInstructions[plotIdx] = 'young'
else:
plotInstructions[plotIdx] = 'field'
# If all plot instructions are "exclude", then stop procedure
allExcl = True
for instr in plotInstructions:
if instr != 'exclude':
allExcl = False
if allExcl:
if not uniqueSpec:
print 'No spectral data to plot based on your request.'
return
# 12. PLOT DATA --------------------------------------------------------------------
# Gather info on each object (for legend purposes)
objInfo = [None] * len(refs)
for posIdx,spIdx in enumerate(specIdx[specSortIdx]):
tmpDesig = data[colNameDesig][spIdx]
tmpJK = data[colNameJK][spIdx]
tmpSPtype = data[colNameType][spIdx]
tmpSPtype = tmpSPtype + ' ' * (5 - len(tmpSPtype)) # For alignment purposes
objInfo[posIdx] = (tmpDesig + ' ' + tmpSPtype + ' ' + '%.2f' %tmpJK)
# Create Figure with Subplots
figObj = plotspec(spectraN, BAND_NAME, BAND_LIMS, objInfo, spTypeInput, grav, \
plotInstructions)
figObj.savefig(FOLDER_ROOT + FOLDER_OUT + spTypeInput + grav + '_fan.pdf', \
dpi=800)
# 3. GET SPECTRAL NIR STANDARDS & TEMPLATES -----------------------------------
spTypes = []
spectra = {}.fromkeys(BANDS)
for band in BANDS:
spectra[band] = []
for idxTp, spTp in enumerate(SPTYPES):
# Fetch standard
tmpStd = nocs.main(spTp, GRAV, plot=False, std=True, normalize=False)
# Normalize standard
for bdIdx, band in enumerate(BANDS):
if idxTp in [1,2,3] and band == 'H':
norm_lims = SPECIAL_H_NORM_LIM
else:
norm_lims = NORM_LIMS[band]['lim']
stdToPlot = at.norm_spec(tmpStd[bdIdx], norm_lims)[0]
spectra[band].append(stdToPlot)
#spectra[band].append(tmpStd[bdIdx])
# Fetch template from ascii files generated by make_templ.py in templates/ folder.
for bdIdx, band in enumerate(BANDS):
fileNm = spTp + band + '_' + GRAV + '.txt'
templRaw = ad.read(FOLDER_OUT_TMPL + fileNm, delimiter='\t', \
format='no_header')
templRawToPlot = np.array([templRaw.columns[i] for i in range(5)])
# Normalize when necessary
if len(templRawToPlot) <= 3:
if idxTp in [1,2,3] and band == 'H':
norm_lims = SPECIAL_H_NORM_LIM
else:
def main(spInput,
grav='',
plot=True,
templ=False,
std=False,
lbl=False,
normalize=True):
# 1. LOAD RELEVANT MODULES ------------------------------------------------
#import asciidata
import astrotools as at
import numpy as np
import sys
import pdb
import matplotlib.pyplot as plt
from astropy.io import ascii
# 2. SET UP VARIABLES -----------------------------------------------------
# Customizable variables <><><><><><><><><><><><><><><><><><><><><><><><><><><>
FOLDER_ROOT = '/Users/alejo/Dropbox/Project_0/more data/' # Location of NIR and OPT folders
FOLDER_IN = '/Users/alejo/Dropbox/Project_0/data/' # Location of input files
FOLDER_OUT = '/Users/alejo/Dropbox/Project_0/plots/' # Location to save output figures
FILE_IN = 'nir_spex_prism_with_optical.txt' # ASCII file w/ data
# <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
# For TXT objects list file
HDR_FILE_IN = ('Ref','Designation`','J','H','K','SpType','SpType_T','NIRFobs',\
'NIRFtel','NIRfile','OPTobs','OPTtel','OPTinst','OPTfile',\
'Young?','Dusty?','Blue?','Binary?','Pec?')
# For TXT standards file
FILE_IN_STD = 'NIR_Standards.txt' # ASCII file w/ standards
HDR_FILE_IN_STD = ('Ref', 'Designation', 'NIR SpType', 'OPT SpType')
colNameNIRS = HDR_FILE_IN_STD[2]
colNameOPTS = HDR_FILE_IN_STD[3]
# For TXT exclude-objects file
EXCL_FILE = 'Exclude_Objs.txt' # ASCII file w/ unums of objects to exclude
OPTNIR_KEYS = ['OPT', 'NIR']
BANDS_NAMES = ['K', 'H', 'J', 'OPT']
data = ''
dataRaw = ''
specFiles = ''
spectraRaw = ''
spectra = ''
colNameRef = HDR_FILE_IN[0]
colNameDesig = HDR_FILE_IN[1]
colNameJ = HDR_FILE_IN[2]
colNameK = HDR_FILE_IN[4]
colNameJK = 'J-K'
colNameType = HDR_FILE_IN[6]
colNameNIRfile = HDR_FILE_IN[9]
colNameYng = HDR_FILE_IN[14]
colNameDust = HDR_FILE_IN[15]
colNameBlue = HDR_FILE_IN[16]
colNameBin = HDR_FILE_IN[17]
colNamePec = HDR_FILE_IN[18]
# Initialize dictionary to store NIR bands limits and normalizing sections
BAND_LIMS = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
BAND_LIMS[bandKey] = dict(lim=[None] * 2, limN=[None] * 2)
# Set wavelength limits for bands
# Limits are in microns
BAND_LIMS['OPT']['lim'][0] = 0.65
BAND_LIMS['OPT']['lim'][1] = 0.90
BAND_LIMS['J']['lim'][0] = 0.8
BAND_LIMS['J']['lim'][1] = 1.4
BAND_LIMS['H']['lim'][0] = 1.4
BAND_LIMS['H']['lim'][1] = 1.9
BAND_LIMS['K']['lim'][0] = 1.9
BAND_LIMS['K']['lim'][1] = 2.4
# Set wl limits for normalizing sections
# Limits are in microns
BAND_LIMS['OPT']['limN'][0] = 0.66
BAND_LIMS['OPT']['limN'][1] = 0.89
BAND_LIMS['J']['limN'][0] = 0.87
BAND_LIMS['J']['limN'][1] = 1.39
BAND_LIMS['H']['limN'][0] = 1.41
BAND_LIMS['H']['limN'][1] = 1.89
BAND_LIMS['K']['limN'][0] = 1.91
BAND_LIMS['K']['limN'][1] = 2.39
# 3. READ DATA FROM INPUT FILES -------------------------------------------
DELL_CHAR = '\t' # Delimiter character
COMM_CHAR = '#' # Comment character
# File with objects (source: query in Access)
dataRaw = ascii.read(FOLDER_IN + FILE_IN, format='no_header', \
delimiter=DELL_CHAR, comment=COMM_CHAR, data_start=1)
# Store data in a dictionary-type object
data = {}.fromkeys(HDR_FILE_IN)
for colIdx, colname in enumerate(dataRaw.colnames):
data[HDR_FILE_IN[colIdx]] = np.array(dataRaw[colname])
# File with standards (source: manually generated)
dataRawS = ascii.read(FOLDER_IN + FILE_IN_STD, data_start=0)
# Store standard data in a dictionary-type object
dataS = {}.fromkeys(HDR_FILE_IN_STD)
for colIdx, colname in enumerate(dataRawS.colnames):
dataS[HDR_FILE_IN_STD[colIdx]] = np.array(dataRawS[colname])
# 4. FORMAT SOME ASCII COLUMNS --------------------------------------------
# 4.1 Convert into unicode the Spectral Type-Text column
uniSpType = [None] * len(data[colNameType])
for sIdx, sType in enumerate(data[colNameType]):
uniSpType[sIdx] = sType #.decode('utf-8')
data[colNameType] = np.array(uniSpType)
# 4.2 Calculate J-K Color
data[colNameJK] = data[colNameJ] - data[colNameK]
# 4.3 Format Designation Number in Designation Column
# (From "XX XX XX.X +XX XX XX.X" to "XXXX+XXXX")
for desigIdx, desig in enumerate(data[colNameDesig]):
desig = ''.join(desig.split())
signType = '+'
signPos = desig.find(signType)
if signPos == -1:
signType = '-'
signPos = desig.find(signType)
desigProper = desig[:4] + signType + desig[signPos + 1:signPos + 5]
data[colNameDesig][desigIdx] = desigProper
# 5. FILTER DATA BY USER INPUT IN spInput ---------------------------------
uniqueSpec = False
specIdx = []
if spInput.upper().startswith('L'):
# If input is a spectral type, then find all spectra of same spectral type
for spIdx, spType in enumerate(data[colNameType]):
if spType.upper().startswith(spInput.upper()):
specIdx.append(spIdx)
if not specIdx:
print('No targets found for given input.')
if std is False:
return
spTypeInput = spInput.upper()
else:
# If input is one single spectrum, then find it
for spIdx, spType in enumerate(data[colNameRef]):
if str(spType) == spInput.upper():
specIdx.append(spIdx)
if not specIdx:
print('Requested target not found.')
if std is False:
return
else:
spTypeInput = data[colNameType][specIdx[0]][0:2]
uniqueSpec = True
# Find NIR standard target that matches user's spectral type
stdIdx = []
for spIdx, spType in enumerate(dataS[colNameNIRS]):
if spType.upper().startswith(spTypeInput):
stdIdx.append(spIdx)
# Add NIR standard target to list of filtered objects if not there already
# (It may not be included in first filter because OPT SpT != NIR SpT)
if not uniqueSpec:
if dataS[colNameNIRS][stdIdx] != dataS[colNameOPTS][stdIdx]:
for spIdx, spRef in enumerate(data[colNameRef]):
if spRef == dataS[colNameRef][stdIdx][0]:
if spIdx not in specIdx:
specIdx.append(spIdx)
# Sort relevant objects by JKmag value
specIdx = np.array(specIdx)
specSortIdx = data[colNameJK][specIdx].argsort()
# 6. READ SPECTRAL DATA FROM SPECTRAL FILES -------------------------------
spectraRaw = {}.fromkeys(
OPTNIR_KEYS) # Used to store the raw data from fits files
specFilesDict = {}.fromkeys(OPTNIR_KEYS) # Used for reference purposes
for key in OPTNIR_KEYS:
specFiles = [None] * len(specSortIdx)
for sortIdx, specSort in enumerate(specSortIdx):
if data[key + 'file'][specIdx[specSort]][-4:] == '.dat': continue
if data[key + 'file'][specIdx[specSort]] == 'include': continue
tmpFullName = FOLDER_ROOT + key + '/' + data[key \
+ 'file'][specIdx[specSort]]
specFiles[sortIdx] = tmpFullName
specFilesDict[key] = specFiles
spectraRaw[key] = at.read_spec(specFiles, atomicron=True, negtonan=True, \
errors=True, verbose=False)
# Clear out spectral data for objects missing either OPT or NIR data
allNone = True
for spIdx in range(0, len(spectraRaw['OPT'])):
if spectraRaw['OPT'][spIdx] is None:
spectraRaw['NIR'][spIdx] = None
elif spectraRaw['NIR'][spIdx] is None:
spectraRaw['OPT'][spIdx] = None
else:
allNone = False
if allNone:
print('No spectral data found for objects of the given spectral type.')
if std is False:
return
# Convert spectraRaw contents into lists if only one spectral data
# (This reduces the dimensions of the object holding the data)
for key in spectraRaw.keys():
if spectraRaw[key][0] is not None:
if len(spectraRaw[key][0]) > 3:
spectraRaw[key] = [
spectraRaw[key],
]
# 7. GATHER OBJECTS' NAMES ------------------------------------------------
# Filtered objects
refs = [None] * len(specSortIdx)
for idx, spIdx in enumerate(specSortIdx):
tmpRef = data[colNameRef][specIdx[spIdx]]
refs[idx] = str(tmpRef)
# Standard objects
refsStd = [None] * len(dataS[colNameRef])
for idx, spIdx in enumerate(dataS[colNameRef]):
tmpRef = dataS[colNameRef][idx]
refsStd[idx] = str(tmpRef)
# Gather reference numbers of objects
objRef = data[colNameRef][specIdx[specSortIdx]]
#8. SMOOTH SPECTRA --------------------------------------------------------
# Smooth the flux data to a reasonable resolution
spectraS = {}.fromkeys(OPTNIR_KEYS)
tmpSpOPT = at.smooth_spec(spectraRaw['OPT'], specFile=specFilesDict['OPT'], \
winWidth=10)
tmpSpNIR = at.smooth_spec(spectraRaw['NIR'], specFile=specFilesDict['NIR'], \
winWidth=0)
spectraS['OPT'] = tmpSpOPT
spectraS['NIR'] = tmpSpNIR
# 9. SELECT SPECTRAL DATA FOR THE DIFFERENT BANDS -------------------------
# Initialize variables
spectra = {}.fromkeys(BANDS_NAMES)
spectraN = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
if bandKey == 'OPT':
optNIR = 'OPT'
else:
optNIR = 'NIR'
# Select band
spectra[bandKey] = at.sel_band(spectraS[optNIR], BAND_LIMS[bandKey]['lim'], \
objRef)
if spectra[bandKey] is None:
break
# Normalize band
spectraN[bandKey], flagN = at.norm_spec(spectra[bandKey], \
BAND_LIMS[bandKey]['limN'], flag=True)
if flagN:
print('LIMITS for normalization changed!')
if spectraN[bandKey] is None:
break
# 10. CHARACTERIZE TARGETS (i.e. identify young, blue, to exclude...) -----
# Determine which targets to exclude
# (source: file manually generated)
toExclude = [False] * len(refs) # FORCE TO INCLUDE ALL TARGETS
# dataExcl = ascii.read(FOLDER_IN + EXCL_FILE, data_start=0, delimiter=DELL_CHAR, \
# comment=COMM_CHAR, names=['ID'])
# if len(dataExcl['ID']) > 0:
# # Extract data from "Exclude_Objs" file
# excludeObjs = np.array(dataExcl['ID'], dtype='string')
#
# # Find intersection of exclude-obj list and filtered targets list
# setExclude = set(excludeObjs).intersection(set(refs))
#
# # Create list with intersection targets
# if len(setExclude) != 0:
# for exclIdx in setExclude:
# tmpExclIdx = np.where(np.array(refs) == exclIdx)
# toExclude[tmpExclIdx[0]] = True
# Determine which target is the NIR Standard object
O_standard = [None] * 3 # Holds standard for output
stdObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameRef][spIdx] == dataS[colNameRef][stdIdx]:
stdObjs[idx] = True
if normalize:
O_standard[0] = spectraN['J'][idx]
O_standard[1] = spectraN['H'][idx]
O_standard[2] = spectraN['K'][idx]
else:
O_standard[0] = spectra['J'][idx]
O_standard[1] = spectra['H'][idx]
O_standard[2] = spectra['K'][idx]
# Determine which targets are blue
blueObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBlue][spIdx].upper() == 'YES':
blueObjs[idx] = True
# Determine which targets are dusty
dustyObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameDust][spIdx].upper() == 'YES':
dustyObjs[idx] = True
# Determine which targets are binary
binObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBin][spIdx].upper() == 'YES':
binObjs[idx] = True
# Determine which targets are peculiar
pecObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNamePec][spIdx].upper() == 'YES':
pecObjs[idx] = True
# Determine which targets are young
youngObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameYng][spIdx].upper() == 'YES':
youngObjs[idx] = True
# Determine which targets are GAMMA
gammaObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb3"
if utcA == '\xce' and utcB == '\xb3':
gammaObjs[idx] = True
# Determine which targets are BETA
betaObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb2"
if utcA == '\xce' and utcB == '\xb2':
betaObjs[idx] = True
# Determine which targets to include in plots (based on user input)
# Consolidate plotting & template-flux instructions
grav = grav.upper()
plotInstructions = ['exclude'] * len(refs)
templInstructions = [False] * len(refs)
if grav == 'Y': # If plot request is Young, include gamma, beta & young targets
for plotIdx in range(len(refs)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx] or betaObjs[plotIdx] or youngObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] \
or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
templInstructions[plotIdx] = True
elif grav == 'G': # If plot request is Gamma, include only gamma targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] \
or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
templInstructions[plotIdx] = True
elif grav == 'B': # If plot request is Beta, include only beta targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] \
or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
templInstructions[plotIdx] = True
elif grav == 'F': # If plot request is Field, include Field & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx] or gammaObjs[plotIdx] or youngObjs[plotIdx]:
continue
#if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] \
# or binObjs[plotIdx]:
# continue
if stdObjs[plotIdx]:
plotInstructions[plotIdx] = 'standard'
else:
plotInstructions[plotIdx] = 'field'
templInstructions[plotIdx] = True
else: # Otherwise, print Field, gamma, beta, young & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] \
or binObjs[plotIdx]:
continue
if youngObjs[plotIdx]:
plotInstructions[plotIdx] = 'young'
elif stdObjs[plotIdx]:
plotInstructions[plotIdx] = 'standard'
else:
plotInstructions[plotIdx] = 'field'
templInstructions[plotIdx] = True
# If all plot instructions are "exclude", then stop procedure (for spectral types)
allExcl = True
for instr in plotInstructions:
if instr != 'exclude':
allExcl = False
if allExcl:
if std:
return O_standard
if not uniqueSpec:
print('No spectral data to plot based on your request.')
return
# 11. CALCULATE TEMPLATE SPECTRA FOR SELECTED SET OF SPECTRA -----------------------
# Gather spectra to use to calculate template spectrum
# if not allExcl:
# O_template = [None] * 3 # Holds calculated template for output
# templCalculated = False
# for bandIdx, bandKey in enumerate(BANDS_NAMES):
# if bandKey == 'OPT':
# continue
#
# templSpecs = []
# for spIdx, spex in enumerate(spectraN[bandKey]):
# if templInstructions[spIdx]:
# # Check that spectrum exists
# if spex is None:
# templInstructions[spIdx] = False
# continue
#
# if bandKey == 'OPT':
# templSpecs.append(spex)
# else:
# # Check that spectrum comes with error values (NIR bands only)
# notNansBool = np.isfinite(spex[2])
# notNans = np.any(notNansBool)
# if notNans:
# templSpecs.append(spex)
# else:
# print(str(objRef[spIdx]) + ' excluded from template')
#
# # Calculate template spectrum
# if len(templSpecs) > 1:
# template = at.mean_comb(templSpecs)
# templCalculated = True
#
# # Append template to list of spectra to plot in the next step
# spectraN[bandKey].append(template)
# # Append template to output object
# if bandIdx == 0:
# tempIdx = 2
# elif bandIdx == 2:
# tempIdx = 0
# else:
# tempIdx = 1
# O_template[tempIdx] = template
#
# if templCalculated:
# refs.append('template')
# plotInstructions.append('template')
# else:
# O_template = None
# 12. PLOT DATA -----------------------------------------------------------
if lbl or plot:
# Gather info on each target
objInfo = [None] * len(refs)
for posIdx, spIdx in enumerate(specIdx[specSortIdx]):
tmpDesig = data[colNameDesig][spIdx]
tmpJK = data[colNameJK][spIdx]
tmpSPtype = data[colNameType][spIdx]
tmpSPtype = tmpSPtype + ' ' * (5 - len(tmpSPtype)
) # For alignment purposes
objInfo[posIdx] = (tmpDesig + ' ' + tmpSPtype + ' ' +
'%.2f' % tmpJK)
if objInfo[-1] is None:
objInfo[-1] = 'template'
if plot:
# Create Figure with Subplots and Annotations
tmpspectraN = {key: spectraN[key] for key in ['J', 'H', 'K']}
tmpBANDS_NAMES = BANDS_NAMES[:-1]
tmpBAND_LIMS = {key: BAND_LIMS[key] for key in ['J', 'H', 'K']}
figObj = plotspec(tmpspectraN, tmpBANDS_NAMES, tmpBAND_LIMS, objInfo, \
spTypeInput, grav, plotInstructions)
figObj.savefig(FOLDER_OUT + spTypeInput + grav + '.pdf', dpi=600)
# 13. DETERMINE OUTPUT ----------------------------------------------------
if templ:
if std:
return O_template, O_standard
else:
return O_template
elif std:
return O_standard
else:
if lbl:
return spectraN, objInfo
else:
return spectraN
if spTp in SPSTDNM:
if spTp == 'L2':
isp = 0
elif spTp == 'L5':
isp = 1
else:
isp = 2
tmpspec = at.read_spec('../more data/NIR/' + SPSTD[isp], errors=False, \
atomicron=True, negtonan=True)
for band in BANDS:
tmpband = at.sel_band(tmpspec, BAND_LIMS[band]['lim'])
if idxTp in [1,2,3] and band == 'H':
norm_lims = SPECIAL_H_NORM_LIM
else:
norm_lims = NORM_LIMS[band]['lim']
stdToPlot = at.norm_spec(tmpband, norm_lims)[0]
spectra[band].append(stdToPlot)
spTypes.append(spTp)
# # Fetch standard
# tmpStd = nocs.main(spTp, GRAV, plot=False, std=True, normalize=False)
# # Normalize standard
# for bdIdx, band in enumerate(BANDS):
# if idxTp in [1,2,3] and band == 'H':
# norm_lims = SPECIAL_H_NORM_LIM
# else:
# norm_lims = NORM_LIMS[band]['lim']
# stdToPlot = at.norm_spec(tmpStd[bdIdx], norm_lims)[0]
# spectra[band].append(stdToPlot)
# #spectra[band].append(tmpStd[bdIdx])
def main(spInput, grav=''):
# 1. LOAD RELEVANT MODULES ---------------------------------------------------------
import astrotools as at
from astropy.io import ascii
import matplotlib.pyplot as plt
import numpy as np
import sys
import pdb
# 2. SET UP VARIABLES --------------------------------------------------------------
# Customizable variables <><><><><><><><><><><><><><><><><><><><><><><><><><><>
FOLDER_ROOT = '/Users/alejo/Dropbox/Project_0/more data/' # Location of NIR and OPT folders
FOLDER_IN = '/Users/alejo/Dropbox/Project_0/data/' # Location of input files
FOLDER_OUT = '/Users/alejo/Dropbox/Project_0/plots/' # Location to save output figures
FILE_IN = 'nir_spex_prism_with_optical.txt' # ASCII file w/ data
# <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
# For TXT objects file (updatable here directly)
HDR_FILE_IN = ('Ref','Designation`','J','H','K','SpType','SpType_T','NIRFobs',\
'NIRFtel','NIRfile','OPTobs','OPTtel','OPTinst','OPTfile',\
'Young?','Dusty?','Blue?','Binary?','Pec?')
OPTNIR_KEYS = ['OPT', 'NIR']
BAND_NAME = ['NIR']
data = ''
dataRaw = ''
specFiles = ''
spectraRaw = ''
spectra = ''
colNameRef = HDR_FILE_IN[0]
colNameDesig = HDR_FILE_IN[1]
colNameJ = HDR_FILE_IN[2]
colNameK = HDR_FILE_IN[4]
colNameJK = 'J-K'
colNameType = HDR_FILE_IN[6]
colNameNIRfile = HDR_FILE_IN[9]
colNameYng = HDR_FILE_IN[14]
colNameDust = HDR_FILE_IN[15]
colNameBlue = HDR_FILE_IN[16]
colNameBin = HDR_FILE_IN[17]
colNamePec = HDR_FILE_IN[18]
# Initialize dictionary to store NIR bands limits and normalizing sections
BAND_LIMS = {}.fromkeys(BAND_NAME)
for bandKey in BAND_NAME:
BAND_LIMS[bandKey] = dict(lim = [None] * 2, limN = [None] * 2)
# Set wl limits for band
# Limits are in microns
BAND_LIMS['NIR']['lim'][0] = 0.8
BAND_LIMS['NIR']['lim'][1] = 2.4
# Set wl limits for normalizing sections; this is the peak of the J band
# Limits are in microns
BAND_LIMS['NIR']['limN'][0] = 1.28
BAND_LIMS['NIR']['limN'][1] = 1.32
# 3. READ DATA FROM INPUT FILES ----------------------------------------------------
DELL_CHAR = '\t' # Delimiter character
COMM_CHAR = '#' # Comment character
# File with objects (source: query in Access)
dataRaw = ascii.read(FOLDER_IN + FILE_IN, format='no_header', \
delimiter=DELL_CHAR, comment=COMM_CHAR, data_start=1)
# Store data in a dictionary-type object
data = {}.fromkeys(HDR_FILE_IN)
for colIdx,colname in enumerate(dataRaw.colnames):
data[HDR_FILE_IN[colIdx]] = np.array(dataRaw[colname])
# 4. FORMAT SOME ASCII COLUMNS -----------------------------------------------------
# 4.1 Convert into unicode the Spectral Type-Text column
uniSpType = [None] * len(data[colNameType])
for sIdx,sType in enumerate(data[colNameType]):
uniSpType[sIdx] = sType#.decode('utf-8')
data[colNameType] = np.array(uniSpType)
# 4.2 Calculate J-K Color
data[colNameJK] = data[colNameJ] - data[colNameK]
# 4.3 Format Designation Number from Designation Column
for desigIdx,desig in enumerate(data[colNameDesig]):
desig = ''.join(desig.split())
signType = '+'
signPos = desig.find(signType)
if signPos == -1:
signType = '-'
signPos = desig.find(signType)
desigProper = desig[:4] + signType + desig[signPos+1:signPos+5]
data[colNameDesig][desigIdx] = desigProper
# 5. FILTER DATA BY USER INPUT IN spInput ------------------------------------------
# Find all spectra of same spectral type
specIdx = []
for spIdx,spType in enumerate(data[colNameType]):
if spType.upper().startswith(spInput.upper()):
specIdx.append(spIdx)
if not specIdx:
print('No target found for given input.')
return
spTypeInput = spInput.upper()
# Sort relevant objects by JKmag value
specIdx = np.array(specIdx)
specSortIdx = data[colNameJK][specIdx].argsort()
# 6. READ SPECTRAL DATA FROM SPECTRAL FILES ----------------------------------------
spectraRaw = {}.fromkeys(OPTNIR_KEYS) # Used to store the raw data from fits files
specFilesDict = {}.fromkeys(OPTNIR_KEYS) # Used for reference purposes
for key in OPTNIR_KEYS:
specFiles = [None] * len(specSortIdx)
for sortIdx,specSort in enumerate(specSortIdx):
if data[key + 'file'][specIdx[specSort]][-4:] == '.dat': continue
if data[key + 'file'][specIdx[specSort]] == 'include': continue
tmpFullName = FOLDER_ROOT + key + '/' + data[key \
+ 'file'][specIdx[specSort]]
specFiles[sortIdx] = tmpFullName
specFilesDict[key] = specFiles
spectraRaw[key] = at.read_spec(specFiles, atomicron=True, negtonan=True, \
errors=True, verbose=False)
# Clear out spectral data for objects missing either OPT or NIR data
allNone = True
for spIdx in range(0,len(spectraRaw['OPT'])):
if spectraRaw['OPT'][spIdx] is None:
spectraRaw['NIR'][spIdx] = None
elif spectraRaw['NIR'][spIdx] is None:
spectraRaw['OPT'][spIdx] = None
else:
allNone = False
if allNone:
print('No spectral data found for objects of the given spectral type.')
return
# Convert spectraRaw contents into lists if only one spectral data
for key in spectraRaw.keys():
if spectraRaw[key][0] is not None:
if len(spectraRaw[key][0]) > 3:
spectraRaw[key] = [spectraRaw[key],]
# 7. GATHER OBJECTS' NAMES ---------------------------------------------------------
# Filtered objects
refs = [None] * len(specSortIdx)
for idx,spIdx in enumerate(specSortIdx):
tmpRef = data[colNameRef][specIdx[spIdx]]
refs[idx] = str(int(tmpRef))
#8. SMOOTH SPECTRA -----------------------------------------------------------------
# Smooth the flux data to a reasonable resolution
spectraS = at.smooth_spec(spectraRaw['NIR'], specFile=specFilesDict['NIR'], \
winWidth=0)
# 9. SELECT SPECTRAL DATA FOR NIR BAND --------------------------------------------
# Initialize variables
spectraN = {}.fromkeys(BAND_NAME)
# Gather reference numbers of objects
objRef = data[colNameRef][specIdx[specSortIdx]]
# Select band
spectra = at.sel_band(spectraS, BAND_LIMS['NIR']['lim'], objRef)
# Normalize band
spectraN['NIR'] = at.norm_spec(spectra, BAND_LIMS['NIR']['limN'])
# 11. CHARACTERIZE TARGETS (i.e. identify young, blue, to exclude...) --------------
# Force to include all targerts
toExclude = [False] * len(refs)
# Determine which targets are blue
blueObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBlue][spIdx].upper() == 'YES':
blueObjs[idx] = True
# Determine which targets are dusty
dustyObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameDust][spIdx].upper() == 'YES':
dustyObjs[idx] = True
# Determine which targets are peculiar
pecObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNamePec][spIdx].upper() == 'YES':
pecObjs[idx] = True
# Determine which plots are young objects
youngObjs = [False] * len(refs)
for idx,spIdx in enumerate(specSortIdx):
if data[colNameYng][specIdx[spIdx]].upper() == 'YES':
youngObjs[idx] = True
# Determine which targets are GAMMA
gammaObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb3"
if utcA == '\xce' and utcB == '\xb3':
gammaObjs[idx] = True
# Determine which targets are BETA
betaObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode('utf-8')
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb2"
if utcA == '\xce' and utcB == '\xb2':
betaObjs[idx] = True
# Determine which targets to include in plots (based on user input)
# Consolidate plotting instructions
grav = grav.upper()
plotInstructions = ['exclude'] * len(refs)
if grav == 'Y': # If plot request is Young, include gamma, beta & young targets
for plotIdx in range(len(refs)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx] or betaObjs[plotIdx] or youngObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'G': # If plot request is Gamma, include only gamma targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'B': # If plot request is Beta, include only beta targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'young'
elif grav == 'F': # If plot request is Field, include Field & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx] or gammaObjs[plotIdx] or youngObjs[plotIdx]:
continue
plotInstructions[plotIdx] = 'field'
else: # Otherwise, print Field, gamma, beta, young & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx]:
continue
if youngObjs[plotIdx]:
plotInstructions[plotIdx] = 'young'
else:
plotInstructions[plotIdx] = 'field'
# If all plot instructions are "exclude", then stop procedure
allExcl = True
for instr in plotInstructions:
if instr != 'exclude':
allExcl = False
if allExcl:
if not uniqueSpec:
print('No spectral data to plot based on your request.')
return
# 11. PLOT DATA --------------------------------------------------------------------
# Gather info on each object (for legend purposes)
objInfo = [None] * len(refs)
for posIdx,spIdx in enumerate(specIdx[specSortIdx]):
tmpDesig = data[colNameDesig][spIdx]
tmpJK = data[colNameJK][spIdx]
tmpSPtype = data[colNameType][spIdx]
tmpSPtype = tmpSPtype + ' ' * (5 - len(tmpSPtype)) # For alignment purposes
objInfo[posIdx] = (tmpDesig + ' ' + tmpSPtype + ' ' + '%.2f' %tmpJK)
# Create Figure with Subplots
figObj = plotspec(spectraN, BAND_NAME, BAND_LIMS, objInfo, spTypeInput, grav, \
plotInstructions)
figObj.savefig(FOLDER_OUT + spTypeInput + grav + '_fan.pdf', \
dpi=800)
def main(spInput, grav='', plot=True, templ=False, std=False, excluded=False, normalize=True):
# 1. LOAD RELEVANT MODULES ------------------------------------------------
from astropy.io import ascii
import astrotools as at
import numpy as np
import sys
import os
import pdb
import matplotlib.pyplot as plt
# 2. SET UP VARIABLES -----------------------------------------------------
# Customizable variables <><><><><><><><><><><><><><><><><><><><><><><><><><><>
FOLDER_ROOT = '/Users/alejo/Dropbox/Project_0/more data/' # Location of NIR and OPT folders
FOLDER_IN = '/Users/alejo/Dropbox/Project_0/data/' # Location of input files
FOLDER_OUT = '/Users/alejo/Dropbox/Project_0/plots/' # Location to save output figures
FILE_IN = 'nir_spex_prism_with_optical.txt' # ASCII file w/ data
# <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
# For TXT objects file (updatable here directly)
HDR_FILE_IN = ('Ref','Designation`','J','H','K','SpType','SpType_T','NIRFobs',\
'NIRFtel','NIRfile','OPTobs','OPTtel','OPTinst','OPTfile',\
'Young?','Dusty?','Blue?','Binary?','Pec?')
# For TXT standards file
FILE_IN_STD = 'NIR_Standards_K10.txt' # ASCII file w/ standards
HDR_FILE_IN_STD = ('Ref','Designation','NIR SpType','OPT SpType')
colNameNIRS = HDR_FILE_IN_STD[2]
colNameOPTS = HDR_FILE_IN_STD[3]
OPTNIR_KEYS = ['OPT','NIR']
BANDS_NAMES = ['K','H','J','OPT']
data = ''
dataRaw = ''
specFiles = ''
spectraRaw = ''
spectra = ''
colNameRef = HDR_FILE_IN[0]
colNameDesig = HDR_FILE_IN[1]
colNameJ = HDR_FILE_IN[2]
colNameK = HDR_FILE_IN[4]
colNameJK = 'J-K'
colNameType = HDR_FILE_IN[6]
colNameNIRfile = HDR_FILE_IN[9]
colNameYng = HDR_FILE_IN[14]
colNameDust = HDR_FILE_IN[15]
colNameBlue = HDR_FILE_IN[16]
colNameBin = HDR_FILE_IN[17]
colNamePec = HDR_FILE_IN[18]
# Initialize dictionary to store NIR bands limits and normalizing sections
BAND_LIMS = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
BAND_LIMS[bandKey] = dict(lim = [None] * 2, limN = [None] * 2)
# Set wavelength limits for bands
# Limits are in microns
BAND_LIMS['OPT']['lim'][0] = 0.65
BAND_LIMS['OPT']['lim'][1] = 0.90
BAND_LIMS['J' ]['lim'][0] = 0.8
BAND_LIMS['J' ]['lim'][1] = 1.4
BAND_LIMS['H' ]['lim'][0] = 1.4
BAND_LIMS['H' ]['lim'][1] = 1.9
BAND_LIMS['K' ]['lim'][0] = 1.9
BAND_LIMS['K' ]['lim'][1] = 2.4
# Set wl limits for normalizing sections
# Limits are in microns
BAND_LIMS['OPT']['limN'][0] = 0.66
BAND_LIMS['OPT']['limN'][1] = 0.89
BAND_LIMS['J' ]['limN'][0] = 0.87
BAND_LIMS['J' ]['limN'][1] = 1.39
BAND_LIMS['H' ]['limN'][0] = 1.41
BAND_LIMS['H' ]['limN'][1] = 1.89
BAND_LIMS['K' ]['limN'][0] = 1.91
BAND_LIMS['K' ]['limN'][1] = 2.39
# 3. READ DATA FROM MAIN INPUT FILE ---------------------------------------
DELL_CHAR = '\t' # Delimiter character
COMM_CHAR = '#' # Comment character
# File with ALL objects (source: query in Access)
dataRaw = ascii.read(FOLDER_IN + FILE_IN, format='no_header', \
delimiter=DELL_CHAR, comment=COMM_CHAR, data_start=1)
# Store data in a dictionary-type object
data = {}.fromkeys(HDR_FILE_IN)
for colIdx,colname in enumerate(dataRaw.colnames):
data[HDR_FILE_IN[colIdx]] = np.array(dataRaw[colname])
# File with standards (source: manually generated)
dataRawS = ascii.read(FOLDER_IN + FILE_IN_STD, data_start=0)
# Store standard data in a dictionary-type object
dataS = {}.fromkeys(HDR_FILE_IN_STD)
for colIdx,colname in enumerate(dataRawS.colnames):
dataS[HDR_FILE_IN_STD[colIdx]] = np.array(dataRawS[colname])
# 4. FORMAT SOME ASCII COLUMNS --------------------------------------------
# 4.1 Convert into unicode the Spectral Type-Text column
uniSpType = [None] * len(data[colNameType])
for sIdx,sType in enumerate(data[colNameType]):
uniSpType[sIdx] = sType #.decode('utf-8')
data[colNameType] = np.array(uniSpType)
# 4.2 Calculate J-K Color
data[colNameJK] = data[colNameJ] - data[colNameK]
# 4.3 Format Designation Number from Designation Column
# (From "XX XX XX.X +XX XX XX.X" to "XXXX+XXXX")
for desigIdx,desig in enumerate(data[colNameDesig]):
desig = ''.join(desig.split())
signType = '+'
signPos = desig.find(signType)
if signPos == -1:
signType = '-'
signPos = desig.find(signType)
desigProper = desig[:4] + signType + desig[signPos+1:signPos+5]
data[colNameDesig][desigIdx] = desigProper
# 5. FILTER DATA BY USER INPUT IN spInput ---------------------------------
specIdx = []
# Find all spectra of same spectral type
for spIdx,spType in enumerate(data[colNameType]):
if spType.upper().startswith(spInput.upper()):
specIdx.append(spIdx)
if not specIdx:
print('No targets found for given input.')
if std is False:
return
spTypeInput = spInput.upper()
# Find NIR standard target that matches user's spectral type
stdIdx = []
for spIdx,spType in enumerate(dataS[colNameNIRS]):
if spType.upper().startswith(spTypeInput):
stdIdx.append(spIdx)
# Add NIR standard target to list of filtered objects if not there already
# (It may not be included in first filter because OPT SpT != NIR SpT)
if dataS[colNameNIRS][stdIdx] != dataS[colNameOPTS][stdIdx]:
for spIdx,spRef in enumerate(data[colNameRef]):
if spRef == int(dataS[colNameRef][stdIdx][0]):
if spIdx not in specIdx:
specIdx.append(spIdx)
# Sort relevant objects by JKmag value
specIdx = np.array(specIdx)
specSortIdx = data[colNameJK][specIdx].argsort()
# 6. READ SPECTRAL DATA FROM SPECTRAL FILES -------------------------------
spectraRaw = {}.fromkeys(OPTNIR_KEYS) # Used to store the raw data from fits files
specFilesDict = {}.fromkeys(OPTNIR_KEYS) # Used for reference purposes
for key in OPTNIR_KEYS:
specFiles = [None] * len(specSortIdx)
for sortIdx,specSort in enumerate(specSortIdx):
if data[key + 'file'][specIdx[specSort]][-4:] == '.dat': continue
if data[key + 'file'][specIdx[specSort]] == 'include': continue
tmpFullName = FOLDER_ROOT + key + '/' + data[key + 'file'][specIdx[specSort]]
specFiles[sortIdx] = tmpFullName
specFilesDict[key] = specFiles
spectraRaw[key] = at.read_spec(specFiles, atomicron=True, negtonan=True, \
errors=True, verbose=False)
# Clear out spectral data for objects missing either OPT or NIR data
allNone = True
for spIdx in range(0,len(spectraRaw['OPT'])):
if spectraRaw['OPT'][spIdx] is None:
spectraRaw['NIR'][spIdx] = None
elif spectraRaw['NIR'][spIdx] is None:
spectraRaw['OPT'][spIdx] = None
else:
allNone = False
if allNone:
print('No spectral data found for objects of the given spectral type.')
if std is False:
return
# Convert spectraRaw contents into lists if only one spectral data
# (This reduces the dimensions of the object holding the data)
for key in spectraRaw.keys():
if spectraRaw[key][0] is not None:
if len(spectraRaw[key][0]) > 3:
spectraRaw[key] = [spectraRaw[key],]
# 7. GATHER OBJECTS' NAMES ------------------------------------------------
# Filtered objects
refs = [None] * len(specSortIdx)
NIRfilenames = [None] * len(specSortIdx)
for idx,spIdx in enumerate(specSortIdx):
tmpRef = data[colNameRef][specIdx[spIdx]]
refs[idx] = str(int(tmpRef))
NIRfilenames[idx] = data[colNameNIRfile][specIdx[spIdx]]
# Standard objects
refsStd = [None] * len(dataS[colNameRef])
for idx,spIdx in enumerate(dataS[colNameRef]):
tmpRef = dataS[colNameRef][idx]
refsStd[idx] = str(int(tmpRef))
# Gather reference numbers of objects
objRef = data[colNameRef][specIdx[specSortIdx]]
# 8. SMOOTH SPECTRA -------------------------------------------------------
# Smooth the flux data to a reasonable resolution
spectraS = {}.fromkeys(OPTNIR_KEYS)
tmpSpOPT = at.smooth_spec(spectraRaw['OPT'], specFile=specFilesDict['OPT'], \
winWidth=10)
tmpSpNIR = at.smooth_spec(spectraRaw['NIR'], specFile=specFilesDict['NIR'], \
winWidth=0)
spectraS['OPT'] = tmpSpOPT
spectraS['NIR'] = tmpSpNIR
# 9. SELECT SPECTRAL DATA FOR THE DIFFERENT BANDS -------------------------
# Initialize variables
spectra = {}.fromkeys(BANDS_NAMES)
spectraN = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
if bandKey == 'OPT':
optNIR = 'OPT'
else:
optNIR = 'NIR'
# Select band
spectra[bandKey] = at.sel_band(spectraS[optNIR], BAND_LIMS[bandKey]['lim'], \
objRef)
if spectra[bandKey] is None:
break
# Normalize band
spectraN[bandKey], flagN = at.norm_spec(spectra[bandKey], \
BAND_LIMS[bandKey]['limN'], flag=True)
if flagN:
print(bandKey + ' LIMITS for normalization changed!')
if spectraN[bandKey] is None:
break
# 10. CHARACTERIZE TARGETS (i.e. identify young, field, and excluded) -----
grav = grav.lower()
toInclude = [False] * len(refs)
# toInclude_LG = [False] * len(refs)
toExclude = [False] * len(refs)
dataIncl = []
# dataIncl_LG = []
dataExcl = []
# 10.1 Extract NIR file names from "keepers" file
fileslist = os.listdir(FOLDER_IN)
inclFile = ''
for fl in fileslist:
if fl.find('keepers') == -1 or fl.find(spInput) == -1: continue
tmpfl = fl.split('_')
if len(tmpfl[1]) == 2 and grav == 'f':
inclFile = fl
break
elif len(tmpfl[1]) == 3:
if tmpfl[1][-1] == grav:
inclFile = fl
break
if inclFile != '':
dataIncl = ascii.read(FOLDER_IN + inclFile, format='no_header', \
delimiter=DELL_CHAR, comment=COMM_CHAR)
if len(dataIncl) > 0:
includeObjs = np.array(dataIncl['col1']).astype(object)
includeObjs = includeObjs + np.repeat('.fits', len(dataIncl))
# Find intersection of include-obj list and filtered targets list
setInclude = set(includeObjs).intersection(set(NIRfilenames))
# Create list with intersection targets
if len(setInclude) != 0:
for inclIdx in setInclude:
tmpInclIdx = np.where(np.array(NIRfilenames) == inclIdx)[0]
toInclude[tmpInclIdx] = True
# 10.2 Extract NIR file names from "rejects" file
exclFile = ''
for fl in fileslist:
if fl.find('rejects') == -1 or fl.find(spInput) == -1: continue
tmpfl = fl.split('_')
if len(tmpfl[1]) == 2 and grav == 'f':
exclFile = fl
break
elif len(tmpfl[1]) == 3:
if tmpfl[1][-1] == grav:
exclFile = fl
break
if exclFile != '':
try:
dataExcl = ascii.read(FOLDER_IN + exclFile, format='no_header', \
delimiter=DELL_CHAR, comment=COMM_CHAR)
except:
dataExcl = []
if len(dataExcl) == 0 and excluded:
print('No objects found in REJECTS file. Nothing to plot.')
return
elif len(dataExcl) > 0:
excludeObjs = np.array(dataExcl['col1']).astype(object)
excludeObjs = excludeObjs + np.repeat('.fits', len(dataExcl))
# Find intersection of exclude-obj list and filtered targets list
setExclude = set(excludeObjs).intersection(set(NIRfilenames))
# Create list with intersection targets
if len(setExclude) != 0:
for exclIdx in setExclude:
tmpExclIdx = np.where(np.array(NIRfilenames) == exclIdx)[0]
toExclude[tmpExclIdx] = True
# 10.3 Determine which target is the NIR Standard object
O_standard = [None] * 3 # Holds standard for output
stdObjs = [False] * len(refs)
for idx,spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameRef][spIdx] == dataS[colNameRef][stdIdx]:
stdObjs[idx] = True
if normalize:
O_standard[0] = spectraN['J'][idx]
O_standard[1] = spectraN['H'][idx]
O_standard[2] = spectraN['K'][idx]
else:
O_standard[0] = spectra['J'][idx]
O_standard[1] = spectra['H'][idx]
O_standard[2] = spectra['K'][idx]
# 10.4 Determine which targets to include in plots (based on user input)
# Consolidate plotting & template-flux instructions
plotInstructions = ['no'] * len(refs)
templInstructions = [False] * len(refs)
if grav == 'f':
plotinstlbl = 'field'
elif grav == 'lg':
plotinstlbl = 'low'
elif grav == 'g':
plotinstlbl = 'gamma'
elif grav == 'b':
plotinstlbl = 'beta'
else:
print('Wrong gravity input.')
return
for plotIdx in range(len(refs)):
if toInclude[plotIdx]:
plotInstructions[plotIdx] = plotinstlbl
templInstructions[plotIdx] = True
if toExclude[plotIdx] and excluded:
plotInstructions[plotIdx] = 'excluded'
# If all plot instructions are "no", then stop procedure (for spectral types)
allExcl = True
for instr in plotInstructions:
if instr != 'no':
allExcl = False
if allExcl:
print('No spectral data to plot based on your request.')
return
# 11. CALCULATE TEMPLATE SPECTRA FOR SELECTED SET OF SPECTRA ---------------------
# Gather spectra to use to calculate template spectrum
if not allExcl:
O_template = [None] * 3 # Holds calculated template for output
templCalculated = False
for bandIdx, bandKey in enumerate(BANDS_NAMES):
template = None
templSpecs = []
for spIdx, spex in enumerate(spectraN[bandKey]):
if templInstructions[spIdx]:
# Check that spectrum exists
if spex is None:
templInstructions[spIdx] = False
continue
if bandKey == 'OPT':
# Manually skip including OPT spectrum of some specific targets
# which use the same NIR fits file as both OPT and NIR spectrum
# so OPT spectrum is very bad
if refs[spIdx] == '50246':
continue
if refs[spIdx] == '50061':
continue
elif refs[spIdx] == '50188':
continue
templSpecs.append(spex)
else:
# Check that spectrum comes with error values (NIR bands only)
notNansBool = np.isfinite(spex[2])
notNans = np.any(notNansBool)
if notNans:
templSpecs.append(spex)
else:
print(str(objRef[spIdx]) + ' excluded from template')
templInstructions[spIdx] = False
# Calculate template spectrum using spec uncertainties as weights
if len(templSpecs) > 1:
if bandKey == 'OPT':
template = at.mean_comb(templSpecs, extremes=True)
else:
template_first, renormSpecs = at.mean_comb(templSpecs, renormalize=True)
# Re-calculate template using re-normalized spectra
template = at.mean_comb(renormSpecs, extremes=True)
# To calculate simple standard deviation, recalculate template without
# any weights, and just use the simple variance that comes out of that.
tmptempl = at.mean_comb(renormSpecs, forcesimple=True)
template[2] = tmptempl[2].copy()
templCalculated = True
# Append template to list of spectra to plot in the next step
if templCalculated:
spectraN[bandKey].append(template)
# Append template to output object
if bandIdx == 0:
tempIdx = 2
elif bandIdx == 2:
tempIdx = 0
elif bandIdx == 1:
tempIdx = 1
else:
tempIdx = None
if tempIdx is not None:
O_template[tempIdx] = template
if templCalculated:
refs.append('template')
plotInstructions.append('template')
else:
O_template = None
# 12. PLOT DATA -----------------------------------------------------------
if plot:
# Gather info on each target
objInfo = [None] * len(refs)
for posIdx,spIdx in enumerate(specIdx[specSortIdx]):
tmpDesig = data[colNameDesig][spIdx]
tmpJK = data[colNameJK][spIdx]
# Append description of special object to its spectral type when missing
spDesc = ''
try:
loc = data[colNameType][spIdx].index(spDesc)
except ValueError:
loc = None
if loc is None:
tmpSPtype = data[colNameType][spIdx] + spDesc
else:
tmpSPtype = data[colNameType][spIdx]
tmpSPtype = tmpSPtype + ' ' * (7 - len(tmpSPtype)) # For alignment purposes
if tmpDesig == '1126-5003':
objInfo[posIdx] = (tmpDesig + ' ' + tmpSPtype + '%.2f' %tmpJK)
else:
objInfo[posIdx] = (tmpDesig + ' ' + tmpSPtype + ' ' + '%.2f' %tmpJK)
if objInfo[-1] is None:
objInfo[-1] = 'template'
# Create Figure with Subplots and Annotations
figObj = plotspec(spectraN, BANDS_NAMES, BAND_LIMS, objInfo, spTypeInput, \
grav, plotInstructions, excluded)
if plot:
if excluded:
sptxt = '_excluded'
else:
sptxt = ''
figObj.savefig(FOLDER_OUT + spTypeInput + 'strip_' + \
grav + sptxt + '.pdf', dpi=300)
# 13. DETERMINE OUTPUT ----------------------------------------------------
if templ:
if std:
return O_template, O_standard
else:
return O_template
elif std:
return O_standard
else:
return spectraN
tgtNames.append(tgt[0])
fitsNames.append(FOLDER_NIR + tgt[1])
# Break into several plots if too many objects
numData = len(dataOrg)
if numData > 6:
numPlots = int(np.ceil(numData / 6.))
else:
numPlots = 1
# 3. GET SPECTRA --------------------------------------------------------------
spectra = at.read_spec(fitsNames, aToMicron=True, negToZero=True, errors=False, plot=False)
# 4. CLEAN AND NORMALIZE SPECTRA ----------------------------------------------
spectraC = at.sel_band(spectra, BAND_LIMS)
spectraN = at.norm_spec(spectraC, BAND_LIMS)
# 6. PLOT ALL SPECTRA ---------------------------------------------------------
# Parameter to space spectra more
if FILE_LBL == 'emln_galaxies':
broad = True
else:
broad = False
start = 0
stop = 6
for plIdx in range(numPlots):
if stop > numData:
plSpecs = spectraN[start:]
plNames = tgtNames[start:]
else:
plSpecs = spectraN[start:stop]
def main(spInput, grav="", plot=True, templ=False, std=False, lbl=False):
# 1. LOAD RELEVANT MODULES ---------------------------------------------------------
import asciidata
import astrotools as at
import pyfits
import numpy
import sys
import pdb
import matplotlib.pyplot as plt
# 2. SET UP VARIABLES --------------------------------------------------------------
# General variables
FOLDER_ROOT = "/Users/alejo/KCData/" # Location of NIR and OPT folders
FOLDER_OUT = "Output/NOC/"
OPTNIR_KEYS = ["OPT", "NIR"]
BANDS_NAMES = ["K", "H", "J", "OPT"]
data = ""
dataRaw = ""
specFiles = ""
spectraRaw = ""
spectra = ""
# For TXT objects file (updatable here directly)
FILE_IN = "nir_spex_prism_with_optical_12aug15.txt" # ASCII file w/ data
HDR_FILE_IN = (
"Ref",
"Designation",
"J",
"H",
"K",
"SpType",
"SpType_T",
"NIRFobs",
"NIRFtel",
"NIRfile",
"OPTobs",
"OPTtel",
"OPTinst",
"OPTfile",
"Young?",
"Dusty?",
"Blue?",
"Multiple?",
"Pec?",
)
colNameRef = HDR_FILE_IN[0]
colNameDesig = HDR_FILE_IN[1]
colNameJ = HDR_FILE_IN[2]
colNameK = HDR_FILE_IN[4]
colNameJK = "J-K"
colNameType = HDR_FILE_IN[6]
colNameYng = HDR_FILE_IN[14]
colNameDust = HDR_FILE_IN[15]
colNameBlue = HDR_FILE_IN[16]
colNameBin = HDR_FILE_IN[17]
colNamePec = HDR_FILE_IN[18]
# For TXT standards file
FILE_IN_STD = "NIR_Standards.txt" # ASCII file w/ standards
HDR_FILE_IN_STD = ("Ref", "Designation", "NIR SpType", "OPT SpType")
colNameNIRS = HDR_FILE_IN_STD[2]
colNameOPTS = HDR_FILE_IN_STD[3]
# For TXT exclude-objects file
EXCL_FILE = "Exclude_Objs.txt" # ASCII file w/ unums of objects to exclude
# 3. READ DATA FROM INPUT FILES-----------------------------------------------------
NULL_CHAR = "" # Null character
DELL_CHAR = "\t" # Delimiter character
COMM_CHAR = "#" # Comment character
# File with objects (query in Access)
dataRaw = asciidata.open(FOLDER_ROOT + FILE_IN, NULL_CHAR, DELL_CHAR, COMM_CHAR)
# Store data in a dictionary-type object
data = {}.fromkeys(HDR_FILE_IN)
for colIdx, colData in enumerate(dataRaw):
data[HDR_FILE_IN[colIdx]] = colData.tonumpy()
# File with standards
dataRawS = asciidata.open(FOLDER_ROOT + FILE_IN_STD, NULL_CHAR, DELL_CHAR, COMM_CHAR)
# Store standard data in a dictionary-type object
dataS = {}.fromkeys(HDR_FILE_IN_STD)
for colIdx, colData in enumerate(dataRawS):
dataS[HDR_FILE_IN_STD[colIdx]] = colData.tonumpy()
# 4. FORMAT SOME ASCII COLUMNS -----------------------------------------------------
# 4.1 Convert into unicode the Spectral Type-Text column
uniSpType = [None] * len(data[colNameType])
for sIdx, sType in enumerate(data[colNameType]):
uniSpType[sIdx] = sType.decode("utf-8")
data[colNameType] = numpy.array(uniSpType)
# 4.2 Calculate J-K Color And Add J-K Column
data[colNameJK] = data[colNameJ] - data[colNameK]
# 4.3 Format Designation Number from Designation Column
# (From "XX XX XX.X +XX XX XX.X" to "XXXX+XXXX")
for desigIdx, desig in enumerate(data[colNameDesig]):
desig = "".join(desig.split())
signType = "+"
signPos = desig.find(signType)
if signPos == -1:
signType = "-"
signPos = desig.find(signType)
desigProper = desig[:4] + signType + desig[signPos + 1 : signPos + 5]
data[colNameDesig][desigIdx] = desigProper
# 5. FILTER DATA BY USER INPUT IN spInput -------------------------------------------
uniqueSpec = False
specIdx = []
if spInput.upper().startswith("L"):
# If input is a spectral type, then find all spectra of same spectral type
for spIdx, spType in enumerate(data[colNameType]):
if spType.upper().startswith(spInput.upper()):
specIdx.append(spIdx)
if not specIdx:
print "No targets found for given input."
if std is False:
return
spTypeInput = spInput.upper()
else:
# If input is one single spectrum, then find it
for spIdx, spType in enumerate(data[colNameRef]):
if str(spType) == spInput.upper():
specIdx.append(spIdx)
if not specIdx:
print "Requested target not found."
if std is False:
return
else:
spTypeInput = data[colNameType][specIdx[0]][0:2]
uniqueSpec = True
# Find NIR standard target that matches user's spectral type
stdIdx = []
for spIdx, spType in enumerate(dataS[colNameNIRS]):
if spType.upper().startswith(spTypeInput):
stdIdx.append(spIdx)
# Add NIR standard target to list of filtered objects if not there already
# (It may not be included in first filter because OPT SpT != NIR SpT)
if not uniqueSpec:
if dataS[colNameNIRS][stdIdx] != dataS[colNameOPTS][stdIdx]:
for spIdx, spRef in enumerate(data[colNameRef]):
if spRef == int(dataS[colNameRef][stdIdx][0]):
if spIdx not in specIdx:
specIdx.append(spIdx)
# Sort relevant objects by JKmag value
specIdx = numpy.array(specIdx)
specSortIdx = data[colNameJK][specIdx].argsort()
# 6. READ SPECTRAL DATA FROM SPECTRAL FILES ----------------------------------------
spectraRaw = {}.fromkeys(OPTNIR_KEYS) # Used to store the raw data from fits files
specFilesDict = {}.fromkeys(OPTNIR_KEYS) # Used for reference purposes
for key in OPTNIR_KEYS:
specFiles = [None] * len(specSortIdx)
for sortIdx, specSort in enumerate(specSortIdx):
tmpFullName = FOLDER_ROOT + key + "/" + data[key + "file"][specIdx[specSort]]
specFiles[sortIdx] = tmpFullName
specFilesDict[key] = specFiles
spectraRaw[key] = at.read_spec(specFiles, atomicron=True, negtonan=True, errors=True, verbose=False)
# Clear out spectral data for objects missing either OPT or NIR data
allNone = True
for spIdx in range(0, len(spectraRaw["OPT"])):
if spectraRaw["OPT"][spIdx] is None:
spectraRaw["NIR"][spIdx] = None
elif spectraRaw["NIR"][spIdx] is None:
spectraRaw["OPT"][spIdx] = None
else:
allNone = False
if allNone:
print "No spectral data found for objects of the given spectral type."
if std is False:
return
# Convert spectraRaw contents into lists if only one spectral data
# (This reduces the dimensions of the object holding the data)
for key in spectraRaw.keys():
if spectraRaw[key][0] is not None:
if len(spectraRaw[key][0]) > 3:
spectraRaw[key] = [spectraRaw[key]]
# 7. GATHER OBJECTS' NAMES----------------------------------------------------------
# Filtered objects
refs = [None] * len(specSortIdx)
for idx, spIdx in enumerate(specSortIdx):
tmpRef = data[colNameRef][specIdx[spIdx]]
refs[idx] = str(int(tmpRef))
# Standard objects
refsStd = [None] * len(dataS[colNameRef])
for idx, spIdx in enumerate(dataS[colNameRef]):
tmpRef = dataS[colNameRef][idx]
refsStd[idx] = str(int(tmpRef))
# Gather reference numbers of objects
objRef = data[colNameRef][specIdx[specSortIdx]]
# 8. SMOOTH SPECTRA -----------------------------------------------------------------
# Smooth the flux data to a reasonable resolution
spectraS = {}.fromkeys(OPTNIR_KEYS)
tmpSpOPT = at.smooth_spec(spectraRaw["OPT"], specFile=specFilesDict["OPT"], winWidth=10)
tmpSpNIR = at.smooth_spec(spectraRaw["NIR"], specFile=specFilesDict["NIR"], winWidth=0)
spectraS["OPT"] = tmpSpOPT
spectraS["NIR"] = tmpSpNIR
# 9. SET LIMITS FOR BANDS AND NORMALIZING SECTIONS----------------------------------
# Initialize dictionary to store limits
BAND_LIMS = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
BAND_LIMS[bandKey] = dict(lim=[None] * 2, limN=[None] * 2)
# Set wavelength limits for bands
# Limits are in microns
BAND_LIMS["OPT"]["lim"][0] = 0.65
BAND_LIMS["OPT"]["lim"][1] = 0.90
BAND_LIMS["J"]["lim"][0] = 0.8
BAND_LIMS["J"]["lim"][1] = 1.4
BAND_LIMS["H"]["lim"][0] = 1.4
BAND_LIMS["H"]["lim"][1] = 1.9
BAND_LIMS["K"]["lim"][0] = 1.9
BAND_LIMS["K"]["lim"][1] = 2.4
# Set wl limits for normalizing sections
# Limits are in microns
BAND_LIMS["OPT"]["limN"][0] = 0.66
BAND_LIMS["OPT"]["limN"][1] = 0.89
BAND_LIMS["J"]["limN"][0] = 0.87
BAND_LIMS["J"]["limN"][1] = 1.39
BAND_LIMS["H"]["limN"][0] = 1.41
BAND_LIMS["H"]["limN"][1] = 1.89
BAND_LIMS["K"]["limN"][0] = 1.91
BAND_LIMS["K"]["limN"][1] = 2.39
# 10. SELECT SPECTRAL DATA FOR OPTICAL, J-BAND, H-BAND, & K-BAND--------------------
# Initialize variables
spectra = {}.fromkeys(BANDS_NAMES)
spectraN = {}.fromkeys(BANDS_NAMES)
for bandKey in BANDS_NAMES:
if bandKey == "OPT":
optNIR = "OPT"
else:
optNIR = "NIR"
# Select band
spectra[bandKey] = at.sel_band(spectraS[optNIR], BAND_LIMS[bandKey]["lim"], objRef)
if spectra[bandKey] is None:
break
# Normalize band
spectraN[bandKey], flagN = at.norm_spec(spectra[bandKey], BAND_LIMS[bandKey]["limN"], flag=True)
if flagN:
print "LIMITS for normalization changed!"
if spectraN[bandKey] is None:
break
# 11. CHARACTERIZE TARGETS (i.e. identify young, blue, to exclude...)---------------
# Determine which targets to exclude using the "Exclude_Objs" file
toExclude = [False] * len(refs)
dataExcl = asciidata.open(FOLDER_ROOT + EXCL_FILE, NULL_CHAR, DELL_CHAR, COMM_CHAR)
if len(dataExcl[0]) > 0:
# Extract data from "Exclude_Objs" file
excludeObjs = [None] * len(dataExcl[0])
for rowIdx, rowData in enumerate(dataExcl[0]):
excludeObjs[rowIdx] = str(rowData)
# Find intersection of exclude-obj list and filtered targets list
setExclude = set(excludeObjs).intersection(set(refs))
# Create list with intersection targets
if len(setExclude) != 0:
for exclIdx in setExclude:
tmpExclIdx = numpy.where(numpy.array(refs) == exclIdx)
toExclude[tmpExclIdx[0]] = True
# Determine which target is the NIR Standard object
O_standard = [None] * 3 # Holds standard for output
stdObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameRef][spIdx] == dataS[colNameRef][stdIdx]:
stdObjs[idx] = True
O_standard[0] = spectraN["J"][idx]
O_standard[1] = spectraN["H"][idx]
O_standard[2] = spectraN["K"][idx]
# Determine which targets are blue
blueObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBlue][spIdx].upper() == "YES":
blueObjs[idx] = True
# Determine which targets are dusty
dustyObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameDust][spIdx].upper() == "YES":
dustyObjs[idx] = True
# Determine which targets are binary
binObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameBin][spIdx].upper() == "YES":
binObjs[idx] = True
# Determine which targets are peculiar
pecObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNamePec][spIdx].upper() == "YES":
pecObjs[idx] = True
# Determine which targets are young
youngObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
if data[colNameYng][spIdx].upper() == "YES":
youngObjs[idx] = True
# Determine which targets are GAMMA
gammaObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode("utf-8")
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb3"
if utcA == "\xce" and utcB == "\xb3":
gammaObjs[idx] = True
# Determine which targets are BETA
betaObjs = [False] * len(refs)
for idx, spIdx in enumerate(specIdx[specSortIdx]):
tmpType = data[colNameType][spIdx].encode("utf-8")
tmpLen = len(tmpType)
utcA = tmpType[tmpLen - 2]
utcB = tmpType[tmpLen - 1]
# GAMMA in utf-8 code is "\xce\xb2"
if utcA == "\xce" and utcB == "\xb2":
betaObjs[idx] = True
# Determine which targets to include in plots (based on user input)
# Consolidate plotting & template-flux instructions
grav = grav.upper()
plotInstructions = ["exclude"] * len(refs)
templInstructions = [False] * len(refs)
if grav == "Y": # If plot request is Young, include gamma, beta & young targets
for plotIdx in range(len(refs)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx] or betaObjs[plotIdx] or youngObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = "young"
templInstructions[plotIdx] = True
elif grav == "G": # If plot request is Gamma, include only gamma targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if gammaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = "young"
templInstructions[plotIdx] = True
elif grav == "B": # If plot request is Beta, include only beta targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx]:
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] or binObjs[plotIdx]:
continue
plotInstructions[plotIdx] = "young"
templInstructions[plotIdx] = True
elif grav == "F": # If plot request is Field, include Field & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if betaObjs[plotIdx] or gammaObjs[plotIdx] or youngObjs[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] or binObjs[plotIdx]:
continue
if stdObjs[plotIdx]:
plotInstructions[plotIdx] = "standard"
else:
plotInstructions[plotIdx] = "field"
templInstructions[plotIdx] = True
else: # Otherwise, print Field, gamma, beta, young & Standard targets
for plotIdx in range(len(plotInstructions)):
if toExclude[plotIdx]:
continue
if blueObjs[plotIdx] or dustyObjs[plotIdx] or pecObjs[plotIdx] or binObjs[plotIdx]:
continue
if youngObjs[plotIdx]:
plotInstructions[plotIdx] = "young"
elif stdObjs[plotIdx]:
plotInstructions[plotIdx] = "standard"
else:
plotInstructions[plotIdx] = "field"
templInstructions[plotIdx] = True
# If all plot instructions are "exclude", then stop procedure (for spectral types)
allExcl = True
for instr in plotInstructions:
if instr != "exclude":
allExcl = False
if allExcl:
if std:
return O_standard
if not uniqueSpec:
print "No spectral data to plot based on your request."
return
# 12. CALCULATE TEMPLATE SPECTRA FOR SELECTED SET OF SPECTRA -----------------------
# Gather spectra to use to calculate template spectrum
if not allExcl:
O_template = [None] * 3 # Holds calculated template for output
templCalculated = False
for bandIdx, bandKey in enumerate(BANDS_NAMES):
if bandKey == "OPT":
continue
templSpecs = []
for spIdx, spex in enumerate(spectraN[bandKey]):
if templInstructions[spIdx]:
# Check that spectrum exists
if spex is None:
templInstructions[spIdx] = False
continue
if bandKey == "OPT":
templSpecs.append(spex)
else:
# Check that spectrum comes with error values (NIR bands only)
notNansBool = numpy.isfinite(spex[2])
notNans = numpy.any(notNansBool)
if notNans:
templSpecs.append(spex)
else:
print str(objRef[spIdx]) + " excluded from template"
# Calculate template spectrum
if len(templSpecs) > 1:
template = at.mean_comb(templSpecs)
templCalculated = True
# Append template to list of spectra to plot in the next step
spectraN[bandKey].append(template)
# Append template to output object
if bandIdx == 0:
tempIdx = 2
elif bandIdx == 2:
tempIdx = 0
else:
tempIdx = 1
O_template[tempIdx] = template
if templCalculated:
refs.append("template")
plotInstructions.append("template")
else:
O_template = None
# 13. PLOT DATA --------------------------------------------------------------------
if lbl or plot:
# Gather info on each target
objInfo = [None] * len(refs)
for posIdx, spIdx in enumerate(specIdx[specSortIdx]):
tmpDesig = data[colNameDesig][spIdx]
tmpJK = data[colNameJK][spIdx]
tmpSPtype = data[colNameType][spIdx]
tmpSPtype = tmpSPtype + " " * (5 - len(tmpSPtype)) # For alignment purposes
objInfo[posIdx] = tmpDesig + " " + tmpSPtype + " " + "%.2f" % tmpJK
if objInfo[-1] is None:
objInfo[-1] = "template"
if plot:
# Create Figure with Subplots and Annotations
figObj = plotspec(spectraN, BANDS_NAMES, BAND_LIMS, objInfo, spTypeInput, grav, plotInstructions)
figObj.savefig(FOLDER_ROOT + FOLDER_OUT + spTypeInput + grav + ".pdf", dpi=600)
# 14. DETERMINE OUTPUT -------------------------------------------------------------
if templ:
if std:
return O_template, O_standard
else:
return O_template
elif std:
return O_standard
else:
if lbl:
return spectraN, objInfo
else:
return spectraN
# 3. GET SPECTRAL NIR STANDARDS & TEMPLATES -----------------------------------
spTypes = []
spectra = {}.fromkeys(BANDS)
for band in BANDS:
spectra[band] = []
for idxTp, spTp in enumerate(SPTYPES):
# Fetch standard
tmpStd = nocs.main(spTp, GRAV, plot=False, std=True, normalize=False)
# Normalize standard
for bdIdx, band in enumerate(BANDS):
if idxTp in [1, 2, 3] and band == "H":
norm_lims = SPECIAL_H_NORM_LIM
else:
norm_lims = NORM_LIMS[band]["lim"]
stdToPlot = at.norm_spec(tmpStd[bdIdx], norm_lims)[0]
spectra[band].append(stdToPlot)
# spectra[band].append(tmpStd[bdIdx])
# Fetch template from ascii files generated by make_templ.py in templates/ folder.
for bdIdx, band in enumerate(BANDS):
fileNm = spTp + band + "_" + GRAV + ".txt"
templRaw = ad.read(FOLDER_OUT_TMPL + fileNm, delimiter="\t", format="no_header")
templRawToPlot = np.array([templRaw.columns[i] for i in range(5)])
# Normalize when necessary
if len(templRawToPlot) <= 3:
if idxTp in [1, 2, 3] and band == "H":
norm_lims = SPECIAL_H_NORM_LIM
else:
norm_lims = NORM_LIMS[band]["lim"]
if spTp in SPSTDNM:
if spTp == 'L2':
isp = 0
elif spTp == 'L5':
isp = 1
else:
isp = 2
tmpspec = at.read_spec(FOLDER_DATASPEC + 'NIR/' + SPSTD[isp], errors=False, \
atomicron=True, negtonan=True)
for band in BANDS:
tmpband = at.sel_band(tmpspec, BAND_LIMS[band]['lim'])
if idxTp in [1,2,3] and band == 'H':
norm_lims = SPECIAL_H_NORM_LIM
else:
norm_lims = NORM_LIMS[band]['lim']
stdToPlot = at.norm_spec(tmpband, norm_lims)[0]
spectra[band].append(stdToPlot)
spTypes.append(spTp)
# # Fetch standard
# tmpStd = nocs.main(spTp, GRAV, plot=False, std=True, normalize=False)
# # Normalize standard
# for bdIdx, band in enumerate(BANDS):
# if idxTp in [1,2,3] and band == 'H':
# norm_lims = SPECIAL_H_NORM_LIM
# else:
# norm_lims = NORM_LIMS[band]['lim']
# stdToPlot = at.norm_spec(tmpStd[bdIdx], norm_lims)[0]
# spectra[band].append(stdToPlot)
# #spectra[band].append(tmpStd[bdIdx])
BAND_NORMS['K'][1] = 2.39
# 3. GET SPECTRUM OF TARGET ---------------------------------------------------
specRaw = at.read_spec(interestObject)[0]
specRaw = np.array(specRaw)
if specRaw is None:
print 'Could not get spectrum from file.'
sys.exit(0)
# Separate spectrum by bands
specSep = separate_bands(specRaw, BANDS, BAND_LIMS)
# Normalize spectrum
specN = [None] * 3
for bIdx, band in enumerate(BANDS):
specN[bIdx] = at.norm_spec(specSep[bIdx], BAND_NORMS[band])[0]
if specN[0] is None:
'Error normalizing spectra.'
sys.exit(0)
# 4. GET NIR TEMPLATES --------------------------------------------------------
# 4.1 Initialize holder of template spectra and strips
templ = [None] * len(SP_TYPES)
for tp in range(len(SP_TYPES)):
templ[tp] = [[] for i in range(3)]
# 4.2 Fetch templates
for spIdx, sp in enumerate(SP_TYPES):
for bdIdx, band in enumerate(BANDS):
if AdamTemplates: