def testReddeningException(self):
"""Test that if reddening=True in matchToObserved CatRA & CatDec are defined or exception is raised"""
testException = selectStarSED(sEDDir = self.testSpecDir, kuruczDir = self.testKDir,
mltDir = self.testMLTDir, wdDir = self.testWDDir)
testSEDList = testException.loadKuruczSEDs()
magnitudes = [[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]
self.assertRaises(RuntimeError, testException.findSED, testSEDList, magnitudes,
reddening = True)
def testReddeningException(self):
"""Test that if reddening=True in matchToObserved CatRA & CatDec are defined or exception is raised"""
testException = selectStarSED(sEDDir = self.testSpecDir, kuruczDir = self.testKDir,
mltDir = self.testMLTDir, wdDir = self.testWDDir)
testSEDList = testException.loadKuruczSEDs()
magnitudes = [[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]
self.assertRaises(RuntimeError, testException.findSED, testSEDList, magnitudes,
reddening = True)
def testLoadWD(self):
"""Test SED loading algorithm by making sure SEDs are all accounted for and
that there are separate lists for H and HE."""
# Test Matching to WD SEDs
loadTestWD = matchStar(wdDir = self.testWDDir)
testSEDsH, testSEDsHE = loadTestWD.loadwdSEDs()
# Add extra step because WD SEDs are separated into helium and hydrogen
testNames = []
for testH in testSEDsH:
testNames.append(testH.name)
for testHE in testSEDsHE:
testNames.append(testHE.name)
# Read in a list of the SEDs in the wd test sed directory
testWDList = os.listdir(self.testWDDir)
# First make sure that all SEDs are correctly accounted for if no subset provided
# Python 3 replaces assertItemsEqual() with assertCountEqual()
if hasattr(self, 'assertItemsEqual'):
self.assertItemsEqual(testNames, testWDList)
else:
self.assertCountEqual(testNames, testWDList)
# Test same condition if subset is provided
testSubsetList = ['bergeron_10000_75.dat_10100.gz', 'bergeron_He_9000_80.dat_9400.gz']
testSEDsSubsetH, testSEDsSubsetHE = selectStarSED(wdDir=
self.testWDDir).loadwdSEDs(subset=
testSubsetList)
testNamesSubset = []
for testH in testSEDsSubsetH:
testNamesSubset.append(testH.name)
for testHE in testSEDsSubsetHE:
testNamesSubset.append(testHE.name)
# Next make sure that correct subset loads if subset is provided
# Python 3 replaces assertItemsEqual() with assertCountEqual()
if hasattr(self, 'assertItemsEqual'):
self.assertItemsEqual(testNamesSubset, testSubsetList)
else:
self.assertCountEqual(testNamesSubset, testSubsetList)
# Make sure that the names get separated into correct wd type
self.assertEqual(testSEDsSubsetH[0].name, testSubsetList[0])
self.assertEqual(testSEDsSubsetHE[0].name, testSubsetList[1])
def testLoadWD(self):
"""Test SED loading algorithm by making sure SEDs are all accounted for and
that there are separate lists for H and HE."""
#Test Matching to WD SEDs
loadTestWD = matchStar(wdDir = self.testWDDir)
testSEDsH, testSEDsHE = loadTestWD.loadwdSEDs()
#Add extra step because WD SEDs are separated into helium and hydrogen
testNames = []
for testH in testSEDsH:
testNames.append(testH.name)
for testHE in testSEDsHE:
testNames.append(testHE.name)
#Read in a list of the SEDs in the wd test sed directory
testWDList = os.listdir(self.testWDDir)
#First make sure that all SEDs are correctly accounted for if no subset provided
self.assertItemsEqual(testNames, testWDList)
#Test same condition if subset is provided
testSubsetList = ['bergeron_10000_75.dat_10100.gz', 'bergeron_He_9000_80.dat_9400.gz']
testSEDsSubsetH, testSEDsSubsetHE = selectStarSED().loadwdSEDs(subset = testSubsetList)
testNamesSubset = []
for testH in testSEDsSubsetH:
testNamesSubset.append(testH.name)
for testHE in testSEDsSubsetHE:
testNamesSubset.append(testHE.name)
#Next make sure that correct subset loads if subset is provided
self.assertItemsEqual(testNamesSubset, testSubsetList)
#Make sure that the names get separated into correct wd type
self.assertEqual(testSEDsSubsetH[0].name, testSubsetList[0])
self.assertEqual(testSEDsSubsetHE[0].name, testSubsetList[1])
def loadGalfast(self,
filenameList,
outFileList,
sEDPath=None,
kuruczPath=None,
mltPath=None,
wdPath=None,
kuruczSubset=None,
mltSubset=None,
wdSubset=None,
chunkSize=10000):
"""
This is customized for the outputs we currently need for the purposes of consistent output
It will read in a galfast output file and output desired values for database input into a file
@param [in] filenameList is a list of the galfast output files that will be loaded and processed.
Can process fits, gzipped, or txt output from galfast.
@param [in] outFileList is a list of the names of the output files that will be created. If gzipped
output is desired simply write the filenames with .gz at the end.
@param [in] kuruczPath is a place to specify a path to kurucz SED files
@param [in] mltPath is the same as kuruczPath except that it specifies a directory for the
mlt SEDs
@param [in] wdPath is the same as the previous two except that it specifies a path to a
white dwarf SED directory.
@param [in] kuruczSubset is a list which provides a subset of the kurucz files within the
kurucz folder that one wants to use
@param [in] mltSubset is a list which provides a subset of the mlt files within the
mlt folder that one wants to use
@param [in] wdSubset is a list which provides a subset of the wd files within the
wd folder that one wants to use
@param [in] chunkSize is the size of chunks of lines to be read from the catalog at one time.
"""
for filename in filenameList:
#Make sure input file exists and is readable format before doing anything else
if os.path.isfile(filename) == False:
raise RuntimeError('*** File does not exist')
#Process various possible galfast outputs
if filename.endswith(('.txt', '.gz', '.fits')):
continue
else:
raise RuntimeError(
str('*** Unsupported File Format in file: ' +
str(filename)))
#If all files exist and are in proper formats then load seds
selectStarSED0 = selectStarSED(kuruczDir=kuruczPath,
mltDir=mltPath,
wdDir=wdPath)
if kuruczSubset is None:
kuruczList = selectStarSED0.loadKuruczSEDs()
else:
kuruczList = selectStarSED0.loadKuruczSEDs(subset=kuruczSubset)
#Only need one dictionary since none of the names overlap
positionDict = {}
for kNum, kuruczSED in enumerate(kuruczList):
positionDict[kuruczSED.name] = kNum
if mltSubset is None:
mltList = selectStarSED0.loadmltSEDs()
else:
mltList = selectStarSED0.loadmltSEDs(subset=mltSubset)
for mltNum, mltSED in enumerate(mltList):
positionDict[mltSED.name] = mltNum
if wdSubset is None:
wdListH, wdListHE = selectStarSED0.loadwdSEDs()
else:
wdListH, wdListHE = selectStarSED0.loadwdSEDs(subset=wdSubset)
for hNum, hSED in enumerate(wdListH):
positionDict[hSED.name] = hNum
for heNum, heSED in enumerate(wdListHE):
positionDict[heSED.name] = heNum
#For adding/subtracting extinction when calculating colors
#Numbers below come from Schlafly and Finkbeiner (2011) (ApJ, 737, 103)
#normalized by SDSS r mag value
sdssExtCoeffs = [1.8551, 1.4455, 1.0, 0.7431, 0.5527]
lsstExtCoeffs = [1.8140, 1.4166, 0.9947, 0.7370, 0.5790, 0.4761]
sdssPhot = BandpassDict.loadTotalBandpassesFromFiles(
['u', 'g', 'r', 'i', 'z'],
bandpassDir=os.path.join(lsst.utils.getPackageDir('throughputs'),
'sdss'),
bandpassRoot='sdss_')
#Load Bandpasses for LSST colors to get colors from matched SEDs
lsstFilterList = ('u', 'g', 'r', 'i', 'z', 'y')
lsstPhot = BandpassDict.loadTotalBandpassesFromFiles(lsstFilterList)
imSimBand = Bandpass()
imSimBand.imsimBandpass()
#Calculate colors and add them to the SED objects
kuruczColors = selectStarSED0.calcBasicColors(kuruczList, sdssPhot)
mltColors = selectStarSED0.calcBasicColors(mltList, sdssPhot)
hColors = selectStarSED0.calcBasicColors(wdListH, sdssPhot)
heColors = selectStarSED0.calcBasicColors(wdListHE, sdssPhot)
listDict = {
'kurucz': kuruczList,
'mlt': mltList,
'H': wdListH,
'HE': wdListHE
}
colorDict = {
'kurucz': kuruczColors,
'mlt': mltColors,
'H': hColors,
'HE': heColors
}
for filename, outFile in zip(filenameList, outFileList):
if filename.endswith('.txt'):
galfastIn = open(filename, 'rt')
inFits = False
gzFile = False
num_lines = sum(1 for line in open(filename))
elif filename.endswith('.gz'):
galfastIn = gzip.open(filename, 'rt')
inFits = False
gzFile = True
num_lines = sum(1 for line in gzip.open(filename))
elif filename.endswith('fits'):
hdulist = fits.open(filename)
galfastIn = hdulist[1].data
num_lines = len(galfastIn)
gzFile = False
inFits = True
if outFile.endswith('.txt'):
fOut = open(outFile, 'wt')
elif outFile.endswith('.gz'):
fOut = gzip.open(outFile, 'wt')
fOut.write('#oID, ra, dec, gall, galb, coordX, coordY, coordZ, sEDName, magNorm, ' +\
'LSSTugrizy, SDSSugriz, absSDSSr, pmRA, pmDec, vRad, pml, pmb, vRadlb, ' +\
'vR, vPhi, vZ, FeH, pop, distKpc, ebv, ebvInf\n')
header_length = 0
numChunks = 0
if inFits == False:
galfastDict = self.parseGalfast(galfastIn.readline())
header_length += 1
header_status = True
while header_status == True:
newLine = galfastIn.readline()
if newLine[0] != '#':
header_status = False
else:
header_length += 1
print('Total objects = %i' % (num_lines - header_length))
numChunks = ((num_lines - header_length) // chunkSize) + 1
for chunk in range(0, numChunks):
if chunk == numChunks - 1:
lastChunkSize = (num_lines - header_length) % chunkSize
readSize = lastChunkSize
else:
readSize = chunkSize
oID = np.arange(readSize * chunk, readSize * (chunk + 1))
if inFits:
starData = galfastIn[readSize * chunk:(readSize * chunk +
readSize)]
sDSS = starData.field('SDSSugriz')
gall, galb = np.transpose(starData.field('lb'))
ra, dec = np.transpose(starData.field('radec'))
coordX, coordY, coordZ = np.transpose(
starData.field('XYZ'))
DM = starData.field('DM')
absSDSSr = starData.field('absSDSSr')
pop = starData.field('comp')
FeH = starData.field('FeH')
vR, vPhi, vZ = np.transpose(starData.field('vcyl'))
pml, pmb, vRadlb = np.transpose(starData.field('pmlb'))
pmRA, pmDec, vRad = np.transpose(starData.field('pmradec'))
am = starData.field('Am')
amInf = starData.field('AmInf')
sdssPhotoFlags = starData.field('SDSSugrizPhotoFlags')
else:
if gzFile == False:
with open(filename) as t_in:
starData = np.loadtxt(
itertools.islice(
t_in, ((readSize * chunk) + header_length),
((readSize *
(chunk + 1)) + header_length)))
else:
with gzip.open(filename) as t_in:
starData = np.loadtxt(
itertools.islice(
t_in, ((readSize * chunk) + header_length),
((readSize *
(chunk + 1)) + header_length)))
starData = np.transpose(starData)
gall = starData[galfastDict['l']]
galb = starData[galfastDict['b']]
ra = starData[galfastDict['ra']]
dec = starData[galfastDict['dec']]
coordX = starData[galfastDict['X']]
coordY = starData[galfastDict['Y']]
coordZ = starData[galfastDict['Z']]
DM = starData[galfastDict['DM']]
absSDSSr = starData[galfastDict['absSDSSr']]
pop = starData[galfastDict['comp']]
FeH = starData[galfastDict['FeH']]
vR = starData[galfastDict['Vr']]
vPhi = starData[galfastDict['Vphi']]
vZ = starData[galfastDict['Vz']]
pml = starData[galfastDict['pml']]
pmb = starData[galfastDict['pmb']]
vRadlb = starData[galfastDict['vRadlb']]
pmRA = starData[galfastDict['pmra']]
pmDec = starData[galfastDict['pmdec']]
vRad = starData[galfastDict['vRad']]
am = starData[galfastDict['Am']]
amInf = starData[galfastDict['AmInf']]
sDSS = np.transpose(
starData[galfastDict['SDSSu']:galfastDict['SDSSz'] +
1])
sDSSPhotoFlags = starData[galfastDict['SDSSPhotoFlags']]
#End of input, now onto processing and output
sDSSunred = selectStarSED0.deReddenMags(
am, sDSS, sdssExtCoeffs)
if readSize == 1:
ra = np.array([ra])
dec = np.array([dec])
"""
Info about the following population cuts:
From Zeljko: "This color corresponds to the temperature (roughly spectral type M0) where
Kurucz models become increasingly bad, and thus we switch to empirical SEDs (the problem
is that for M and later stars, the effective surface temperature is low enough for
molecules to form, and their opacity is too complex to easily model, especially TiO)."
"""
mIn = np.where(((pop < 10) | (pop >= 20))
& (sDSSunred[:, 2] - sDSSunred[:, 3] > 0.59))
kIn = np.where(((pop < 10) | (pop >= 20))
& (sDSSunred[:, 2] - sDSSunred[:, 3] <= 0.59))
hIn = np.where((pop >= 10) & (pop < 15))
heIn = np.where((pop >= 15) & (pop < 20))
sEDNameK, magNormK, matchErrorK = selectStarSED0.findSED(
listDict['kurucz'],
sDSSunred[kIn],
ra[kIn],
dec[kIn],
reddening=False,
colors=colorDict['kurucz'])
sEDNameM, magNormM, matchErrorM = selectStarSED0.findSED(
listDict['mlt'],
sDSSunred[mIn],
ra[mIn],
dec[mIn],
reddening=False,
colors=colorDict['mlt'])
sEDNameH, magNormH, matchErrorH = selectStarSED0.findSED(
listDict['H'],
sDSSunred[hIn],
ra[hIn],
dec[hIn],
reddening=False,
colors=colorDict['H'])
sEDNameHE, magNormHE, matchErrorHE = selectStarSED0.findSED(
listDict['HE'],
sDSSunred[heIn],
ra[heIn],
dec[heIn],
reddening=False,
colors=colorDict['HE'])
chunkNames = np.empty(readSize, dtype='S32')
chunkTypes = np.empty(readSize, dtype='S8')
chunkMagNorms = np.zeros(readSize)
chunkMatchErrors = np.zeros(readSize)
chunkNames[kIn] = sEDNameK
chunkTypes[kIn] = 'kurucz'
chunkMagNorms[kIn] = magNormK
chunkMatchErrors[kIn] = matchErrorK
chunkNames[mIn] = sEDNameM
chunkTypes[mIn] = 'mlt'
chunkMagNorms[mIn] = magNormM
chunkMatchErrors[mIn] = matchErrorM
chunkNames[hIn] = sEDNameH
chunkTypes[hIn] = 'H'
chunkMagNorms[hIn] = magNormH
chunkMatchErrors[hIn] = matchErrorH
chunkNames[heIn] = sEDNameHE
chunkTypes[heIn] = 'HE'
chunkMagNorms[heIn] = magNormHE
chunkMatchErrors[heIn] = matchErrorHE
lsstMagsUnred = []
for sedName, sedType, magNorm, matchError in zip(
chunkNames.astype(str), chunkTypes.astype(str),
chunkMagNorms, chunkMatchErrors):
testSED = Sed()
testSED.setSED(
listDict[sedType][positionDict[sedName]].wavelen,
flambda=listDict[sedType][
positionDict[sedName]].flambda)
fluxNorm = testSED.calcFluxNorm(magNorm, imSimBand)
testSED.multiplyFluxNorm(fluxNorm)
lsstMagsUnred.append(lsstPhot.magListForSed(testSED))
#If the extinction value is negative then it will add the reddening back in
lsstMags = selectStarSED0.deReddenMags(
(-1.0 * am), lsstMagsUnred, lsstExtCoeffs)
distKpc = self.convDMtoKpc(DM)
ebv = am / 2.285 #From Schlafly and Finkbeiner 2011, (ApJ, 737, 103) for sdssr
ebvInf = amInf / 2.285
for line in range(0, readSize):
outFmt = '%i,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%s,' +\
'%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%i,%3.7f,%3.7f,%3.7f\n'
if readSize == 1:
if inFits == True:
sDSS = sDSS[0]
outDat = (oID, ra[line], dec[line], gall, galb, coordX,
coordY, coordZ, chunkNames, chunkMagNorms,
chunkMatchErrors, lsstMags[line][0],
lsstMags[line][1], lsstMags[line][2],
lsstMags[line][3], lsstMags[line][4],
lsstMags[line][5], sDSS[0], sDSS[1], sDSS[2],
sDSS[3], sDSS[4], absSDSSr, pmRA, pmDec,
vRad, pml, pmb, vRadlb, vR, vPhi, vZ, FeH,
pop, distKpc, ebv, ebvInf)
else:
outDat = (oID[line], ra[line], dec[line], gall[line],
galb[line], coordX[line], coordY[line],
coordZ[line], chunkNames[line],
chunkMagNorms[line], chunkMatchErrors[line],
lsstMags[line][0], lsstMags[line][1],
lsstMags[line][2], lsstMags[line][3],
lsstMags[line][4], lsstMags[line][5],
sDSS[line][0], sDSS[line][1], sDSS[line][2],
sDSS[line][3], sDSS[line][4], absSDSSr[line],
pmRA[line], pmDec[line], vRad[line],
pml[line], pmb[line], vRadlb[line], vR[line],
vPhi[line], vZ[line], FeH[line], pop[line],
distKpc[line], ebv[line], ebvInf[line])
fOut.write(outFmt % outDat)
print('Chunk Num Done = %i out of %i' % (chunk + 1, numChunks))
def testFindSED(self):
"""Pull SEDs from each type and make sure that each SED gets matched to itself.
Includes testing with extinction and passing in only colors."""
rng = np.random.RandomState(42)
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'), 'sdss')
starPhot = BandpassDict.loadTotalBandpassesFromFiles(('u', 'g', 'r', 'i', 'z'),
bandpassDir = bandpassDir,
bandpassRoot = 'sdss_')
imSimBand = Bandpass()
imSimBand.imsimBandpass()
testMatching = selectStarSED(kuruczDir=self.testKDir,
mltDir=self.testMLTDir,
wdDir=self.testWDDir)
testSEDList = []
testSEDList.append(testMatching.loadKuruczSEDs())
testSEDList.append(testMatching.loadmltSEDs())
testSEDListH, testSEDListHE = testMatching.loadwdSEDs()
testSEDList.append(testSEDListH)
testSEDList.append(testSEDListHE)
testSEDNames = []
testMags = []
testMagNormList = []
magNormStep = 1
for typeList in testSEDList:
if len(typeList) != 0:
typeSEDNames = []
typeMags = []
typeMagNorms = []
for testSED in typeList:
getSEDMags = Sed()
typeSEDNames.append(testSED.name)
getSEDMags.setSED(wavelen = testSED.wavelen, flambda = testSED.flambda)
testMagNorm = np.round(rng.uniform(20.0, 22.0), magNormStep)
typeMagNorms.append(testMagNorm)
fluxNorm = getSEDMags.calcFluxNorm(testMagNorm, imSimBand)
getSEDMags.multiplyFluxNorm(fluxNorm)
typeMags.append(starPhot.magListForSed(getSEDMags))
testSEDNames.append(typeSEDNames)
testMags.append(typeMags)
testMagNormList.append(typeMagNorms)
# Since default bandpassDict should be SDSS ugrizy shouldn't need to specify it
# Substitute in nan values to simulate incomplete data.
for typeList, names, mags, magNorms in zip(testSEDList, testSEDNames, testMags, testMagNormList):
if len(typeList) > 2:
nanMags = np.array(mags)
nanMags[0][0] = np.nan
nanMags[0][2] = np.nan
nanMags[0][3] = np.nan
nanMags[1][1] = np.nan
testMatchingResults = testMatching.findSED(typeList, nanMags, reddening = False)
self.assertEqual(None, testMatchingResults[0][0])
self.assertEqual(names[1:], testMatchingResults[0][1:])
self.assertEqual(None, testMatchingResults[1][0])
np.testing.assert_almost_equal(magNorms[1:], testMatchingResults[1][1:],
decimal = magNormStep)
else:
testMatchingResults = testMatching.findSED(typeList, mags, reddening = False)
self.assertEqual(names, testMatchingResults[0])
np.testing.assert_almost_equal(magNorms, testMatchingResults[1], decimal = magNormStep)
# Test Null Values option
nullMags = np.array(testMags[0])
nullMags[0][0] = -99.
nullMags[0][4] = -99.
nullMags[1][0] = -99.
nullMags[1][1] = -99.
testMatchingResultsNull = testMatching.findSED(testSEDList[0], nullMags,
nullValues = -99., reddening = False)
self.assertEqual(testSEDNames[0], testMatchingResultsNull[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsNull[1],
decimal = magNormStep)
# Test Error Output
errMags = np.array((testMags[0][0], testMags[0][0], testMags[0][0], testMags[0][0]))
errMags[1, 1] += 1. # Total MSE will be 2/(4 colors) = 0.5
errMags[2, 0:2] = np.nan
errMags[2, 3] += 1. # Total MSE will be 2/(2 colors) = 1.0
errMags[3, :] = None
errSED = testSEDList[0][0]
testMatchingResultsErrors = testMatching.findSED([errSED], errMags, reddening = False)
np.testing.assert_almost_equal(np.array((0.0, 0.5, 1.0)), testMatchingResultsErrors[2][0:3],
decimal = 3)
self.assertEqual(None, testMatchingResultsErrors[2][3])
# Now test what happens if we pass in a bandpassDict
testMatchingResultsNoDefault = testMatching.findSED(testSEDList[0], testMags[0],
bandpassDict = starPhot,
reddening = False)
self.assertEqual(testSEDNames[0], testMatchingResultsNoDefault[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsNoDefault[1],
decimal = magNormStep)
# Test Reddening
testRA = rng.uniform(10, 170, len(testSEDList[0]))
testDec = rng.uniform(10, 80, len(testSEDList[0]))
extFactor = .5
raDec = np.array((testRA, testDec))
ebvVals = ebv().calculateEbv(equatorialCoordinates = raDec)
extVals = ebvVals*extFactor
testRedMags = []
for extVal, testMagSet in zip(extVals, testMags[0]):
testRedMags.append(testMagSet + extVal)
testMatchingResultsRed = testMatching.findSED(testSEDList[0], testRedMags, catRA = testRA,
catDec = testDec, reddening = True,
extCoeffs = np.ones(5)*extFactor)
self.assertEqual(testSEDNames[0], testMatchingResultsRed[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsRed[1],
decimal = magNormStep)
# Finally, test color input
testColors = []
for testMagSet in testMags[0]:
testColorSet = []
for filtNum in range(0, len(starPhot)-1):
testColorSet.append(testMagSet[filtNum] - testMagSet[filtNum+1])
testColors.append(testColorSet)
testMatchingColorsInput = testMatching.findSED(testSEDList[0], testMags[0],
reddening = False, colors = testColors)
self.assertEqual(testSEDNames[0], testMatchingColorsInput[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingColorsInput[1],
decimal = magNormStep)
def testFindSED(self):
"""Pull SEDs from each type and make sure that each SED gets matched to itself.
Includes testing with extinction and passing in only colors."""
np.random.seed(42)
starPhot = BandpassDict.loadTotalBandpassesFromFiles(('u','g','r','i','z'),
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'),'sdss'),
bandpassRoot = 'sdss_')
imSimBand = Bandpass()
imSimBand.imsimBandpass()
testMatching = selectStarSED(sEDDir = self.testSpecDir, kuruczDir = self.testKDir,
mltDir = self.testMLTDir, wdDir = self.testWDDir)
testSEDList = []
testSEDList.append(testMatching.loadKuruczSEDs())
testSEDList.append(testMatching.loadmltSEDs())
testSEDListH, testSEDListHE = testMatching.loadwdSEDs()
testSEDList.append(testSEDListH)
testSEDList.append(testSEDListHE)
testSEDNames = []
testMags = []
testMagNormList = []
magNormStep = 1
for typeList in testSEDList:
if len(typeList) != 0:
typeSEDNames = []
typeMags = []
typeMagNorms = []
for testSED in typeList:
getSEDMags = Sed()
typeSEDNames.append(testSED.name)
getSEDMags.setSED(wavelen = testSED.wavelen, flambda = testSED.flambda)
testMagNorm = np.round(np.random.uniform(20.0,22.0),magNormStep)
typeMagNorms.append(testMagNorm)
fluxNorm = getSEDMags.calcFluxNorm(testMagNorm, imSimBand)
getSEDMags.multiplyFluxNorm(fluxNorm)
typeMags.append(starPhot.magListForSed(getSEDMags))
testSEDNames.append(typeSEDNames)
testMags.append(typeMags)
testMagNormList.append(typeMagNorms)
fakeRA = np.ones(len(testSEDList[0]))
fakeDec = np.ones(len(testSEDList[0]))
#Since default bandpassDict should be SDSS ugrizy shouldn't need to specify it
#Substitute in nan values to simulate incomplete data.
for typeList, names, mags, magNorms in zip(testSEDList, testSEDNames, testMags, testMagNormList):
if len(typeList) > 2:
nanMags = np.array(mags)
nanMags[0][0] = np.nan
nanMags[0][2] = np.nan
nanMags[0][3] = np.nan
nanMags[1][1] = np.nan
testMatchingResults = testMatching.findSED(typeList, nanMags, reddening = False)
self.assertEqual(None, testMatchingResults[0][0])
self.assertEqual(names[1:], testMatchingResults[0][1:])
self.assertEqual(None, testMatchingResults[1][0])
np.testing.assert_almost_equal(magNorms[1:], testMatchingResults[1][1:],
decimal = magNormStep)
else:
testMatchingResults = testMatching.findSED(typeList, mags, reddening = False)
self.assertEqual(names, testMatchingResults[0])
np.testing.assert_almost_equal(magNorms, testMatchingResults[1], decimal = magNormStep)
#Test Null Values option
nullMags = np.array(testMags[0])
nullMags[0][0] = -99.
nullMags[0][4] = -99.
nullMags[1][0] = -99.
nullMags[1][1] = -99.
testMatchingResultsNull = testMatching.findSED(testSEDList[0], nullMags,
nullValues = -99., reddening = False)
self.assertEqual(testSEDNames[0], testMatchingResultsNull[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsNull[1],
decimal = magNormStep)
#Test Error Output
errMags = np.array((testMags[0][0], testMags[0][0], testMags[0][0], testMags[0][0]))
errMags[1,1] += 1. #Total MSE will be 2/(4 colors) = 0.5
errMags[2, 0:2] = np.nan
errMags[2, 3] += 1. #Total MSE will be 2/(2 colors) = 1.0
errMags[3, :] = None
errSED = testSEDList[0][0]
testMatchingResultsErrors = testMatching.findSED([errSED], errMags, reddening = False)
np.testing.assert_almost_equal(np.array((0.0, 0.5, 1.0)), testMatchingResultsErrors[2][0:3],
decimal = 3)
self.assertEqual(None, testMatchingResultsErrors[2][3])
#Now test what happens if we pass in a bandpassDict
testMatchingResultsNoDefault = testMatching.findSED(testSEDList[0], testMags[0],
bandpassDict = starPhot,
reddening = False)
self.assertEqual(testSEDNames[0], testMatchingResultsNoDefault[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsNoDefault[1],
decimal = magNormStep)
#Test Reddening
testRA = np.random.uniform(10,170,len(testSEDList[0]))
testDec = np.random.uniform(10,80,len(testSEDList[0]))
extFactor = .5
raDec = np.array((testRA, testDec))
ebvVals = ebv().calculateEbv(equatorialCoordinates = raDec)
extVals = ebvVals*extFactor
testRedMags = []
for extVal, testMagSet in zip(extVals, testMags[0]):
testRedMags.append(testMagSet + extVal)
testMatchingResultsRed = testMatching.findSED(testSEDList[0], testRedMags, catRA = testRA,
catDec = testDec, reddening = True,
extCoeffs = np.ones(5)*extFactor)
self.assertEqual(testSEDNames[0], testMatchingResultsRed[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsRed[1],
decimal = magNormStep)
#Finally, test color input
testColors = []
for testMagSet in testMags[0]:
testColorSet = []
for filtNum in range(0, len(starPhot)-1):
testColorSet.append(testMagSet[filtNum] - testMagSet[filtNum+1])
testColors.append(testColorSet)
testMatchingColorsInput = testMatching.findSED(testSEDList[0], testMags[0],
reddening = False, colors = testColors)
self.assertEqual(testSEDNames[0], testMatchingColorsInput[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingColorsInput[1],
decimal = magNormStep)
def loadGalfast(self, filenameList, outFileList, sEDPath = None, kuruczPath = None,
mltPath = None, wdPath = None, kuruczSubset = None,
mltSubset = None, wdSubset = None, chunkSize = 10000):
"""
This is customized for the outputs we currently need for the purposes of consistent output
It will read in a galfast output file and output desired values for database input into a file
@param [in] filenameList is a list of the galfast output files that will be loaded and processed.
Can process fits, gzipped, or txt output from galfast.
@param [in] outFileList is a list of the names of the output files that will be created. If gzipped
output is desired simply write the filenames with .gz at the end.
@param [in] kuruczPath is a place to specify a different path to kurucz SED files than the
files in the LSST sims_sed_library. If set to None it will default to the LSST library.
Will probably be most useful for those who want to use loadGalfast without downloading the
entire LSST sims_sed_library which contains much more than just the star SEDs.
@param [in] mltPath is the same as kuruczPath except that it specifies a directory for the
mlt SEDs
@param [in] wdPath is the same as the previous two except that it specifies a path to an
alternate white dwarf SED directory.
@param [in] kuruczSubset is a list which provides a subset of the kurucz files within the
kurucz folder that one wants to use
@param [in] mltSubset is a list which provides a subset of the mlt files within the
mlt folder that one wants to use
@param [in] wdSubset is a list which provides a subset of the wd files within the
wd folder that one wants to use
@param [in] chunkSize is the size of chunks of lines to be read from the catalog at one time.
"""
for filename in filenameList:
#Make sure input file exists and is readable format before doing anything else
if os.path.isfile(filename) == False:
raise RuntimeError, '*** File does not exist'
#Process various possible galfast outputs
if filename.endswith(('.txt', '.gz', '.fits')):
continue
else:
raise RuntimeError, str('*** Unsupported File Format in file: ' + str(filename))
#If all files exist and are in proper formats then load seds
selectStarSED0 = selectStarSED(sEDDir = sEDPath, kuruczDir = kuruczPath,
mltDir = mltPath, wdDir = wdPath)
if kuruczSubset is None:
kuruczList = selectStarSED0.loadKuruczSEDs()
else:
kuruczList = selectStarSED0.loadKuruczSEDs(subset = kuruczSubset)
#Only need one dictionary since none of the names overlap
positionDict = {}
for kNum, kuruczSED in enumerate(kuruczList):
positionDict[kuruczSED.name] = kNum
if mltSubset is None:
mltList = selectStarSED0.loadmltSEDs()
else:
mltList = selectStarSED0.loadmltSEDs(subset = mltSubset)
for mltNum, mltSED in enumerate(mltList):
positionDict[mltSED.name] = mltNum
if wdSubset is None:
wdListH, wdListHE = selectStarSED0.loadwdSEDs()
else:
wdListH, wdListHE = selectStarSED0.loadwdSEDs(subset = wdSubset)
for hNum, hSED in enumerate(wdListH):
positionDict[hSED.name] = hNum
for heNum, heSED in enumerate(wdListHE):
positionDict[heSED.name] = heNum
#For adding/subtracting extinction when calculating colors
#Numbers below come from Schlafly and Finkbeiner (2011) (ApJ, 737, 103)
#normalized by SDSS r mag value
sdssExtCoeffs = [1.8551, 1.4455, 1.0, 0.7431, 0.5527]
lsstExtCoeffs = [1.8140, 1.4166, 0.9947, 0.7370, 0.5790, 0.4761]
sdssPhot = BandpassDict.loadTotalBandpassesFromFiles(['u','g','r','i','z'],
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'),
'sdss'),
bandpassRoot = 'sdss_')
#Load Bandpasses for LSST colors to get colors from matched SEDs
lsstFilterList = ('u', 'g', 'r', 'i', 'z', 'y')
lsstPhot = BandpassDict.loadTotalBandpassesFromFiles(lsstFilterList)
imSimBand = Bandpass()
imSimBand.imsimBandpass()
#Calculate colors and add them to the SED objects
kuruczColors = selectStarSED0.calcBasicColors(kuruczList, sdssPhot)
mltColors = selectStarSED0.calcBasicColors(mltList, sdssPhot)
hColors = selectStarSED0.calcBasicColors(wdListH, sdssPhot)
heColors = selectStarSED0.calcBasicColors(wdListHE, sdssPhot)
listDict = {'kurucz':kuruczList, 'mlt':mltList, 'H':wdListH, 'HE':wdListHE}
colorDict = {'kurucz':kuruczColors, 'mlt':mltColors, 'H':hColors, 'HE':heColors}
for filename, outFile in zip(filenameList, outFileList):
if filename.endswith('.txt'):
galfastIn = open(filename, 'r')
inFits = False
gzFile = False
num_lines = sum(1 for line in open(filename))
elif filename.endswith('.gz'):
galfastIn = gzip.open(filename, 'r')
inFits = False
gzFile = True
num_lines = sum(1 for line in gzip.open(filename))
elif filename.endswith('fits'):
hdulist = pyfits.open(filename)
galfastIn = hdulist[1].data
num_lines = len(galfastIn)
gzFile = False
inFits = True
if outFile.endswith('.txt'):
fOut = open(outFile, 'w')
elif outFile.endswith('.gz'):
fOut = gzip.open(outFile, 'w')
fOut.write('#oID, ra, dec, gall, galb, coordX, coordY, coordZ, sEDName, magNorm, ' +\
'LSSTugrizy, SDSSugriz, absSDSSr, pmRA, pmDec, vRad, pml, pmb, vRadlb, ' +\
'vR, vPhi, vZ, FeH, pop, distKpc, ebv, ebvInf\n')
header_length = 0
numChunks = 0
if inFits == False:
galfastDict = self.parseGalfast(galfastIn.readline())
header_length += 1
header_status = True
while header_status == True:
newLine = galfastIn.readline()
if newLine[0] != '#':
header_status = False
else:
header_length += 1
print 'Total objects = %i' % (num_lines - header_length)
numChunks = ((num_lines-header_length)/chunkSize) + 1
for chunk in range(0,numChunks):
if chunk == numChunks-1:
lastChunkSize = (num_lines - header_length) % chunkSize
readSize = lastChunkSize
else:
readSize = chunkSize
oID = np.arange(readSize*chunk, readSize*(chunk+1))
if inFits:
starData = galfastIn[readSize*chunk:(readSize*chunk + readSize)]
sDSS = starData.field('SDSSugriz')
gall, galb = np.transpose(starData.field('lb'))
ra, dec = np.transpose(starData.field('radec'))
coordX, coordY, coordZ = np.transpose(starData.field('XYZ'))
DM = starData.field('DM')
absSDSSr = starData.field('absSDSSr')
pop = starData.field('comp')
FeH = starData.field('FeH')
vR, vPhi, vZ = np.transpose(starData.field('vcyl'))
pml, pmb, vRadlb = np.transpose(starData.field('pmlb'))
pmRA, pmDec, vRad = np.transpose(starData.field('pmradec'))
am = starData.field('Am')
amInf = starData.field('AmInf')
sdssPhotoFlags = starData.field('SDSSugrizPhotoFlags')
else:
if gzFile == False:
with open(filename) as t_in:
starData = np.loadtxt(itertools.islice(t_in,((readSize*chunk)+header_length),
((readSize*(chunk+1))+header_length)))
else:
with gzip.open(filename) as t_in:
starData = np.loadtxt(itertools.islice(t_in,((readSize*chunk)+header_length),
((readSize*(chunk+1))+header_length)))
starData = np.transpose(starData)
gall = starData[galfastDict['l']]
galb = starData[galfastDict['b']]
ra = starData[galfastDict['ra']]
dec = starData[galfastDict['dec']]
coordX = starData[galfastDict['X']]
coordY = starData[galfastDict['Y']]
coordZ = starData[galfastDict['Z']]
DM = starData[galfastDict['DM']]
absSDSSr = starData[galfastDict['absSDSSr']]
pop = starData[galfastDict['comp']]
FeH = starData[galfastDict['FeH']]
vR = starData[galfastDict['Vr']]
vPhi = starData[galfastDict['Vphi']]
vZ = starData[galfastDict['Vz']]
pml = starData[galfastDict['pml']]
pmb = starData[galfastDict['pmb']]
vRadlb = starData[galfastDict['vRadlb']]
pmRA = starData[galfastDict['pmra']]
pmDec = starData[galfastDict['pmdec']]
vRad = starData[galfastDict['vRad']]
am = starData[galfastDict['Am']]
amInf = starData[galfastDict['AmInf']]
sDSS = np.transpose(starData[galfastDict['SDSSu']:galfastDict['SDSSz']+1])
sDSSPhotoFlags = starData[galfastDict['SDSSPhotoFlags']]
#End of input, now onto processing and output
sDSSunred = selectStarSED0.deReddenMags(am, sDSS, sdssExtCoeffs)
if readSize == 1:
ra = np.array([ra])
dec = np.array([dec])
"""
Info about the following population cuts:
From Zeljko: "This color corresponds to the temperature (roughly spectral type M0) where
Kurucz models become increasingly bad, and thus we switch to empirical SEDs (the problem
is that for M and later stars, the effective surface temperature is low enough for
molecules to form, and their opacity is too complex to easily model, especially TiO)."
"""
mIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:,2] - sDSSunred[:,3] > 0.59))
kIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:,2] - sDSSunred[:,3] <= 0.59))
hIn = np.where((pop >= 10) & (pop < 15))
heIn = np.where((pop >= 15) & (pop < 20))
sEDNameK, magNormK, matchErrorK = selectStarSED0.findSED(listDict['kurucz'],
sDSSunred[kIn], ra[kIn], dec[kIn],
reddening = False,
colors = colorDict['kurucz'])
sEDNameM, magNormM, matchErrorM = selectStarSED0.findSED(listDict['mlt'],
sDSSunred[mIn], ra[mIn], dec[mIn],
reddening = False,
colors = colorDict['mlt'])
sEDNameH, magNormH, matchErrorH = selectStarSED0.findSED(listDict['H'],
sDSSunred[hIn], ra[hIn], dec[hIn],
reddening = False,
colors = colorDict['H'])
sEDNameHE, magNormHE, matchErrorHE = selectStarSED0.findSED(listDict['HE'],
sDSSunred[heIn],
ra[heIn], dec[heIn],
reddening = False,
colors = colorDict['HE'])
chunkNames = np.empty(readSize, dtype = 'S32')
chunkTypes = np.empty(readSize, dtype = 'S8')
chunkMagNorms = np.zeros(readSize)
chunkMatchErrors = np.zeros(readSize)
chunkNames[kIn] = sEDNameK
chunkTypes[kIn] = 'kurucz'
chunkMagNorms[kIn] = magNormK
chunkMatchErrors[kIn] = matchErrorK
chunkNames[mIn] = sEDNameM
chunkTypes[mIn] = 'mlt'
chunkMagNorms[mIn] = magNormM
chunkMatchErrors[mIn] = matchErrorM
chunkNames[hIn] = sEDNameH
chunkTypes[hIn] = 'H'
chunkMagNorms[hIn] = magNormH
chunkMatchErrors[hIn] = matchErrorH
chunkNames[heIn] = sEDNameHE
chunkTypes[heIn] = 'HE'
chunkMagNorms[heIn] = magNormHE
chunkMatchErrors[heIn] = matchErrorHE
lsstMagsUnred = []
for sedName, sedType, magNorm, matchError in zip(chunkNames, chunkTypes, chunkMagNorms,
chunkMatchErrors):
testSED = Sed()
testSED.setSED(listDict[sedType][positionDict[sedName]].wavelen,
flambda = listDict[sedType][positionDict[sedName]].flambda)
fluxNorm = testSED.calcFluxNorm(magNorm, imSimBand)
testSED.multiplyFluxNorm(fluxNorm)
lsstMagsUnred.append(lsstPhot.magListForSed(testSED))
#If the extinction value is negative then it will add the reddening back in
lsstMags = selectStarSED0.deReddenMags((-1.0*am), lsstMagsUnred,
lsstExtCoeffs)
distKpc = self.convDMtoKpc(DM)
ebv = am / 2.285 #From Schlafly and Finkbeiner 2011, (ApJ, 737, 103) for sdssr
ebvInf = amInf / 2.285
for line in range(0, readSize):
outFmt = '%i,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%s,' +\
'%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%i,%3.7f,%3.7f,%3.7f\n'
if readSize == 1:
if inFits == True:
sDSS = sDSS[0]
outDat = (oID, ra[line], dec[line], gall, galb, coordX,
coordY, coordZ, chunkNames,
chunkMagNorms, chunkMatchErrors,
lsstMags[line][0], lsstMags[line][1], lsstMags[line][2],
lsstMags[line][3], lsstMags[line][4], lsstMags[line][5],
sDSS[0], sDSS[1], sDSS[2], sDSS[3],
sDSS[4], absSDSSr, pmRA, pmDec, vRad,
pml, pmb, vRadlb, vR, vPhi, vZ,
FeH, pop, distKpc, ebv, ebvInf)
else:
outDat = (oID[line], ra[line], dec[line], gall[line], galb[line], coordX[line],
coordY[line], coordZ[line], chunkNames[line],
chunkMagNorms[line], chunkMatchErrors[line],
lsstMags[line][0], lsstMags[line][1], lsstMags[line][2],
lsstMags[line][3], lsstMags[line][4], lsstMags[line][5],
sDSS[line][0], sDSS[line][1], sDSS[line][2], sDSS[line][3],
sDSS[line][4], absSDSSr[line], pmRA[line], pmDec[line], vRad[line],
pml[line], pmb[line], vRadlb[line], vR[line], vPhi[line], vZ[line],
FeH[line], pop[line], distKpc[line], ebv[line], ebvInf[line])
fOut.write(outFmt % outDat)
print 'Chunk Num Done = %i out of %i' % (chunk+1, numChunks)
