def testReddeningException(self): """Test that if reddening=True in matchToObserved CatRA & CatDec are defined or exception is raised""" testException = selectStarSED(sEDDir = self.testSpecDir, kuruczDir = self.testKDir, mltDir = self.testMLTDir, wdDir = self.testWDDir) testSEDList = testException.loadKuruczSEDs() magnitudes = [[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]] self.assertRaises(RuntimeError, testException.findSED, testSEDList, magnitudes, reddening = True)
def testLoadWD(self): """Test SED loading algorithm by making sure SEDs are all accounted for and that there are separate lists for H and HE.""" # Test Matching to WD SEDs loadTestWD = matchStar(wdDir = self.testWDDir) testSEDsH, testSEDsHE = loadTestWD.loadwdSEDs() # Add extra step because WD SEDs are separated into helium and hydrogen testNames = [] for testH in testSEDsH: testNames.append(testH.name) for testHE in testSEDsHE: testNames.append(testHE.name) # Read in a list of the SEDs in the wd test sed directory testWDList = os.listdir(self.testWDDir) # First make sure that all SEDs are correctly accounted for if no subset provided # Python 3 replaces assertItemsEqual() with assertCountEqual() if hasattr(self, 'assertItemsEqual'): self.assertItemsEqual(testNames, testWDList) else: self.assertCountEqual(testNames, testWDList) # Test same condition if subset is provided testSubsetList = ['bergeron_10000_75.dat_10100.gz', 'bergeron_He_9000_80.dat_9400.gz'] testSEDsSubsetH, testSEDsSubsetHE = selectStarSED(wdDir= self.testWDDir).loadwdSEDs(subset= testSubsetList) testNamesSubset = [] for testH in testSEDsSubsetH: testNamesSubset.append(testH.name) for testHE in testSEDsSubsetHE: testNamesSubset.append(testHE.name) # Next make sure that correct subset loads if subset is provided # Python 3 replaces assertItemsEqual() with assertCountEqual() if hasattr(self, 'assertItemsEqual'): self.assertItemsEqual(testNamesSubset, testSubsetList) else: self.assertCountEqual(testNamesSubset, testSubsetList) # Make sure that the names get separated into correct wd type self.assertEqual(testSEDsSubsetH[0].name, testSubsetList[0]) self.assertEqual(testSEDsSubsetHE[0].name, testSubsetList[1])
def testLoadWD(self): """Test SED loading algorithm by making sure SEDs are all accounted for and that there are separate lists for H and HE.""" #Test Matching to WD SEDs loadTestWD = matchStar(wdDir = self.testWDDir) testSEDsH, testSEDsHE = loadTestWD.loadwdSEDs() #Add extra step because WD SEDs are separated into helium and hydrogen testNames = [] for testH in testSEDsH: testNames.append(testH.name) for testHE in testSEDsHE: testNames.append(testHE.name) #Read in a list of the SEDs in the wd test sed directory testWDList = os.listdir(self.testWDDir) #First make sure that all SEDs are correctly accounted for if no subset provided self.assertItemsEqual(testNames, testWDList) #Test same condition if subset is provided testSubsetList = ['bergeron_10000_75.dat_10100.gz', 'bergeron_He_9000_80.dat_9400.gz'] testSEDsSubsetH, testSEDsSubsetHE = selectStarSED().loadwdSEDs(subset = testSubsetList) testNamesSubset = [] for testH in testSEDsSubsetH: testNamesSubset.append(testH.name) for testHE in testSEDsSubsetHE: testNamesSubset.append(testHE.name) #Next make sure that correct subset loads if subset is provided self.assertItemsEqual(testNamesSubset, testSubsetList) #Make sure that the names get separated into correct wd type self.assertEqual(testSEDsSubsetH[0].name, testSubsetList[0]) self.assertEqual(testSEDsSubsetHE[0].name, testSubsetList[1])
def loadGalfast(self, filenameList, outFileList, sEDPath=None, kuruczPath=None, mltPath=None, wdPath=None, kuruczSubset=None, mltSubset=None, wdSubset=None, chunkSize=10000): """ This is customized for the outputs we currently need for the purposes of consistent output It will read in a galfast output file and output desired values for database input into a file @param [in] filenameList is a list of the galfast output files that will be loaded and processed. Can process fits, gzipped, or txt output from galfast. @param [in] outFileList is a list of the names of the output files that will be created. If gzipped output is desired simply write the filenames with .gz at the end. @param [in] kuruczPath is a place to specify a path to kurucz SED files @param [in] mltPath is the same as kuruczPath except that it specifies a directory for the mlt SEDs @param [in] wdPath is the same as the previous two except that it specifies a path to a white dwarf SED directory. @param [in] kuruczSubset is a list which provides a subset of the kurucz files within the kurucz folder that one wants to use @param [in] mltSubset is a list which provides a subset of the mlt files within the mlt folder that one wants to use @param [in] wdSubset is a list which provides a subset of the wd files within the wd folder that one wants to use @param [in] chunkSize is the size of chunks of lines to be read from the catalog at one time. """ for filename in filenameList: #Make sure input file exists and is readable format before doing anything else if os.path.isfile(filename) == False: raise RuntimeError('*** File does not exist') #Process various possible galfast outputs if filename.endswith(('.txt', '.gz', '.fits')): continue else: raise RuntimeError( str('*** Unsupported File Format in file: ' + str(filename))) #If all files exist and are in proper formats then load seds selectStarSED0 = selectStarSED(kuruczDir=kuruczPath, mltDir=mltPath, wdDir=wdPath) if kuruczSubset is None: kuruczList = selectStarSED0.loadKuruczSEDs() else: kuruczList = selectStarSED0.loadKuruczSEDs(subset=kuruczSubset) #Only need one dictionary since none of the names overlap positionDict = {} for kNum, kuruczSED in enumerate(kuruczList): positionDict[kuruczSED.name] = kNum if mltSubset is None: mltList = selectStarSED0.loadmltSEDs() else: mltList = selectStarSED0.loadmltSEDs(subset=mltSubset) for mltNum, mltSED in enumerate(mltList): positionDict[mltSED.name] = mltNum if wdSubset is None: wdListH, wdListHE = selectStarSED0.loadwdSEDs() else: wdListH, wdListHE = selectStarSED0.loadwdSEDs(subset=wdSubset) for hNum, hSED in enumerate(wdListH): positionDict[hSED.name] = hNum for heNum, heSED in enumerate(wdListHE): positionDict[heSED.name] = heNum #For adding/subtracting extinction when calculating colors #Numbers below come from Schlafly and Finkbeiner (2011) (ApJ, 737, 103) #normalized by SDSS r mag value sdssExtCoeffs = [1.8551, 1.4455, 1.0, 0.7431, 0.5527] lsstExtCoeffs = [1.8140, 1.4166, 0.9947, 0.7370, 0.5790, 0.4761] sdssPhot = BandpassDict.loadTotalBandpassesFromFiles( ['u', 'g', 'r', 'i', 'z'], bandpassDir=os.path.join(lsst.utils.getPackageDir('throughputs'), 'sdss'), bandpassRoot='sdss_') #Load Bandpasses for LSST colors to get colors from matched SEDs lsstFilterList = ('u', 'g', 'r', 'i', 'z', 'y') lsstPhot = BandpassDict.loadTotalBandpassesFromFiles(lsstFilterList) imSimBand = Bandpass() imSimBand.imsimBandpass() #Calculate colors and add them to the SED objects kuruczColors = selectStarSED0.calcBasicColors(kuruczList, sdssPhot) mltColors = selectStarSED0.calcBasicColors(mltList, sdssPhot) hColors = selectStarSED0.calcBasicColors(wdListH, sdssPhot) heColors = selectStarSED0.calcBasicColors(wdListHE, sdssPhot) listDict = { 'kurucz': kuruczList, 'mlt': mltList, 'H': wdListH, 'HE': wdListHE } colorDict = { 'kurucz': kuruczColors, 'mlt': mltColors, 'H': hColors, 'HE': heColors } for filename, outFile in zip(filenameList, outFileList): if filename.endswith('.txt'): galfastIn = open(filename, 'rt') inFits = False gzFile = False num_lines = sum(1 for line in open(filename)) elif filename.endswith('.gz'): galfastIn = gzip.open(filename, 'rt') inFits = False gzFile = True num_lines = sum(1 for line in gzip.open(filename)) elif filename.endswith('fits'): hdulist = fits.open(filename) galfastIn = hdulist[1].data num_lines = len(galfastIn) gzFile = False inFits = True if outFile.endswith('.txt'): fOut = open(outFile, 'wt') elif outFile.endswith('.gz'): fOut = gzip.open(outFile, 'wt') fOut.write('#oID, ra, dec, gall, galb, coordX, coordY, coordZ, sEDName, magNorm, ' +\ 'LSSTugrizy, SDSSugriz, absSDSSr, pmRA, pmDec, vRad, pml, pmb, vRadlb, ' +\ 'vR, vPhi, vZ, FeH, pop, distKpc, ebv, ebvInf\n') header_length = 0 numChunks = 0 if inFits == False: galfastDict = self.parseGalfast(galfastIn.readline()) header_length += 1 header_status = True while header_status == True: newLine = galfastIn.readline() if newLine[0] != '#': header_status = False else: header_length += 1 print('Total objects = %i' % (num_lines - header_length)) numChunks = ((num_lines - header_length) // chunkSize) + 1 for chunk in range(0, numChunks): if chunk == numChunks - 1: lastChunkSize = (num_lines - header_length) % chunkSize readSize = lastChunkSize else: readSize = chunkSize oID = np.arange(readSize * chunk, readSize * (chunk + 1)) if inFits: starData = galfastIn[readSize * chunk:(readSize * chunk + readSize)] sDSS = starData.field('SDSSugriz') gall, galb = np.transpose(starData.field('lb')) ra, dec = np.transpose(starData.field('radec')) coordX, coordY, coordZ = np.transpose( starData.field('XYZ')) DM = starData.field('DM') absSDSSr = starData.field('absSDSSr') pop = starData.field('comp') FeH = starData.field('FeH') vR, vPhi, vZ = np.transpose(starData.field('vcyl')) pml, pmb, vRadlb = np.transpose(starData.field('pmlb')) pmRA, pmDec, vRad = np.transpose(starData.field('pmradec')) am = starData.field('Am') amInf = starData.field('AmInf') sdssPhotoFlags = starData.field('SDSSugrizPhotoFlags') else: if gzFile == False: with open(filename) as t_in: starData = np.loadtxt( itertools.islice( t_in, ((readSize * chunk) + header_length), ((readSize * (chunk + 1)) + header_length))) else: with gzip.open(filename) as t_in: starData = np.loadtxt( itertools.islice( t_in, ((readSize * chunk) + header_length), ((readSize * (chunk + 1)) + header_length))) starData = np.transpose(starData) gall = starData[galfastDict['l']] galb = starData[galfastDict['b']] ra = starData[galfastDict['ra']] dec = starData[galfastDict['dec']] coordX = starData[galfastDict['X']] coordY = starData[galfastDict['Y']] coordZ = starData[galfastDict['Z']] DM = starData[galfastDict['DM']] absSDSSr = starData[galfastDict['absSDSSr']] pop = starData[galfastDict['comp']] FeH = starData[galfastDict['FeH']] vR = starData[galfastDict['Vr']] vPhi = starData[galfastDict['Vphi']] vZ = starData[galfastDict['Vz']] pml = starData[galfastDict['pml']] pmb = starData[galfastDict['pmb']] vRadlb = starData[galfastDict['vRadlb']] pmRA = starData[galfastDict['pmra']] pmDec = starData[galfastDict['pmdec']] vRad = starData[galfastDict['vRad']] am = starData[galfastDict['Am']] amInf = starData[galfastDict['AmInf']] sDSS = np.transpose( starData[galfastDict['SDSSu']:galfastDict['SDSSz'] + 1]) sDSSPhotoFlags = starData[galfastDict['SDSSPhotoFlags']] #End of input, now onto processing and output sDSSunred = selectStarSED0.deReddenMags( am, sDSS, sdssExtCoeffs) if readSize == 1: ra = np.array([ra]) dec = np.array([dec]) """ Info about the following population cuts: From Zeljko: "This color corresponds to the temperature (roughly spectral type M0) where Kurucz models become increasingly bad, and thus we switch to empirical SEDs (the problem is that for M and later stars, the effective surface temperature is low enough for molecules to form, and their opacity is too complex to easily model, especially TiO)." """ mIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:, 2] - sDSSunred[:, 3] > 0.59)) kIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:, 2] - sDSSunred[:, 3] <= 0.59)) hIn = np.where((pop >= 10) & (pop < 15)) heIn = np.where((pop >= 15) & (pop < 20)) sEDNameK, magNormK, matchErrorK = selectStarSED0.findSED( listDict['kurucz'], sDSSunred[kIn], ra[kIn], dec[kIn], reddening=False, colors=colorDict['kurucz']) sEDNameM, magNormM, matchErrorM = selectStarSED0.findSED( listDict['mlt'], sDSSunred[mIn], ra[mIn], dec[mIn], reddening=False, colors=colorDict['mlt']) sEDNameH, magNormH, matchErrorH = selectStarSED0.findSED( listDict['H'], sDSSunred[hIn], ra[hIn], dec[hIn], reddening=False, colors=colorDict['H']) sEDNameHE, magNormHE, matchErrorHE = selectStarSED0.findSED( listDict['HE'], sDSSunred[heIn], ra[heIn], dec[heIn], reddening=False, colors=colorDict['HE']) chunkNames = np.empty(readSize, dtype='S32') chunkTypes = np.empty(readSize, dtype='S8') chunkMagNorms = np.zeros(readSize) chunkMatchErrors = np.zeros(readSize) chunkNames[kIn] = sEDNameK chunkTypes[kIn] = 'kurucz' chunkMagNorms[kIn] = magNormK chunkMatchErrors[kIn] = matchErrorK chunkNames[mIn] = sEDNameM chunkTypes[mIn] = 'mlt' chunkMagNorms[mIn] = magNormM chunkMatchErrors[mIn] = matchErrorM chunkNames[hIn] = sEDNameH chunkTypes[hIn] = 'H' chunkMagNorms[hIn] = magNormH chunkMatchErrors[hIn] = matchErrorH chunkNames[heIn] = sEDNameHE chunkTypes[heIn] = 'HE' chunkMagNorms[heIn] = magNormHE chunkMatchErrors[heIn] = matchErrorHE lsstMagsUnred = [] for sedName, sedType, magNorm, matchError in zip( chunkNames.astype(str), chunkTypes.astype(str), chunkMagNorms, chunkMatchErrors): testSED = Sed() testSED.setSED( listDict[sedType][positionDict[sedName]].wavelen, flambda=listDict[sedType][ positionDict[sedName]].flambda) fluxNorm = testSED.calcFluxNorm(magNorm, imSimBand) testSED.multiplyFluxNorm(fluxNorm) lsstMagsUnred.append(lsstPhot.magListForSed(testSED)) #If the extinction value is negative then it will add the reddening back in lsstMags = selectStarSED0.deReddenMags( (-1.0 * am), lsstMagsUnred, lsstExtCoeffs) distKpc = self.convDMtoKpc(DM) ebv = am / 2.285 #From Schlafly and Finkbeiner 2011, (ApJ, 737, 103) for sdssr ebvInf = amInf / 2.285 for line in range(0, readSize): outFmt = '%i,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%s,' +\ '%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%i,%3.7f,%3.7f,%3.7f\n' if readSize == 1: if inFits == True: sDSS = sDSS[0] outDat = (oID, ra[line], dec[line], gall, galb, coordX, coordY, coordZ, chunkNames, chunkMagNorms, chunkMatchErrors, lsstMags[line][0], lsstMags[line][1], lsstMags[line][2], lsstMags[line][3], lsstMags[line][4], lsstMags[line][5], sDSS[0], sDSS[1], sDSS[2], sDSS[3], sDSS[4], absSDSSr, pmRA, pmDec, vRad, pml, pmb, vRadlb, vR, vPhi, vZ, FeH, pop, distKpc, ebv, ebvInf) else: outDat = (oID[line], ra[line], dec[line], gall[line], galb[line], coordX[line], coordY[line], coordZ[line], chunkNames[line], chunkMagNorms[line], chunkMatchErrors[line], lsstMags[line][0], lsstMags[line][1], lsstMags[line][2], lsstMags[line][3], lsstMags[line][4], lsstMags[line][5], sDSS[line][0], sDSS[line][1], sDSS[line][2], sDSS[line][3], sDSS[line][4], absSDSSr[line], pmRA[line], pmDec[line], vRad[line], pml[line], pmb[line], vRadlb[line], vR[line], vPhi[line], vZ[line], FeH[line], pop[line], distKpc[line], ebv[line], ebvInf[line]) fOut.write(outFmt % outDat) print('Chunk Num Done = %i out of %i' % (chunk + 1, numChunks))
def testFindSED(self): """Pull SEDs from each type and make sure that each SED gets matched to itself. Includes testing with extinction and passing in only colors.""" rng = np.random.RandomState(42) bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'), 'sdss') starPhot = BandpassDict.loadTotalBandpassesFromFiles(('u', 'g', 'r', 'i', 'z'), bandpassDir = bandpassDir, bandpassRoot = 'sdss_') imSimBand = Bandpass() imSimBand.imsimBandpass() testMatching = selectStarSED(kuruczDir=self.testKDir, mltDir=self.testMLTDir, wdDir=self.testWDDir) testSEDList = [] testSEDList.append(testMatching.loadKuruczSEDs()) testSEDList.append(testMatching.loadmltSEDs()) testSEDListH, testSEDListHE = testMatching.loadwdSEDs() testSEDList.append(testSEDListH) testSEDList.append(testSEDListHE) testSEDNames = [] testMags = [] testMagNormList = [] magNormStep = 1 for typeList in testSEDList: if len(typeList) != 0: typeSEDNames = [] typeMags = [] typeMagNorms = [] for testSED in typeList: getSEDMags = Sed() typeSEDNames.append(testSED.name) getSEDMags.setSED(wavelen = testSED.wavelen, flambda = testSED.flambda) testMagNorm = np.round(rng.uniform(20.0, 22.0), magNormStep) typeMagNorms.append(testMagNorm) fluxNorm = getSEDMags.calcFluxNorm(testMagNorm, imSimBand) getSEDMags.multiplyFluxNorm(fluxNorm) typeMags.append(starPhot.magListForSed(getSEDMags)) testSEDNames.append(typeSEDNames) testMags.append(typeMags) testMagNormList.append(typeMagNorms) # Since default bandpassDict should be SDSS ugrizy shouldn't need to specify it # Substitute in nan values to simulate incomplete data. for typeList, names, mags, magNorms in zip(testSEDList, testSEDNames, testMags, testMagNormList): if len(typeList) > 2: nanMags = np.array(mags) nanMags[0][0] = np.nan nanMags[0][2] = np.nan nanMags[0][3] = np.nan nanMags[1][1] = np.nan testMatchingResults = testMatching.findSED(typeList, nanMags, reddening = False) self.assertEqual(None, testMatchingResults[0][0]) self.assertEqual(names[1:], testMatchingResults[0][1:]) self.assertEqual(None, testMatchingResults[1][0]) np.testing.assert_almost_equal(magNorms[1:], testMatchingResults[1][1:], decimal = magNormStep) else: testMatchingResults = testMatching.findSED(typeList, mags, reddening = False) self.assertEqual(names, testMatchingResults[0]) np.testing.assert_almost_equal(magNorms, testMatchingResults[1], decimal = magNormStep) # Test Null Values option nullMags = np.array(testMags[0]) nullMags[0][0] = -99. nullMags[0][4] = -99. nullMags[1][0] = -99. nullMags[1][1] = -99. testMatchingResultsNull = testMatching.findSED(testSEDList[0], nullMags, nullValues = -99., reddening = False) self.assertEqual(testSEDNames[0], testMatchingResultsNull[0]) np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsNull[1], decimal = magNormStep) # Test Error Output errMags = np.array((testMags[0][0], testMags[0][0], testMags[0][0], testMags[0][0])) errMags[1, 1] += 1. # Total MSE will be 2/(4 colors) = 0.5 errMags[2, 0:2] = np.nan errMags[2, 3] += 1. # Total MSE will be 2/(2 colors) = 1.0 errMags[3, :] = None errSED = testSEDList[0][0] testMatchingResultsErrors = testMatching.findSED([errSED], errMags, reddening = False) np.testing.assert_almost_equal(np.array((0.0, 0.5, 1.0)), testMatchingResultsErrors[2][0:3], decimal = 3) self.assertEqual(None, testMatchingResultsErrors[2][3]) # Now test what happens if we pass in a bandpassDict testMatchingResultsNoDefault = testMatching.findSED(testSEDList[0], testMags[0], bandpassDict = starPhot, reddening = False) self.assertEqual(testSEDNames[0], testMatchingResultsNoDefault[0]) np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsNoDefault[1], decimal = magNormStep) # Test Reddening testRA = rng.uniform(10, 170, len(testSEDList[0])) testDec = rng.uniform(10, 80, len(testSEDList[0])) extFactor = .5 raDec = np.array((testRA, testDec)) ebvVals = ebv().calculateEbv(equatorialCoordinates = raDec) extVals = ebvVals*extFactor testRedMags = [] for extVal, testMagSet in zip(extVals, testMags[0]): testRedMags.append(testMagSet + extVal) testMatchingResultsRed = testMatching.findSED(testSEDList[0], testRedMags, catRA = testRA, catDec = testDec, reddening = True, extCoeffs = np.ones(5)*extFactor) self.assertEqual(testSEDNames[0], testMatchingResultsRed[0]) np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsRed[1], decimal = magNormStep) # Finally, test color input testColors = [] for testMagSet in testMags[0]: testColorSet = [] for filtNum in range(0, len(starPhot)-1): testColorSet.append(testMagSet[filtNum] - testMagSet[filtNum+1]) testColors.append(testColorSet) testMatchingColorsInput = testMatching.findSED(testSEDList[0], testMags[0], reddening = False, colors = testColors) self.assertEqual(testSEDNames[0], testMatchingColorsInput[0]) np.testing.assert_almost_equal(testMagNormList[0], testMatchingColorsInput[1], decimal = magNormStep)
def testFindSED(self): """Pull SEDs from each type and make sure that each SED gets matched to itself. Includes testing with extinction and passing in only colors.""" np.random.seed(42) starPhot = BandpassDict.loadTotalBandpassesFromFiles(('u','g','r','i','z'), bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'),'sdss'), bandpassRoot = 'sdss_') imSimBand = Bandpass() imSimBand.imsimBandpass() testMatching = selectStarSED(sEDDir = self.testSpecDir, kuruczDir = self.testKDir, mltDir = self.testMLTDir, wdDir = self.testWDDir) testSEDList = [] testSEDList.append(testMatching.loadKuruczSEDs()) testSEDList.append(testMatching.loadmltSEDs()) testSEDListH, testSEDListHE = testMatching.loadwdSEDs() testSEDList.append(testSEDListH) testSEDList.append(testSEDListHE) testSEDNames = [] testMags = [] testMagNormList = [] magNormStep = 1 for typeList in testSEDList: if len(typeList) != 0: typeSEDNames = [] typeMags = [] typeMagNorms = [] for testSED in typeList: getSEDMags = Sed() typeSEDNames.append(testSED.name) getSEDMags.setSED(wavelen = testSED.wavelen, flambda = testSED.flambda) testMagNorm = np.round(np.random.uniform(20.0,22.0),magNormStep) typeMagNorms.append(testMagNorm) fluxNorm = getSEDMags.calcFluxNorm(testMagNorm, imSimBand) getSEDMags.multiplyFluxNorm(fluxNorm) typeMags.append(starPhot.magListForSed(getSEDMags)) testSEDNames.append(typeSEDNames) testMags.append(typeMags) testMagNormList.append(typeMagNorms) fakeRA = np.ones(len(testSEDList[0])) fakeDec = np.ones(len(testSEDList[0])) #Since default bandpassDict should be SDSS ugrizy shouldn't need to specify it #Substitute in nan values to simulate incomplete data. for typeList, names, mags, magNorms in zip(testSEDList, testSEDNames, testMags, testMagNormList): if len(typeList) > 2: nanMags = np.array(mags) nanMags[0][0] = np.nan nanMags[0][2] = np.nan nanMags[0][3] = np.nan nanMags[1][1] = np.nan testMatchingResults = testMatching.findSED(typeList, nanMags, reddening = False) self.assertEqual(None, testMatchingResults[0][0]) self.assertEqual(names[1:], testMatchingResults[0][1:]) self.assertEqual(None, testMatchingResults[1][0]) np.testing.assert_almost_equal(magNorms[1:], testMatchingResults[1][1:], decimal = magNormStep) else: testMatchingResults = testMatching.findSED(typeList, mags, reddening = False) self.assertEqual(names, testMatchingResults[0]) np.testing.assert_almost_equal(magNorms, testMatchingResults[1], decimal = magNormStep) #Test Null Values option nullMags = np.array(testMags[0]) nullMags[0][0] = -99. nullMags[0][4] = -99. nullMags[1][0] = -99. nullMags[1][1] = -99. testMatchingResultsNull = testMatching.findSED(testSEDList[0], nullMags, nullValues = -99., reddening = False) self.assertEqual(testSEDNames[0], testMatchingResultsNull[0]) np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsNull[1], decimal = magNormStep) #Test Error Output errMags = np.array((testMags[0][0], testMags[0][0], testMags[0][0], testMags[0][0])) errMags[1,1] += 1. #Total MSE will be 2/(4 colors) = 0.5 errMags[2, 0:2] = np.nan errMags[2, 3] += 1. #Total MSE will be 2/(2 colors) = 1.0 errMags[3, :] = None errSED = testSEDList[0][0] testMatchingResultsErrors = testMatching.findSED([errSED], errMags, reddening = False) np.testing.assert_almost_equal(np.array((0.0, 0.5, 1.0)), testMatchingResultsErrors[2][0:3], decimal = 3) self.assertEqual(None, testMatchingResultsErrors[2][3]) #Now test what happens if we pass in a bandpassDict testMatchingResultsNoDefault = testMatching.findSED(testSEDList[0], testMags[0], bandpassDict = starPhot, reddening = False) self.assertEqual(testSEDNames[0], testMatchingResultsNoDefault[0]) np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsNoDefault[1], decimal = magNormStep) #Test Reddening testRA = np.random.uniform(10,170,len(testSEDList[0])) testDec = np.random.uniform(10,80,len(testSEDList[0])) extFactor = .5 raDec = np.array((testRA, testDec)) ebvVals = ebv().calculateEbv(equatorialCoordinates = raDec) extVals = ebvVals*extFactor testRedMags = [] for extVal, testMagSet in zip(extVals, testMags[0]): testRedMags.append(testMagSet + extVal) testMatchingResultsRed = testMatching.findSED(testSEDList[0], testRedMags, catRA = testRA, catDec = testDec, reddening = True, extCoeffs = np.ones(5)*extFactor) self.assertEqual(testSEDNames[0], testMatchingResultsRed[0]) np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsRed[1], decimal = magNormStep) #Finally, test color input testColors = [] for testMagSet in testMags[0]: testColorSet = [] for filtNum in range(0, len(starPhot)-1): testColorSet.append(testMagSet[filtNum] - testMagSet[filtNum+1]) testColors.append(testColorSet) testMatchingColorsInput = testMatching.findSED(testSEDList[0], testMags[0], reddening = False, colors = testColors) self.assertEqual(testSEDNames[0], testMatchingColorsInput[0]) np.testing.assert_almost_equal(testMagNormList[0], testMatchingColorsInput[1], decimal = magNormStep)
def loadGalfast(self, filenameList, outFileList, sEDPath = None, kuruczPath = None, mltPath = None, wdPath = None, kuruczSubset = None, mltSubset = None, wdSubset = None, chunkSize = 10000): """ This is customized for the outputs we currently need for the purposes of consistent output It will read in a galfast output file and output desired values for database input into a file @param [in] filenameList is a list of the galfast output files that will be loaded and processed. Can process fits, gzipped, or txt output from galfast. @param [in] outFileList is a list of the names of the output files that will be created. If gzipped output is desired simply write the filenames with .gz at the end. @param [in] kuruczPath is a place to specify a different path to kurucz SED files than the files in the LSST sims_sed_library. If set to None it will default to the LSST library. Will probably be most useful for those who want to use loadGalfast without downloading the entire LSST sims_sed_library which contains much more than just the star SEDs. @param [in] mltPath is the same as kuruczPath except that it specifies a directory for the mlt SEDs @param [in] wdPath is the same as the previous two except that it specifies a path to an alternate white dwarf SED directory. @param [in] kuruczSubset is a list which provides a subset of the kurucz files within the kurucz folder that one wants to use @param [in] mltSubset is a list which provides a subset of the mlt files within the mlt folder that one wants to use @param [in] wdSubset is a list which provides a subset of the wd files within the wd folder that one wants to use @param [in] chunkSize is the size of chunks of lines to be read from the catalog at one time. """ for filename in filenameList: #Make sure input file exists and is readable format before doing anything else if os.path.isfile(filename) == False: raise RuntimeError, '*** File does not exist' #Process various possible galfast outputs if filename.endswith(('.txt', '.gz', '.fits')): continue else: raise RuntimeError, str('*** Unsupported File Format in file: ' + str(filename)) #If all files exist and are in proper formats then load seds selectStarSED0 = selectStarSED(sEDDir = sEDPath, kuruczDir = kuruczPath, mltDir = mltPath, wdDir = wdPath) if kuruczSubset is None: kuruczList = selectStarSED0.loadKuruczSEDs() else: kuruczList = selectStarSED0.loadKuruczSEDs(subset = kuruczSubset) #Only need one dictionary since none of the names overlap positionDict = {} for kNum, kuruczSED in enumerate(kuruczList): positionDict[kuruczSED.name] = kNum if mltSubset is None: mltList = selectStarSED0.loadmltSEDs() else: mltList = selectStarSED0.loadmltSEDs(subset = mltSubset) for mltNum, mltSED in enumerate(mltList): positionDict[mltSED.name] = mltNum if wdSubset is None: wdListH, wdListHE = selectStarSED0.loadwdSEDs() else: wdListH, wdListHE = selectStarSED0.loadwdSEDs(subset = wdSubset) for hNum, hSED in enumerate(wdListH): positionDict[hSED.name] = hNum for heNum, heSED in enumerate(wdListHE): positionDict[heSED.name] = heNum #For adding/subtracting extinction when calculating colors #Numbers below come from Schlafly and Finkbeiner (2011) (ApJ, 737, 103) #normalized by SDSS r mag value sdssExtCoeffs = [1.8551, 1.4455, 1.0, 0.7431, 0.5527] lsstExtCoeffs = [1.8140, 1.4166, 0.9947, 0.7370, 0.5790, 0.4761] sdssPhot = BandpassDict.loadTotalBandpassesFromFiles(['u','g','r','i','z'], bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'), 'sdss'), bandpassRoot = 'sdss_') #Load Bandpasses for LSST colors to get colors from matched SEDs lsstFilterList = ('u', 'g', 'r', 'i', 'z', 'y') lsstPhot = BandpassDict.loadTotalBandpassesFromFiles(lsstFilterList) imSimBand = Bandpass() imSimBand.imsimBandpass() #Calculate colors and add them to the SED objects kuruczColors = selectStarSED0.calcBasicColors(kuruczList, sdssPhot) mltColors = selectStarSED0.calcBasicColors(mltList, sdssPhot) hColors = selectStarSED0.calcBasicColors(wdListH, sdssPhot) heColors = selectStarSED0.calcBasicColors(wdListHE, sdssPhot) listDict = {'kurucz':kuruczList, 'mlt':mltList, 'H':wdListH, 'HE':wdListHE} colorDict = {'kurucz':kuruczColors, 'mlt':mltColors, 'H':hColors, 'HE':heColors} for filename, outFile in zip(filenameList, outFileList): if filename.endswith('.txt'): galfastIn = open(filename, 'r') inFits = False gzFile = False num_lines = sum(1 for line in open(filename)) elif filename.endswith('.gz'): galfastIn = gzip.open(filename, 'r') inFits = False gzFile = True num_lines = sum(1 for line in gzip.open(filename)) elif filename.endswith('fits'): hdulist = pyfits.open(filename) galfastIn = hdulist[1].data num_lines = len(galfastIn) gzFile = False inFits = True if outFile.endswith('.txt'): fOut = open(outFile, 'w') elif outFile.endswith('.gz'): fOut = gzip.open(outFile, 'w') fOut.write('#oID, ra, dec, gall, galb, coordX, coordY, coordZ, sEDName, magNorm, ' +\ 'LSSTugrizy, SDSSugriz, absSDSSr, pmRA, pmDec, vRad, pml, pmb, vRadlb, ' +\ 'vR, vPhi, vZ, FeH, pop, distKpc, ebv, ebvInf\n') header_length = 0 numChunks = 0 if inFits == False: galfastDict = self.parseGalfast(galfastIn.readline()) header_length += 1 header_status = True while header_status == True: newLine = galfastIn.readline() if newLine[0] != '#': header_status = False else: header_length += 1 print 'Total objects = %i' % (num_lines - header_length) numChunks = ((num_lines-header_length)/chunkSize) + 1 for chunk in range(0,numChunks): if chunk == numChunks-1: lastChunkSize = (num_lines - header_length) % chunkSize readSize = lastChunkSize else: readSize = chunkSize oID = np.arange(readSize*chunk, readSize*(chunk+1)) if inFits: starData = galfastIn[readSize*chunk:(readSize*chunk + readSize)] sDSS = starData.field('SDSSugriz') gall, galb = np.transpose(starData.field('lb')) ra, dec = np.transpose(starData.field('radec')) coordX, coordY, coordZ = np.transpose(starData.field('XYZ')) DM = starData.field('DM') absSDSSr = starData.field('absSDSSr') pop = starData.field('comp') FeH = starData.field('FeH') vR, vPhi, vZ = np.transpose(starData.field('vcyl')) pml, pmb, vRadlb = np.transpose(starData.field('pmlb')) pmRA, pmDec, vRad = np.transpose(starData.field('pmradec')) am = starData.field('Am') amInf = starData.field('AmInf') sdssPhotoFlags = starData.field('SDSSugrizPhotoFlags') else: if gzFile == False: with open(filename) as t_in: starData = np.loadtxt(itertools.islice(t_in,((readSize*chunk)+header_length), ((readSize*(chunk+1))+header_length))) else: with gzip.open(filename) as t_in: starData = np.loadtxt(itertools.islice(t_in,((readSize*chunk)+header_length), ((readSize*(chunk+1))+header_length))) starData = np.transpose(starData) gall = starData[galfastDict['l']] galb = starData[galfastDict['b']] ra = starData[galfastDict['ra']] dec = starData[galfastDict['dec']] coordX = starData[galfastDict['X']] coordY = starData[galfastDict['Y']] coordZ = starData[galfastDict['Z']] DM = starData[galfastDict['DM']] absSDSSr = starData[galfastDict['absSDSSr']] pop = starData[galfastDict['comp']] FeH = starData[galfastDict['FeH']] vR = starData[galfastDict['Vr']] vPhi = starData[galfastDict['Vphi']] vZ = starData[galfastDict['Vz']] pml = starData[galfastDict['pml']] pmb = starData[galfastDict['pmb']] vRadlb = starData[galfastDict['vRadlb']] pmRA = starData[galfastDict['pmra']] pmDec = starData[galfastDict['pmdec']] vRad = starData[galfastDict['vRad']] am = starData[galfastDict['Am']] amInf = starData[galfastDict['AmInf']] sDSS = np.transpose(starData[galfastDict['SDSSu']:galfastDict['SDSSz']+1]) sDSSPhotoFlags = starData[galfastDict['SDSSPhotoFlags']] #End of input, now onto processing and output sDSSunred = selectStarSED0.deReddenMags(am, sDSS, sdssExtCoeffs) if readSize == 1: ra = np.array([ra]) dec = np.array([dec]) """ Info about the following population cuts: From Zeljko: "This color corresponds to the temperature (roughly spectral type M0) where Kurucz models become increasingly bad, and thus we switch to empirical SEDs (the problem is that for M and later stars, the effective surface temperature is low enough for molecules to form, and their opacity is too complex to easily model, especially TiO)." """ mIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:,2] - sDSSunred[:,3] > 0.59)) kIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:,2] - sDSSunred[:,3] <= 0.59)) hIn = np.where((pop >= 10) & (pop < 15)) heIn = np.where((pop >= 15) & (pop < 20)) sEDNameK, magNormK, matchErrorK = selectStarSED0.findSED(listDict['kurucz'], sDSSunred[kIn], ra[kIn], dec[kIn], reddening = False, colors = colorDict['kurucz']) sEDNameM, magNormM, matchErrorM = selectStarSED0.findSED(listDict['mlt'], sDSSunred[mIn], ra[mIn], dec[mIn], reddening = False, colors = colorDict['mlt']) sEDNameH, magNormH, matchErrorH = selectStarSED0.findSED(listDict['H'], sDSSunred[hIn], ra[hIn], dec[hIn], reddening = False, colors = colorDict['H']) sEDNameHE, magNormHE, matchErrorHE = selectStarSED0.findSED(listDict['HE'], sDSSunred[heIn], ra[heIn], dec[heIn], reddening = False, colors = colorDict['HE']) chunkNames = np.empty(readSize, dtype = 'S32') chunkTypes = np.empty(readSize, dtype = 'S8') chunkMagNorms = np.zeros(readSize) chunkMatchErrors = np.zeros(readSize) chunkNames[kIn] = sEDNameK chunkTypes[kIn] = 'kurucz' chunkMagNorms[kIn] = magNormK chunkMatchErrors[kIn] = matchErrorK chunkNames[mIn] = sEDNameM chunkTypes[mIn] = 'mlt' chunkMagNorms[mIn] = magNormM chunkMatchErrors[mIn] = matchErrorM chunkNames[hIn] = sEDNameH chunkTypes[hIn] = 'H' chunkMagNorms[hIn] = magNormH chunkMatchErrors[hIn] = matchErrorH chunkNames[heIn] = sEDNameHE chunkTypes[heIn] = 'HE' chunkMagNorms[heIn] = magNormHE chunkMatchErrors[heIn] = matchErrorHE lsstMagsUnred = [] for sedName, sedType, magNorm, matchError in zip(chunkNames, chunkTypes, chunkMagNorms, chunkMatchErrors): testSED = Sed() testSED.setSED(listDict[sedType][positionDict[sedName]].wavelen, flambda = listDict[sedType][positionDict[sedName]].flambda) fluxNorm = testSED.calcFluxNorm(magNorm, imSimBand) testSED.multiplyFluxNorm(fluxNorm) lsstMagsUnred.append(lsstPhot.magListForSed(testSED)) #If the extinction value is negative then it will add the reddening back in lsstMags = selectStarSED0.deReddenMags((-1.0*am), lsstMagsUnred, lsstExtCoeffs) distKpc = self.convDMtoKpc(DM) ebv = am / 2.285 #From Schlafly and Finkbeiner 2011, (ApJ, 737, 103) for sdssr ebvInf = amInf / 2.285 for line in range(0, readSize): outFmt = '%i,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%s,' +\ '%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\ '%3.7f,%i,%3.7f,%3.7f,%3.7f\n' if readSize == 1: if inFits == True: sDSS = sDSS[0] outDat = (oID, ra[line], dec[line], gall, galb, coordX, coordY, coordZ, chunkNames, chunkMagNorms, chunkMatchErrors, lsstMags[line][0], lsstMags[line][1], lsstMags[line][2], lsstMags[line][3], lsstMags[line][4], lsstMags[line][5], sDSS[0], sDSS[1], sDSS[2], sDSS[3], sDSS[4], absSDSSr, pmRA, pmDec, vRad, pml, pmb, vRadlb, vR, vPhi, vZ, FeH, pop, distKpc, ebv, ebvInf) else: outDat = (oID[line], ra[line], dec[line], gall[line], galb[line], coordX[line], coordY[line], coordZ[line], chunkNames[line], chunkMagNorms[line], chunkMatchErrors[line], lsstMags[line][0], lsstMags[line][1], lsstMags[line][2], lsstMags[line][3], lsstMags[line][4], lsstMags[line][5], sDSS[line][0], sDSS[line][1], sDSS[line][2], sDSS[line][3], sDSS[line][4], absSDSSr[line], pmRA[line], pmDec[line], vRad[line], pml[line], pmb[line], vRadlb[line], vR[line], vPhi[line], vZ[line], FeH[line], pop[line], distKpc[line], ebv[line], ebvInf[line]) fOut.write(outFmt % outDat) print 'Chunk Num Done = %i out of %i' % (chunk+1, numChunks)