def get_quiescent_lsst_magnitudes(self):
if not hasattr(self, 'lsstBandpassDict'):
self.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
return self._quiescentMagnitudeGetter(self.lsstBandpassDict,
self.get_quiescent_lsst_magnitudes._colnames)
def testLSSTmags(self):
"""
Test that PhotometrySSM properly calculates LSST magnitudes
"""
cat = LSST_SSM_photCat(self.photDB)
dtype = np.dtype([('id', np.int), ('u', np.float), ('g', np.float),
('r', np.float), ('i', np.float), ('z', np.float),
('y', np.float)])
with lsst.utils.tests.getTempFilePath('.txt') as catName:
cat.write_catalog(catName)
testData = np.genfromtxt(catName, dtype=dtype, delimiter=',')
self.assertGreater(len(testData), 0)
controlData = np.genfromtxt(self.dbFile, dtype=self.dtype)
self.assertGreater(len(controlData), 0)
LSSTbandpasses = BandpassDict.loadTotalBandpassesFromFiles()
controlSedList = SedList(controlData['sedFilename'], controlData['magNorm'],
wavelenMatch=LSSTbandpasses.wavelenMatch,
fileDir=getPackageDir('sims_sed_library'),
specMap=defaultSpecMap)
controlMags = LSSTbandpasses.magListForSedList(controlSedList)
for ii in range(len(controlMags)):
for jj, bpName in enumerate(['u', 'g', 'r', 'i', 'z', 'y']):
self.assertAlmostEqual(controlMags[ii][jj], testData[bpName][ii], 10)
def get_test_disk_mags(self):
if not hasattr(self, 'testBandpassDict'):
self.testBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
return self._magnitudeGetter('disk', self.testBandpassDict,
self.get_test_disk_mags._colnames)
def testLoadTotalBandpassesFromFiles(self):
"""
Test that the class method loadTotalBandpassesFromFiles produces the
expected result
"""
bandpassDir = os.path.join(getPackageDir('sims_photUtils'), 'tests', 'cartoonSedTestData')
bandpassNames = ['g', 'r', 'u']
bandpassRoot = 'test_bandpass_'
bandpassDict = BandpassDict.loadTotalBandpassesFromFiles(bandpassNames=bandpassNames,
bandpassDir=bandpassDir,
bandpassRoot = bandpassRoot)
controlBandpassList = []
for bpn in bandpassNames:
dummyBp = Bandpass()
dummyBp.readThroughput(os.path.join(bandpassDir,bandpassRoot+bpn+'.dat'))
controlBandpassList.append(dummyBp)
wMin = controlBandpassList[0].wavelen[0]
wMax = controlBandpassList[0].wavelen[-1]
wStep = controlBandpassList[0].wavelen[1]-controlBandpassList[0].wavelen[0]
for bp in controlBandpassList:
bp.resampleBandpass(wavelen_min=wMin, wavelen_max=wMax, wavelen_step=wStep)
for test, control in zip(bandpassDict.values(), controlBandpassList):
numpy.testing.assert_array_almost_equal(test.wavelen, control.wavelen, 19)
numpy.testing.assert_array_almost_equal(test.sb, control.sb, 19)
def testLSSTmags(self):
"""
Test that PhotometrySSM properly calculates LSST magnitudes
"""
catName = os.path.join(getPackageDir('sims_catUtils'), 'tests', 'scratchSpace', 'lsstSsmPhotCat.txt')
cat=LSST_SSM_photCat(self.photDB)
cat.write_catalog(catName)
dtype = np.dtype([('id', np.int), ('u', np.float), ('g', np.float),
('r', np.float), ('i', np.float), ('z', np.float),
('y', np.float)])
testData = np.genfromtxt(catName, dtype=dtype, delimiter=',')
self.assertGreater(len(testData), 0)
controlData = np.genfromtxt(self.dbFile, dtype=self.dtype)
self.assertGreater(len(controlData), 0)
LSSTbandpasses = BandpassDict.loadTotalBandpassesFromFiles()
controlSedList = SedList(controlData['sedFilename'], controlData['magNorm'],
wavelenMatch=LSSTbandpasses.wavelenMatch)
controlMags = LSSTbandpasses.magListForSedList(controlSedList)
for ii in range(len(controlMags)):
for jj, bpName in enumerate(['u', 'g', 'r', 'i', 'z', 'y']):
self.assertAlmostEqual(controlMags[ii][jj], testData[bpName][ii], 10)
if os.path.exists(catName):
os.unlink(catName)
def get_lsst_magnitudes(self):
"""
getter for LSST magnitudes of solar system objects
"""
if not hasattr(self, 'lsstBandpassDict'):
self.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
return self._quiescentMagnitudeGetter(self.lsstBandpassDict, self.get_lsst_magnitudes._colnames)
def get_test_mags(self):
if not hasattr(self, 'variabilitybandpassDict'):
self.variabilityBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
self._loadSedList(self.variabilityBandpassDict.wavelenMatch)
if not hasattr(self, '_sedList'):
return numpy.ones((6,0))
return self._magnitudeGetter(self.variabilityBandpassDict, self.get_test_mags._colnames)
def get_test_agn_mags(self):
if not hasattr(self, 'testBandpassDict'):
self.testBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
mag = self._quiescentMagnitudeGetter('agn', self.testBandpassDict,
self.get_test_agn_mags._colnames)
mag += self._variabilityGetter(self.get_test_agn_mags._colnames)
return mag
def testManyMagSystems(self):
"""
Test that the SSM photometry mixin can simultaneously calculate magnitudes
in multiple bandpass systems
"""
catName = os.path.join(getPackageDir('sims_catUtils'), 'tests', 'scratchSpace', 'compoundSsmPhotCat.txt')
cat=Compound_SSM_photCat(self.photDB)
cat.write_catalog(catName)
dtype = np.dtype([('id', np.int), ('lsst_u', np.float), ('lsst_g', np.float),
('lsst_r', np.float), ('lsst_i', np.float), ('lsst_z', np.float),
('lsst_y', np.float),
('cartoon_u', np.float), ('cartoon_g', np.float),
('cartoon_r', np.float), ('cartoon_i', np.float),
('cartoon_z', np.float)])
testData = np.genfromtxt(catName, dtype=dtype, delimiter=',')
self.assertGreater(len(testData), 0)
controlData = np.genfromtxt(self.dbFile, dtype=self.dtype)
self.assertGreater(len(controlData), 0)
LSSTbandpasses = BandpassDict.loadTotalBandpassesFromFiles()
cartoonBandpasses = BandpassDict.loadTotalBandpassesFromFiles(
['u', 'g', 'r', 'i', 'z'],
bandpassDir = os.path.join(getPackageDir('sims_photUtils'), 'tests', 'cartoonSedTestData'),
bandpassRoot = 'test_bandpass_'
)
controlSedList = SedList(controlData['sedFilename'], controlData['magNorm'],
wavelenMatch=LSSTbandpasses.wavelenMatch)
controlLsstMags = LSSTbandpasses.magListForSedList(controlSedList)
controlCartoonMags = cartoonBandpasses.magListForSedList(controlSedList)
for ii in range(len(controlLsstMags)):
for jj, bpName in enumerate(['lsst_u', 'lsst_g', 'lsst_r', 'lsst_i', 'lsst_z', 'lsst_y']):
self.assertAlmostEqual(controlLsstMags[ii][jj], testData[bpName][ii], 10)
for jj, bpName in enumerate(['cartoon_u', 'cartoon_g', 'cartoon_r', 'cartoon_i', 'cartoon_z']):
self.assertAlmostEqual(controlCartoonMags[ii][jj], testData[bpName][ii], 10)
if os.path.exists(catName):
os.unlink(catName)
def get_magnitudes(self):
"""
Example photometry getter for alternative (i.e. non-LSST) bandpasses
"""
if not hasattr(self, 'cartoonBandpassDict'):
bandpassNames = ['u','g','r','i','z']
bandpassDir = os.path.join(getPackageDir('sims_photUtils'), 'tests', 'cartoonSedTestData')
self.cartoonBandpassDict = BandpassDict.loadTotalBandpassesFromFiles(bandpassNames,bandpassDir = bandpassDir,
bandpassRoot = 'test_bandpass_')
return self._quiescentMagnitudeGetter(self.cartoonBandpassDict, self.get_magnitudes._colnames)
def get_cartoon_mags(self):
if not hasattr(self, 'cartoonBandpassDict'):
bandpassDir = os.path.join(getPackageDir('sims_photUtils'), 'tests', 'cartoonSedTestData')
self.cartoonBandpassDict = \
BandpassDict.loadTotalBandpassesFromFiles(bandpassNames = ['u', 'g', 'r', 'i', 'z'],
bandpassDir = bandpassDir,
bandpassRoot = 'test_bandpass_')
return self._quiescentMagnitudeGetter(self.cartoonBandpassDict, self.get_cartoon_mags._colnames)
def testAlternateBandpassesStars(self):
"""
This will test our ability to do photometry using non-LSST bandpasses.
It will first calculate the magnitudes using the getters in cartoonPhotometryStars.
It will then load the alternate bandpass files 'by hand' and re-calculate the magnitudes
and make sure that the magnitude values agree. This is guarding against the possibility
that some default value did not change and the code actually ended up loading the
LSST bandpasses.
"""
obs_metadata_pointed = ObservationMetaData(
mjd=2013.23, boundType="circle", unrefractedRA=200.0, unrefractedDec=-30.0, boundLength=1.0
)
bandpassDir = os.path.join(lsst.utils.getPackageDir("sims_photUtils"), "tests", "cartoonSedTestData")
cartoon_dict = BandpassDict.loadTotalBandpassesFromFiles(
["u", "g", "r", "i", "z"], bandpassDir=bandpassDir, bandpassRoot="test_bandpass_"
)
testBandPasses = {}
keys = ["u", "g", "r", "i", "z"]
bplist = []
for kk in keys:
testBandPasses[kk] = Bandpass()
testBandPasses[kk].readThroughput(os.path.join(bandpassDir, "test_bandpass_%s.dat" % kk))
bplist.append(testBandPasses[kk])
sedObj = Sed()
phiArray, waveLenStep = sedObj.setupPhiArray(bplist)
sedFileName = os.path.join(lsst.utils.getPackageDir("sims_sed_library"), "starSED", "kurucz")
sedFileName = os.path.join(sedFileName, "km20_5750.fits_g40_5790.gz")
ss = Sed()
ss.readSED_flambda(sedFileName)
controlBandpass = Bandpass()
controlBandpass.imsimBandpass()
ff = ss.calcFluxNorm(22.0, controlBandpass)
ss.multiplyFluxNorm(ff)
testMags = cartoon_dict.magListForSed(ss)
ss.resampleSED(wavelen_match=bplist[0].wavelen)
ss.flambdaTofnu()
mags = -2.5 * numpy.log10(numpy.sum(phiArray * ss.fnu, axis=1) * waveLenStep) - ss.zp
self.assertTrue(len(mags) == len(testMags))
self.assertTrue(len(mags) > 0)
for j in range(len(mags)):
self.assertAlmostEqual(mags[j], testMags[j], 10)
def setUpClass(cls):
cls.scratchDir = os.path.join(getPackageDir('sims_GalSimInterface'), 'tests', 'scratchSpace')
cls.obs = ObservationMetaData(pointingRA=122.0, pointingDec=-29.1,
mjd=57381.2, rotSkyPos=43.2)
cls.camera = camTestUtils.CameraWrapper().camera
cls.dbFileName = os.path.join(cls.scratchDir, 'allowed_chips_test_db.txt')
if os.path.exists(cls.dbFileName):
os.unlink(cls.dbFileName)
cls.controlSed = Sed()
cls.controlSed.readSED_flambda(os.path.join(getPackageDir('sims_sed_library'),
'flatSED','sed_flat.txt.gz'))
cls.magNorm = 18.1
imsim = Bandpass()
imsim.imsimBandpass()
ff = cls.controlSed.calcFluxNorm(cls.magNorm, imsim)
cls.controlSed.multiplyFluxNorm(ff)
a_x, b_x = cls.controlSed.setupCCMab()
cls.controlSed.addCCMDust(a_x, b_x, A_v=0.1, R_v=3.1)
bpd = BandpassDict.loadTotalBandpassesFromFiles()
pp = PhotometricParameters()
cls.controlADU = cls.controlSed.calcADU(bpd['u'], pp)
cls.countSigma = np.sqrt(cls.controlADU/pp.gain)
cls.x_pix = 50
cls.y_pix = 50
x_list = []
y_list = []
name_list = []
for dd in cls.camera:
x_list.append(cls.x_pix)
y_list.append(cls.y_pix)
name_list.append(dd.getName())
x_list = np.array(x_list)
y_list = np.array(y_list)
ra_list, dec_list = raDecFromPixelCoords(x_list, y_list, name_list,
camera=cls.camera, obs_metadata=cls.obs,
epoch=2000.0)
dra_list = 3600.0*(ra_list-cls.obs.pointingRA)
ddec_list = 3600.0*(dec_list-cls.obs.pointingDec)
create_text_catalog(cls.obs, cls.dbFileName, dra_list, ddec_list,
mag_norm=[cls.magNorm]*len(dra_list))
cls.db = allowedChipsFileDBObj(cls.dbFileName, runtable='test')
def testMatchToRestFrame(self):
"""Test that Galaxies with no effects added into catalog mags are matched correctly."""
np.random.seed(42)
galPhot = BandpassDict.loadTotalBandpassesFromFiles()
imSimBand = Bandpass()
imSimBand.imsimBandpass()
testMatching = selectGalaxySED(galDir = self.testSpecDir)
testSEDList = testMatching.loadBC03()
testSEDNames = []
testMags = []
testMagNormList = []
magNormStep = 1
for testSED in testSEDList:
getSEDMags = Sed()
testSEDNames.append(testSED.name)
getSEDMags.setSED(wavelen = testSED.wavelen, flambda = testSED.flambda)
testMagNorm = np.round(np.random.uniform(20.0,22.0),magNormStep)
testMagNormList.append(testMagNorm)
fluxNorm = getSEDMags.calcFluxNorm(testMagNorm, imSimBand)
getSEDMags.multiplyFluxNorm(fluxNorm)
testMags.append(galPhot.magListForSed(getSEDMags))
#Also testing to make sure passing in non-default bandpasses works
#Substitute in nan values to simulate incomplete data.
testMags[0][1] = np.nan
testMags[0][2] = np.nan
testMags[0][4] = np.nan
testMags[1][1] = np.nan
testMatchingResults = testMatching.matchToRestFrame(testSEDList, testMags,
bandpassDict = galPhot)
self.assertEqual(None, testMatchingResults[0][0])
self.assertEqual(testSEDNames[1:], testMatchingResults[0][1:])
self.assertEqual(None, testMatchingResults[1][0])
np.testing.assert_almost_equal(testMagNormList[1:], testMatchingResults[1][1:], decimal = magNormStep)
#Test Match Errors
errMags = np.array((testMags[2], testMags[2], testMags[2], testMags[2]))
errMags[1,1] += 1. #Total MSE will be 2/(5 colors) = 0.4
errMags[2, 0:2] = np.nan
errMags[2, 3] += 1. #Total MSE will be 2/(3 colors) = 0.667
errMags[3, :] = None
errSED = testSEDList[2]
testMatchingResultsErrors = testMatching.matchToRestFrame([errSED], errMags,
bandpassDict = galPhot)
np.testing.assert_almost_equal(np.array((0.0, 0.4, 2./3.)), testMatchingResultsErrors[2][0:3],
decimal = 3)
self.assertEqual(None, testMatchingResultsErrors[2][3])
def get_lsst_agn_mags(self):
"""
Getter for AGN magnitudes in the LSST bandpasses
"""
# load a BandpassDict of LSST bandpasses, if not done already
if not hasattr(self, 'lsstBandpassDict'):
self.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
# actually calculate the magnitudes
mag = self._quiescentMagnitudeGetter('agn', self.lsstBandpassDict,
self.get_lsst_agn_mags._colnames)
mag += self._variabilityGetter(self.get_lsst_agn_mags._colnames)
return mag
def testSystematicUncertainty(self):
"""
Test that systematic uncertainty is added correctly.
"""
sigmaSys = 0.002
m5 = [23.5, 24.3, 22.1, 20.0, 19.5, 21.7]
photParams = PhotometricParameters(sigmaSys=sigmaSys)
bandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
obs_metadata = ObservationMetaData(unrefractedRA=23.0, unrefractedDec=45.0, m5=m5, bandpassName=self.bandpasses)
magnitudes = bandpassDict.magListForSed(self.starSED)
skySeds = []
for i in range(len(self.bandpasses)):
skyDummy = Sed()
skyDummy.readSED_flambda(os.path.join(lsst.utils.getPackageDir("throughputs"), "baseline", "darksky.dat"))
normalizedSkyDummy = setM5(
obs_metadata.m5[self.bandpasses[i]],
skyDummy,
self.totalBandpasses[i],
self.hardwareBandpasses[i],
seeing=LSSTdefaults().seeing(self.bandpasses[i]),
photParams=PhotometricParameters(),
)
skySeds.append(normalizedSkyDummy)
for i in range(len(self.bandpasses)):
snr = calcSNR_sed(
self.starSED,
self.totalBandpasses[i],
skySeds[i],
self.hardwareBandpasses[i],
seeing=LSSTdefaults().seeing(self.bandpasses[i]),
photParams=PhotometricParameters(),
)
testSNR, gamma = calcSNR_m5(
numpy.array([magnitudes[i]]),
[self.totalBandpasses[i]],
numpy.array([m5[i]]),
photParams=PhotometricParameters(sigmaSys=0.0),
)
self.assertAlmostEqual(snr, testSNR[0], 10, msg="failed on calcSNR_m5 test %e != %e " % (snr, testSNR[0]))
control = numpy.sqrt(numpy.power(magErrorFromSNR(testSNR), 2) + numpy.power(sigmaSys, 2))
def get_sdss_magnitudes(self):
"""
An example getter for stellar magnitudes in SDSS bands
"""
# Load a BandpassDict of SDSS bandpasses, if not done already
if not hasattr(self, 'sdssBandpassDict'):
bandpassNames = ['u','g','r','i','z']
bandpassDir = os.path.join(getPackageDir('throughputs'),'sdss')
bandpassRoot = 'sdss_'
self.sdssBandpassDict = BandpassDict.loadTotalBandpassesFromFiles(bandpassNames,
bandpassRoot = bandpassRoot,
bandpassDir = bandpassDir)
# Actually calculate the magnitudes
return self._magnitudeGetter(self.sdssBandpassDict, self.get_sdss_magnitudes._colnames)
def get_magnitudes(self):
"""
Example photometry getter for alternative (i.e. non-LSST) bandpasses
"""
idNames = self.column_by_name('id')
columnNames = [name for name in self.get_magnitudes._colnames]
bandpassNames = ['u','g','r','i','z']
bandpassDir = os.path.join(getPackageDir('sims_photUtils'), 'tests', 'cartoonSedTestData')
if not hasattr(self, 'cartoonBandpassDict'):
self.cartoonBandpassDict = BandpassDict.loadTotalBandpassesFromFiles(bandpassNames,bandpassDir = bandpassDir,
bandpassRoot = 'test_bandpass_')
output = self._quiescentMagnitudeGetter(self.cartoonBandpassDict, self.get_magnitudes._colnames)
#############################################################################
#Everything below this comment exists solely for the purposes of the unit test
#if you need to write a customized getter for photometry that uses non-LSST
#bandpasses, you only need to emulate the code above this comment.
magNormList = self.column_by_name('magNorm')
sedNames = self.column_by_name('sedFilename')
av = self.column_by_name('galacticAv')
#the two variables below will allow us to get at the SED and magnitude
#data from within the unit test class, so that we can be sure
#that the mixin loaded the correct bandpasses
sublist = SedList(sedNames, magNormList, galacticAvList=av,
fileDir=getPackageDir('sims_sed_library'),
specMap=defaultSpecMap)
for ss in sublist:
self.sedMasterList.append(ss)
if len(output) > 0:
for i in range(len(output[0])):
subList = []
for j in range(len(output)):
subList.append(output[j][i])
self.magnitudeMasterList.append(subList)
return output
def dflux_for_mlt(chunk, filter_obs, mjd_obs, variability_cache, dflux_out):
valid_obj = np.where(chunk['var_type'] == 2)
n_mlt = len(valid_obj[0])
if n_mlt == 0:
return
lc_id_map = {}
for ii in range(4):
lc_id_map[10+ii] = 'early_inactive_%d' % ii
lc_id_map[20+ii] = 'early_active_%d' % ii
lc_id_map[30+ii] = 'mid_inactive_%d' % ii
lc_id_map[40+ii] = 'mid_active_%d' % ii
lc_id_map[50+ii] = 'late_active_%d' % ii
mlt_model = MLTflaringMixin()
mlt_model.photParams = PhotometricParameters(nexp=1, exptime=30.0)
mlt_model.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
mlt_model._actually_calculated_columns = []
for bp in 'ugrizy':
mlt_model._actually_calculated_columns.append('lsst_%s' % bp)
params = {}
params['lc'] = np.array([lc_id_map[ii] for ii in chunk['lc_id'][valid_obj]])
params['t0'] = chunk['t0'][valid_obj]
q_mags = {}
for bp in 'ugrizy':
q_mags[bp] = chunk['%smag' % bp][valid_obj]
for i_bp, bp in enumerate('ugrizy'):
valid_bp = np.where(filter_obs==i_bp)
if len(valid_bp[0]) == 0:
continue
results = mlt_model.applyMLTflaring(np.array([True]*n_mlt),
params, mjd_obs[valid_bp],
parallax=chunk['parallax'][valid_obj],
ebv=chunk['ebv'][valid_obj],
quiescent_mags=q_mags,
variability_cache=variability_cache,
do_mags=False,
mag_name_tuple=(bp,))
for i_local, i_global in enumerate(valid_obj[0]):
dflux_out[i_global][valid_bp] = results[0][i_local]
def testCalcBasicColors(self):
"""Tests the calculation of the colors of an SED in given bandpasses."""
testUtils = matchBase()
testSED = Sed()
testPhot = BandpassDict.loadTotalBandpassesFromFiles(self.filterList,
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'),'sdss'),
bandpassRoot = 'sdss_')
testSED.readSED_flambda(str(self.galDir + os.listdir(self.galDir)[0]))
testMags = testPhot.magListForSed(testSED)
testColors = []
for filtNum in range(0, len(self.filterList)-1):
testColors.append(testMags[filtNum] - testMags[filtNum+1])
testOutput = testUtils.calcBasicColors([testSED], testPhot)
np.testing.assert_equal([testColors], testOutput)
def testCalcBasicColors(self):
"""Tests the calculation of the colors of an SED in given bandpasses."""
testUtils = matchBase()
testSED = Sed()
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'),
'sdss')
testPhot = BandpassDict.loadTotalBandpassesFromFiles(
self.filterList, bandpassDir=bandpassDir, bandpassRoot='sdss_')
testSED.readSED_flambda(str(self.galDir + os.listdir(self.galDir)[0]))
testMags = testPhot.magListForSed(testSED)
testColors = []
for filtNum in range(0, len(self.filterList) - 1):
testColors.append(testMags[filtNum] - testMags[filtNum + 1])
testOutput = testUtils.calcBasicColors([testSED], testPhot)
np.testing.assert_equal([testColors], testOutput)
def testPhotometricIndicesRaw(self):
"""
Use manMagCalc_list with specified indices on an Sed. Make sure
that the appropriate magnitudes are or are not Nan
"""
starName = os.path.join(getPackageDir('sims_sed_library'), defaultSpecMap['km20_5750.fits_g40_5790'])
starPhot = BandpassDict.loadTotalBandpassesFromFiles()
testSed = Sed()
testSed.readSED_flambda(starName)
indices = [1, 3]
mags = starPhot.magListForSed(testSed, indices=indices)
np.testing.assert_equal(mags[0], np.NaN)
self.assertFalse(np.isnan(mags[1]), msg='mags[1] is NaN; should not be')
np.testing.assert_equal(mags[2], np.NaN)
self.assertFalse(np.isnan(mags[3]), msg='mags[3] is NaN; should not be')
np.testing.assert_equal(mags[4], np.NaN)
np.testing.assert_equal(mags[5], np.NaN)
self.assertEqual(len(mags), 6)
def get_sdss_agn_mags(self):
"""
An example getter for SDSS AGN magnitudes
"""
# load a BandpassDict of SDSS bandpasses, if not done already
if not hasattr(self, 'sdssBandpassDict'):
bandpassNames = ['u', 'g', 'r', 'i', 'z']
bandpassDir = os.path.join(getPackageDir('throughputs'), 'sdss')
bandpassRoot = 'sdss_'
self.sdssBandpassDict = BandpassDict.loadTotalBandpassesFromFiles(
bandpassNames,
bandpassRoot=bandpassRoot,
bandpassDir=bandpassDir)
# actually calculate the magnitudes
return self._magnitudeGetter('agn', self.sdssBandpassDict,
self.get_sdss_agn_mags._colnames)
def compare_to_imsim(phosim_commands):
bandpass = bandpass_all[int(phosim_commands['filter'].values[0])]
obs_md = ObservationMetaData(
pointingRA=float(phosim_commands['rightascension'].values[0]),
pointingDec=float(phosim_commands['declination'].values[0]),
mjd=float(phosim_commands['mjd'].values[0]),
rotSkyPos=float(phosim_commands['rotskypos'].values[0]),
bandpassName=bandpass,
m5=LSSTdefaults().m5(bandpass),
seeing=float(phosim_commands['seeing'].values[0]))
noise_and_background = ESOSkyModel(obs_md,
addNoise=True,
addBackground=True)
phot_params = PhotometricParameters(
exptime=float(phosim_commands['vistime'].values[0]),
nexp=int(phosim_commands['nsnap'].values[0]),
gain=1,
readnoise=0,
darkcurrent=0,
bandpass=bandpass)
# We are going to check one sensor only
detector_list = [
make_galsim_detector(camera_wrapper, "R:2,2 S:1,1", phot_params,
obs_md)
]
bp_dict = BandpassDict.loadTotalBandpassesFromFiles(
bandpassNames=obs_md.bandpass)
gs_interpreter = GalSimInterpreter(obs_metadata=obs_md,
epoch=2000.0,
detectors=detector_list,
bandpassDict=bp_dict,
noiseWrapper=noise_and_background,
seed=1234)
image = gs_interpreter.blankImage(detector=detector_list[0])
image_2 = noise_and_background.addNoiseAndBackground(
image,
bandpass=obs_md.bandpass,
m5=obs_md.m5,
FWHMeff=obs_md.seeing,
photParams=phot_params,
chipName=detector_list[0].name)
return compute_bkg(image_2.array)
def testPhotometricIndicesRaw(self):
"""
Use manMagCalc_list with specified indices on an Sed. Make sure
that the appropriate magnitudes are or are not Nan
"""
starName = os.path.join(getPackageDir('sims_sed_library'),
defaultSpecMap['km20_5750.fits_g40_5790'])
starPhot = BandpassDict.loadTotalBandpassesFromFiles()
testSed = Sed()
testSed.readSED_flambda(starName)
indices = [1, 3]
mags = starPhot.magListForSed(testSed, indices=indices)
np.testing.assert_equal(mags[0], np.NaN)
self.assertFalse(np.isnan(mags[1]),
msg='mags[1] is NaN; should not be')
np.testing.assert_equal(mags[2], np.NaN)
self.assertFalse(np.isnan(mags[3]),
msg='mags[3] is NaN; should not be')
np.testing.assert_equal(mags[4], np.NaN)
np.testing.assert_equal(mags[5], np.NaN)
self.assertEqual(len(mags), 6)
def test_skycounts_function(self):
"""
Test that the SkyCountsPerSec class gives the right result for the previously
calculated zero points. (This is defined as the number of counts per second for
a 24 magnitude source.) Here we set magNorm=24 to calculate the zero points
but when calculating the sky background from the sky brightness
model magNorm=None as above.
"""
desc.imsim.read_config()
instcat_file = os.path.join(os.environ['IMSIM_DIR'], 'tests',
'tiny_instcat.txt')
_, phot_params, _ = desc.imsim.parsePhoSimInstanceFile(instcat_file)
skyModel = skybrightness.SkyModel(mags=False)
skyModel.setRaDecMjd(0., 90., 58000, azAlt=True, degrees=True)
bandPassdic = BandpassDict.loadTotalBandpassesFromFiles(['u', 'g', 'r', 'i', 'z', 'y'])
skycounts_persec = desc.imsim.skyModel.SkyCountsPerSec(skyModel, phot_params, bandPassdic)
skycounts_persec_u = skycounts_persec('u', 24)
self.assertAlmostEqual(skycounts_persec_u.value, self.zp_u)
def setUp(self):
"""
Setup tests
SN_blank: A SNObject with no MW extinction
"""
mydir = get_config_dir()
print('===============================')
print('===============================')
print(mydir)
print('===============================')
print('===============================')
# A range of wavelengths in Ang
self.wave = np.arange(3000., 12000., 50.)
# Equivalent wavelenths in nm
self.wavenm = self.wave / 10.
# Time to be used as Peak
self.mjdobs = 571190
# Check that we can set up a SED
# with no extinction
self.SN_blank = SNObject()
self.SN_blank.setCoords(ra=30., dec=-60.)
self.SN_blank.set(z=0.96, t0=571181, x1=2.66, c=0.353, x0=1.796e-6)
self.SN_blank.set_MWebv(0.)
self.SN_extincted = SNObject(ra=30., dec=-60.)
self.SN_extincted.set(z=0.96,
t0=571181,
x1=2.66,
c=0.353,
x0=1.796112e-06)
self.SNCosmoModel = self.SN_extincted.equivalentSNCosmoModel()
self.rectify_photParams = PhotometricParameters()
self.lsstBandPass = BandpassDict.loadTotalBandpassesFromFiles()
self.SNCosmoBP = sncosmo.Bandpass(wave=self.lsstBandPass['r'].wavelen,
trans=self.lsstBandPass['r'].sb,
wave_unit=astropy.units.Unit('nm'),
name='lsst_r')
def setUp(self):
"""
Setup tests
SN_blank: A SNObject with no MW extinction
"""
from astropy.config import get_config_dir
mydir = get_config_dir()
print '==============================='
print '==============================='
print (mydir)
print '==============================='
print '==============================='
# A range of wavelengths in Ang
self.wave = np.arange(3000., 12000., 50.)
# Equivalent wavelenths in nm
self.wavenm = self.wave / 10.
# Time to be used as Peak
self.mjdobs = 571190
# Check that we can set up a SED
# with no extinction
self.SN_blank = SNObject()
self.SN_blank.setCoords(ra=30., dec=-60.)
self.SN_blank.set(z=0.96, t0=571181, x1=2.66, c=0.353, x0=1.796e-6)
self.SN_blank.set_MWebv(0.)
self.SN_extincted = SNObject(ra=30., dec=-60.)
self.SN_extincted.set(z=0.96, t0=571181, x1=2.66, c=0.353,
x0=1.796112e-06)
self.SNCosmoModel = self.SN_extincted.equivalentSNCosmoModel()
self.lsstBandPass = BandpassDict.loadTotalBandpassesFromFiles()
self.SNCosmoBP = sncosmo.Bandpass(wave=self.lsstBandPass['r'].wavelen,
trans=self.lsstBandPass['r'].sb,
wave_unit=astropy.units.Unit('nm'),
name='lsst_r')
def testSEDCopyBasicColors(self):
"""Tests that when makeCopy=True in calcBasicColors the SED object is unchanged after calling
and that colors are still accurately calculated"""
testUtils = matchBase()
testSED = Sed()
copyTest = Sed()
testPhot = BandpassDict.loadTotalBandpassesFromFiles(self.filterList,
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'),'sdss'),
bandpassRoot = 'sdss_')
testSED.readSED_flambda(str(self.galDir + os.listdir(self.galDir)[0]))
copyTest.setSED(wavelen = testSED.wavelen, flambda = testSED.flambda)
testLambda = copyTest.wavelen[0]
testMags = testPhot.magListForSed(testSED)
testColors = []
for filtNum in range(0, len(self.filterList)-1):
testColors.append(testMags[filtNum] - testMags[filtNum+1])
testOutput = testUtils.calcBasicColors([copyTest], testPhot, makeCopy=True)
self.assertEqual(testLambda, copyTest.wavelen[0])
np.testing.assert_equal([testColors], testOutput)
def testSEDCopyBasicColors(self):
"""Tests that when makeCopy=True in calcBasicColors the SED object is unchanged after calling
and that colors are still accurately calculated"""
testUtils = matchBase()
testSED = Sed()
copyTest = Sed()
testPhot = BandpassDict.loadTotalBandpassesFromFiles(self.filterList,
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'),'sdss'),
bandpassRoot = 'sdss_')
testSED.readSED_flambda(str(self.galDir + os.listdir(self.galDir)[0]))
copyTest.setSED(wavelen = testSED.wavelen, flambda = testSED.flambda)
testLambda = copyTest.wavelen[0]
testMags = testPhot.magListForSed(testSED)
testColors = []
for filtNum in range(0, len(self.filterList)-1):
testColors.append(testMags[filtNum] - testMags[filtNum+1])
testOutput = testUtils.calcBasicColors([copyTest], testPhot, makeCopy=True)
self.assertEqual(testLambda, copyTest.wavelen[0])
np.testing.assert_equal([testColors], testOutput)
def testCalcMagNorm(self):
"""Tests the calculation of magnitude normalization for an SED with the given magnitudes
in the given bandpasses."""
testUtils = matchBase()
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'), 'sdss')
testPhot = BandpassDict.loadTotalBandpassesFromFiles(self.filterList,
bandpassDir = bandpassDir,
bandpassRoot = 'sdss_')
unChangedSED = Sed()
unChangedSED.readSED_flambda(str(self.galDir + os.listdir(self.galDir)[0]))
imSimBand = Bandpass()
imSimBand.imsimBandpass()
testSED = Sed()
testSED.setSED(unChangedSED.wavelen, flambda = unChangedSED.flambda)
magNorm = 20.0
redVal = 0.1
testSED.redshiftSED(redVal)
fluxNorm = testSED.calcFluxNorm(magNorm, imSimBand)
testSED.multiplyFluxNorm(fluxNorm)
sedMags = testPhot.magListForSed(testSED)
stepSize = 0.001
testMagNorm = testUtils.calcMagNorm(sedMags, unChangedSED, testPhot, redshift = redVal)
# Test adding in mag_errors. If an array of np.ones is passed in we should get same result
testMagNormWithErr = testUtils.calcMagNorm(sedMags, unChangedSED, testPhot,
mag_error = np.ones(len(sedMags)), redshift = redVal)
# Also need to add in test for filtRange
sedMagsIncomp = sedMags
sedMagsIncomp[1] = None
filtRangeTest = [0, 2, 3, 4]
testMagNormFiltRange = testUtils.calcMagNorm(sedMagsIncomp, unChangedSED, testPhot,
redshift = redVal, filtRange = filtRangeTest)
self.assertAlmostEqual(magNorm, testMagNorm, delta = stepSize)
self.assertAlmostEqual(magNorm, testMagNormWithErr, delta = stepSize)
self.assertAlmostEqual(magNorm, testMagNormFiltRange, delta = stepSize)
def testCalcMagNorm(self):
"""Tests the calculation of magnitude normalization for an SED with the given magnitudes
in the given bandpasses."""
testUtils = matchBase()
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'), 'sdss')
testPhot = BandpassDict.loadTotalBandpassesFromFiles(self.filterList,
bandpassDir = bandpassDir,
bandpassRoot = 'sdss_')
unChangedSED = Sed()
unChangedSED.readSED_flambda(str(self.galDir + os.listdir(self.galDir)[0]))
imSimBand = Bandpass()
imSimBand.imsimBandpass()
testSED = Sed()
testSED.setSED(unChangedSED.wavelen, flambda = unChangedSED.flambda)
magNorm = 20.0
redVal = 0.1
testSED.redshiftSED(redVal)
fluxNorm = testSED.calcFluxNorm(magNorm, imSimBand)
testSED.multiplyFluxNorm(fluxNorm)
sedMags = testPhot.magListForSed(testSED)
stepSize = 0.001
testMagNorm = testUtils.calcMagNorm(sedMags, unChangedSED, testPhot, redshift = redVal)
# Test adding in mag_errors. If an array of np.ones is passed in we should get same result
testMagNormWithErr = testUtils.calcMagNorm(sedMags, unChangedSED, testPhot,
mag_error = np.ones(len(sedMags)), redshift = redVal)
# Also need to add in test for filtRange
sedMagsIncomp = sedMags
sedMagsIncomp[1] = None
filtRangeTest = [0, 2, 3, 4]
testMagNormFiltRange = testUtils.calcMagNorm(sedMagsIncomp, unChangedSED, testPhot,
redshift = redVal, filtRange = filtRangeTest)
self.assertAlmostEqual(magNorm, testMagNorm, delta = stepSize)
self.assertAlmostEqual(magNorm, testMagNormWithErr, delta = stepSize)
self.assertAlmostEqual(magNorm, testMagNormFiltRange, delta = stepSize)
def __init__(self, R_v=3.1, bandpassDict=None, ref_ebv=1.):
# Calculate dust extinction values
self.Ax1 = {}
if bandpassDict is None:
bandpassDict = BandpassDict.loadTotalBandpassesFromFiles(
['u', 'g', 'r', 'i', 'z', 'y'])
for filtername in bandpassDict:
wavelen_min = bandpassDict[filtername].wavelen.min()
wavelen_max = bandpassDict[filtername].wavelen.max()
testsed = Sed()
testsed.setFlatSED(wavelen_min=wavelen_min,
wavelen_max=wavelen_max,
wavelen_step=1.0)
self.ref_ebv = ref_ebv
# Calculate non-dust-extincted magnitude
flatmag = testsed.calcMag(bandpassDict[filtername])
# Add dust
a, b = testsed.setupCCM_ab()
testsed.addDust(a, b, ebv=self.ref_ebv, R_v=R_v)
# Calculate difference due to dust when EBV=1.0 (m_dust = m_nodust - Ax, Ax > 0)
self.Ax1[filtername] = testsed.calcMag(
bandpassDict[filtername]) - flatmag
def test_k_correction(self):
"""
Test that the K correction correctly converts absolute magnitude
to observed magnitude.
"""
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
rng = np.random.RandomState(41321)
sed_dir = os.path.join(getPackageDir('sims_sed_library'), 'galaxySED')
list_of_sed_files = os.listdir(sed_dir)
list_of_sed_files.sort()
sed_to_check = rng.choice(list_of_sed_files, size=10)
redshift_arr = rng.random_sample(len(sed_to_check)) * 2.0 + 0.1
bp = bp_dict['g']
for sed_name, zz in zip(sed_to_check, redshift_arr):
full_name = os.path.join(sed_dir, sed_name)
ss = Sed()
ss.readSED_flambda(full_name)
true_rest_mag = ss.calcMag(bp)
ss.redshiftSED(zz, dimming=True)
obs_mag = ss.calcMag(bp)
k_corr = k_correction(ss, bp, zz)
self.assertLess(np.abs(true_rest_mag - obs_mag + k_corr), 0.001)
def add_flux(self, sed_name, redshift, magnorm_dict, av, rv, bp_dict=None):
if bp_dict is None:
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
result_dict_no_mw = {}
result_dict_mw = {}
sed_obj = Sed()
sed_obj.readSED_flambda(os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'], sed_name))
a_x, b_x = sed_obj.setupCCM_ab()
for bandpass_name in bp_dict.keys():
sed_copy = deepcopy(sed_obj)
flux_norm = getImsimFluxNorm(sed_copy, magnorm_dict[bandpass_name])
sed_copy.multiplyFluxNorm(flux_norm)
sed_copy.redshiftSED(redshift, dimming=True)
band_flux = sed_copy.calcFlux(bp_dict[bandpass_name])
result_dict_no_mw[bandpass_name] = band_flux
sed_copy.addDust(a_x, b_x, A_v=av, R_v=rv)
band_flux_mw = sed_copy.calcFlux(bp_dict[bandpass_name])
result_dict_mw[bandpass_name] = band_flux_mw
return result_dict_no_mw, result_dict_mw
def create_k_corr_grid(redshift):
"""
(Edited from original because of error)
Returns a grid of redshifts and K corrections on the
LSST Simulations AGN SED that can be used for K correction
interpolation.
"""
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
bp_i = bp_dict['i']
sed_dir = os.path.join(getPackageDir('sims_sed_library'), 'agnSED')
sed_name = os.path.join(sed_dir, 'agn.spec.gz')
if not os.path.exists(sed_name):
raise RuntimeError('\n\n%s\n\nndoes not exist\n\n' % sed_name)
base_sed = Sed()
base_sed.readSED_flambda(sed_name)
z_grid = np.arange(0.0, redshift.max(), 0.01)
k_grid = np.zeros(len(z_grid), dtype=float)
for i_z, zz in enumerate(z_grid):
ss = Sed(flambda=base_sed.flambda, wavelen=base_sed.wavelen)
ss.redshiftSED(zz, dimming=True)
k = k_correction(ss, bp_i, zz)
k_grid[i_z] = k
return z_grid, k_grid
def __init__(self,
opsimdb,
descqa_catalog,
dither=True,
min_mag=10,
minsource=100,
proper_motion=False,
protoDC2_ra=0,
protoDC2_dec=0,
star_db_name=None,
sed_lookup_dir=None,
agn_db_name=None,
agn_threads=1,
sn_db_name=None,
sprinkler=False,
host_image_dir=None,
host_data_dir=None,
config_dict=None,
gzip_threads=3,
objects_to_skip=()):
"""
Parameters
----------
opsimdb: str
OpSim db filename.
descqa_catalog: str
Name of the DESCQA galaxy catalog.
dither: bool [True]
Flag to enable the dithering included in the opsim db file.
min_mag: float [10]
Minimum value of the star magnitude at 500nm to include.
minsource: int [100]
Minimum number of objects for phosim.py to simulate a chip.
proper_motion: bool [True]
Flag to enable application of proper motion to stars.
protoDC2_ra: float [0]
Desired RA (J2000 degrees) of protoDC2 center.
protoDC2_dec: float [0]
Desired Dec (J2000 degrees) of protoDC2 center.
star_db_name: str [None]
Filename of the database containing stellar sources
sed_lookup_dir: str [None]
Directory where the SED lookup tables reside.
agn_db_name: str [None]
Filename of the agn parameter sqlite db file.
agn_threads: int [1]
Number of threads to use when simulating AGN variability
sn_db_name: str [None]
Filename of the supernova parameter sqlite db file.
sprinkler: bool [False]
Flag to enable the Sprinkler.
host_image_dir: string
The location of the FITS images of lensed AGN/SNe hosts produced by generate_lensed_hosts_***.py
host_data_dir: string
Location of csv file of lensed host data created by the sprinkler
gzip_threads: int
The number of gzip jobs that can be started in parallel after
catalogs are written (default=3)
objects_to_skip: set-like or list-like [()]
Collection of object types to skip, e.g., stars, knots, bulges, disks, sne, agn
"""
self.t_start = time.time()
if not os.path.exists(opsimdb):
raise RuntimeError('%s does not exist' % opsimdb)
self.gzip_threads = gzip_threads
# load the data for the parametrized light
# curve stellar variability model into a
# global cache
plc = ParametrizedLightCurveMixin()
plc.load_parametrized_light_curves()
self.config_dict = config_dict if config_dict is not None else {}
self.descqa_catalog = descqa_catalog
self.dither = dither
self.min_mag = min_mag
self.minsource = minsource
self.proper_motion = proper_motion
self.protoDC2_ra = protoDC2_ra
self.protoDC2_dec = protoDC2_dec
self.phot_params = PhotometricParameters(nexp=1, exptime=30)
self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
self.obs_gen = ObservationMetaDataGenerator(database=opsimdb,
driver='sqlite')
if star_db_name is None:
raise IOError("Need to specify star_db_name")
if not os.path.isfile(star_db_name):
raise IOError("%s is not a file\n" % star_db_name +
"(This is what you specified for star_db_name")
self.star_db = DC2StarObj(database=star_db_name, driver='sqlite')
self.sprinkler = sprinkler
if self.sprinkler and not HAS_TWINKLES:
raise RuntimeError("You are trying to enable the sprinkler; "
"but Twinkles cannot be imported")
if not os.path.isdir(sed_lookup_dir):
raise IOError("\n%s\nis not a dir" % sed_lookup_dir)
self.sed_lookup_dir = sed_lookup_dir
self._agn_threads = agn_threads
if agn_db_name is not None:
if os.path.exists(agn_db_name):
self.agn_db_name = agn_db_name
else:
raise IOError("Path to Proto DC2 AGN database does not exist.")
else:
self.agn_db_name = None
self.sn_db_name = None
if sn_db_name is not None:
if os.path.isfile(sn_db_name):
self.sn_db_name = sn_db_name
else:
raise IOError("%s is not a file" % sn_db_name)
if host_image_dir is None and self.sprinkler is not False:
raise IOError(
"Need to specify the name of the host image directory.")
elif self.sprinkler is not False:
if os.path.exists(host_image_dir):
self.host_image_dir = host_image_dir
else:
raise IOError("Path to host image directory" +
"\n\n%s\n\n" % host_image_dir +
"does not exist.")
if host_data_dir is None and self.sprinkler is not False:
raise IOError(
"Need to specify the name of the host data directory.")
elif self.sprinkler is not False:
if os.path.exists(host_data_dir):
self.host_data_dir = host_data_dir
else:
raise IOError(
"Path to host data directory does not exist.\n\n",
"%s\n\n" % host_data_dir)
self.instcats = get_instance_catalogs()
object_types = 'stars knots bulges disks sprinkled hosts sne agn'.split(
)
if any([_ not in object_types for _ in objects_to_skip]):
raise RuntimeError(f'objects_to_skip ({objects_to_skip}) '
'contains invalid object types')
self.do_obj_type = {_: _ not in objects_to_skip for _ in object_types}
def test_alert_data_generation(self):
dmag_cutoff = 0.005
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5}
_max_var_param_str = self.max_str_len
class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject):
objid = 'star_alert'
tableid = 'stars'
idColKey = 'simobjid'
raColName = 'ra'
decColName = 'dec'
objectTypeId = 0
columns = [('raJ2000', 'ra*0.01745329252'),
('decJ2000', 'dec*0.01745329252'),
('parallax', 'px*0.01745329252/3600.0'),
('properMotionRa', 'pmra*0.01745329252/3600.0'),
('properMotionDec', 'pmdec*0.01745329252/3600.0'),
('radialVelocity', 'vrad'),
('variabilityParameters', 'varParamStr', str, _max_var_param_str)]
class TestAlertsVarCatMixin(object):
@register_method('alert_test')
def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None):
if len(params) == 0:
return np.array([[], [], [], [], [], []])
if isinstance(expmjd, numbers.Number):
dmags_out = np.zeros((6, self.num_variable_obj(params)))
else:
dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd)))
for i_star in range(self.num_variable_obj(params)):
if params['amp'][i_star] is not None:
dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd)
for i_filter in range(6):
dmags_out[i_filter][i_star] = dmags
return dmags_out
class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog):
pass
class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoords, AstrometryStars,
Variability, InstanceCatalog):
column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert']
camera = obs_lsst_phosim.PhosimMapper().camera
@compound('delta_umag', 'delta_gmag', 'delta_rmag',
'delta_imag', 'delta_zmag', 'delta_ymag')
def get_TruthVariability(self):
return self.applyVariability(self.column_by_name('varParamStr'))
@cached
def get_dmagAlert(self):
return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass)
@cached
def get_magAlert(self):
return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
self.column_by_name('dmagAlert')
star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite')
# assemble the true light curves for each object; we need to figure out
# if their np.max(dMag) ever goes over dmag_cutoff; then we will know if
# we are supposed to simulate them
true_lc_dict = {}
true_lc_obshistid_dict = {}
is_visible_dict = {}
obs_dict = {}
max_obshistid = -1
n_total_observations = 0
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid > max_obshistid:
max_obshistid = obshistid
cat = TestAlertsTruthCat(star_db, obs_metadata=obs)
for line in cat.iter_catalog():
if line[1] is None:
continue
n_total_observations += 1
if line[0] not in true_lc_dict:
true_lc_dict[line[0]] = {}
true_lc_obshistid_dict[line[0]] = []
true_lc_dict[line[0]][obshistid] = line[2]
true_lc_obshistid_dict[line[0]].append(obshistid)
if line[0] not in is_visible_dict:
is_visible_dict[line[0]] = False
if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]:
is_visible_dict[line[0]] = True
obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2)))
skipped_due_to_mag = 0
objects_to_simulate = []
obshistid_unqid_set = set()
for obj_id in true_lc_dict:
dmag_max = -1.0
for obshistid in true_lc_dict[obj_id]:
if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max:
dmag_max = np.abs(true_lc_dict[obj_id][obshistid])
if dmag_max >= dmag_cutoff:
if not is_visible_dict[obj_id]:
skipped_due_to_mag += 1
continue
objects_to_simulate.append(obj_id)
for obshistid in true_lc_obshistid_dict[obj_id]:
obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid)
self.assertGreater(len(objects_to_simulate), 10)
self.assertGreater(skipped_due_to_mag, 0)
log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat,
output_prefix='alert_test',
output_dir=self.output_dir,
dmag_cutoff=dmag_cutoff,
log_file_name=log_file_name)
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
phot_params = PhotometricParameters()
# First, verify that the contents of the sqlite files are all correct
n_tot_simulated = 0
alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, '
alert_query += 'meta.band, quiescent.flux, alert.dflux, '
alert_query += 'quiescent.snr, alert.snr, '
alert_query += 'alert.ra, alert.dec, alert.chipNum, '
alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, '
alert_query += 'ast.parallax '
alert_query += 'FROM alert_data AS alert '
alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId '
alert_query += 'INNER JOIN quiescent_flux AS quiescent '
alert_query += 'ON quiescent.uniqueId=alert.uniqueId '
alert_query += 'AND quiescent.band=meta.band '
alert_query += 'INNER JOIN baseline_astrometry AS ast '
alert_query += 'ON ast.uniqueId=alert.uniqueId'
alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int),
('TAI', float), ('band', int),
('q_flux', float), ('dflux', float),
('q_snr', float), ('tot_snr', float),
('ra', float), ('dec', float),
('chipNum', int), ('xPix', float), ('yPix', float),
('pmRA', float), ('pmDec', float), ('parallax', float)])
sqlite_file_list = os.listdir(self.output_dir)
n_tot_simulated = 0
obshistid_unqid_simulated_set = set()
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype)
if len(alert_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI'])
for i_obj in range(len(alert_data)):
n_tot_simulated += 1
obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) +
alert_data['obshistId'][i_obj])
unq = alert_data['uniqueId'][i_obj]
obj_dex = (unq//1024)-1
self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4)
self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4)
self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4)
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
alert_data['ra'][i_obj], alert_data['dec'][i_obj])
distance_arcsec = 3600.0*distance
msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth,
alert_data['ra'][i_obj],
alert_data['dec'][i_obj])
self.assertLess(distance_arcsec, 0.0005, msg=msg)
obs = obs_dict[alert_data['obshistId'][i_obj]]
chipname = chipNameFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
camera=self.camera)
chipnum = int(chipname.replace('R', '').replace('S', '').
replace(' ', '').replace(';', '').replace(',', '').
replace(':', ''))
self.assertEqual(chipnum, alert_data['chipNum'][i_obj])
xpix, ypix = pixelCoordsFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
camera=self.camera)
self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4)
self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4)
dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj])
self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]],
dmag_sim, 3)
mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]]
m5 = obs.m5[mag_name]
q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj])
self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
q_mag, 4)
snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
bp_dict[mag_name],
self.obs_mag_cutoff[alert_data['band'][i_obj]],
phot_params)
self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4)
tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \
true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]]
snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name],
m5, phot_params)
self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4)
for val in obshistid_unqid_set:
self.assertIn(val, obshistid_unqid_simulated_set)
self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set))
astrometry_query = 'SELECT uniqueId, ra, dec, TAI '
astrometry_query += 'FROM baseline_astrometry'
astrometry_dtype = np.dtype([('uniqueId', int),
('ra', float),
('dec', float),
('TAI', float)])
tai_list = []
for obs in self.obs_list:
tai_list.append(obs.mjd.TAI)
tai_list = np.array(tai_list)
n_tot_ast_simulated = 0
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype)
if len(astrometry_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI'])
for i_obj in range(len(astrometry_data)):
n_tot_ast_simulated += 1
obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
astrometry_data['ra'][i_obj],
astrometry_data['dec'][i_obj])
self.assertLess(3600.0*distance, 0.0005)
del alert_gen
gc.collect()
self.assertGreater(n_tot_simulated, 10)
self.assertGreater(len(obshistid_unqid_simulated_set), 10)
self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations)
self.assertGreater(n_tot_ast_simulated, 0)
def __init__(self,
catsim_cat,
visit_mjd,
specFileMap,
sed_path,
om10_cat='twinkles_lenses_v2.fits',
sne_cat='dc2_sne_cat.csv',
density_param=1.,
cached_sprinkling=False,
agn_cache_file=None,
sne_cache_file=None,
defs_file=None):
"""
Parameters
----------
catsim_cat: catsim catalog
The results array from an instance catalog.
visit_mjd: float
The mjd of the visit
specFileMap:
This will tell the instance catalog where to write the files
om10_cat: optional, defaults to 'twinkles_lenses_v2.fits
fits file with OM10 catalog
sne_cat: optional, defaults to 'dc2_sne_cat.csv'
density_param: `np.float`, optioanl, defaults to 1.0
the fraction of eligible agn objects that become lensed and should
be between 0.0 and 1.0.
cached_sprinkling: boolean
If true then pick from a preselected list of galtileids
agn_cache_file: str
sne_cache_file: str
defs_file: str
Returns
-------
updated_catalog:
A new results array with lens systems added.
"""
twinklesDir = getPackageDir('Twinkles')
om10_cat = os.path.join(twinklesDir, 'data', om10_cat)
self.catalog = catsim_cat
# ****** THIS ASSUMES THAT THE ENVIRONMENT VARIABLE OM10_DIR IS SET *******
lensdb = om10.DB(catalog=om10_cat, vb=False)
self.lenscat = lensdb.lenses.copy()
self.density_param = density_param
self.bandpassDict = BandpassDict.loadTotalBandpassesFromFiles(
bandpassNames=['i'])
self.sne_catalog = pd.read_csv(
os.path.join(twinklesDir, 'data', sne_cat))
#self.sne_catalog = self.sne_catalog.iloc[:101] ### Remove this after testing
self.used_systems = []
self.visit_mjd = visit_mjd
self.sn_obj = SNObject(0., 0.)
self.write_dir = specFileMap.subdir_map['(^specFileGLSN)']
self.sed_path = sed_path
self.cached_sprinkling = cached_sprinkling
if self.cached_sprinkling is True:
if ((agn_cache_file is None) | (sne_cache_file is None)):
raise AttributeError(
'Must specify cache files if using cached_sprinkling.')
#agn_cache_file = os.path.join(twinklesDir, 'data', 'test_agn_galtile_cache.csv')
self.agn_cache = pd.read_csv(agn_cache_file)
#sne_cache_file = os.path.join(twinklesDir, 'data', 'test_sne_galtile_cache.csv')
self.sne_cache = pd.read_csv(sne_cache_file)
else:
self.agn_cache = None
self.sne_cache = None
if defs_file is None:
self.defs_file = os.path.join(twinklesDir, 'data',
'catsim_defs.csv')
else:
self.defs_file = defs_file
specFileStart = 'Burst'
for key, val in sorted(iteritems(SpecMap.subdir_map)):
if re.match(key, specFileStart):
galSpecDir = str(val)
self.galDir = str(
getPackageDir('sims_sed_library') + '/' + galSpecDir + '/')
self.imSimBand = Bandpass()
self.imSimBand.imsimBandpass()
#self.LRG_name = 'Burst.25E09.1Z.spec'
#self.LRG = Sed()
#self.LRG.readSED_flambda(str(galDir + self.LRG_name))
#return
#Calculate imsimband magnitudes of source galaxies for matching
agn_fname = str(
getPackageDir('sims_sed_library') + '/agnSED/agn.spec.gz')
src_iband = self.lenscat['MAGI_IN']
src_z = self.lenscat['ZSRC']
self.src_mag_norm = []
for src, s_z in zip(src_iband, src_z):
agn_sed = Sed()
agn_sed.readSED_flambda(agn_fname)
agn_sed.redshiftSED(s_z, dimming=True)
self.src_mag_norm.append(matchBase().calcMagNorm(
[src], agn_sed, self.bandpassDict))
#self.src_mag_norm = matchBase().calcMagNorm(src_iband,
# [agn_sed]*len(src_iband),
#
# self.bandpassDict)
self.defs_dict = {}
with open(self.defs_file, 'r') as f:
for line in f:
line_defs = line.split(',')
if len(line_defs) > 1:
self.defs_dict[line_defs[0]] = line_defs[1].split('\n')[0]
def test_airmassdep_bandpass_fid():
adb = AirmassDependentBandpass.fromThroughputs()
tot = BandpassDict.loadTotalBandpassesFromFiles()
bp = adb.bandpassForAirmass(bandname='r', airmass=1.200)
assert_allclose(bp.wavelen, tot['r'].wavelen, rtol=1.0e-3, atol=1.0e-8)
assert_allclose(bp.sb, tot['r'].sb, rtol=1.0e-1, atol=1.0e-8)
def matchToRestFrame(self, sedList, catMags, mag_error = None, bandpassDict = None, makeCopy = False):
"""
This will find the closest match to the magnitudes of a galaxy catalog if those magnitudes are in
the rest frame. Objects without magnitudes in at least two adjacent bandpasses will return as none
and print out a message.
@param [in] sedList is the set of spectral objects from the models SEDs provided by loadBC03
or other custom loader routine.
@param [in] catMags is an array of the magnitudes of catalog objects to be matched with a model SED.
It should be organized so that there is one object's magnitudes along each row.
@param [in] mag_error are provided error values for magnitudes in objectMags. If none provided
then this defaults to 1.0. This should be an array of the same size as catMags.
@param [in] bandpassDict is an OrderedDict of bandpass objects with which to calculate magnitudes.
If left equal to None it will by default load the SDSS [u,g,r,i,z] bandpasses.
@param [in] makeCopy indicates whether or not to operate on copies of the SED objects in sedList
since this method will change the wavelength grid.
@param [out] sedMatches is a list with the name of a model SED that matches most closely to each
object in the catalog.
@param [out] magNormMatches are the magnitude normalizations for the given magnitudes and
matched SED.
@param [out] matchErrors contains the Mean Squared Error between the colors of each object and
the colors of the matched SED.
"""
#Set up photometry to calculate model Mags
if bandpassDict is None:
galPhot = BandpassDict.loadTotalBandpassesFromFiles(['u','g','r','i','z'],
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'),'sdss'),
bandpassRoot = 'sdss_')
else:
galPhot = bandpassDict
modelColors = []
sedMatches = []
magNormMatches = []
#Find the colors for all model SEDs
modelColors = self.calcBasicColors(sedList, galPhot, makeCopy = makeCopy)
modelColors = np.transpose(modelColors)
#Match the catalog colors to models
numCatMags = len(catMags)
numOn = 0
notMatched = 0
matchColors = []
matchErrors = []
for filtNum in range(0, len(galPhot)-1):
matchColors.append(np.transpose(catMags)[filtNum] - np.transpose(catMags)[filtNum+1])
matchColors = np.transpose(matchColors)
for catObject in matchColors:
#This is done to handle objects with incomplete magnitude data
colorRange = np.arange(0, len(galPhot)-1)
filtNums = np.arange(0, len(galPhot))
if np.isnan(np.amin(catObject))==True:
colorRange = np.where(np.isnan(catObject)==False)[0]
filtNums = np.unique([colorRange, colorRange+1]) #To pick out right filters in calcMagNorm
if len(colorRange) == 0:
print 'Could not match object #%i. No magnitudes for two adjacent bandpasses.' % (numOn)
notMatched += 1
sedMatches.append(None)
magNormMatches.append(None)
matchErrors.append(None)
else:
distanceArray = np.zeros(len(sedList))
for colorNum in colorRange:
distanceArray += np.power((modelColors[colorNum] - catObject[colorNum]),2)
matchedSEDNum = np.nanargmin(distanceArray)
sedMatches.append(sedList[matchedSEDNum].name)
magNorm = self.calcMagNorm(np.array(catMags[numOn]), sedList[matchedSEDNum],
galPhot, mag_error = mag_error, filtRange = filtNums)
magNormMatches.append(magNorm)
matchErrors.append(distanceArray[matchedSEDNum]/len(colorRange))
numOn += 1
if numOn % 10000 == 0:
print 'Matched %i of %i catalog objects to SEDs' % (numOn-notMatched, numCatMags)
print 'Done Matching. Matched %i of %i catalog objects to SEDs' % (numCatMags-notMatched, numCatMags)
if notMatched > 0:
print '%i objects did not get matched' % (notMatched)
return sedMatches, magNormMatches, matchErrors
def findSED(self,
sedList,
catMags,
catRA=None,
catDec=None,
mag_error=None,
reddening=True,
bandpassDict=None,
colors=None,
extCoeffs=(4.239, 3.303, 2.285, 1.698, 1.263),
makeCopy=False):
"""
This will find the SEDs that are the closest match to the magnitudes of a star catalog.
It can also correct for reddening from within the milky way. Objects without magnitudes in at least
two adjacent bandpasses will return as none and print out a message.
@param [in] sedList is the set of spectral objects from the models SEDs provided by loaders in rgStar
in rgUtils.py or other custom loader routine.
@param [in] catMags is an array of the magnitudes of catalog objects to be matched with a model SED.
It should be organized so that there is one object's magnitudes along each row.
@param [in] catRA is an array of the RA positions for each catalog object.
@param [in] catDec is an array of the Dec position for each catalog object.
@param [in] mag_error are provided error values for magnitudes in objectMags. If none provided
then this defaults to 1.0. This should be an array of the same length as objectMags.
@param [in] reddening is a boolean that determines whether to correct catalog magnitudes for
dust in the milky way. By default, it is True.
If true, this uses calculateEBV from EBV.py to find an EBV value for the object's
ra and dec coordinates and then uses the coefficients provided by extCoeffs which should come
from Schlafly and Finkbeiner (2011) for the correct filters and in the same order as provided
in bandpassDict.
If false, this means it will not run the dereddening procedure.
@param [in] bandpassDict is a BandpassDict with which to calculate magnitudes. If left
equal to None it will by default load the SDSS [u,g,r,i,z] bandpasses and therefore agree with
default extCoeffs.
@param [in] colors is None if you are just providing a list of SED objects to match, but is the
array holding the colors of those SED models (each row should be the colors for one model in the
same order as sedList) if you have already calculated the colors.
@param [in] extCoeffs are the Schlafly and Finkbeiner (2011) (ApJ, 737, 103) coefficients for the
given filters from bandpassDict and need to be in the same order as bandpassDict. The default given
are the SDSS [u,g,r,i,z] values.
@param [in] makeCopy indicates whether or not to operate on copies of the SED objects in sedList
since this method will change the wavelength grid.
@param [out] sedMatches is a list with the name of a model SED that matches most closely to each
object in the catalog.
@param [out] magNormMatches are the magnitude normalizations for the given magnitudes and
matched SED.
@param [out] matchErrors contains the Mean Squared Error between the colors of each object and
the colors of the matched SED.
"""
if bandpassDict is None:
starPhot = BandpassDict.loadTotalBandpassesFromFiles(
['u', 'g', 'r', 'i', 'z'],
bandpassDir=os.path.join(
lsst.utils.getPackageDir('throughputs'), 'sdss'),
bandpassRoot='sdss_')
else:
starPhot = bandpassDict
if colors is None:
modelColors = self.calcBasicColors(sedList,
starPhot,
makeCopy=makeCopy)
else:
modelColors = colors
#Transpose so that all values for one color are in one row as needed for the matching loop below
modelColors = np.transpose(modelColors)
if reddening == True:
#Check that catRA and catDec are included
if catRA is None or catDec is None:
raise RuntimeError(
"Reddening is True, but catRA and catDec are not included."
)
calcEBV = ebv()
raDec = np.array((catRA, catDec))
#If only matching one object need to reshape for calculateEbv
if len(raDec.shape) == 1:
raDec = raDec.reshape((2, 1))
ebvVals = calcEBV.calculateEbv(equatorialCoordinates=raDec)
objMags = self.deReddenMags(ebvVals, catMags, extCoeffs)
else:
objMags = catMags
objMags = np.array(objMags)
matchColors = []
for filtNum in range(0, len(starPhot) - 1):
matchColors.append(
np.transpose(objMags)[filtNum] -
np.transpose(objMags)[filtNum + 1])
matchColors = np.transpose(matchColors)
numCatMags = len(catMags)
numOn = 0
sedMatches = []
magNormMatches = []
notMatched = 0
matchErrors = []
for catObject in matchColors:
#This is done to handle objects with incomplete magnitude data
colorRange = np.arange(0, len(starPhot) - 1)
filtNums = np.arange(0, len(starPhot))
if np.isnan(np.amin(catObject)) == True:
colorRange = np.where(np.isnan(catObject) == False)[0]
filtNums = np.unique([colorRange, colorRange + 1
]) #Pick right filters in calcMagNorm
if len(colorRange) == 0:
print 'Could not match object #%i. No magnitudes for two adjacent bandpasses.' % (
numOn)
notMatched += 1
sedMatches.append(None)
magNormMatches.append(None)
matchErrors.append(None)
else:
distanceArray = np.zeros(len(sedList))
for colorNum in colorRange:
distanceArray += np.power(
(modelColors[colorNum] - catObject[colorNum]), 2)
matchedSEDNum = np.nanargmin(distanceArray)
sedMatches.append(sedList[matchedSEDNum].name)
magNorm = self.calcMagNorm(objMags[numOn],
sedList[matchedSEDNum],
starPhot,
filtRange=filtNums)
magNormMatches.append(magNorm)
matchErrors.append(distanceArray[matchedSEDNum] /
len(colorRange)) #Mean Squared Error
numOn += 1
if numOn % 10000 == 0:
print 'Matched %i of %i catalog objects to SEDs' % (
numOn - notMatched, numCatMags)
if numCatMags > 1:
print 'Done Matching. Matched %i of %i catalog objects to SEDs' % (
numCatMags - notMatched, numCatMags)
if notMatched > 0:
print '%i objects did not get matched' % (notMatched)
return sedMatches, magNormMatches, matchErrors
def loadGalfast(self,
filenameList,
outFileList,
sEDPath=None,
kuruczPath=None,
mltPath=None,
wdPath=None,
kuruczSubset=None,
mltSubset=None,
wdSubset=None,
chunkSize=10000):
"""
This is customized for the outputs we currently need for the purposes of consistent output
It will read in a galfast output file and output desired values for database input into a file
@param [in] filenameList is a list of the galfast output files that will be loaded and processed.
Can process fits, gzipped, or txt output from galfast.
@param [in] outFileList is a list of the names of the output files that will be created. If gzipped
output is desired simply write the filenames with .gz at the end.
@param [in] kuruczPath is a place to specify a path to kurucz SED files
@param [in] mltPath is the same as kuruczPath except that it specifies a directory for the
mlt SEDs
@param [in] wdPath is the same as the previous two except that it specifies a path to a
white dwarf SED directory.
@param [in] kuruczSubset is a list which provides a subset of the kurucz files within the
kurucz folder that one wants to use
@param [in] mltSubset is a list which provides a subset of the mlt files within the
mlt folder that one wants to use
@param [in] wdSubset is a list which provides a subset of the wd files within the
wd folder that one wants to use
@param [in] chunkSize is the size of chunks of lines to be read from the catalog at one time.
"""
for filename in filenameList:
#Make sure input file exists and is readable format before doing anything else
if os.path.isfile(filename) == False:
raise RuntimeError('*** File does not exist')
#Process various possible galfast outputs
if filename.endswith(('.txt', '.gz', '.fits')):
continue
else:
raise RuntimeError(
str('*** Unsupported File Format in file: ' +
str(filename)))
#If all files exist and are in proper formats then load seds
selectStarSED0 = selectStarSED(kuruczDir=kuruczPath,
mltDir=mltPath,
wdDir=wdPath)
if kuruczSubset is None:
kuruczList = selectStarSED0.loadKuruczSEDs()
else:
kuruczList = selectStarSED0.loadKuruczSEDs(subset=kuruczSubset)
#Only need one dictionary since none of the names overlap
positionDict = {}
for kNum, kuruczSED in enumerate(kuruczList):
positionDict[kuruczSED.name] = kNum
if mltSubset is None:
mltList = selectStarSED0.loadmltSEDs()
else:
mltList = selectStarSED0.loadmltSEDs(subset=mltSubset)
for mltNum, mltSED in enumerate(mltList):
positionDict[mltSED.name] = mltNum
if wdSubset is None:
wdListH, wdListHE = selectStarSED0.loadwdSEDs()
else:
wdListH, wdListHE = selectStarSED0.loadwdSEDs(subset=wdSubset)
for hNum, hSED in enumerate(wdListH):
positionDict[hSED.name] = hNum
for heNum, heSED in enumerate(wdListHE):
positionDict[heSED.name] = heNum
#For adding/subtracting extinction when calculating colors
#Numbers below come from Schlafly and Finkbeiner (2011) (ApJ, 737, 103)
#normalized by SDSS r mag value
sdssExtCoeffs = [1.8551, 1.4455, 1.0, 0.7431, 0.5527]
lsstExtCoeffs = [1.8140, 1.4166, 0.9947, 0.7370, 0.5790, 0.4761]
sdssPhot = BandpassDict.loadTotalBandpassesFromFiles(
['u', 'g', 'r', 'i', 'z'],
bandpassDir=os.path.join(lsst.utils.getPackageDir('throughputs'),
'sdss'),
bandpassRoot='sdss_')
#Load Bandpasses for LSST colors to get colors from matched SEDs
lsstFilterList = ('u', 'g', 'r', 'i', 'z', 'y')
lsstPhot = BandpassDict.loadTotalBandpassesFromFiles(lsstFilterList)
imSimBand = Bandpass()
imSimBand.imsimBandpass()
#Calculate colors and add them to the SED objects
kuruczColors = selectStarSED0.calcBasicColors(kuruczList, sdssPhot)
mltColors = selectStarSED0.calcBasicColors(mltList, sdssPhot)
hColors = selectStarSED0.calcBasicColors(wdListH, sdssPhot)
heColors = selectStarSED0.calcBasicColors(wdListHE, sdssPhot)
listDict = {
'kurucz': kuruczList,
'mlt': mltList,
'H': wdListH,
'HE': wdListHE
}
colorDict = {
'kurucz': kuruczColors,
'mlt': mltColors,
'H': hColors,
'HE': heColors
}
for filename, outFile in zip(filenameList, outFileList):
if filename.endswith('.txt'):
galfastIn = open(filename, 'rt')
inFits = False
gzFile = False
num_lines = sum(1 for line in open(filename))
elif filename.endswith('.gz'):
galfastIn = gzip.open(filename, 'rt')
inFits = False
gzFile = True
num_lines = sum(1 for line in gzip.open(filename))
elif filename.endswith('fits'):
hdulist = fits.open(filename)
galfastIn = hdulist[1].data
num_lines = len(galfastIn)
gzFile = False
inFits = True
if outFile.endswith('.txt'):
fOut = open(outFile, 'wt')
elif outFile.endswith('.gz'):
fOut = gzip.open(outFile, 'wt')
fOut.write('#oID, ra, dec, gall, galb, coordX, coordY, coordZ, sEDName, magNorm, ' +\
'LSSTugrizy, SDSSugriz, absSDSSr, pmRA, pmDec, vRad, pml, pmb, vRadlb, ' +\
'vR, vPhi, vZ, FeH, pop, distKpc, ebv, ebvInf\n')
header_length = 0
numChunks = 0
if inFits == False:
galfastDict = self.parseGalfast(galfastIn.readline())
header_length += 1
header_status = True
while header_status == True:
newLine = galfastIn.readline()
if newLine[0] != '#':
header_status = False
else:
header_length += 1
print('Total objects = %i' % (num_lines - header_length))
numChunks = ((num_lines - header_length) // chunkSize) + 1
for chunk in range(0, numChunks):
if chunk == numChunks - 1:
lastChunkSize = (num_lines - header_length) % chunkSize
readSize = lastChunkSize
else:
readSize = chunkSize
oID = np.arange(readSize * chunk, readSize * (chunk + 1))
if inFits:
starData = galfastIn[readSize * chunk:(readSize * chunk +
readSize)]
sDSS = starData.field('SDSSugriz')
gall, galb = np.transpose(starData.field('lb'))
ra, dec = np.transpose(starData.field('radec'))
coordX, coordY, coordZ = np.transpose(
starData.field('XYZ'))
DM = starData.field('DM')
absSDSSr = starData.field('absSDSSr')
pop = starData.field('comp')
FeH = starData.field('FeH')
vR, vPhi, vZ = np.transpose(starData.field('vcyl'))
pml, pmb, vRadlb = np.transpose(starData.field('pmlb'))
pmRA, pmDec, vRad = np.transpose(starData.field('pmradec'))
am = starData.field('Am')
amInf = starData.field('AmInf')
sdssPhotoFlags = starData.field('SDSSugrizPhotoFlags')
else:
if gzFile == False:
with open(filename) as t_in:
starData = np.loadtxt(
itertools.islice(
t_in, ((readSize * chunk) + header_length),
((readSize *
(chunk + 1)) + header_length)))
else:
with gzip.open(filename) as t_in:
starData = np.loadtxt(
itertools.islice(
t_in, ((readSize * chunk) + header_length),
((readSize *
(chunk + 1)) + header_length)))
starData = np.transpose(starData)
gall = starData[galfastDict['l']]
galb = starData[galfastDict['b']]
ra = starData[galfastDict['ra']]
dec = starData[galfastDict['dec']]
coordX = starData[galfastDict['X']]
coordY = starData[galfastDict['Y']]
coordZ = starData[galfastDict['Z']]
DM = starData[galfastDict['DM']]
absSDSSr = starData[galfastDict['absSDSSr']]
pop = starData[galfastDict['comp']]
FeH = starData[galfastDict['FeH']]
vR = starData[galfastDict['Vr']]
vPhi = starData[galfastDict['Vphi']]
vZ = starData[galfastDict['Vz']]
pml = starData[galfastDict['pml']]
pmb = starData[galfastDict['pmb']]
vRadlb = starData[galfastDict['vRadlb']]
pmRA = starData[galfastDict['pmra']]
pmDec = starData[galfastDict['pmdec']]
vRad = starData[galfastDict['vRad']]
am = starData[galfastDict['Am']]
amInf = starData[galfastDict['AmInf']]
sDSS = np.transpose(
starData[galfastDict['SDSSu']:galfastDict['SDSSz'] +
1])
sDSSPhotoFlags = starData[galfastDict['SDSSPhotoFlags']]
#End of input, now onto processing and output
sDSSunred = selectStarSED0.deReddenMags(
am, sDSS, sdssExtCoeffs)
if readSize == 1:
ra = np.array([ra])
dec = np.array([dec])
"""
Info about the following population cuts:
From Zeljko: "This color corresponds to the temperature (roughly spectral type M0) where
Kurucz models become increasingly bad, and thus we switch to empirical SEDs (the problem
is that for M and later stars, the effective surface temperature is low enough for
molecules to form, and their opacity is too complex to easily model, especially TiO)."
"""
mIn = np.where(((pop < 10) | (pop >= 20))
& (sDSSunred[:, 2] - sDSSunred[:, 3] > 0.59))
kIn = np.where(((pop < 10) | (pop >= 20))
& (sDSSunred[:, 2] - sDSSunred[:, 3] <= 0.59))
hIn = np.where((pop >= 10) & (pop < 15))
heIn = np.where((pop >= 15) & (pop < 20))
sEDNameK, magNormK, matchErrorK = selectStarSED0.findSED(
listDict['kurucz'],
sDSSunred[kIn],
ra[kIn],
dec[kIn],
reddening=False,
colors=colorDict['kurucz'])
sEDNameM, magNormM, matchErrorM = selectStarSED0.findSED(
listDict['mlt'],
sDSSunred[mIn],
ra[mIn],
dec[mIn],
reddening=False,
colors=colorDict['mlt'])
sEDNameH, magNormH, matchErrorH = selectStarSED0.findSED(
listDict['H'],
sDSSunred[hIn],
ra[hIn],
dec[hIn],
reddening=False,
colors=colorDict['H'])
sEDNameHE, magNormHE, matchErrorHE = selectStarSED0.findSED(
listDict['HE'],
sDSSunred[heIn],
ra[heIn],
dec[heIn],
reddening=False,
colors=colorDict['HE'])
chunkNames = np.empty(readSize, dtype='S32')
chunkTypes = np.empty(readSize, dtype='S8')
chunkMagNorms = np.zeros(readSize)
chunkMatchErrors = np.zeros(readSize)
chunkNames[kIn] = sEDNameK
chunkTypes[kIn] = 'kurucz'
chunkMagNorms[kIn] = magNormK
chunkMatchErrors[kIn] = matchErrorK
chunkNames[mIn] = sEDNameM
chunkTypes[mIn] = 'mlt'
chunkMagNorms[mIn] = magNormM
chunkMatchErrors[mIn] = matchErrorM
chunkNames[hIn] = sEDNameH
chunkTypes[hIn] = 'H'
chunkMagNorms[hIn] = magNormH
chunkMatchErrors[hIn] = matchErrorH
chunkNames[heIn] = sEDNameHE
chunkTypes[heIn] = 'HE'
chunkMagNorms[heIn] = magNormHE
chunkMatchErrors[heIn] = matchErrorHE
lsstMagsUnred = []
for sedName, sedType, magNorm, matchError in zip(
chunkNames.astype(str), chunkTypes.astype(str),
chunkMagNorms, chunkMatchErrors):
testSED = Sed()
testSED.setSED(
listDict[sedType][positionDict[sedName]].wavelen,
flambda=listDict[sedType][
positionDict[sedName]].flambda)
fluxNorm = testSED.calcFluxNorm(magNorm, imSimBand)
testSED.multiplyFluxNorm(fluxNorm)
lsstMagsUnred.append(lsstPhot.magListForSed(testSED))
#If the extinction value is negative then it will add the reddening back in
lsstMags = selectStarSED0.deReddenMags(
(-1.0 * am), lsstMagsUnred, lsstExtCoeffs)
distKpc = self.convDMtoKpc(DM)
ebv = am / 2.285 #From Schlafly and Finkbeiner 2011, (ApJ, 737, 103) for sdssr
ebvInf = amInf / 2.285
for line in range(0, readSize):
outFmt = '%i,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%s,' +\
'%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%i,%3.7f,%3.7f,%3.7f\n'
if readSize == 1:
if inFits == True:
sDSS = sDSS[0]
outDat = (oID, ra[line], dec[line], gall, galb, coordX,
coordY, coordZ, chunkNames, chunkMagNorms,
chunkMatchErrors, lsstMags[line][0],
lsstMags[line][1], lsstMags[line][2],
lsstMags[line][3], lsstMags[line][4],
lsstMags[line][5], sDSS[0], sDSS[1], sDSS[2],
sDSS[3], sDSS[4], absSDSSr, pmRA, pmDec,
vRad, pml, pmb, vRadlb, vR, vPhi, vZ, FeH,
pop, distKpc, ebv, ebvInf)
else:
outDat = (oID[line], ra[line], dec[line], gall[line],
galb[line], coordX[line], coordY[line],
coordZ[line], chunkNames[line],
chunkMagNorms[line], chunkMatchErrors[line],
lsstMags[line][0], lsstMags[line][1],
lsstMags[line][2], lsstMags[line][3],
lsstMags[line][4], lsstMags[line][5],
sDSS[line][0], sDSS[line][1], sDSS[line][2],
sDSS[line][3], sDSS[line][4], absSDSSr[line],
pmRA[line], pmDec[line], vRad[line],
pml[line], pmb[line], vRadlb[line], vR[line],
vPhi[line], vZ[line], FeH[line], pop[line],
distKpc[line], ebv[line], ebvInf[line])
fOut.write(outFmt % outDat)
print('Chunk Num Done = %i out of %i' % (chunk + 1, numChunks))
def testMatchToObserved(self):
"""Test that Galaxy SEDs with extinction or redshift are matched correctly"""
rng = np.random.RandomState(42)
galPhot = BandpassDict.loadTotalBandpassesFromFiles()
imSimBand = Bandpass()
imSimBand.imsimBandpass()
testMatching = selectGalaxySED(galDir = self.testSpecDir)
testSEDList = testMatching.loadBC03()
testSEDNames = []
testRA = []
testDec = []
testRedshifts = []
testMagNormList = []
magNormStep = 1
extCoeffs = [1.8140, 1.4166, 0.9947, 0.7370, 0.5790, 0.4761]
testMags = []
testMagsRedshift = []
testMagsExt = []
for testSED in testSEDList:
# As a check make sure that it matches when no extinction and no redshift are present
getSEDMags = Sed()
testSEDNames.append(testSED.name)
getSEDMags.setSED(wavelen = testSED.wavelen, flambda = testSED.flambda)
testMags.append(galPhot.magListForSed(getSEDMags))
# Check Extinction corrections
sedRA = rng.uniform(10, 170)
sedDec = rng.uniform(10, 80)
testRA.append(sedRA)
testDec.append(sedDec)
raDec = np.array((sedRA, sedDec)).reshape((2, 1))
ebvVal = ebv().calculateEbv(equatorialCoordinates = raDec)
extVal = ebvVal*extCoeffs
testMagsExt.append(galPhot.magListForSed(getSEDMags) + extVal)
# Setup magnitudes for testing matching to redshifted values
getRedshiftMags = Sed()
testZ = np.round(rng.uniform(1.1, 1.3), 3)
testRedshifts.append(testZ)
testMagNorm = np.round(rng.uniform(20.0, 22.0), magNormStep)
testMagNormList.append(testMagNorm)
getRedshiftMags.setSED(wavelen = testSED.wavelen, flambda = testSED.flambda)
getRedshiftMags.redshiftSED(testZ)
fluxNorm = getRedshiftMags.calcFluxNorm(testMagNorm, imSimBand)
getRedshiftMags.multiplyFluxNorm(fluxNorm)
testMagsRedshift.append(galPhot.magListForSed(getRedshiftMags))
# Will also test in passing of non-default bandpass
testNoExtNoRedshift = testMatching.matchToObserved(testSEDList, testMags, np.zeros(8),
reddening = False,
bandpassDict = galPhot)
testMatchingEbvVals = testMatching.matchToObserved(testSEDList, testMagsExt, np.zeros(8),
catRA = testRA, catDec = testDec,
reddening = True, extCoeffs = extCoeffs,
bandpassDict = galPhot)
# Substitute in nan values to simulate incomplete data and make sure magnorm works too.
testMagsRedshift[0][1] = np.nan
testMagsRedshift[0][3] = np.nan
testMagsRedshift[0][4] = np.nan
testMagsRedshift[1][1] = np.nan
testMatchingRedshift = testMatching.matchToObserved(testSEDList, testMagsRedshift, testRedshifts,
dzAcc = 3, reddening = False,
bandpassDict = galPhot)
self.assertEqual(testSEDNames, testNoExtNoRedshift[0])
self.assertEqual(testSEDNames, testMatchingEbvVals[0])
self.assertEqual(None, testMatchingRedshift[0][0])
self.assertEqual(testSEDNames[1:], testMatchingRedshift[0][1:])
self.assertEqual(None, testMatchingRedshift[1][0])
np.testing.assert_almost_equal(testMagNormList[1:], testMatchingRedshift[1][1:],
decimal = magNormStep)
# Test Match Errors
errMag = testMagsRedshift[2]
errRedshift = testRedshifts[2]
errMags = np.array((errMag, errMag, errMag, errMag))
errRedshifts = np.array((errRedshift, errRedshift, errRedshift, errRedshift))
errMags[1, 1] += 1. # Total MSE will be 2/(5 colors) = 0.4
errMags[2, 0:2] = np.nan
errMags[2, 3] += 1. # Total MSE will be 2/(3 colors) = 0.667
errMags[3, :] = None
errSED = testSEDList[2]
testMatchingResultsErrors = testMatching.matchToObserved([errSED], errMags, errRedshifts,
reddening = False,
bandpassDict = galPhot,
dzAcc = 3)
np.testing.assert_almost_equal(np.array((0.0, 0.4, 2./3.)), testMatchingResultsErrors[2][0:3],
decimal = 2) # Give a little more leeway due to redshifting effects
self.assertEqual(None, testMatchingResultsErrors[2][3])
if ii>0 and ii%100 == 0:
duration = (time.time()-t_start)/3600.0
pred = n_samples*duration/ii
print('%d in %e hrs; predict %e' % (ii,duration,pred))
out_dict[dict_key] = (mag_interp, mag_truth)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--n_samples', type=int, default=1000000)
parser.add_argument('--n_processes', type=int, default=30)
args = parser.parse_args()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
cosmo = CosmologyObject()
grid_name = '../data/sne_interp_models.h5'
rng = np.random.RandomState(44)
c0_range = 0.6
x1_range = 6.0
z_range = 1.2
abs_mag_mean = -19.3
abs_mag_std = 0.3
d_samples = args.n_samples//args.n_processes
with h5py.File(grid_name, 'r') as in_file:
def test_selective_MLT_magnitudes(self):
"""
This test will verify that applyMLTflaring responds correctly
when mag_name_tuple is set to simulate only a subset of bandpasses
"""
rng =np.random.RandomState(87124)
n_stars = 1000
n_time = 5
quiescent_mags = {}
quiescent_mags['u'] = rng.random_sample(n_stars)*5.0+17.0
quiescent_mags['g'] = rng.random_sample(n_stars)*5.0+17.0
ebv = rng.random_sample(n_stars)*4.0
parallax = rng.random_sample(n_stars)*np.radians(0.001/3600.0)
available_lc = np.array(['lc_1', 'lc_2'])
params = {}
params['lc'] = rng.choice(available_lc, size=n_stars)
params['t0'] = rng.random_sample(n_stars)*1000.0
mjd_arr = rng.random_sample(n_time)*500.0*59580.0
mlt_obj = MLTflaringMixin()
mlt_obj.photParams = PhotometricParameters()
mlt_obj.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
mlt_obj._mlt_lc_file = self.mlt_lc_name
mlt_obj._actually_calculated_columns = ['delta_lsst_u', 'delta_lsst_g']
valid_dex = [] # applyMLT does not actually use valid_dex; it looks for non-None params['lc']
valid_obj = rng.choice(np.arange(n_stars, dtype=int), size=50)
for ix in range(n_stars):
if ix not in valid_obj:
params['lc'][ix] = None
delta_mag_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags)
n_time = len(mjd_arr)
self.assertEqual(delta_mag_vector.shape, (6, n_stars, n_time))
# test that delta_flux is correctly calculated
delta_flux_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags,
do_mags=False)
self.assertEqual(delta_flux_vector.shape, delta_mag_vector.shape)
dummy_sed = Sed()
for i_band in range(6):
if i_band>=2:
# because we only implemented models of variability for u, g
np.testing.assert_array_equal(delta_flux_vector[i_band],
np.zeros(delta_flux_vector[i_band].shape, dtype=float))
continue
for i_obj in range(n_stars):
mag0 = quiescent_mags['ug'[i_band]][i_obj]
flux0 = dummy_sed.fluxFromMag(mag0)
delta_flux_control = dummy_sed.fluxFromMag(mag0 + delta_mag_vector[i_band][i_obj])-flux0
denom = np.abs(delta_flux_control)
denom = np.where(denom>0.0, denom, 1.0e-20)
err = np.abs(delta_flux_vector[i_band][i_obj]-delta_flux_control)/denom
self.assertLess(err.max(), 1.0e-6)
# see that we get the same results as before when we request a subset
# of magnitudes
test_mag_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags,
mag_name_tuple=('g','i'))
non_zero = np.where(test_mag_vector[0]<0.0)
self.assertGreater(len(non_zero[0]), 0)
np.testing.assert_array_equal(test_mag_vector[0],
delta_mag_vector[1])
np.testing.assert_array_equal(test_mag_vector[1],
delta_mag_vector[3])
test_flux_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags,
do_mags=False,
mag_name_tuple=('u','z','g'))
non_zero = np.where(test_flux_vector[2]>0.0)
self.assertGreater(len(non_zero[0]), 0)
np.testing.assert_array_equal(test_flux_vector[0],
delta_flux_vector[0])
np.testing.assert_array_equal(test_flux_vector[1],
delta_flux_vector[4])
np.testing.assert_array_equal(test_flux_vector[2],
delta_flux_vector[1])
def test_alert_data_generation(self):
dmag_cutoff = 0.005
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5}
_max_var_param_str = self.max_str_len
class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject):
objid = 'star_alert'
tableid = 'stars'
idColKey = 'simobjid'
raColName = 'ra'
decColName = 'dec'
objectTypeId = 0
columns = [('raJ2000', 'ra*0.01745329252'),
('decJ2000', 'dec*0.01745329252'),
('parallax', 'px*0.01745329252/3600.0'),
('properMotionRa', 'pmra*0.01745329252/3600.0'),
('properMotionDec', 'pmdec*0.01745329252/3600.0'),
('radialVelocity', 'vrad'),
('variabilityParameters', 'varParamStr', str, _max_var_param_str)]
class TestAlertsVarCatMixin(object):
@register_method('alert_test')
def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None):
if len(params) == 0:
return np.array([[], [], [], [], [], []])
if isinstance(expmjd, numbers.Number):
dmags_out = np.zeros((6, self.num_variable_obj(params)))
else:
dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd)))
for i_star in range(self.num_variable_obj(params)):
if params['amp'][i_star] is not None:
dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd)
for i_filter in range(6):
dmags_out[i_filter][i_star] = dmags
return dmags_out
class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog):
pass
class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoordsLSST, AstrometryStars,
Variability, InstanceCatalog):
column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert']
@compound('delta_umag', 'delta_gmag', 'delta_rmag',
'delta_imag', 'delta_zmag', 'delta_ymag')
def get_TruthVariability(self):
return self.applyVariability(self.column_by_name('varParamStr'))
@cached
def get_dmagAlert(self):
return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass)
@cached
def get_magAlert(self):
return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
self.column_by_name('dmagAlert')
star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite')
# assemble the true light curves for each object; we need to figure out
# if their np.max(dMag) ever goes over dmag_cutoff; then we will know if
# we are supposed to simulate them
true_lc_dict = {}
true_lc_obshistid_dict = {}
is_visible_dict = {}
obs_dict = {}
max_obshistid = -1
n_total_observations = 0
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid > max_obshistid:
max_obshistid = obshistid
cat = TestAlertsTruthCat(star_db, obs_metadata=obs)
for line in cat.iter_catalog():
if line[1] is None:
continue
n_total_observations += 1
if line[0] not in true_lc_dict:
true_lc_dict[line[0]] = {}
true_lc_obshistid_dict[line[0]] = []
true_lc_dict[line[0]][obshistid] = line[2]
true_lc_obshistid_dict[line[0]].append(obshistid)
if line[0] not in is_visible_dict:
is_visible_dict[line[0]] = False
if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]:
is_visible_dict[line[0]] = True
obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2)))
skipped_due_to_mag = 0
objects_to_simulate = []
obshistid_unqid_set = set()
for obj_id in true_lc_dict:
dmag_max = -1.0
for obshistid in true_lc_dict[obj_id]:
if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max:
dmag_max = np.abs(true_lc_dict[obj_id][obshistid])
if dmag_max >= dmag_cutoff:
if not is_visible_dict[obj_id]:
skipped_due_to_mag += 1
continue
objects_to_simulate.append(obj_id)
for obshistid in true_lc_obshistid_dict[obj_id]:
obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid)
self.assertGreater(len(objects_to_simulate), 10)
self.assertGreater(skipped_due_to_mag, 0)
log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat,
output_prefix='alert_test',
output_dir=self.output_dir,
dmag_cutoff=dmag_cutoff,
log_file_name=log_file_name)
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
phot_params = PhotometricParameters()
# First, verify that the contents of the sqlite files are all correct
n_tot_simulated = 0
alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, '
alert_query += 'meta.band, quiescent.flux, alert.dflux, '
alert_query += 'quiescent.snr, alert.snr, '
alert_query += 'alert.ra, alert.dec, alert.chipNum, '
alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, '
alert_query += 'ast.parallax '
alert_query += 'FROM alert_data AS alert '
alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId '
alert_query += 'INNER JOIN quiescent_flux AS quiescent '
alert_query += 'ON quiescent.uniqueId=alert.uniqueId '
alert_query += 'AND quiescent.band=meta.band '
alert_query += 'INNER JOIN baseline_astrometry AS ast '
alert_query += 'ON ast.uniqueId=alert.uniqueId'
alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int),
('TAI', float), ('band', int),
('q_flux', float), ('dflux', float),
('q_snr', float), ('tot_snr', float),
('ra', float), ('dec', float),
('chipNum', int), ('xPix', float), ('yPix', float),
('pmRA', float), ('pmDec', float), ('parallax', float)])
sqlite_file_list = os.listdir(self.output_dir)
n_tot_simulated = 0
obshistid_unqid_simulated_set = set()
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype)
if len(alert_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI'])
for i_obj in range(len(alert_data)):
n_tot_simulated += 1
obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) +
alert_data['obshistId'][i_obj])
unq = alert_data['uniqueId'][i_obj]
obj_dex = (unq//1024)-1
self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4)
self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4)
self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4)
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
alert_data['ra'][i_obj], alert_data['dec'][i_obj])
distance_arcsec = 3600.0*distance
msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth,
alert_data['ra'][i_obj],
alert_data['dec'][i_obj])
self.assertLess(distance_arcsec, 0.0005, msg=msg)
obs = obs_dict[alert_data['obshistId'][i_obj]]
chipname = chipNameFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
chipnum = int(chipname.replace('R', '').replace('S', '').
replace(' ', '').replace(';', '').replace(',', '').
replace(':', ''))
self.assertEqual(chipnum, alert_data['chipNum'][i_obj])
xpix, ypix = pixelCoordsFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4)
self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4)
dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj])
self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]],
dmag_sim, 3)
mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]]
m5 = obs.m5[mag_name]
q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj])
self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
q_mag, 4)
snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
bp_dict[mag_name],
self.obs_mag_cutoff[alert_data['band'][i_obj]],
phot_params)
self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4)
tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \
true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]]
snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name],
m5, phot_params)
self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4)
for val in obshistid_unqid_set:
self.assertIn(val, obshistid_unqid_simulated_set)
self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set))
astrometry_query = 'SELECT uniqueId, ra, dec, TAI '
astrometry_query += 'FROM baseline_astrometry'
astrometry_dtype = np.dtype([('uniqueId', int),
('ra', float),
('dec', float),
('TAI', float)])
tai_list = []
for obs in self.obs_list:
tai_list.append(obs.mjd.TAI)
tai_list = np.array(tai_list)
n_tot_ast_simulated = 0
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype)
if len(astrometry_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI'])
for i_obj in range(len(astrometry_data)):
n_tot_ast_simulated += 1
obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
astrometry_data['ra'][i_obj],
astrometry_data['dec'][i_obj])
self.assertLess(3600.0*distance, 0.0005)
del alert_gen
gc.collect()
self.assertGreater(n_tot_simulated, 10)
self.assertGreater(len(obshistid_unqid_simulated_set), 10)
self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations)
self.assertGreater(n_tot_ast_simulated, 0)
def __init__(self,
opsimdb,
descqa_catalog,
dither=True,
min_mag=10,
minsource=100,
proper_motion=False,
imsim_catalog=False,
protoDC2_ra=0,
protoDC2_dec=0,
agn_db_name=None,
sprinkler=False):
"""
Parameters
----------
obsimdb: str
OpSim db filename.
descqa_catalog: str
Name of the DESCQA galaxy catalog.
dither: bool [True]
Flag to enable the dithering included in the opsim db file.
min_mag: float [10]
Minimum value of the star magnitude at 500nm to include.
minsource: int [100]
Minimum number of objects for phosim.py to simulate a chip.
proper_motion: bool [True]
Flag to enable application of proper motion to stars.
imsim_catalog: bool [False]
Flag to write an imsim-style object catalog.
protoDC2_ra: float [0]
Desired RA (J2000 degrees) of protoDC2 center.
protoDC2_dec: float [0]
Desired Dec (J2000 degrees) of protoDC2 center.
agn_db_name: str [None]
Filename of the agn parameter sqlite db file.
sprinkler: bool [False]
Flag to enable the Sprinkler.
"""
if not os.path.exists(opsimdb):
raise RuntimeError('%s does not exist' % opsimdb)
# load the data for the parametrized light
# curve stellar variability model into a
# global cache
plc = ParametrizedLightCurveMixin()
plc.load_parametrized_light_curves()
self.descqa_catalog = descqa_catalog
self.dither = dither
self.min_mag = min_mag
self.minsource = minsource
self.proper_motion = proper_motion
self.imsim_catalog = imsim_catalog
self.protoDC2_ra = protoDC2_ra
self.protoDC2_dec = protoDC2_dec
self.phot_params = PhotometricParameters(nexp=1, exptime=30)
self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
self.obs_gen = ObservationMetaDataGenerator(database=opsimdb,
driver='sqlite')
self.star_db = StarObj(database='LSSTCATSIM',
host='fatboy.phys.washington.edu',
port=1433,
driver='mssql+pymssql')
if agn_db_name is None:
raise IOError("Need to specify an Proto DC2 AGN database.")
else:
if os.path.exists(agn_db_name):
self.agn_db_name = agn_db_name
else:
raise IOError("Path to Proto DC2 AGN database does not exist.")
self.sprinkler = sprinkler
self.instcats = get_instance_catalogs(imsim_catalog)
def test_flare_magnitudes_mixed_with_none(self):
"""
Test that we get the expected magnitudes out
"""
db = MLT_test_DB(database=self.db_name, driver='sqlite')
# load the quiescent SEDs of the objects in our catalog
sed_list = SedList(['lte028-5.0+0.5a+0.0.BT-Settl.spec.gz']*4,
[17.1, 17.2, 17.3, 17.4],
galacticAvList = [2.432, 1.876, 2.654, 2.364],
fileDir=getPackageDir('sims_sed_library'),
specMap=defaultSpecMap)
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
# calculate the quiescent fluxes of the objects in our catalog
baseline_fluxes = bp_dict.fluxListForSedList(sed_list)
bb_wavelen = np.arange(100.0, 1600.0, 0.1)
bb = models.BlackBody(temperature=9000.0 * u.K, scale=1.0 * u.erg / (u.cm ** 2 * u.AA * u.s * u.sr))
bb_flambda = bb(bb_wavelen * u.nm).to_value()
# this data is taken from the setUpClass() classmethod above
t0_list = [456.2, 41006.2, 117.2, 10456.2]
av_list = [2.432, 1.876, 2.654, 2.364]
parallax_list = np.array([0.25, 0.15, 0.3, 0.22])
distance_list = 1.0/(206265.0*radiansFromArcsec(0.001*parallax_list))
distance_list *= 3.0857e18 # convert to cm
dtype = np.dtype([('id', int), ('u', float), ('g', float)])
photParams = PhotometricParameters()
ss = Sed()
quiet_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_none_quiet_cat.txt')
flare_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_none_flaring_cat.txt')
# loop over several MJDs and verify that, to within a
# milli-mag, our flaring model gives us the magnitudes
# expected, given the light curves specified in
# setUpClass()
for mjd in (59580.0, 60000.0, 70000.0, 80000.0):
obs = ObservationMetaData(mjd=mjd)
quiet_cat = QuiescentCatalog(db, obs_metadata=obs)
quiet_cat.write_catalog(quiet_cat_name)
flare_cat = FlaringCatalog(db, obs_metadata=obs)
flare_cat._mlt_lc_file = self.mlt_lc_name
flare_cat.write_catalog(flare_cat_name)
quiescent_data = np.genfromtxt(quiet_cat_name, dtype=dtype, delimiter=',')
flaring_data = np.genfromtxt(flare_cat_name, dtype=dtype, delimiter=',')
self.assertGreater(len(flaring_data), 3)
for ix in range(len(flaring_data)):
obj_id = flaring_data['id'][ix]
self.assertEqual(obj_id, ix)
# the models below are as specified in the
# setUpClass() method
if obj_id == 0 or obj_id == 1:
amp = 1.0e42
dt = 3652.5
t_min = flare_cat._survey_start - t0_list[obj_id]
tt = mjd - t_min
while tt > dt:
tt -= dt
u_flux = amp*(1.0+np.power(np.sin(tt/100.0), 2))
g_flux = amp*(1.0+np.power(np.cos(tt/100.0), 2))
elif obj_id==2:
amp = 2.0e41
dt = 365.25
t_min = flare_cat._survey_start - t0_list[obj_id]
tt = mjd - t_min
while tt > dt:
tt -= dt
u_flux = amp*(1.0+np.power(np.sin(tt/50.0), 2))
g_flux = amp*(1.0+np.power(np.cos(tt/50.0), 2))
else:
u_flux = 0.0
g_flux = 0.0
# calculate the multiplicative effect of dust on a 9000K
# black body
bb_sed = Sed(wavelen=bb_wavelen, flambda=bb_flambda)
u_bb_flux = bb_sed.calcFlux(bp_dict['u'])
g_bb_flux = bb_sed.calcFlux(bp_dict['g'])
a_x, b_x = bb_sed.setupCCM_ab()
bb_sed.addDust(a_x, b_x, A_v=av_list[obj_id])
u_bb_dusty_flux = bb_sed.calcFlux(bp_dict['u'])
g_bb_dusty_flux = bb_sed.calcFlux(bp_dict['g'])
dust_u = u_bb_dusty_flux/u_bb_flux
dust_g = g_bb_dusty_flux/g_bb_flux
area = 4.0*np.pi*np.power(distance_list[obj_id], 2)
tot_u_flux = baseline_fluxes[obj_id][0] + u_flux*dust_u/area
tot_g_flux = baseline_fluxes[obj_id][1] + g_flux*dust_g/area
msg = ('failed on object %d; mjd %.2f\n u_quiet %e u_flare %e\n g_quiet %e g_flare %e' %
(obj_id, mjd, quiescent_data['u'][obj_id], flaring_data['u'][obj_id],
quiescent_data['g'][obj_id], flaring_data['g'][obj_id]))
self.assertEqual(quiescent_data['id'][obj_id], flaring_data['id'][obj_id], msg=msg)
self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][0]),
quiescent_data['u'][obj_id], 3, msg=msg)
self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][1]),
quiescent_data['g'][obj_id], 3, msg=msg)
self.assertAlmostEqual(ss.magFromFlux(tot_u_flux), flaring_data['u'][obj_id],
3, msg=msg)
self.assertAlmostEqual(ss.magFromFlux(tot_g_flux), flaring_data['g'][obj_id],
3, msg=msg)
if obj_id != 3:
self.assertGreater(np.abs(flaring_data['g'][obj_id]-quiescent_data['g'][obj_id]),
0.001, msg=msg)
self.assertGreater(np.abs(flaring_data['u'][obj_id]-quiescent_data['u'][obj_id]),
0.001, msg=msg)
else:
self.assertEqual(flaring_data['g'][obj_id]-quiescent_data['g'][obj_id], 0.0, msg=msg)
self.assertEqual(flaring_data['u'][obj_id]-quiescent_data['u'][obj_id], 0.0, msg=msg)
if os.path.exists(quiet_cat_name):
os.unlink(quiet_cat_name)
if os.path.exists(flare_cat_name):
os.unlink(flare_cat_name)
def process_stellar_chunk(chunk, filter_obs, mjd_obs, m5_obs,
coadd_m5, m5_single, obs_md_list, proper_chip,
variability_cache, out_data, lock):
t_start_chunk = time.time()
#print('processing %d' % len(chunk))
ct_first = 0
ct_at_all = 0
ct_tot = 0
n_t = len(filter_obs)
n_obj = len(chunk)
coadd_visits = {}
coadd_visits['u'] = 6
coadd_visits['g'] = 8
coadd_visits['r'] = 18
coadd_visits['i'] = 18
coadd_visits['z'] = 16
coadd_visits['y'] = 16
gamma_coadd = {}
for bp in 'ugrizy':
gamma_coadd[bp] = None
gamma_single = {}
for bp in 'ugrizy':
gamma_single[bp] = [None]*n_t
dflux = np.zeros((n_obj,n_t), dtype=float)
dflux_for_mlt(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dflux_for_kepler(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dflux_for_rrly(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dummy_sed = Sed()
lsst_bp = BandpassDict.loadTotalBandpassesFromFiles()
flux_q = np.zeros((6,n_obj), dtype=float)
flux_coadd = np.zeros((6,n_obj), dtype=float)
mag_coadd = np.zeros((6,n_obj), dtype=float)
snr_coadd = np.zeros((6,n_obj), dtype=float)
snr_single = {}
snr_single_mag_grid = np.arange(14.0, 30.0, 0.05)
phot_params_single = PhotometricParameters(nexp=1,
exptime=30.0)
t_start_snr = time.time()
photometry_mask = np.zeros((n_obj, n_t), dtype=bool)
photometry_mask_1d = np.zeros(n_obj, dtype=bool)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs == i_bp)
phot_params_coadd = PhotometricParameters(nexp=1,
exptime=30.0*coadd_visits[bp])
flux_q[i_bp] = dummy_sed.fluxFromMag(chunk['%smag' % bp])
dflux_sub = dflux[:,valid_obs[0]]
assert dflux_sub.shape == (n_obj, len(valid_obs[0]))
dflux_mean = np.mean(dflux_sub, axis=1)
assert dflux_mean.shape==(n_obj,)
flux_coadd[i_bp] = flux_q[i_bp]+dflux_mean
mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp])
(snr_coadd[i_bp],
gamma) = SNR.calcSNR_m5(mag_coadd[i_bp],
lsst_bp[bp],
coadd_m5[bp],
phot_params_coadd)
(snr_single[bp],
gamma) = SNR.calcSNR_m5(snr_single_mag_grid,
lsst_bp[bp],
m5_single[bp],
phot_params_single)
#print('got all snr in %e' % (time.time()-t_start_snr))
t_start_obj = time.time()
snr_arr = np.zeros((n_obj, n_t), dtype=float)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs==i_bp)
if len(valid_obs[0])==0:
continue
n_bp = len(valid_obs[0])
dflux_bp = dflux[:,valid_obs[0]]
flux0_arr = flux_q[i_bp]
flux_tot = flux0_arr[:,None] + dflux_bp
mag_tot = dummy_sed.magFromFlux(flux_tot)
snr_single_val = np.interp(mag_tot,
snr_single_mag_grid,
snr_single[bp])
noise_coadd = flux_coadd[i_bp]/snr_coadd[i_bp]
noise_single = flux_tot/snr_single_val
noise = np.sqrt(noise_coadd[:,None]**2+noise_single**2)
dflux_thresh = 5.0*noise
dflux_bp = np.abs(dflux_bp)
detected = (dflux_bp>=dflux_thresh)
snr_arr[:,valid_obs[0]] = dflux_bp/noise
for i_obj in range(n_obj):
if detected[i_obj].any():
photometry_mask_1d[i_obj] = True
photometry_mask[i_obj,valid_obs[0]] = detected[i_obj]
t_before_chip = time.time()
chip_mask = apply_focal_plane(chunk['ra'], chunk['decl'],
photometry_mask_1d, obs_md_list,
filter_obs, proper_chip)
duration = (time.time()-t_before_chip)/3600.0
unq_out = -1*np.ones(n_obj, dtype=int)
mjd_out = -1.0*np.ones(n_obj, dtype=float)
snr_out = -1.0*np.ones(n_obj, dtype=float)
var_type_out = -1*np.ones(n_obj, dtype=int)
for i_obj in range(n_obj):
if photometry_mask_1d[i_obj]:
detected = photometry_mask[i_obj,:] & chip_mask[i_obj,:]
if detected.any():
unq_out[i_obj] = chunk['simobjid'][i_obj]
first_dex = np.where(detected)[0].min()
mjd_out[i_obj] = mjd_obs[first_dex]
snr_out[i_obj] = snr_arr[i_obj, first_dex]
var_type_out[i_obj] = chunk['var_type'][i_obj]
valid = np.where(unq_out>=0)
unq_out = unq_out[valid]
mjd_out = mjd_out[valid]
var_type_out = var_type_out[valid]
snr_out = snr_out[valid]
with lock:
existing_keys = list(out_data.keys())
key_val = 0
if len(existing_keys)>0:
key_val = max(existing_keys)
while key_val in existing_keys:
key_val += 1
out_data[key_val] = (None, None, None, None)
out_data[key_val] = (unq_out, mjd_out,
snr_out, var_type_out)
def test_MLT_many_mjd_some_invalid(self):
"""
This test will verify that applyMLTflaring responds properly when given
an array/vector of MJD values in the case where some stars are not
marked as valid MLT stars.
"""
db = MLT_test_DB(database=self.db_name, driver='sqlite')
rng = np.random.RandomState(16)
mjd_arr = rng.random_sample(17)*3653.3+59580.0
objid = []
parallax = []
ebv = []
quiescent_u = []
quiescent_g = []
delta_u = []
delta_g = []
varparams = []
for mjd in mjd_arr:
obs = ObservationMetaData(mjd=mjd)
cat = FlaringCatalog(db, obs_metadata=obs,
column_outputs=['parallax', 'ebv',
'quiescent_lsst_u',
'quiescent_lsst_g',
'varParamStr',
'delta_lsst_u',
'delta_lsst_g'])
cat._mlt_lc_file = self.mlt_lc_name
n_obj = 0
for line in cat.iter_catalog():
n_obj += 1
objid.append(line[0])
parallax.append(line[3])
ebv.append(line[4])
quiescent_u.append(line[5])
quiescent_g.append(line[6])
varparams.append(line[7])
delta_u.append(line[8])
delta_g.append(line[9])
objid = np.array(objid)
parallax = np.array(parallax)
ebv = np.array(ebv)
quiescent_u = np.array(quiescent_u)
quiescent_g = np.array(quiescent_g)
delta_u = np.array(delta_u)
delta_g = np.array(delta_g)
self.assertEqual(len(parallax), n_obj*len(mjd_arr))
np.testing.assert_array_equal(objid, np.array([0,1,2,3]*len(mjd_arr)))
quiescent_mags = {}
quiescent_mags['u'] = quiescent_u
quiescent_mags['g'] = quiescent_g
params = {}
params['lc'] = []
params['t0'] = []
for ix in range(n_obj):
local_dict = json.loads(varparams[ix])
params['lc'].append(local_dict['p']['lc'])
params['t0'].append(local_dict['p']['t0'])
params['lc'] = np.array(params['lc'])
params['t0'] = np.array(params['t0'])
mlt_obj = MLTflaringMixin()
mlt_obj.photParams = PhotometricParameters()
mlt_obj.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
mlt_obj._mlt_lc_file = cat._mlt_lc_file
mlt_obj._actually_calculated_columns = ['delta_lsst_u', 'delta_lsst_g']
valid_dex = [] # applyMLT does not actually use valid_dex; it looks for non-None params['lc']
for ix in range(n_obj):
if ix not in (1,2):
params['lc'][ix] = None
delta_mag_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags)
n_time = len(mjd_arr)
self.assertEqual(delta_mag_vector.shape, (6, n_obj, n_time))
for i_time, mjd in enumerate(mjd_arr):
for i_obj in range(n_obj):
if i_obj not in (1,2):
for i_band in range(6):
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time], 0.0)
continue
for i_band in range(6):
if i_band==0:
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time],
delta_u[i_time*n_obj + i_obj])
elif i_band==1:
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time],
delta_g[i_time*n_obj + i_obj])
else:
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time], 0.0)
# test that delta_flux is correctly calculated
delta_flux_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags,
do_mags=False)
self.assertEqual(delta_flux_vector.shape, delta_mag_vector.shape)
dummy_sed = Sed()
for i_band in range(6):
if i_band>=2:
# because we only implemented models of variability for u, g
np.testing.assert_array_equal(delta_flux_vector[i_band],
np.zeros(delta_flux_vector[i_band].shape, dtype=float))
continue
for i_obj in range(n_obj):
mag0 = quiescent_mags['ug'[i_band]][i_obj]
flux0 = dummy_sed.fluxFromMag(mag0)
delta_flux_control = dummy_sed.fluxFromMag(mag0 + delta_mag_vector[i_band][i_obj])-flux0
denom = np.abs(delta_flux_control)
denom = np.where(denom>0.0, denom, 1.0e-20)
err = np.abs(delta_flux_vector[i_band][i_obj]-delta_flux_control)/denom
self.assertLess(err.max(), 1.0e-10)
def test_flare_magnitudes_mixed_with_dummy(self):
"""
Test that we get the expected magnitudes out
"""
db = MLT_test_DB(database=self.db_name, driver='sqlite')
# load the quiescent SEDs of the objects in our catalog
sed_list = SedList(['lte028-5.0+0.5a+0.0.BT-Settl.spec.gz']*4,
[17.1, 17.2, 17.3, 17.4],
galacticAvList = [2.432, 1.876, 2.654, 2.364],
fileDir=getPackageDir('sims_sed_library'),
specMap=defaultSpecMap)
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
# calculate the quiescent fluxes of the objects in our catalog
baseline_fluxes = bp_dict.fluxListForSedList(sed_list)
bb_wavelen = np.arange(100.0, 1600.0, 0.1)
bb_flambda = blackbody_lambda(bb_wavelen*10.0, 9000.0)
# this data is taken from the setUpClass() classmethod above
t0_list = [456.2, 41006.2, 117.2, 10456.2]
av_list = [2.432, 1.876, 2.654, 2.364]
parallax_list = np.array([0.25, 0.15, 0.3, 0.22])
distance_list = 1.0/(206265.0*radiansFromArcsec(0.001*parallax_list))
distance_list *= 3.0857e18 # convert to cm
dtype = np.dtype([('id', int), ('u', float), ('g', float)])
photParams = PhotometricParameters()
ss = Sed()
quiet_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_dummy_quiet_cat.txt')
flare_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_dummy_flaring_cat.txt')
# loop over several MJDs and verify that, to within a
# milli-mag, our flaring model gives us the magnitudes
# expected, given the light curves specified in
# setUpClass()
for mjd in (59580.0, 60000.0, 70000.0, 80000.0):
obs = ObservationMetaData(mjd=mjd)
quiet_cat = QuiescentCatalog(db, obs_metadata=obs)
quiet_cat.write_catalog(quiet_cat_name)
flare_cat = FlaringCatalogDummy(db, obs_metadata=obs)
flare_cat.scratch_dir = self.scratch_dir
flare_cat._mlt_lc_file = self.mlt_lc_name
flare_cat.write_catalog(flare_cat_name)
quiescent_data = np.genfromtxt(quiet_cat_name, dtype=dtype, delimiter=',')
flaring_data = np.genfromtxt(flare_cat_name, dtype=dtype, delimiter=',')
self.assertGreater(len(quiescent_data), 2)
self.assertEqual(len(quiescent_data), len(flaring_data))
self.assertIn(3, flaring_data['id'])
for ix in range(len(flaring_data)):
obj_id = flaring_data['id'][ix]
self.assertEqual(obj_id, ix)
msg = ('failed on object %d; mjd %.2f\n u_quiet %e u_flare %e\n g_quiet %e g_flare %e' %
(obj_id, mjd, quiescent_data['u'][obj_id], flaring_data['u'][obj_id],
quiescent_data['g'][obj_id], flaring_data['g'][obj_id]))
self.assertEqual(quiescent_data['id'][obj_id], flaring_data['id'][obj_id], msg=msg)
self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][0]),
quiescent_data['u'][obj_id], 3, msg=msg)
self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][1]),
quiescent_data['g'][obj_id], 3, msg=msg)
if obj_id != 3:
# the models below are as specified in the
# setUpClass() method
if obj_id == 0 or obj_id == 1:
amp = 1.0e42
dt = 3652.5
t_min = flare_cat._survey_start - t0_list[obj_id]
tt = mjd - t_min
while tt > dt:
tt -= dt
u_flux = amp*(1.0+np.power(np.sin(tt/100.0), 2))
g_flux = amp*(1.0+np.power(np.cos(tt/100.0), 2))
elif obj_id==2:
amp = 2.0e41
dt = 365.25
t_min = flare_cat._survey_start - t0_list[obj_id]
tt = mjd - t_min
while tt > dt:
tt -= dt
u_flux = amp*(1.0+np.power(np.sin(tt/50.0), 2))
g_flux = amp*(1.0+np.power(np.cos(tt/50.0), 2))
# calculate the multiplicative effect of dust on a 9000K
# black body
bb_sed = Sed(wavelen=bb_wavelen, flambda=bb_flambda)
u_bb_flux = bb_sed.calcFlux(bp_dict['u'])
g_bb_flux = bb_sed.calcFlux(bp_dict['g'])
a_x, b_x = bb_sed.setupCCM_ab()
bb_sed.addDust(a_x, b_x, A_v=av_list[obj_id])
u_bb_dusty_flux = bb_sed.calcFlux(bp_dict['u'])
g_bb_dusty_flux = bb_sed.calcFlux(bp_dict['g'])
dust_u = u_bb_dusty_flux/u_bb_flux
dust_g = g_bb_dusty_flux/g_bb_flux
area = 4.0*np.pi*np.power(distance_list[obj_id], 2)
tot_u_flux = baseline_fluxes[obj_id][0] + u_flux*dust_u/area
tot_g_flux = baseline_fluxes[obj_id][1] + g_flux*dust_g/area
self.assertAlmostEqual(ss.magFromFlux(tot_u_flux), flaring_data['u'][obj_id],
3, msg=msg)
self.assertAlmostEqual(ss.magFromFlux(tot_g_flux), flaring_data['g'][obj_id],
3, msg=msg)
self.assertGreater(np.abs(flaring_data['g'][obj_id]-quiescent_data['g'][obj_id]),
0.001, msg=msg)
self.assertGreater(np.abs(flaring_data['u'][obj_id]-quiescent_data['u'][obj_id]),
0.001, msg=msg)
else:
self.assertAlmostEqual(flaring_data['g'][obj_id],
quiescent_data['g'][obj_id]+3*(mjd-59580.0)/10000.0,
3, msg=msg)
self.assertAlmostEqual(flaring_data['u'][obj_id],
quiescent_data['u'][obj_id]+2*(mjd-59580.0)/10000.0,
3, msg=msg)
if os.path.exists(quiet_cat_name):
os.unlink(quiet_cat_name)
if os.path.exists(flare_cat_name):
os.unlink(flare_cat_name)
def loadGalfast(self, filenameList, outFileList, sEDPath = None, kuruczPath = None,
mltPath = None, wdPath = None, kuruczSubset = None,
mltSubset = None, wdSubset = None, chunkSize = 10000):
"""
This is customized for the outputs we currently need for the purposes of consistent output
It will read in a galfast output file and output desired values for database input into a file
@param [in] filenameList is a list of the galfast output files that will be loaded and processed.
Can process fits, gzipped, or txt output from galfast.
@param [in] outFileList is a list of the names of the output files that will be created. If gzipped
output is desired simply write the filenames with .gz at the end.
@param [in] kuruczPath is a place to specify a path to kurucz SED files
@param [in] mltPath is the same as kuruczPath except that it specifies a directory for the
mlt SEDs
@param [in] wdPath is the same as the previous two except that it specifies a path to a
white dwarf SED directory.
@param [in] kuruczSubset is a list which provides a subset of the kurucz files within the
kurucz folder that one wants to use
@param [in] mltSubset is a list which provides a subset of the mlt files within the
mlt folder that one wants to use
@param [in] wdSubset is a list which provides a subset of the wd files within the
wd folder that one wants to use
@param [in] chunkSize is the size of chunks of lines to be read from the catalog at one time.
"""
for filename in filenameList:
#Make sure input file exists and is readable format before doing anything else
if os.path.isfile(filename) == False:
raise RuntimeError('*** File does not exist')
#Process various possible galfast outputs
if filename.endswith(('.txt', '.gz', '.fits')):
continue
else:
raise RuntimeError(str('*** Unsupported File Format in file: ' + str(filename)))
#If all files exist and are in proper formats then load seds
selectStarSED0 = selectStarSED(kuruczDir=kuruczPath,
mltDir=mltPath,
wdDir=wdPath)
if kuruczSubset is None:
kuruczList = selectStarSED0.loadKuruczSEDs()
else:
kuruczList = selectStarSED0.loadKuruczSEDs(subset = kuruczSubset)
#Only need one dictionary since none of the names overlap
positionDict = {}
for kNum, kuruczSED in enumerate(kuruczList):
positionDict[kuruczSED.name] = kNum
if mltSubset is None:
mltList = selectStarSED0.loadmltSEDs()
else:
mltList = selectStarSED0.loadmltSEDs(subset = mltSubset)
for mltNum, mltSED in enumerate(mltList):
positionDict[mltSED.name] = mltNum
if wdSubset is None:
wdListH, wdListHE = selectStarSED0.loadwdSEDs()
else:
wdListH, wdListHE = selectStarSED0.loadwdSEDs(subset = wdSubset)
for hNum, hSED in enumerate(wdListH):
positionDict[hSED.name] = hNum
for heNum, heSED in enumerate(wdListHE):
positionDict[heSED.name] = heNum
#For adding/subtracting extinction when calculating colors
#Numbers below come from Schlafly and Finkbeiner (2011) (ApJ, 737, 103)
#normalized by SDSS r mag value
sdssExtCoeffs = [1.8551, 1.4455, 1.0, 0.7431, 0.5527]
lsstExtCoeffs = [1.8140, 1.4166, 0.9947, 0.7370, 0.5790, 0.4761]
sdssPhot = BandpassDict.loadTotalBandpassesFromFiles(['u','g','r','i','z'],
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'),
'sdss'),
bandpassRoot = 'sdss_')
#Load Bandpasses for LSST colors to get colors from matched SEDs
lsstFilterList = ('u', 'g', 'r', 'i', 'z', 'y')
lsstPhot = BandpassDict.loadTotalBandpassesFromFiles(lsstFilterList)
imSimBand = Bandpass()
imSimBand.imsimBandpass()
#Calculate colors and add them to the SED objects
kuruczColors = selectStarSED0.calcBasicColors(kuruczList, sdssPhot)
mltColors = selectStarSED0.calcBasicColors(mltList, sdssPhot)
hColors = selectStarSED0.calcBasicColors(wdListH, sdssPhot)
heColors = selectStarSED0.calcBasicColors(wdListHE, sdssPhot)
listDict = {'kurucz':kuruczList, 'mlt':mltList, 'H':wdListH, 'HE':wdListHE}
colorDict = {'kurucz':kuruczColors, 'mlt':mltColors, 'H':hColors, 'HE':heColors}
for filename, outFile in zip(filenameList, outFileList):
if filename.endswith('.txt'):
galfastIn = open(filename, 'rt')
inFits = False
gzFile = False
num_lines = sum(1 for line in open(filename))
elif filename.endswith('.gz'):
galfastIn = gzip.open(filename, 'rt')
inFits = False
gzFile = True
num_lines = sum(1 for line in gzip.open(filename))
elif filename.endswith('fits'):
hdulist = fits.open(filename)
galfastIn = hdulist[1].data
num_lines = len(galfastIn)
gzFile = False
inFits = True
if outFile.endswith('.txt'):
fOut = open(outFile, 'wt')
elif outFile.endswith('.gz'):
fOut = gzip.open(outFile, 'wt')
fOut.write('#oID, ra, dec, gall, galb, coordX, coordY, coordZ, sEDName, magNorm, ' +\
'LSSTugrizy, SDSSugriz, absSDSSr, pmRA, pmDec, vRad, pml, pmb, vRadlb, ' +\
'vR, vPhi, vZ, FeH, pop, distKpc, ebv, ebvInf\n')
header_length = 0
numChunks = 0
if inFits == False:
galfastDict = self.parseGalfast(galfastIn.readline())
header_length += 1
header_status = True
while header_status == True:
newLine = galfastIn.readline()
if newLine[0] != '#':
header_status = False
else:
header_length += 1
print('Total objects = %i' % (num_lines - header_length))
numChunks = ((num_lines-header_length)//chunkSize) + 1
for chunk in range(0,numChunks):
if chunk == numChunks-1:
lastChunkSize = (num_lines - header_length) % chunkSize
readSize = lastChunkSize
else:
readSize = chunkSize
oID = np.arange(readSize*chunk, readSize*(chunk+1))
if inFits:
starData = galfastIn[readSize*chunk:(readSize*chunk + readSize)]
sDSS = starData.field('SDSSugriz')
gall, galb = np.transpose(starData.field('lb'))
ra, dec = np.transpose(starData.field('radec'))
coordX, coordY, coordZ = np.transpose(starData.field('XYZ'))
DM = starData.field('DM')
absSDSSr = starData.field('absSDSSr')
pop = starData.field('comp')
FeH = starData.field('FeH')
vR, vPhi, vZ = np.transpose(starData.field('vcyl'))
pml, pmb, vRadlb = np.transpose(starData.field('pmlb'))
pmRA, pmDec, vRad = np.transpose(starData.field('pmradec'))
am = starData.field('Am')
amInf = starData.field('AmInf')
sdssPhotoFlags = starData.field('SDSSugrizPhotoFlags')
else:
if gzFile == False:
with open(filename) as t_in:
starData = np.loadtxt(itertools.islice(t_in,((readSize*chunk)+header_length),
((readSize*(chunk+1))+header_length)))
else:
with gzip.open(filename) as t_in:
starData = np.loadtxt(itertools.islice(t_in,((readSize*chunk)+header_length),
((readSize*(chunk+1))+header_length)))
starData = np.transpose(starData)
gall = starData[galfastDict['l']]
galb = starData[galfastDict['b']]
ra = starData[galfastDict['ra']]
dec = starData[galfastDict['dec']]
coordX = starData[galfastDict['X']]
coordY = starData[galfastDict['Y']]
coordZ = starData[galfastDict['Z']]
DM = starData[galfastDict['DM']]
absSDSSr = starData[galfastDict['absSDSSr']]
pop = starData[galfastDict['comp']]
FeH = starData[galfastDict['FeH']]
vR = starData[galfastDict['Vr']]
vPhi = starData[galfastDict['Vphi']]
vZ = starData[galfastDict['Vz']]
pml = starData[galfastDict['pml']]
pmb = starData[galfastDict['pmb']]
vRadlb = starData[galfastDict['vRadlb']]
pmRA = starData[galfastDict['pmra']]
pmDec = starData[galfastDict['pmdec']]
vRad = starData[galfastDict['vRad']]
am = starData[galfastDict['Am']]
amInf = starData[galfastDict['AmInf']]
sDSS = np.transpose(starData[galfastDict['SDSSu']:galfastDict['SDSSz']+1])
sDSSPhotoFlags = starData[galfastDict['SDSSPhotoFlags']]
#End of input, now onto processing and output
sDSSunred = selectStarSED0.deReddenMags(am, sDSS, sdssExtCoeffs)
if readSize == 1:
ra = np.array([ra])
dec = np.array([dec])
"""
Info about the following population cuts:
From Zeljko: "This color corresponds to the temperature (roughly spectral type M0) where
Kurucz models become increasingly bad, and thus we switch to empirical SEDs (the problem
is that for M and later stars, the effective surface temperature is low enough for
molecules to form, and their opacity is too complex to easily model, especially TiO)."
"""
mIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:,2] - sDSSunred[:,3] > 0.59))
kIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:,2] - sDSSunred[:,3] <= 0.59))
hIn = np.where((pop >= 10) & (pop < 15))
heIn = np.where((pop >= 15) & (pop < 20))
sEDNameK, magNormK, matchErrorK = selectStarSED0.findSED(listDict['kurucz'],
sDSSunred[kIn], ra[kIn], dec[kIn],
reddening = False,
colors = colorDict['kurucz'])
sEDNameM, magNormM, matchErrorM = selectStarSED0.findSED(listDict['mlt'],
sDSSunred[mIn], ra[mIn], dec[mIn],
reddening = False,
colors = colorDict['mlt'])
sEDNameH, magNormH, matchErrorH = selectStarSED0.findSED(listDict['H'],
sDSSunred[hIn], ra[hIn], dec[hIn],
reddening = False,
colors = colorDict['H'])
sEDNameHE, magNormHE, matchErrorHE = selectStarSED0.findSED(listDict['HE'],
sDSSunred[heIn],
ra[heIn], dec[heIn],
reddening = False,
colors = colorDict['HE'])
chunkNames = np.empty(readSize, dtype = 'S32')
chunkTypes = np.empty(readSize, dtype = 'S8')
chunkMagNorms = np.zeros(readSize)
chunkMatchErrors = np.zeros(readSize)
chunkNames[kIn] = sEDNameK
chunkTypes[kIn] = 'kurucz'
chunkMagNorms[kIn] = magNormK
chunkMatchErrors[kIn] = matchErrorK
chunkNames[mIn] = sEDNameM
chunkTypes[mIn] = 'mlt'
chunkMagNorms[mIn] = magNormM
chunkMatchErrors[mIn] = matchErrorM
chunkNames[hIn] = sEDNameH
chunkTypes[hIn] = 'H'
chunkMagNorms[hIn] = magNormH
chunkMatchErrors[hIn] = matchErrorH
chunkNames[heIn] = sEDNameHE
chunkTypes[heIn] = 'HE'
chunkMagNorms[heIn] = magNormHE
chunkMatchErrors[heIn] = matchErrorHE
lsstMagsUnred = []
for sedName, sedType, magNorm, matchError in zip(chunkNames.astype(str),
chunkTypes.astype(str),
chunkMagNorms,
chunkMatchErrors):
testSED = Sed()
testSED.setSED(listDict[sedType][positionDict[sedName]].wavelen,
flambda = listDict[sedType][positionDict[sedName]].flambda)
fluxNorm = testSED.calcFluxNorm(magNorm, imSimBand)
testSED.multiplyFluxNorm(fluxNorm)
lsstMagsUnred.append(lsstPhot.magListForSed(testSED))
#If the extinction value is negative then it will add the reddening back in
lsstMags = selectStarSED0.deReddenMags((-1.0*am), lsstMagsUnred,
lsstExtCoeffs)
distKpc = self.convDMtoKpc(DM)
ebv = am / 2.285 #From Schlafly and Finkbeiner 2011, (ApJ, 737, 103) for sdssr
ebvInf = amInf / 2.285
for line in range(0, readSize):
outFmt = '%i,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%s,' +\
'%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
'%3.7f,%i,%3.7f,%3.7f,%3.7f\n'
if readSize == 1:
if inFits == True:
sDSS = sDSS[0]
outDat = (oID, ra[line], dec[line], gall, galb, coordX,
coordY, coordZ, chunkNames,
chunkMagNorms, chunkMatchErrors,
lsstMags[line][0], lsstMags[line][1], lsstMags[line][2],
lsstMags[line][3], lsstMags[line][4], lsstMags[line][5],
sDSS[0], sDSS[1], sDSS[2], sDSS[3],
sDSS[4], absSDSSr, pmRA, pmDec, vRad,
pml, pmb, vRadlb, vR, vPhi, vZ,
FeH, pop, distKpc, ebv, ebvInf)
else:
outDat = (oID[line], ra[line], dec[line], gall[line], galb[line], coordX[line],
coordY[line], coordZ[line], chunkNames[line],
chunkMagNorms[line], chunkMatchErrors[line],
lsstMags[line][0], lsstMags[line][1], lsstMags[line][2],
lsstMags[line][3], lsstMags[line][4], lsstMags[line][5],
sDSS[line][0], sDSS[line][1], sDSS[line][2], sDSS[line][3],
sDSS[line][4], absSDSSr[line], pmRA[line], pmDec[line], vRad[line],
pml[line], pmb[line], vRadlb[line], vR[line], vPhi[line], vZ[line],
FeH[line], pop[line], distKpc[line], ebv[line], ebvInf[line])
fOut.write(outFmt % outDat)
print('Chunk Num Done = %i out of %i' % (chunk+1, numChunks))
def findSED(
self,
sedList,
catMags,
catRA=None,
catDec=None,
mag_error=None,
reddening=True,
bandpassDict=None,
colors=None,
extCoeffs=(4.239, 3.303, 2.285, 1.698, 1.263),
makeCopy=False,
):
"""
This will find the SEDs that are the closest match to the magnitudes of a star catalog.
It can also correct for reddening from within the milky way. Objects without magnitudes in at least
two adjacent bandpasses will return as none and print out a message.
@param [in] sedList is the set of spectral objects from the models SEDs provided by loaders in rgStar
in rgUtils.py or other custom loader routine.
@param [in] catMags is an array of the magnitudes of catalog objects to be matched with a model SED.
It should be organized so that there is one object's magnitudes along each row.
@param [in] catRA is an array of the RA positions for each catalog object.
@param [in] catDec is an array of the Dec position for each catalog object.
@param [in] mag_error are provided error values for magnitudes in objectMags. If none provided
then this defaults to 1.0. This should be an array of the same length as objectMags.
@param [in] reddening is a boolean that determines whether to correct catalog magnitudes for
dust in the milky way. By default, it is True.
If true, this uses calculateEBV from EBV.py to find an EBV value for the object's
ra and dec coordinates and then uses the coefficients provided by extCoeffs which should come
from Schlafly and Finkbeiner (2011) for the correct filters and in the same order as provided
in bandpassDict.
If false, this means it will not run the dereddening procedure.
@param [in] bandpassDict is a BandpassDict with which to calculate magnitudes. If left
equal to None it will by default load the SDSS [u,g,r,i,z] bandpasses and therefore agree with
default extCoeffs.
@param [in] colors is None if you are just providing a list of SED objects to match, but is the
array holding the colors of those SED models (each row should be the colors for one model in the
same order as sedList) if you have already calculated the colors.
@param [in] extCoeffs are the Schlafly and Finkbeiner (2011) (ApJ, 737, 103) coefficients for the
given filters from bandpassDict and need to be in the same order as bandpassDict. The default given
are the SDSS [u,g,r,i,z] values.
@param [in] makeCopy indicates whether or not to operate on copies of the SED objects in sedList
since this method will change the wavelength grid.
@param [out] sedMatches is a list with the name of a model SED that matches most closely to each
object in the catalog.
@param [out] magNormMatches are the magnitude normalizations for the given magnitudes and
matched SED.
@param [out] matchErrors contains the Mean Squared Error between the colors of each object and
the colors of the matched SED.
"""
if bandpassDict is None:
starPhot = BandpassDict.loadTotalBandpassesFromFiles(
["u", "g", "r", "i", "z"],
bandpassDir=os.path.join(lsst.utils.getPackageDir("throughputs"), "sdss"),
bandpassRoot="sdss_",
)
else:
starPhot = bandpassDict
if colors is None:
modelColors = self.calcBasicColors(sedList, starPhot, makeCopy=makeCopy)
else:
modelColors = colors
# Transpose so that all values for one color are in one row as needed for the matching loop below
modelColors = np.transpose(modelColors)
if reddening == True:
# Check that catRA and catDec are included
if catRA is None or catDec is None:
raise RuntimeError("Reddening is True, but catRA and catDec are not included.")
calcEBV = ebv()
raDec = np.array((catRA, catDec))
# If only matching one object need to reshape for calculateEbv
if len(raDec.shape) == 1:
raDec = raDec.reshape((2, 1))
ebvVals = calcEBV.calculateEbv(equatorialCoordinates=raDec)
objMags = self.deReddenMags(ebvVals, catMags, extCoeffs)
else:
objMags = catMags
objMags = np.array(objMags)
matchColors = []
for filtNum in range(0, len(starPhot) - 1):
matchColors.append(np.transpose(objMags)[filtNum] - np.transpose(objMags)[filtNum + 1])
matchColors = np.transpose(matchColors)
numCatMags = len(catMags)
numOn = 0
sedMatches = []
magNormMatches = []
notMatched = 0
matchErrors = []
for catObject in matchColors:
# This is done to handle objects with incomplete magnitude data
colorRange = np.arange(0, len(starPhot) - 1)
filtNums = np.arange(0, len(starPhot))
if np.isnan(np.amin(catObject)) == True:
colorRange = np.where(np.isnan(catObject) == False)[0]
filtNums = np.unique([colorRange, colorRange + 1]) # Pick right filters in calcMagNorm
if len(colorRange) == 0:
print "Could not match object #%i. No magnitudes for two adjacent bandpasses." % (numOn)
notMatched += 1
sedMatches.append(None)
magNormMatches.append(None)
matchErrors.append(None)
else:
distanceArray = np.zeros(len(sedList))
for colorNum in colorRange:
distanceArray += np.power((modelColors[colorNum] - catObject[colorNum]), 2)
matchedSEDNum = np.nanargmin(distanceArray)
sedMatches.append(sedList[matchedSEDNum].name)
magNorm = self.calcMagNorm(objMags[numOn], sedList[matchedSEDNum], starPhot, filtRange=filtNums)
magNormMatches.append(magNorm)
matchErrors.append(distanceArray[matchedSEDNum] / len(colorRange)) # Mean Squared Error
numOn += 1
if numOn % 10000 == 0:
print "Matched %i of %i catalog objects to SEDs" % (numOn - notMatched, numCatMags)
if numCatMags > 1:
print "Done Matching. Matched %i of %i catalog objects to SEDs" % (numCatMags - notMatched, numCatMags)
if notMatched > 0:
print "%i objects did not get matched" % (notMatched)
return sedMatches, magNormMatches, matchErrors
def main():
"""
Drive GalSim to simulate the LSST.
"""
# Setup a parser to take command line arguments
parser = argparse.ArgumentParser()
parser.add_argument('file', help="The instance catalog")
parser.add_argument('-n',
'--numrows',
default=None,
type=int,
help="Read the first numrows of the file.")
parser.add_argument('--outdir',
type=str,
default='fits',
help='Output directory for eimage file')
parser.add_argument(
'--sensor',
type=str,
default=None,
help='Sensor to simulate, e.g., "R:2,2 S:1,1".' +
'If None, then simulate all sensors with sources on them')
parser.add_argument(
'--config_file',
type=str,
default=None,
help="Config file. If None, the default config will be used.")
parser.add_argument('--log_level',
type=str,
choices=['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'],
default='INFO',
help='Logging level. Default: "INFO"')
parser.add_argument(
'--psf',
type=str,
default='Kolmogorov',
choices=['DoubleGaussian', 'Kolmogorov'],
help="PSF model to use; either the double Gaussian "
"from LSE=40 (equation 30), or the Kolmogorov convolved "
"with a Gaussian proposed by David Kirkby at the "
"23 March 2017 SSims telecon")
parser.add_argument('--checkpoint_file',
type=str,
default=None,
help='Checkpoint file name.')
parser.add_argument('--nobj_checkpoint',
type=int,
default=1000,
help='# objects to process between checkpoints')
parser.add_argument('--seed',
type=int,
default=267,
help='integer used to seed random number generator')
arguments = parser.parse_args()
config = desc.imsim.read_config(arguments.config_file)
logger = desc.imsim.get_logger(arguments.log_level)
# Get the number of rows to read from the instance file. Use
# default if not specified.
numRows = arguments.numrows
if numRows is not None:
logger.info("Reading %i rows from the instance catalog %s.", numRows,
arguments.file)
else:
logger.info("Reading all rows from the instance catalog %s.",
arguments.file)
camera_wrapper = LSSTCameraWrapper()
catalog_contents = desc.imsim.parsePhoSimInstanceFile(arguments.file,
numRows=numRows)
obs_md = catalog_contents.obs_metadata
phot_params = catalog_contents.phot_params
sources = catalog_contents.sources
gs_object_arr = sources[0]
gs_object_dict = sources[1]
# Sub-divide the source dataframe into stars and galaxies.
if arguments.sensor is not None:
detector_list = [
make_galsim_detector(camera_wrapper, arguments.sensor, phot_params,
obs_md)
]
else:
detector_list = []
for det in camera_wrapper.camera:
det_type = det.getType()
if det_type != WAVEFRONT and det_type != GUIDER:
detector_list.append(
make_galsim_detector(camera_wrapper, det.getName(),
phot_params, obs_md))
# Add noise and sky background
# The simple code using the default lsst-GalSim interface would be:
#
# PhoSimStarCatalog.noise_and_background = ExampleCCDNoise(addNoise=True,
# addBackground=True)
#
# But, we need a more realistic sky model and we need to pass more than
# this basic info to use Peter Y's ESO sky model.
# We must pass obs_metadata, chip information etc...
noise_and_background \
= ESOSkyModel(obs_md, addNoise=True, addBackground=True)
bp_dict = BandpassDict.loadTotalBandpassesFromFiles(
bandpassNames=obs_md.bandpass)
gs_interpreter = GalSimInterpreter(obs_metadata=obs_md,
epoch=2000.0,
detectors=detector_list,
bandpassDict=bp_dict,
noiseWrapper=noise_and_background,
seed=arguments.seed)
gs_interpreter.checkpoint_file = arguments.checkpoint_file
gs_interpreter.nobj_checkpoint = arguments.nobj_checkpoint
gs_interpreter.restore_checkpoint(camera_wrapper, phot_params, obs_md)
# Add a PSF.
if arguments.psf.lower() == "doublegaussian":
# This one is taken from equation 30 of
# www.astro.washington.edu/users/ivezic/Astr511/LSST_SNRdoc.pdf .
#
# Set seeing from self.obs_metadata.
local_PSF = \
SNRdocumentPSF(obs_md.OpsimMetaData['FWHMgeom'])
elif arguments.psf.lower() == "kolmogorov":
# This PSF was presented by David Kirkby at the 23 March 2017
# Survey Simulations Working Group telecon
#
# https://confluence.slac.stanford.edu/pages/viewpage.action?spaceKey=LSSTDESC&title=SSim+2017-03-23
# equation 3 of Krisciunas and Schaefer 1991
airmass = 1.0 / np.sqrt(
1.0 - 0.96 *
(np.sin(0.5 * np.pi - obs_md.OpsimMetaData['altitude']))**2)
local_PSF = \
Kolmogorov_and_Gaussian_PSF(airmass=airmass,
rawSeeing=obs_md.OpsimMetaData['rawSeeing'],
band=obs_md.bandpass)
else:
raise RuntimeError("Do not know what to do with psf model: "
"%s" % arguments.psf)
gs_interpreter.setPSF(PSF=local_PSF)
if arguments.sensor is not None:
gs_objects_to_draw = gs_object_dict[arguments.sensor]
else:
gs_objects_to_draw = gs_object_arr
for gs_obj in gs_objects_to_draw:
if gs_obj.uniqueId in gs_interpreter.drawn_objects:
continue
gs_interpreter.drawObject(gs_obj)
desc.imsim.add_cosmic_rays(gs_interpreter, phot_params)
# Write out the fits files
outdir = arguments.outdir
if not os.path.isdir(outdir):
os.makedirs(outdir)
prefix = config['persistence']['eimage_prefix']
gs_interpreter.writeImages(nameRoot=os.path.join(outdir, prefix) +
str(obs_md.OpsimMetaData['obshistID']))
def matchToObserved(self, sedList, catMags, catRedshifts, catRA = None, catDec = None,
mag_error = None, bandpassDict = None, dzAcc = 2, reddening = True,
extCoeffs = (4.239, 3.303, 2.285, 1.698, 1.263)):
"""
This will find the closest match to the magnitudes of a galaxy catalog if those magnitudes are in
the observed frame and can correct for reddening from within the milky way as well if needed.
In order to make things faster it first calculates colors for all model SEDs at redshifts between
the minimum and maximum redshifts of the catalog objects provided with a grid spacing in redshift
defined by the parameter dzAcc. Objects without magnitudes in at least two adjacent bandpasses will
return as none and print out a message.
@param [in] sedList is the set of spectral objects from the models SEDs provided by loadBC03
or other custom loader routine.
@param [in] catMags is an array of the magnitudes of catalog objects to be matched with a model SED.
It should be organized so that there is one object's magnitudes along each row.
@param [in] catRedshifts is an array of the redshifts of each catalog object.
@param [in] catRA is an array of the RA positions for each catalog object.
@param [in] catDec is an array of the Dec position for each catalog object.
@param [in] mag_error are provided error values for magnitudes in objectMags. If none provided
then this defaults to 1.0. This should be an array of the same size as catMags.
@param [in] bandpassDict is a BandpassDict with which to calculate magnitudes.
If left equal to None it will by default load the SDSS [u,g,r,i,z] bandpasses and therefore agree with
default extCoeffs.
@param [in] dzAcc is the number of decimal places you want to use when building the redshift grid.
For example, dzAcc = 2 will create a grid between the minimum and maximum redshifts with colors
calculated at every 0.01 change in redshift.
@param [in] reddening is a boolean that determines whether to correct catalog magnitudes for
dust in the milky way. By default, it is True.
If true, this uses calculateEBV from EBV.py to find an EBV value for the object's
ra and dec coordinates and then uses the coefficients provided by extCoeffs which should come
from Schlafly and Finkbeiner (2011) for the correct filters and in the same order as provided
in bandpassDict.
If false, this means it will not run the dereddening procedure.
@param [in] extCoeffs are the Schlafly and Finkbeiner (2011) (ApJ, 737, 103) coefficients for the
given filters from bandpassDict and need to be in the same order as bandpassDict. The default given
are the SDSS [u,g,r,i,z] values.
@param [out] sedMatches is a list with the name of a model SED that matches most closely to each
object in the catalog.
@param [out] magNormMatches are the magnitude normalizations for the given magnitudes and
matched SED.
@param [out] matchErrors contains the Mean Squared Error between the colors of each object and
the colors of the matched SED.
"""
#Set up photometry to calculate model Mags
if bandpassDict is None:
galPhot = BandpassDict.loadTotalBandpassesFromFiles(['u','g','r','i','z'],
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'),'sdss'),
bandpassRoot = 'sdss_')
else:
galPhot = bandpassDict
#Calculate ebv from ra, dec coordinates if needed
if reddening == True:
#Check that catRA and catDec are included
if catRA is None or catDec is None:
raise RuntimeError("Reddening is True, but catRA and catDec are not included.")
calcEBV = ebv()
raDec = np.array((catRA,catDec))
#If only matching one object need to reshape for calculateEbv
if len(raDec.shape) == 1:
raDec = raDec.reshape((2,1))
ebvVals = calcEBV.calculateEbv(equatorialCoordinates = raDec)
objMags = self.deReddenMags(ebvVals, catMags, extCoeffs)
else:
objMags = catMags
minRedshift = np.round(np.min(catRedshifts), dzAcc)
maxRedshift = np.round(np.max(catRedshifts), dzAcc)
dz = np.power(10., (-1*dzAcc))
redshiftRange = np.round(np.arange(minRedshift - dz, maxRedshift + (2*dz), dz), dzAcc)
numRedshifted = 0
sedMatches = [None] * len(catRedshifts)
magNormMatches = [None] * len(catRedshifts)
matchErrors = [None] * len(catRedshifts)
redshiftIndex = np.argsort(catRedshifts)
numOn = 0
notMatched = 0
lastRedshift = -100
print 'Starting Matching. Arranged by redshift value.'
for redshift in redshiftRange:
if numRedshifted % 10 == 0:
print '%i out of %i redshifts gone through' % (numRedshifted, len(redshiftRange))
numRedshifted += 1
colorSet = []
for galSpec in sedList:
sedColors = []
fileSED = Sed()
fileSED.setSED(wavelen = galSpec.wavelen, flambda = galSpec.flambda)
fileSED.redshiftSED(redshift)
sedColors = self.calcBasicColors([fileSED], galPhot, makeCopy = True)
colorSet.append(sedColors)
colorSet = np.transpose(colorSet)
for currentIndex in redshiftIndex[numOn:]:
matchMags = objMags[currentIndex]
if lastRedshift < np.round(catRedshifts[currentIndex],dzAcc) <= redshift:
colorRange = np.arange(0, len(galPhot)-1)
matchColors = []
for colorNum in colorRange:
matchColors.append(matchMags[colorNum] - matchMags[colorNum+1])
#This is done to handle objects with incomplete magnitude data
filtNums = np.arange(0, len(galPhot))
if np.isnan(np.amin(matchColors))==True:
colorRange = np.where(np.isnan(matchColors)==False)[0]
filtNums = np.unique([colorRange, colorRange+1]) #Pick right filters in calcMagNorm
if len(colorRange) == 0:
print 'Could not match object #%i. No magnitudes for two adjacent bandpasses.' \
% (currentIndex)
notMatched += 1
#Don't need to assign 'None' here in result array, b/c 'None' is default value
else:
distanceArray = [np.zeros(len(sedList))]
for colorNum in colorRange:
distanceArray += np.power((colorSet[colorNum] - matchColors[colorNum]),2)
matchedSEDNum = np.nanargmin(distanceArray)
sedMatches[currentIndex] = sedList[matchedSEDNum].name
magNormVal = self.calcMagNorm(np.array(matchMags), sedList[matchedSEDNum],
galPhot, mag_error = mag_error,
redshift = catRedshifts[currentIndex],
filtRange = filtNums)
magNormMatches[currentIndex] = magNormVal
matchErrors[currentIndex] = (distanceArray[0,matchedSEDNum]/len(colorRange))
numOn += 1
else:
break
lastRedshift = redshift
print 'Done Matching. Matched %i of %i catalog objects to SEDs' % (len(catMags)-notMatched,
len(catMags))
if notMatched > 0:
print '%i objects did not get matched.' % (notMatched)
return sedMatches, magNormMatches, matchErrors
def testFindSED(self):
"""Pull SEDs from each type and make sure that each SED gets matched to itself.
Includes testing with extinction and passing in only colors."""
rng = np.random.RandomState(42)
bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'), 'sdss')
starPhot = BandpassDict.loadTotalBandpassesFromFiles(('u', 'g', 'r', 'i', 'z'),
bandpassDir = bandpassDir,
bandpassRoot = 'sdss_')
imSimBand = Bandpass()
imSimBand.imsimBandpass()
testMatching = selectStarSED(kuruczDir=self.testKDir,
mltDir=self.testMLTDir,
wdDir=self.testWDDir)
testSEDList = []
testSEDList.append(testMatching.loadKuruczSEDs())
testSEDList.append(testMatching.loadmltSEDs())
testSEDListH, testSEDListHE = testMatching.loadwdSEDs()
testSEDList.append(testSEDListH)
testSEDList.append(testSEDListHE)
testSEDNames = []
testMags = []
testMagNormList = []
magNormStep = 1
for typeList in testSEDList:
if len(typeList) != 0:
typeSEDNames = []
typeMags = []
typeMagNorms = []
for testSED in typeList:
getSEDMags = Sed()
typeSEDNames.append(testSED.name)
getSEDMags.setSED(wavelen = testSED.wavelen, flambda = testSED.flambda)
testMagNorm = np.round(rng.uniform(20.0, 22.0), magNormStep)
typeMagNorms.append(testMagNorm)
fluxNorm = getSEDMags.calcFluxNorm(testMagNorm, imSimBand)
getSEDMags.multiplyFluxNorm(fluxNorm)
typeMags.append(starPhot.magListForSed(getSEDMags))
testSEDNames.append(typeSEDNames)
testMags.append(typeMags)
testMagNormList.append(typeMagNorms)
# Since default bandpassDict should be SDSS ugrizy shouldn't need to specify it
# Substitute in nan values to simulate incomplete data.
for typeList, names, mags, magNorms in zip(testSEDList, testSEDNames, testMags, testMagNormList):
if len(typeList) > 2:
nanMags = np.array(mags)
nanMags[0][0] = np.nan
nanMags[0][2] = np.nan
nanMags[0][3] = np.nan
nanMags[1][1] = np.nan
testMatchingResults = testMatching.findSED(typeList, nanMags, reddening = False)
self.assertEqual(None, testMatchingResults[0][0])
self.assertEqual(names[1:], testMatchingResults[0][1:])
self.assertEqual(None, testMatchingResults[1][0])
np.testing.assert_almost_equal(magNorms[1:], testMatchingResults[1][1:],
decimal = magNormStep)
else:
testMatchingResults = testMatching.findSED(typeList, mags, reddening = False)
self.assertEqual(names, testMatchingResults[0])
np.testing.assert_almost_equal(magNorms, testMatchingResults[1], decimal = magNormStep)
# Test Null Values option
nullMags = np.array(testMags[0])
nullMags[0][0] = -99.
nullMags[0][4] = -99.
nullMags[1][0] = -99.
nullMags[1][1] = -99.
testMatchingResultsNull = testMatching.findSED(testSEDList[0], nullMags,
nullValues = -99., reddening = False)
self.assertEqual(testSEDNames[0], testMatchingResultsNull[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsNull[1],
decimal = magNormStep)
# Test Error Output
errMags = np.array((testMags[0][0], testMags[0][0], testMags[0][0], testMags[0][0]))
errMags[1, 1] += 1. # Total MSE will be 2/(4 colors) = 0.5
errMags[2, 0:2] = np.nan
errMags[2, 3] += 1. # Total MSE will be 2/(2 colors) = 1.0
errMags[3, :] = None
errSED = testSEDList[0][0]
testMatchingResultsErrors = testMatching.findSED([errSED], errMags, reddening = False)
np.testing.assert_almost_equal(np.array((0.0, 0.5, 1.0)), testMatchingResultsErrors[2][0:3],
decimal = 3)
self.assertEqual(None, testMatchingResultsErrors[2][3])
# Now test what happens if we pass in a bandpassDict
testMatchingResultsNoDefault = testMatching.findSED(testSEDList[0], testMags[0],
bandpassDict = starPhot,
reddening = False)
self.assertEqual(testSEDNames[0], testMatchingResultsNoDefault[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsNoDefault[1],
decimal = magNormStep)
# Test Reddening
testRA = rng.uniform(10, 170, len(testSEDList[0]))
testDec = rng.uniform(10, 80, len(testSEDList[0]))
extFactor = .5
raDec = np.array((testRA, testDec))
ebvVals = ebv().calculateEbv(equatorialCoordinates = raDec)
extVals = ebvVals*extFactor
testRedMags = []
for extVal, testMagSet in zip(extVals, testMags[0]):
testRedMags.append(testMagSet + extVal)
testMatchingResultsRed = testMatching.findSED(testSEDList[0], testRedMags, catRA = testRA,
catDec = testDec, reddening = True,
extCoeffs = np.ones(5)*extFactor)
self.assertEqual(testSEDNames[0], testMatchingResultsRed[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingResultsRed[1],
decimal = magNormStep)
# Finally, test color input
testColors = []
for testMagSet in testMags[0]:
testColorSet = []
for filtNum in range(0, len(starPhot)-1):
testColorSet.append(testMagSet[filtNum] - testMagSet[filtNum+1])
testColors.append(testColorSet)
testMatchingColorsInput = testMatching.findSED(testSEDList[0], testMags[0],
reddening = False, colors = testColors)
self.assertEqual(testSEDNames[0], testMatchingColorsInput[0])
np.testing.assert_almost_equal(testMagNormList[0], testMatchingColorsInput[1],
decimal = magNormStep)
np.testing.assert_array_equal(phosim_objid_arr, objid_arr)
return (objid_arr, flux_arr, phosim_flux_arr, redshift_arr, magnorm_arr,
x_arr, y_arr, chip_name_arr, filter_name)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--instcat_dir', type=str)
parser.add_argument('--centroid_dir', type=str)
parser.add_argument('--throughputs_dir', type=str)
parser.add_argument('--fig_dir', type=str)
args = parser.parse_args()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles(
bandpassDir=args.throughputs_dir)
instcat_list = os.listdir(args.instcat_dir)
phosim_flux_dict = {}
objid_dict = {}
catsim_flux_dict = {}
magnorm_dict = {}
redshift_dict = {}
xpix_dict = {}
ypix_dict = {}
chip_name_dict = {}
for instcat in instcat_list:
instcat_file = os.path.join(args.instcat_dir, instcat)
print('processing %s' % instcat)
def test_sne(grid_name, x1_vals, c0_vals,
z_vals, abs_mag_vals, t_vals,
dict_key, out_dict):
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
cosmo = CosmologyObject()
n_samples = len(z_vals)
mag_truth = np.zeros((6,n_samples), dtype=float)
mag_interp = np.zeros((6,n_samples), dtype=float)
mag_grid_dict = {}
with h5py.File(grid_name, 'r') as in_file:
param_mins = in_file['param_mins'].value
d_params = in_file['d_params'].value
t_grid = in_file['t_grid'].value
for name in in_file.keys():
if name == 'param_mins':
continue
if name == 'd_params':
continue
if name == 't_grid':
continue
mag_grid_dict[name] = in_file[name].value
t_start = time.time()
for ii in range(n_samples):
x1 = x1_vals[ii]
c0 = c0_vals[ii]
z = z_vals[ii]
abs_mag = abs_mag_vals[ii]
t = t_vals[ii]
sn = sncosmo.Model(source='salt2-extended')
sn.set(x1=x1,c=c0,z=z)
sn.source.set_peakmag(abs_mag+cosmo.distanceModulus(z),
band='bessellb', magsys='ab')
flambda = 10.0*sn.flux(time=t, wave=10.0*bp_dict['g'].wavelen)
ss = Sed(flambda=flambda, wavelen=bp_dict['g'].wavelen)
mag_truth[:,ii] = bp_dict.magListForSed(ss)
i_x1 = np.round((x1-param_mins[0])/d_params[0]).astype(int)
i_c0 = np.round((c0-param_mins[1])/d_params[1]).astype(int)
i_z = np.round((z-param_mins[2])/d_params[2]).astype(int)
d_mag = abs_mag-param_mins[3]
tag = i_x1+i_c0*100+i_z*10000
mag_grid = mag_grid_dict['%d' % tag]
interp_mags = np.zeros(6, dtype=float)
for i_bp in range(6):
mm = np.interp(t, t_grid, mag_grid[i_bp])+d_mag
mag_interp[i_bp,ii] = mm
if ii>0 and ii%100 == 0:
duration = (time.time()-t_start)/3600.0
pred = n_samples*duration/ii
print('%d in %e hrs; predict %e' % (ii,duration,pred))
out_dict[dict_key] = (mag_interp, mag_truth)
def test_MLT_many_mjd_some_invalid(self):
"""
This test will verify that applyMLTflaring responds properly when given
an array/vector of MJD values in the case where some stars are not
marked as valid MLT stars.
"""
db = MLT_test_DB(database=self.db_name, driver='sqlite')
rng = np.random.RandomState(16)
mjd_arr = rng.random_sample(17)*3653.3+59580.0
objid = []
parallax = []
ebv = []
quiescent_u = []
quiescent_g = []
delta_u = []
delta_g = []
varparams = []
for mjd in mjd_arr:
obs = ObservationMetaData(mjd=mjd)
cat = FlaringCatalog(db, obs_metadata=obs,
column_outputs=['parallax', 'ebv',
'quiescent_lsst_u',
'quiescent_lsst_g',
'varParamStr',
'delta_lsst_u',
'delta_lsst_g'])
cat._mlt_lc_file = self.mlt_lc_name
n_obj = 0
for line in cat.iter_catalog():
n_obj += 1
objid.append(line[0])
parallax.append(line[3])
ebv.append(line[4])
quiescent_u.append(line[5])
quiescent_g.append(line[6])
varparams.append(line[7])
delta_u.append(line[8])
delta_g.append(line[9])
objid = np.array(objid)
parallax = np.array(parallax)
ebv = np.array(ebv)
quiescent_u = np.array(quiescent_u)
quiescent_g = np.array(quiescent_g)
delta_u = np.array(delta_u)
delta_g = np.array(delta_g)
self.assertEqual(len(parallax), n_obj*len(mjd_arr))
np.testing.assert_array_equal(objid, np.array([0,1,2,3]*len(mjd_arr)))
quiescent_mags = {}
quiescent_mags['u'] = quiescent_u
quiescent_mags['g'] = quiescent_g
params = {}
params['lc'] = []
params['t0'] = []
for ix in range(n_obj):
local_dict = json.loads(varparams[ix])
params['lc'].append(local_dict['p']['lc'])
params['t0'].append(local_dict['p']['t0'])
params['lc'] = np.array(params['lc'])
params['t0'] = np.array(params['t0'])
mlt_obj = MLTflaringMixin()
mlt_obj.photParams = PhotometricParameters()
mlt_obj.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
mlt_obj._mlt_lc_file = cat._mlt_lc_file
mlt_obj._actually_calculated_columns = ['delta_lsst_u', 'delta_lsst_g']
valid_dex = [] # applyMLT does not actually use valid_dex; it looks for non-None params['lc']
for ix in range(n_obj):
if ix not in (1,2):
params['lc'][ix] = None
delta_mag_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags)
n_time = len(mjd_arr)
self.assertEqual(delta_mag_vector.shape, (6, n_obj, n_time))
for i_time, mjd in enumerate(mjd_arr):
for i_obj in range(n_obj):
if i_obj not in (1,2):
for i_band in range(6):
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time], 0.0)
continue
for i_band in range(6):
if i_band==0:
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time],
delta_u[i_time*n_obj + i_obj])
elif i_band==1:
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time],
delta_g[i_time*n_obj + i_obj])
else:
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time], 0.0)