def testSignalToNoise(self):
"""
Test that calcSNR_m5 and calcSNR_sed give similar results
"""
defaults = LSSTdefaults()
photParams = PhotometricParameters()
totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles()
skySED = Sed()
skySED.readSED_flambda(
os.path.join(lsst.utils.getPackageDir('throughputs'), 'baseline',
'darksky.dat'))
m5 = []
for filt in totalDict:
m5.append(
calcM5(skySED,
totalDict[filt],
hardwareDict[filt],
photParams,
seeing=defaults.seeing(filt)))
sedDir = lsst.utils.getPackageDir('sims_sed_library')
sedDir = os.path.join(sedDir, 'starSED', 'kurucz')
fileNameList = os.listdir(sedDir)
numpy.random.seed(42)
offset = numpy.random.random_sample(len(fileNameList)) * 2.0
for ix, name in enumerate(fileNameList):
if ix > 100:
break
spectrum = Sed()
spectrum.readSED_flambda(os.path.join(sedDir, name))
ff = spectrum.calcFluxNorm(m5[2] - offset[ix],
totalDict.values()[2])
spectrum.multiplyFluxNorm(ff)
magList = []
controlList = []
magList = []
for filt in totalDict:
controlList.append(
calcSNR_sed(spectrum, totalDict[filt], skySED,
hardwareDict[filt], photParams,
defaults.seeing(filt)))
magList.append(spectrum.calcMag(totalDict[filt]))
testList, gammaList = calcSNR_m5(numpy.array(magList),
numpy.array(totalDict.values()),
numpy.array(m5), photParams)
for tt, cc in zip(controlList, testList):
msg = '%e != %e ' % (tt, cc)
self.assertTrue(numpy.abs(tt / cc - 1.0) < 0.001, msg=msg)
def testSystematicUncertainty(self):
"""
Test that systematic uncertainty is added correctly.
"""
sigmaSys = 0.002
m5 = [23.5, 24.3, 22.1, 20.0, 19.5, 21.7]
photParams = PhotometricParameters(sigmaSys=sigmaSys)
bandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
obs_metadata = ObservationMetaData(unrefractedRA=23.0,
unrefractedDec=45.0,
m5=m5,
bandpassName=self.bandpasses)
magnitudes = bandpassDict.magListForSed(self.starSED)
skySeds = []
for i in range(len(self.bandpasses)):
skyDummy = Sed()
skyDummy.readSED_flambda(
os.path.join(lsst.utils.getPackageDir('throughputs'),
'baseline', 'darksky.dat'))
normalizedSkyDummy = setM5(obs_metadata.m5[self.bandpasses[i]],
skyDummy,
self.totalBandpasses[i],
self.hardwareBandpasses[i],
seeing=LSSTdefaults().seeing(
self.bandpasses[i]),
photParams=PhotometricParameters())
skySeds.append(normalizedSkyDummy)
for i in range(len(self.bandpasses)):
snr = calcSNR_sed(self.starSED,
self.totalBandpasses[i],
skySeds[i],
self.hardwareBandpasses[i],
seeing=LSSTdefaults().seeing(self.bandpasses[i]),
photParams=PhotometricParameters())
testSNR, gamma = calcSNR_m5(
numpy.array([magnitudes[i]]), [self.totalBandpasses[i]],
numpy.array([m5[i]]),
photParams=PhotometricParameters(sigmaSys=0.0))
self.assertAlmostEqual(snr, testSNR[0], 10, msg = 'failed on calcSNR_m5 test %e != %e ' \
% (snr, testSNR[0]))
control = numpy.sqrt(
numpy.power(magErrorFromSNR(testSNR), 2) +
numpy.power(sigmaSys, 2))
def testUncertaintyExceptions(self):
"""
Test that calcSNR_m5 raises exceptions when it needs to
"""
totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles()
magnitudes = numpy.array([22.0, 23.0, 24.0, 25.0, 26.0, 27.0])
shortMagnitudes = numpy.array([22.0])
photParams = PhotometricParameters()
shortGamma = numpy.array([1.0, 1.0])
self.assertRaises(RuntimeError, calcSNR_m5, magnitudes, totalDict.values(), shortMagnitudes, photParams)
self.assertRaises(RuntimeError, calcSNR_m5, shortMagnitudes, totalDict.values(), magnitudes, photParams)
self.assertRaises(
RuntimeError, calcSNR_m5, magnitudes, totalDict.values(), magnitudes, photParams, gamma=shortGamma
)
snr, gg = calcSNR_m5(magnitudes, totalDict.values(), magnitudes, photParams)
def testSystematicUncertainty(self):
"""
Test that systematic uncertainty is added correctly.
"""
sigmaSys = 0.002
m5 = [23.5, 24.3, 22.1, 20.0, 19.5, 21.7]
photParams = PhotometricParameters(sigmaSys=sigmaSys)
bandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
obs_metadata = ObservationMetaData(unrefractedRA=23.0, unrefractedDec=45.0, m5=m5, bandpassName=self.bandpasses)
magnitudes = bandpassDict.magListForSed(self.starSED)
skySeds = []
for i in range(len(self.bandpasses)):
skyDummy = Sed()
skyDummy.readSED_flambda(os.path.join(lsst.utils.getPackageDir("throughputs"), "baseline", "darksky.dat"))
normalizedSkyDummy = setM5(
obs_metadata.m5[self.bandpasses[i]],
skyDummy,
self.totalBandpasses[i],
self.hardwareBandpasses[i],
seeing=LSSTdefaults().seeing(self.bandpasses[i]),
photParams=PhotometricParameters(),
)
skySeds.append(normalizedSkyDummy)
for i in range(len(self.bandpasses)):
snr = calcSNR_sed(
self.starSED,
self.totalBandpasses[i],
skySeds[i],
self.hardwareBandpasses[i],
seeing=LSSTdefaults().seeing(self.bandpasses[i]),
photParams=PhotometricParameters(),
)
testSNR, gamma = calcSNR_m5(
numpy.array([magnitudes[i]]),
[self.totalBandpasses[i]],
numpy.array([m5[i]]),
photParams=PhotometricParameters(sigmaSys=0.0),
)
self.assertAlmostEqual(snr, testSNR[0], 10, msg="failed on calcSNR_m5 test %e != %e " % (snr, testSNR[0]))
control = numpy.sqrt(numpy.power(magErrorFromSNR(testSNR), 2) + numpy.power(sigmaSys, 2))
def testSignalToNoise(self):
"""
Test that calcSNR_m5 and calcSNR_sed give similar results
"""
defaults = LSSTdefaults()
photParams = PhotometricParameters()
totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles()
skySED = Sed()
skySED.readSED_flambda(os.path.join(lsst.utils.getPackageDir("throughputs"), "baseline", "darksky.dat"))
m5 = []
for filt in totalDict:
m5.append(calcM5(skySED, totalDict[filt], hardwareDict[filt], photParams, seeing=defaults.seeing(filt)))
sedDir = lsst.utils.getPackageDir("sims_sed_library")
sedDir = os.path.join(sedDir, "starSED", "kurucz")
fileNameList = os.listdir(sedDir)
numpy.random.seed(42)
offset = numpy.random.random_sample(len(fileNameList)) * 2.0
for ix, name in enumerate(fileNameList):
if ix > 100:
break
spectrum = Sed()
spectrum.readSED_flambda(os.path.join(sedDir, name))
ff = spectrum.calcFluxNorm(m5[2] - offset[ix], totalDict.values()[2])
spectrum.multiplyFluxNorm(ff)
magList = []
controlList = []
magList = []
for filt in totalDict:
controlList.append(
calcSNR_sed(
spectrum, totalDict[filt], skySED, hardwareDict[filt], photParams, defaults.seeing(filt)
)
)
magList.append(spectrum.calcMag(totalDict[filt]))
testList, gammaList = calcSNR_m5(
numpy.array(magList), numpy.array(totalDict.values()), numpy.array(m5), photParams
)
for tt, cc in zip(controlList, testList):
msg = "%e != %e " % (tt, cc)
self.assertTrue(numpy.abs(tt / cc - 1.0) < 0.001, msg=msg)
def get_magSNR(self):
"""
Calculate the SNR for the observation, given m5 from obs_metadata and the trailing losses.
"""
magFilter = self.column_by_name('magFilter')
bandpass = self.lsstBandpassDict[self.obs_metadata.bandpass]
# Get m5 for the visit
m5 = self.obs_metadata.m5[self.obs_metadata.bandpass]
# Adjust the magnitude of the source for the trailing losses.
dmagSNR = self.column_by_name('dmagTrailing')
magObj = magFilter - dmagSNR
if len(magObj) == 0:
snr = []
else:
snr, gamma = calcSNR_m5(magObj, bandpass, m5, self.photParams)
return snr
def get_magSNR(self):
"""
Calculate the SNR for the observation, given m5 from obs_metadata and the trailing losses.
"""
magFilter = self.column_by_name('magFilter')
bandpass = self.lsstBandpassDict[self.obs_metadata.bandpass]
# Get m5 for the visit
m5 = self.obs_metadata.m5[self.obs_metadata.bandpass]
# Adjust the magnitude of the source for the trailing losses.
dmagSNR = self.column_by_name('dmagTrailing')
magObj = magFilter - dmagSNR
if len(magObj) == 0:
snr = []
else:
snr, gamma = calcSNR_m5(magObj, bandpass, m5, self.photParams)
return snr
def testUncertaintyExceptions(self):
"""
Test that calcSNR_m5 raises exceptions when it needs to
"""
totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles()
magnitudes = numpy.array([22.0, 23.0, 24.0, 25.0, 26.0, 27.0])
shortMagnitudes = numpy.array([22.0])
photParams = PhotometricParameters()
shortGamma = numpy.array([1.0, 1.0])
self.assertRaises(RuntimeError, calcSNR_m5, magnitudes,
totalDict.values(), shortMagnitudes, photParams)
self.assertRaises(RuntimeError, calcSNR_m5, shortMagnitudes,
totalDict.values(), magnitudes, photParams)
self.assertRaises(RuntimeError,
calcSNR_m5,
magnitudes,
totalDict.values(),
magnitudes,
photParams,
gamma=shortGamma)
snr, gg = calcSNR_m5(magnitudes, totalDict.values(), magnitudes,
photParams)
def test_alert_data_generation(self):
dmag_cutoff = 0.005
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5}
_max_var_param_str = self.max_str_len
class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject):
objid = 'star_alert'
tableid = 'stars'
idColKey = 'simobjid'
raColName = 'ra'
decColName = 'dec'
objectTypeId = 0
columns = [('raJ2000', 'ra*0.01745329252'),
('decJ2000', 'dec*0.01745329252'),
('parallax', 'px*0.01745329252/3600.0'),
('properMotionRa', 'pmra*0.01745329252/3600.0'),
('properMotionDec', 'pmdec*0.01745329252/3600.0'),
('radialVelocity', 'vrad'),
('variabilityParameters', 'varParamStr', str, _max_var_param_str)]
class TestAlertsVarCatMixin(object):
@register_method('alert_test')
def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None):
if len(params) == 0:
return np.array([[], [], [], [], [], []])
if isinstance(expmjd, numbers.Number):
dmags_out = np.zeros((6, self.num_variable_obj(params)))
else:
dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd)))
for i_star in range(self.num_variable_obj(params)):
if params['amp'][i_star] is not None:
dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd)
for i_filter in range(6):
dmags_out[i_filter][i_star] = dmags
return dmags_out
class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog):
pass
class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoords, AstrometryStars,
Variability, InstanceCatalog):
column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert']
camera = obs_lsst_phosim.PhosimMapper().camera
@compound('delta_umag', 'delta_gmag', 'delta_rmag',
'delta_imag', 'delta_zmag', 'delta_ymag')
def get_TruthVariability(self):
return self.applyVariability(self.column_by_name('varParamStr'))
@cached
def get_dmagAlert(self):
return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass)
@cached
def get_magAlert(self):
return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
self.column_by_name('dmagAlert')
star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite')
# assemble the true light curves for each object; we need to figure out
# if their np.max(dMag) ever goes over dmag_cutoff; then we will know if
# we are supposed to simulate them
true_lc_dict = {}
true_lc_obshistid_dict = {}
is_visible_dict = {}
obs_dict = {}
max_obshistid = -1
n_total_observations = 0
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid > max_obshistid:
max_obshistid = obshistid
cat = TestAlertsTruthCat(star_db, obs_metadata=obs)
for line in cat.iter_catalog():
if line[1] is None:
continue
n_total_observations += 1
if line[0] not in true_lc_dict:
true_lc_dict[line[0]] = {}
true_lc_obshistid_dict[line[0]] = []
true_lc_dict[line[0]][obshistid] = line[2]
true_lc_obshistid_dict[line[0]].append(obshistid)
if line[0] not in is_visible_dict:
is_visible_dict[line[0]] = False
if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]:
is_visible_dict[line[0]] = True
obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2)))
skipped_due_to_mag = 0
objects_to_simulate = []
obshistid_unqid_set = set()
for obj_id in true_lc_dict:
dmag_max = -1.0
for obshistid in true_lc_dict[obj_id]:
if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max:
dmag_max = np.abs(true_lc_dict[obj_id][obshistid])
if dmag_max >= dmag_cutoff:
if not is_visible_dict[obj_id]:
skipped_due_to_mag += 1
continue
objects_to_simulate.append(obj_id)
for obshistid in true_lc_obshistid_dict[obj_id]:
obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid)
self.assertGreater(len(objects_to_simulate), 10)
self.assertGreater(skipped_due_to_mag, 0)
log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat,
output_prefix='alert_test',
output_dir=self.output_dir,
dmag_cutoff=dmag_cutoff,
log_file_name=log_file_name)
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
phot_params = PhotometricParameters()
# First, verify that the contents of the sqlite files are all correct
n_tot_simulated = 0
alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, '
alert_query += 'meta.band, quiescent.flux, alert.dflux, '
alert_query += 'quiescent.snr, alert.snr, '
alert_query += 'alert.ra, alert.dec, alert.chipNum, '
alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, '
alert_query += 'ast.parallax '
alert_query += 'FROM alert_data AS alert '
alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId '
alert_query += 'INNER JOIN quiescent_flux AS quiescent '
alert_query += 'ON quiescent.uniqueId=alert.uniqueId '
alert_query += 'AND quiescent.band=meta.band '
alert_query += 'INNER JOIN baseline_astrometry AS ast '
alert_query += 'ON ast.uniqueId=alert.uniqueId'
alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int),
('TAI', float), ('band', int),
('q_flux', float), ('dflux', float),
('q_snr', float), ('tot_snr', float),
('ra', float), ('dec', float),
('chipNum', int), ('xPix', float), ('yPix', float),
('pmRA', float), ('pmDec', float), ('parallax', float)])
sqlite_file_list = os.listdir(self.output_dir)
n_tot_simulated = 0
obshistid_unqid_simulated_set = set()
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype)
if len(alert_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI'])
for i_obj in range(len(alert_data)):
n_tot_simulated += 1
obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) +
alert_data['obshistId'][i_obj])
unq = alert_data['uniqueId'][i_obj]
obj_dex = (unq//1024)-1
self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4)
self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4)
self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4)
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
alert_data['ra'][i_obj], alert_data['dec'][i_obj])
distance_arcsec = 3600.0*distance
msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth,
alert_data['ra'][i_obj],
alert_data['dec'][i_obj])
self.assertLess(distance_arcsec, 0.0005, msg=msg)
obs = obs_dict[alert_data['obshistId'][i_obj]]
chipname = chipNameFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
camera=self.camera)
chipnum = int(chipname.replace('R', '').replace('S', '').
replace(' ', '').replace(';', '').replace(',', '').
replace(':', ''))
self.assertEqual(chipnum, alert_data['chipNum'][i_obj])
xpix, ypix = pixelCoordsFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
camera=self.camera)
self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4)
self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4)
dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj])
self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]],
dmag_sim, 3)
mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]]
m5 = obs.m5[mag_name]
q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj])
self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
q_mag, 4)
snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
bp_dict[mag_name],
self.obs_mag_cutoff[alert_data['band'][i_obj]],
phot_params)
self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4)
tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \
true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]]
snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name],
m5, phot_params)
self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4)
for val in obshistid_unqid_set:
self.assertIn(val, obshistid_unqid_simulated_set)
self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set))
astrometry_query = 'SELECT uniqueId, ra, dec, TAI '
astrometry_query += 'FROM baseline_astrometry'
astrometry_dtype = np.dtype([('uniqueId', int),
('ra', float),
('dec', float),
('TAI', float)])
tai_list = []
for obs in self.obs_list:
tai_list.append(obs.mjd.TAI)
tai_list = np.array(tai_list)
n_tot_ast_simulated = 0
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype)
if len(astrometry_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI'])
for i_obj in range(len(astrometry_data)):
n_tot_ast_simulated += 1
obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
astrometry_data['ra'][i_obj],
astrometry_data['dec'][i_obj])
self.assertLess(3600.0*distance, 0.0005)
del alert_gen
gc.collect()
self.assertGreater(n_tot_simulated, 10)
self.assertGreater(len(obshistid_unqid_simulated_set), 10)
self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations)
self.assertGreater(n_tot_ast_simulated, 0)
def test_alert_data_generation(self):
dmag_cutoff = 0.005
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5}
_max_var_param_str = self.max_str_len
class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject):
objid = 'star_alert'
tableid = 'stars'
idColKey = 'simobjid'
raColName = 'ra'
decColName = 'dec'
objectTypeId = 0
columns = [('raJ2000', 'ra*0.01745329252'),
('decJ2000', 'dec*0.01745329252'),
('parallax', 'px*0.01745329252/3600.0'),
('properMotionRa', 'pmra*0.01745329252/3600.0'),
('properMotionDec', 'pmdec*0.01745329252/3600.0'),
('radialVelocity', 'vrad'),
('variabilityParameters', 'varParamStr', str, _max_var_param_str)]
class TestAlertsVarCatMixin(object):
@register_method('alert_test')
def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None):
if len(params) == 0:
return np.array([[], [], [], [], [], []])
if isinstance(expmjd, numbers.Number):
dmags_out = np.zeros((6, self.num_variable_obj(params)))
else:
dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd)))
for i_star in range(self.num_variable_obj(params)):
if params['amp'][i_star] is not None:
dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd)
for i_filter in range(6):
dmags_out[i_filter][i_star] = dmags
return dmags_out
class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog):
pass
class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoordsLSST, AstrometryStars,
Variability, InstanceCatalog):
column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert']
@compound('delta_umag', 'delta_gmag', 'delta_rmag',
'delta_imag', 'delta_zmag', 'delta_ymag')
def get_TruthVariability(self):
return self.applyVariability(self.column_by_name('varParamStr'))
@cached
def get_dmagAlert(self):
return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass)
@cached
def get_magAlert(self):
return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
self.column_by_name('dmagAlert')
star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite')
# assemble the true light curves for each object; we need to figure out
# if their np.max(dMag) ever goes over dmag_cutoff; then we will know if
# we are supposed to simulate them
true_lc_dict = {}
true_lc_obshistid_dict = {}
is_visible_dict = {}
obs_dict = {}
max_obshistid = -1
n_total_observations = 0
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid > max_obshistid:
max_obshistid = obshistid
cat = TestAlertsTruthCat(star_db, obs_metadata=obs)
for line in cat.iter_catalog():
if line[1] is None:
continue
n_total_observations += 1
if line[0] not in true_lc_dict:
true_lc_dict[line[0]] = {}
true_lc_obshistid_dict[line[0]] = []
true_lc_dict[line[0]][obshistid] = line[2]
true_lc_obshistid_dict[line[0]].append(obshistid)
if line[0] not in is_visible_dict:
is_visible_dict[line[0]] = False
if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]:
is_visible_dict[line[0]] = True
obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2)))
skipped_due_to_mag = 0
objects_to_simulate = []
obshistid_unqid_set = set()
for obj_id in true_lc_dict:
dmag_max = -1.0
for obshistid in true_lc_dict[obj_id]:
if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max:
dmag_max = np.abs(true_lc_dict[obj_id][obshistid])
if dmag_max >= dmag_cutoff:
if not is_visible_dict[obj_id]:
skipped_due_to_mag += 1
continue
objects_to_simulate.append(obj_id)
for obshistid in true_lc_obshistid_dict[obj_id]:
obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid)
self.assertGreater(len(objects_to_simulate), 10)
self.assertGreater(skipped_due_to_mag, 0)
log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat,
output_prefix='alert_test',
output_dir=self.output_dir,
dmag_cutoff=dmag_cutoff,
log_file_name=log_file_name)
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
phot_params = PhotometricParameters()
# First, verify that the contents of the sqlite files are all correct
n_tot_simulated = 0
alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, '
alert_query += 'meta.band, quiescent.flux, alert.dflux, '
alert_query += 'quiescent.snr, alert.snr, '
alert_query += 'alert.ra, alert.dec, alert.chipNum, '
alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, '
alert_query += 'ast.parallax '
alert_query += 'FROM alert_data AS alert '
alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId '
alert_query += 'INNER JOIN quiescent_flux AS quiescent '
alert_query += 'ON quiescent.uniqueId=alert.uniqueId '
alert_query += 'AND quiescent.band=meta.band '
alert_query += 'INNER JOIN baseline_astrometry AS ast '
alert_query += 'ON ast.uniqueId=alert.uniqueId'
alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int),
('TAI', float), ('band', int),
('q_flux', float), ('dflux', float),
('q_snr', float), ('tot_snr', float),
('ra', float), ('dec', float),
('chipNum', int), ('xPix', float), ('yPix', float),
('pmRA', float), ('pmDec', float), ('parallax', float)])
sqlite_file_list = os.listdir(self.output_dir)
n_tot_simulated = 0
obshistid_unqid_simulated_set = set()
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype)
if len(alert_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI'])
for i_obj in range(len(alert_data)):
n_tot_simulated += 1
obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) +
alert_data['obshistId'][i_obj])
unq = alert_data['uniqueId'][i_obj]
obj_dex = (unq//1024)-1
self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4)
self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4)
self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4)
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
alert_data['ra'][i_obj], alert_data['dec'][i_obj])
distance_arcsec = 3600.0*distance
msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth,
alert_data['ra'][i_obj],
alert_data['dec'][i_obj])
self.assertLess(distance_arcsec, 0.0005, msg=msg)
obs = obs_dict[alert_data['obshistId'][i_obj]]
chipname = chipNameFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
chipnum = int(chipname.replace('R', '').replace('S', '').
replace(' ', '').replace(';', '').replace(',', '').
replace(':', ''))
self.assertEqual(chipnum, alert_data['chipNum'][i_obj])
xpix, ypix = pixelCoordsFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4)
self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4)
dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj])
self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]],
dmag_sim, 3)
mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]]
m5 = obs.m5[mag_name]
q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj])
self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
q_mag, 4)
snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
bp_dict[mag_name],
self.obs_mag_cutoff[alert_data['band'][i_obj]],
phot_params)
self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4)
tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \
true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]]
snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name],
m5, phot_params)
self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4)
for val in obshistid_unqid_set:
self.assertIn(val, obshistid_unqid_simulated_set)
self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set))
astrometry_query = 'SELECT uniqueId, ra, dec, TAI '
astrometry_query += 'FROM baseline_astrometry'
astrometry_dtype = np.dtype([('uniqueId', int),
('ra', float),
('dec', float),
('TAI', float)])
tai_list = []
for obs in self.obs_list:
tai_list.append(obs.mjd.TAI)
tai_list = np.array(tai_list)
n_tot_ast_simulated = 0
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype)
if len(astrometry_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI'])
for i_obj in range(len(astrometry_data)):
n_tot_ast_simulated += 1
obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
astrometry_data['ra'][i_obj],
astrometry_data['dec'][i_obj])
self.assertLess(3600.0*distance, 0.0005)
del alert_gen
gc.collect()
self.assertGreater(n_tot_simulated, 10)
self.assertGreater(len(obshistid_unqid_simulated_set), 10)
self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations)
self.assertGreater(n_tot_ast_simulated, 0)
def test_avro_alert_generation_diff_dmag(self):
"""
Make sure everything works properly when the AlertDataGenerator
and the AvroAlertGenerator have different dmag thresholds
"""
dmag_cutoff_sqlite = 0.005
dmag_cutoff_avro = 0.2
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
star_db = StarAlertTestDBObj_avro(database=self.star_db_name,
driver='sqlite')
# assemble a dict of all of the alerts that need to be generated
obshistid_list = []
for obs in self.obs_list:
obshistid_list.append(obs.OpsimMetaData['obsHistID'])
obshistid_max = max(obshistid_list)
obshistid_bits = int(np.ceil(np.log(obshistid_max) / np.log(2.0)))
true_alert_dict = {}
obs_dict = {}
ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs)
cat.camera = obs_lsst_phosim.PhosimMapper().camera
for line in cat.iter_catalog():
if line[1] is None:
continue
dmag = line[2]
mag = line[3]
if (np.abs(dmag) > dmag_cutoff_avro and mag <=
self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]):
alertId = (line[0] << obshistid_bits) + obshistid
self.assertNotIn(alertId, true_alert_dict)
true_alert_dict[alertId] = {}
true_alert_dict[alertId]['chipName'] = line[1]
true_alert_dict[alertId]['dmag'] = dmag
true_alert_dict[alertId]['mag'] = mag
true_alert_dict[alertId]['ra'] = np.degrees(line[4])
true_alert_dict[alertId]['decl'] = np.degrees(line[5])
true_alert_dict[alertId]['xPix'] = line[6]
true_alert_dict[alertId]['yPix'] = line[7]
elif np.abs(dmag) > dmag_cutoff_sqlite:
ignored_sqlite += 1
self.assertGreater(len(true_alert_dict), 10)
self.assertGreater(ignored_sqlite,
50) # just make sure that some sqlite
# alerts were ignored by the more
# stringent avro cut
log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir,
suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(
htmid,
star_db,
photometry_class=TestAlertsVarCat_avro,
output_prefix='alert_test',
output_dir=self.alert_data_output_dir,
dmag_cutoff=dmag_cutoff_sqlite,
log_file_name=log_file_name)
obshistid_to_htmid = {}
for htmid in alert_gen.htmid_list:
for obs in alert_gen.obs_from_htmid(htmid):
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid not in obshistid_to_htmid:
obshistid_to_htmid[obshistid] = []
obshistid_to_htmid[obshistid].append(htmid)
avro_gen = AvroAlertGenerator()
avro_gen.load_schema(
os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData',
'avroSchema'))
sql_prefix_list = ['alert_test']
out_prefix = 'test_avro'
log_file_name = tempfile.mktemp(dir=self.avro_out_dir,
prefix='test_avro',
suffix='log.txt')
for obshistid in obshistid_list:
avro_gen.write_alerts(obshistid,
self.alert_data_output_dir,
sql_prefix_list,
obshistid_to_htmid[obshistid],
self.avro_out_dir,
out_prefix,
dmag_cutoff_avro,
lock=None,
log_file_name=log_file_name)
list_of_avro_files = os.listdir(self.avro_out_dir)
self.assertGreater(len(list_of_avro_files), 2)
alert_ct = 0
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
photParams = PhotometricParameters()
diasourceId_set = set()
for avro_file_name in list_of_avro_files:
if avro_file_name.endswith('log.txt'):
continue
full_name = os.path.join(self.avro_out_dir, avro_file_name)
with DataFileReader(open(full_name, 'rb'),
DatumReader()) as data_reader:
for alert in data_reader:
alert_ct += 1
obshistid = alert['alertId'] >> 20
obs = obs_dict[obshistid]
uniqueId = alert['diaObject']['diaObjectId']
true_alert_id = (uniqueId << obshistid_bits) + obshistid
self.assertIn(true_alert_id, true_alert_dict)
self.assertEqual(alert['l1dbId'], uniqueId)
true_alert = true_alert_dict[true_alert_id]
diaSource = alert['diaSource']
self.assertAlmostEqual(diaSource['ra'], true_alert['ra'],
10)
self.assertAlmostEqual(diaSource['decl'],
true_alert['decl'], 10)
self.assertAlmostEqual(diaSource['x'], true_alert['xPix'],
3)
self.assertAlmostEqual(diaSource['y'], true_alert['yPix'],
3)
self.assertAlmostEqual(diaSource['midPointTai'],
obs.mjd.TAI, 4)
true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag'])
true_q_mag = true_alert['mag'] - true_alert['dmag']
true_q_flux = dummy_sed.fluxFromMag(true_q_mag)
true_dflux = true_tot_flux - true_q_flux
self.assertAlmostEqual(diaSource['psFlux'] / true_dflux,
1.0, 6)
self.assertAlmostEqual(
diaSource['totFlux'] / true_tot_flux, 1.0, 6)
self.assertAlmostEqual(diaSource['diffFlux'] / true_dflux,
1.0, 6)
true_tot_snr, gamma = calcSNR_m5(true_alert['mag'],
bp_dict[obs.bandpass],
obs.m5[obs.bandpass],
photParams)
true_q_snr, gamma = calcSNR_m5(
true_q_mag, bp_dict[obs.bandpass],
self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]],
photParams)
true_tot_err = true_tot_flux / true_tot_snr
true_q_err = true_q_flux / true_q_snr
true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2)
self.assertAlmostEqual(
diaSource['snr'] / np.abs(true_dflux / true_diff_err),
1.0, 6)
self.assertAlmostEqual(
diaSource['totFluxErr'] / true_tot_err, 1.0, 6)
self.assertAlmostEqual(
diaSource['diffFluxErr'] / true_diff_err, 1.0, 6)
chipnum = int(true_alert['chipName'].replace(
'R', '').replace('S', '').replace(',', '').replace(
':', '').replace(' ', ''))
true_ccdid = (chipnum * 10**7) + obshistid
self.assertEqual(true_ccdid, diaSource['ccdVisitId'])
self.assertEqual(uniqueId, diaSource['diaObjectId'])
self.assertNotIn(diaSource['diaSourceId'], diasourceId_set)
diasourceId_set.add(diaSource['diaSourceId'])
diaObject = alert['diaObject']
obj_dex = (uniqueId // 1024) - 1
self.assertAlmostEqual(
0.001 * diaObject['pmRa'] / self.pmra_truth[obj_dex],
1.0, 5)
self.assertAlmostEqual(
0.001 * diaObject['pmDecl'] /
self.pmdec_truth[obj_dex], 1.0, 5)
self.assertAlmostEqual(
0.001 * diaObject['parallax'] / self.px_truth[obj_dex],
1.0, 5)
(true_ra_base, true_dec_base) = applyProperMotion(
self.ra_truth[obj_dex],
self.dec_truth[obj_dex],
self.pmra_truth[obj_dex],
self.pmdec_truth[obj_dex],
self.px_truth[obj_dex],
self.vrad_truth[obj_dex],
mjd=ModifiedJulianDate(TAI=diaObject['radecTai']))
self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7)
self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7)
self.assertEqual(alert_ct, len(true_alert_dict))
def _light_curves_from_query(self, cat_dict, query_result, grp, lc_per_field=None):
t_dict = {}
gamma_dict = {}
m5_dict = {}
t_min = None
t_max = None
for bp_name in cat_dict:
self.lsstBandpassDict[bp_name].sbTophi()
# generate a 2-D numpy array containing MJDs, m5, and photometric gamma values
# for each observation in the given bandpass
raw_array = np.array([[obs.mjd.TAI, obs.m5[bp_name],
calcGamma(self.lsstBandpassDict[bp_name],
obs.m5[obs.bandpass],
self.phot_params)]
for obs in grp if obs.bandpass == bp_name]).transpose()
if len(raw_array) > 0:
t_dict[bp_name] = raw_array[0]
m5_dict[bp_name] = raw_array[1]
gamma_dict[bp_name] = raw_array[2]
local_t_min = t_dict[bp_name].min()
local_t_max = t_dict[bp_name].max()
if t_min is None or local_t_min < t_min:
t_min = local_t_min
if t_max is None or local_t_max > t_max:
t_max = local_t_max
snobj = SNObject()
cat = cat_dict[list(cat_dict.keys())[0]] # does not need to be associated with a bandpass
dummy_sed = Sed()
n_actual_sn = 0 # how many SN have we actually delivered?
for chunk in query_result:
if lc_per_field is not None and n_actual_sn >= lc_per_field:
break
t_start_chunk = time.time()
for sn in cat.iter_catalog(query_cache=[chunk]):
sn_rng = self.sn_universe.getSN_rng(sn[1])
sn_t0 = self.sn_universe.drawFromT0Dist(sn_rng)
if sn[5] <= self.z_cutoff and np.isfinite(sn_t0) and \
sn_t0 < t_max + cat.maxTimeSNVisible and \
sn_t0 > t_min - cat.maxTimeSNVisible:
sn_c = self.sn_universe.drawFromcDist(sn_rng)
sn_x1 = self.sn_universe.drawFromx1Dist(sn_rng)
sn_x0 = self.sn_universe.drawFromX0Dist(sn_rng, sn_x1, sn_c, sn[4])
snobj.set(t0=sn_t0, c=sn_c, x1=sn_x1, x0=sn_x0, z=sn[5])
for bp_name in t_dict:
t_list = t_dict[bp_name]
m5_list = m5_dict[bp_name]
gamma_list = gamma_dict[bp_name]
bandpass = self.lsstBandpassDict[bp_name]
if len(t_list) == 0:
continue
if snobj.maxtime() >= t_list[0] and snobj.mintime() <= t_list[-1]:
active_dexes = np.where(np.logical_and(t_list >= snobj.mintime(),
t_list <= snobj.maxtime()))
t_active = t_list[active_dexes]
m5_active = m5_list[active_dexes]
gamma_active = gamma_list[active_dexes]
if len(t_active) > 0:
wave_ang = bandpass.wavelen*10.0
mask = np.logical_and(wave_ang > snobj.minwave(),
wave_ang < snobj.maxwave())
wave_ang = wave_ang[mask]
snobj.set(mwebv=sn[6])
sn_ff_buffer = snobj.flux(time=t_active, wave=wave_ang)*10.0
flambda_grid = np.zeros((len(t_active), len(bandpass.wavelen)))
for ff, ff_sn in zip(flambda_grid, sn_ff_buffer):
ff[mask] = np.where(ff_sn > 0.0, ff_sn, 0.0)
fnu_grid = flambda_grid*bandpass.wavelen* \
bandpass.wavelen*dummy_sed._physParams.nm2m* \
dummy_sed._physParams.ergsetc2jansky/dummy_sed._physParams.lightspeed
flux_list = \
(fnu_grid*bandpass.phi).sum(axis=1)*(bandpass.wavelen[1]-bandpass.wavelen[0])
acceptable = np.where(flux_list>0.0)
flux_error_list = flux_list[acceptable]/ \
calcSNR_m5(dummy_sed.magFromFlux(flux_list[acceptable]),
bandpass,
m5_active[acceptable], self.phot_params,
gamma=gamma_active[acceptable])
if len(acceptable) > 0:
n_actual_sn += 1
if lc_per_field is not None and n_actual_sn > lc_per_field:
break
if sn[0] not in self.truth_dict:
self.truth_dict[sn[0]] = {}
self.truth_dict[sn[0]]['t0'] = sn_t0
self.truth_dict[sn[0]]['x1'] = sn_x1
self.truth_dict[sn[0]]['x0'] = sn_x0
self.truth_dict[sn[0]]['c'] = sn_c
self.truth_dict[sn[0]]['z'] = sn[5]
self.truth_dict[sn[0]]['E(B-V)'] = sn[6]
if sn[0] not in self.mjd_dict:
self.mjd_dict[sn[0]] = {}
self.bright_dict[sn[0]] = {}
self.sig_dict[sn[0]] = {}
if bp_name not in self.mjd_dict[sn[0]]:
self.mjd_dict[sn[0]][bp_name] = []
self.bright_dict[sn[0]][bp_name] = []
self.sig_dict[sn[0]][bp_name] = []
for tt, ff, ee in zip(t_active[acceptable], flux_list[acceptable],
flux_error_list[0]):
self.mjd_dict[sn[0]][bp_name].append(tt)
self.bright_dict[sn[0]][bp_name].append(ff/3631.0)
self.sig_dict[sn[0]][bp_name].append(ee/3631.0)
print("chunk of ", len(chunk), " took ", time.time()-t_start_chunk)
def test_avro_alert_generation_diff_dmag(self):
"""
Make sure everything works properly when the AlertDataGenerator
and the AvroAlertGenerator have different dmag thresholds
"""
dmag_cutoff_sqlite = 0.005
dmag_cutoff_avro = 0.2
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite')
# assemble a dict of all of the alerts that need to be generated
obshistid_list = []
for obs in self.obs_list:
obshistid_list.append(obs.OpsimMetaData['obsHistID'])
obshistid_max = max(obshistid_list)
obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0)))
true_alert_dict = {}
obs_dict = {}
ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs)
cat.camera = lsst_camera()
for line in cat.iter_catalog():
if line[1] is None:
continue
dmag = line[2]
mag = line[3]
if (np.abs(dmag) > dmag_cutoff_avro and
mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]):
alertId = (line[0] << obshistid_bits) + obshistid
self.assertNotIn(alertId, true_alert_dict)
true_alert_dict[alertId] = {}
true_alert_dict[alertId]['chipName'] = line[1]
true_alert_dict[alertId]['dmag'] = dmag
true_alert_dict[alertId]['mag'] = mag
true_alert_dict[alertId]['ra'] = np.degrees(line[4])
true_alert_dict[alertId]['decl'] = np.degrees(line[5])
true_alert_dict[alertId]['xPix'] = line[6]
true_alert_dict[alertId]['yPix'] = line[7]
elif np.abs(dmag) > dmag_cutoff_sqlite:
ignored_sqlite += 1
self.assertGreater(len(true_alert_dict), 10)
self.assertGreater(ignored_sqlite, 50) # just make sure that some sqlite
# alerts were ignored by the more
# stringent avro cut
log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat_avro,
output_prefix='alert_test',
output_dir=self.alert_data_output_dir,
dmag_cutoff=dmag_cutoff_sqlite,
log_file_name=log_file_name)
obshistid_to_htmid = {}
for htmid in alert_gen.htmid_list:
for obs in alert_gen.obs_from_htmid(htmid):
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid not in obshistid_to_htmid:
obshistid_to_htmid[obshistid] = []
obshistid_to_htmid[obshistid].append(htmid)
avro_gen = AvroAlertGenerator()
avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema'))
sql_prefix_list = ['alert_test']
out_prefix = 'test_avro'
log_file_name = tempfile.mktemp(dir=self.avro_out_dir,
prefix='test_avro',
suffix='log.txt')
for obshistid in obshistid_list:
avro_gen.write_alerts(obshistid, self.alert_data_output_dir,
sql_prefix_list,
obshistid_to_htmid[obshistid],
self.avro_out_dir, out_prefix,
dmag_cutoff_avro, lock=None,
log_file_name=log_file_name)
list_of_avro_files = os.listdir(self.avro_out_dir)
self.assertGreater(len(list_of_avro_files), 2)
alert_ct = 0
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
photParams = PhotometricParameters()
diasourceId_set = set()
for avro_file_name in list_of_avro_files:
if avro_file_name.endswith('log.txt'):
continue
full_name = os.path.join(self.avro_out_dir, avro_file_name)
with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader:
for alert in data_reader:
alert_ct += 1
obshistid = alert['alertId'] >> 20
obs = obs_dict[obshistid]
uniqueId = alert['diaObject']['diaObjectId']
true_alert_id = (uniqueId << obshistid_bits) + obshistid
self.assertIn(true_alert_id, true_alert_dict)
self.assertEqual(alert['l1dbId'], uniqueId)
true_alert = true_alert_dict[true_alert_id]
diaSource = alert['diaSource']
self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10)
self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10)
self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3)
self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3)
self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4)
true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag'])
true_q_mag = true_alert['mag'] - true_alert['dmag']
true_q_flux = dummy_sed.fluxFromMag(true_q_mag)
true_dflux = true_tot_flux - true_q_flux
self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6)
self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6)
self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6)
true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass],
obs.m5[obs.bandpass], photParams)
true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass],
self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]],
photParams)
true_tot_err = true_tot_flux/true_tot_snr
true_q_err = true_q_flux/true_q_snr
true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2)
self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err),
1.0, 6)
self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6)
self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6)
chipnum = int(true_alert['chipName'].replace('R', '').replace('S', '').
replace(',', '').replace(':', '').replace(' ', ''))
true_ccdid = (chipnum*10**7)+obshistid
self.assertEqual(true_ccdid, diaSource['ccdVisitId'])
self.assertEqual(uniqueId, diaSource['diaObjectId'])
self.assertNotIn(diaSource['diaSourceId'], diasourceId_set)
diasourceId_set.add(diaSource['diaSourceId'])
diaObject = alert['diaObject']
obj_dex = (uniqueId//1024) - 1
self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5)
self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5)
self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5)
(true_ra_base,
true_dec_base) = applyProperMotion(self.ra_truth[obj_dex],
self.dec_truth[obj_dex],
self.pmra_truth[obj_dex],
self.pmdec_truth[obj_dex],
self.px_truth[obj_dex],
self.vrad_truth[obj_dex],
mjd=ModifiedJulianDate(TAI=diaObject['radecTai']))
self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7)
self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7)
self.assertEqual(alert_ct, len(true_alert_dict))
