def test_alert_data_generation(self):
dmag_cutoff = 0.005
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5}
_max_var_param_str = self.max_str_len
class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject):
objid = 'star_alert'
tableid = 'stars'
idColKey = 'simobjid'
raColName = 'ra'
decColName = 'dec'
objectTypeId = 0
columns = [('raJ2000', 'ra*0.01745329252'),
('decJ2000', 'dec*0.01745329252'),
('parallax', 'px*0.01745329252/3600.0'),
('properMotionRa', 'pmra*0.01745329252/3600.0'),
('properMotionDec', 'pmdec*0.01745329252/3600.0'),
('radialVelocity', 'vrad'),
('variabilityParameters', 'varParamStr', str, _max_var_param_str)]
class TestAlertsVarCatMixin(object):
@register_method('alert_test')
def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None):
if len(params) == 0:
return np.array([[], [], [], [], [], []])
if isinstance(expmjd, numbers.Number):
dmags_out = np.zeros((6, self.num_variable_obj(params)))
else:
dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd)))
for i_star in range(self.num_variable_obj(params)):
if params['amp'][i_star] is not None:
dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd)
for i_filter in range(6):
dmags_out[i_filter][i_star] = dmags
return dmags_out
class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog):
pass
class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoords, AstrometryStars,
Variability, InstanceCatalog):
column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert']
camera = obs_lsst_phosim.PhosimMapper().camera
@compound('delta_umag', 'delta_gmag', 'delta_rmag',
'delta_imag', 'delta_zmag', 'delta_ymag')
def get_TruthVariability(self):
return self.applyVariability(self.column_by_name('varParamStr'))
@cached
def get_dmagAlert(self):
return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass)
@cached
def get_magAlert(self):
return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
self.column_by_name('dmagAlert')
star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite')
# assemble the true light curves for each object; we need to figure out
# if their np.max(dMag) ever goes over dmag_cutoff; then we will know if
# we are supposed to simulate them
true_lc_dict = {}
true_lc_obshistid_dict = {}
is_visible_dict = {}
obs_dict = {}
max_obshistid = -1
n_total_observations = 0
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid > max_obshistid:
max_obshistid = obshistid
cat = TestAlertsTruthCat(star_db, obs_metadata=obs)
for line in cat.iter_catalog():
if line[1] is None:
continue
n_total_observations += 1
if line[0] not in true_lc_dict:
true_lc_dict[line[0]] = {}
true_lc_obshistid_dict[line[0]] = []
true_lc_dict[line[0]][obshistid] = line[2]
true_lc_obshistid_dict[line[0]].append(obshistid)
if line[0] not in is_visible_dict:
is_visible_dict[line[0]] = False
if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]:
is_visible_dict[line[0]] = True
obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2)))
skipped_due_to_mag = 0
objects_to_simulate = []
obshistid_unqid_set = set()
for obj_id in true_lc_dict:
dmag_max = -1.0
for obshistid in true_lc_dict[obj_id]:
if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max:
dmag_max = np.abs(true_lc_dict[obj_id][obshistid])
if dmag_max >= dmag_cutoff:
if not is_visible_dict[obj_id]:
skipped_due_to_mag += 1
continue
objects_to_simulate.append(obj_id)
for obshistid in true_lc_obshistid_dict[obj_id]:
obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid)
self.assertGreater(len(objects_to_simulate), 10)
self.assertGreater(skipped_due_to_mag, 0)
log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat,
output_prefix='alert_test',
output_dir=self.output_dir,
dmag_cutoff=dmag_cutoff,
log_file_name=log_file_name)
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
phot_params = PhotometricParameters()
# First, verify that the contents of the sqlite files are all correct
n_tot_simulated = 0
alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, '
alert_query += 'meta.band, quiescent.flux, alert.dflux, '
alert_query += 'quiescent.snr, alert.snr, '
alert_query += 'alert.ra, alert.dec, alert.chipNum, '
alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, '
alert_query += 'ast.parallax '
alert_query += 'FROM alert_data AS alert '
alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId '
alert_query += 'INNER JOIN quiescent_flux AS quiescent '
alert_query += 'ON quiescent.uniqueId=alert.uniqueId '
alert_query += 'AND quiescent.band=meta.band '
alert_query += 'INNER JOIN baseline_astrometry AS ast '
alert_query += 'ON ast.uniqueId=alert.uniqueId'
alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int),
('TAI', float), ('band', int),
('q_flux', float), ('dflux', float),
('q_snr', float), ('tot_snr', float),
('ra', float), ('dec', float),
('chipNum', int), ('xPix', float), ('yPix', float),
('pmRA', float), ('pmDec', float), ('parallax', float)])
sqlite_file_list = os.listdir(self.output_dir)
n_tot_simulated = 0
obshistid_unqid_simulated_set = set()
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype)
if len(alert_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI'])
for i_obj in range(len(alert_data)):
n_tot_simulated += 1
obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) +
alert_data['obshistId'][i_obj])
unq = alert_data['uniqueId'][i_obj]
obj_dex = (unq//1024)-1
self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4)
self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4)
self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4)
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
alert_data['ra'][i_obj], alert_data['dec'][i_obj])
distance_arcsec = 3600.0*distance
msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth,
alert_data['ra'][i_obj],
alert_data['dec'][i_obj])
self.assertLess(distance_arcsec, 0.0005, msg=msg)
obs = obs_dict[alert_data['obshistId'][i_obj]]
chipname = chipNameFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
camera=self.camera)
chipnum = int(chipname.replace('R', '').replace('S', '').
replace(' ', '').replace(';', '').replace(',', '').
replace(':', ''))
self.assertEqual(chipnum, alert_data['chipNum'][i_obj])
xpix, ypix = pixelCoordsFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
camera=self.camera)
self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4)
self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4)
dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj])
self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]],
dmag_sim, 3)
mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]]
m5 = obs.m5[mag_name]
q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj])
self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
q_mag, 4)
snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
bp_dict[mag_name],
self.obs_mag_cutoff[alert_data['band'][i_obj]],
phot_params)
self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4)
tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \
true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]]
snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name],
m5, phot_params)
self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4)
for val in obshistid_unqid_set:
self.assertIn(val, obshistid_unqid_simulated_set)
self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set))
astrometry_query = 'SELECT uniqueId, ra, dec, TAI '
astrometry_query += 'FROM baseline_astrometry'
astrometry_dtype = np.dtype([('uniqueId', int),
('ra', float),
('dec', float),
('TAI', float)])
tai_list = []
for obs in self.obs_list:
tai_list.append(obs.mjd.TAI)
tai_list = np.array(tai_list)
n_tot_ast_simulated = 0
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype)
if len(astrometry_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI'])
for i_obj in range(len(astrometry_data)):
n_tot_ast_simulated += 1
obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
astrometry_data['ra'][i_obj],
astrometry_data['dec'][i_obj])
self.assertLess(3600.0*distance, 0.0005)
del alert_gen
gc.collect()
self.assertGreater(n_tot_simulated, 10)
self.assertGreater(len(obshistid_unqid_simulated_set), 10)
self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations)
self.assertGreater(n_tot_ast_simulated, 0)
if os.path.exists(args.log_file):
raise RuntimeError('%s already exists' % args.log_file)
if not os.path.exists(args.out_dir):
os.mkdir(args.out_dir)
# get the list of ObservationMetaData to simulate
obs_gen = ObservationMetaDataGenerator(args.opsim_db, driver='sqlite')
obs_list = obs_gen.getObservationMetaData(night=(args.night0, args.night1))
del obs_gen
sims_clean_up()
gc.collect()
# get the list of trixel htmids to simulate
alert_gen = AlertDataGenerator()
alert_gen.subdivide_obs(obs_list, htmid_level=6)
n_tot_obs = 0
for htmid in alert_gen.htmid_list:
n_tot_obs += alert_gen.n_obs(htmid)
with open(args.log_file, 'a') as out_file:
for htmid in alert_gen.htmid_list:
out_file.write('htmid %d n_obs %d\n' %
(htmid, alert_gen.n_obs(htmid)))
out_file.write('n_htmid %d n_obs(total) %d\n' %
(len(alert_gen.htmid_list), n_tot_obs))
# subdivide the trixels into n_proc lists, trying to make sure
# each process has an equal number of observations to simulate
def test_alert_data_generation(self):
dmag_cutoff = 0.005
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5}
_max_var_param_str = self.max_str_len
class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject):
objid = 'star_alert'
tableid = 'stars'
idColKey = 'simobjid'
raColName = 'ra'
decColName = 'dec'
objectTypeId = 0
columns = [('raJ2000', 'ra*0.01745329252'),
('decJ2000', 'dec*0.01745329252'),
('parallax', 'px*0.01745329252/3600.0'),
('properMotionRa', 'pmra*0.01745329252/3600.0'),
('properMotionDec', 'pmdec*0.01745329252/3600.0'),
('radialVelocity', 'vrad'),
('variabilityParameters', 'varParamStr', str, _max_var_param_str)]
class TestAlertsVarCatMixin(object):
@register_method('alert_test')
def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None):
if len(params) == 0:
return np.array([[], [], [], [], [], []])
if isinstance(expmjd, numbers.Number):
dmags_out = np.zeros((6, self.num_variable_obj(params)))
else:
dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd)))
for i_star in range(self.num_variable_obj(params)):
if params['amp'][i_star] is not None:
dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd)
for i_filter in range(6):
dmags_out[i_filter][i_star] = dmags
return dmags_out
class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog):
pass
class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoordsLSST, AstrometryStars,
Variability, InstanceCatalog):
column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert']
@compound('delta_umag', 'delta_gmag', 'delta_rmag',
'delta_imag', 'delta_zmag', 'delta_ymag')
def get_TruthVariability(self):
return self.applyVariability(self.column_by_name('varParamStr'))
@cached
def get_dmagAlert(self):
return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass)
@cached
def get_magAlert(self):
return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
self.column_by_name('dmagAlert')
star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite')
# assemble the true light curves for each object; we need to figure out
# if their np.max(dMag) ever goes over dmag_cutoff; then we will know if
# we are supposed to simulate them
true_lc_dict = {}
true_lc_obshistid_dict = {}
is_visible_dict = {}
obs_dict = {}
max_obshistid = -1
n_total_observations = 0
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid > max_obshistid:
max_obshistid = obshistid
cat = TestAlertsTruthCat(star_db, obs_metadata=obs)
for line in cat.iter_catalog():
if line[1] is None:
continue
n_total_observations += 1
if line[0] not in true_lc_dict:
true_lc_dict[line[0]] = {}
true_lc_obshistid_dict[line[0]] = []
true_lc_dict[line[0]][obshistid] = line[2]
true_lc_obshistid_dict[line[0]].append(obshistid)
if line[0] not in is_visible_dict:
is_visible_dict[line[0]] = False
if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]:
is_visible_dict[line[0]] = True
obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2)))
skipped_due_to_mag = 0
objects_to_simulate = []
obshistid_unqid_set = set()
for obj_id in true_lc_dict:
dmag_max = -1.0
for obshistid in true_lc_dict[obj_id]:
if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max:
dmag_max = np.abs(true_lc_dict[obj_id][obshistid])
if dmag_max >= dmag_cutoff:
if not is_visible_dict[obj_id]:
skipped_due_to_mag += 1
continue
objects_to_simulate.append(obj_id)
for obshistid in true_lc_obshistid_dict[obj_id]:
obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid)
self.assertGreater(len(objects_to_simulate), 10)
self.assertGreater(skipped_due_to_mag, 0)
log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat,
output_prefix='alert_test',
output_dir=self.output_dir,
dmag_cutoff=dmag_cutoff,
log_file_name=log_file_name)
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
phot_params = PhotometricParameters()
# First, verify that the contents of the sqlite files are all correct
n_tot_simulated = 0
alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, '
alert_query += 'meta.band, quiescent.flux, alert.dflux, '
alert_query += 'quiescent.snr, alert.snr, '
alert_query += 'alert.ra, alert.dec, alert.chipNum, '
alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, '
alert_query += 'ast.parallax '
alert_query += 'FROM alert_data AS alert '
alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId '
alert_query += 'INNER JOIN quiescent_flux AS quiescent '
alert_query += 'ON quiescent.uniqueId=alert.uniqueId '
alert_query += 'AND quiescent.band=meta.band '
alert_query += 'INNER JOIN baseline_astrometry AS ast '
alert_query += 'ON ast.uniqueId=alert.uniqueId'
alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int),
('TAI', float), ('band', int),
('q_flux', float), ('dflux', float),
('q_snr', float), ('tot_snr', float),
('ra', float), ('dec', float),
('chipNum', int), ('xPix', float), ('yPix', float),
('pmRA', float), ('pmDec', float), ('parallax', float)])
sqlite_file_list = os.listdir(self.output_dir)
n_tot_simulated = 0
obshistid_unqid_simulated_set = set()
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype)
if len(alert_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI'])
for i_obj in range(len(alert_data)):
n_tot_simulated += 1
obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) +
alert_data['obshistId'][i_obj])
unq = alert_data['uniqueId'][i_obj]
obj_dex = (unq//1024)-1
self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4)
self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4)
self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4)
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
alert_data['ra'][i_obj], alert_data['dec'][i_obj])
distance_arcsec = 3600.0*distance
msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth,
alert_data['ra'][i_obj],
alert_data['dec'][i_obj])
self.assertLess(distance_arcsec, 0.0005, msg=msg)
obs = obs_dict[alert_data['obshistId'][i_obj]]
chipname = chipNameFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
chipnum = int(chipname.replace('R', '').replace('S', '').
replace(' ', '').replace(';', '').replace(',', '').
replace(':', ''))
self.assertEqual(chipnum, alert_data['chipNum'][i_obj])
xpix, ypix = pixelCoordsFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4)
self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4)
dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj])
self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]],
dmag_sim, 3)
mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]]
m5 = obs.m5[mag_name]
q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj])
self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
q_mag, 4)
snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
bp_dict[mag_name],
self.obs_mag_cutoff[alert_data['band'][i_obj]],
phot_params)
self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4)
tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \
true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]]
snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name],
m5, phot_params)
self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4)
for val in obshistid_unqid_set:
self.assertIn(val, obshistid_unqid_simulated_set)
self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set))
astrometry_query = 'SELECT uniqueId, ra, dec, TAI '
astrometry_query += 'FROM baseline_astrometry'
astrometry_dtype = np.dtype([('uniqueId', int),
('ra', float),
('dec', float),
('TAI', float)])
tai_list = []
for obs in self.obs_list:
tai_list.append(obs.mjd.TAI)
tai_list = np.array(tai_list)
n_tot_ast_simulated = 0
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype)
if len(astrometry_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI'])
for i_obj in range(len(astrometry_data)):
n_tot_ast_simulated += 1
obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
astrometry_data['ra'][i_obj],
astrometry_data['dec'][i_obj])
self.assertLess(3600.0*distance, 0.0005)
del alert_gen
gc.collect()
self.assertGreater(n_tot_simulated, 10)
self.assertGreater(len(obshistid_unqid_simulated_set), 10)
self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations)
self.assertGreater(n_tot_ast_simulated, 0)
if os.path.exists(args.log_file):
raise RuntimeError('%s already exists' % args.log_file)
if not os.path.exists(args.out_dir):
os.mkdir(args.out_dir)
# get the list of ObservationMetaData to simulate
obs_gen = ObservationMetaDataGenerator(args.opsim_db, driver='sqlite')
obs_list = obs_gen.getObservationMetaData(night=(args.night0,args.night1))
del obs_gen
sims_clean_up()
gc.collect()
# get the list of trixel htmids to simulate
alert_gen = AlertDataGenerator()
alert_gen.subdivide_obs(obs_list, htmid_level=6)
n_tot_obs=0
for htmid in alert_gen.htmid_list:
n_tot_obs += alert_gen.n_obs(htmid)
with open(args.log_file, 'a') as out_file:
for htmid in alert_gen.htmid_list:
out_file.write('htmid %d n_obs %d\n' % (htmid, alert_gen.n_obs(htmid)))
out_file.write('n_htmid %d n_obs(total) %d\n' % (len(alert_gen.htmid_list), n_tot_obs))
# subdivide the trixels into n_proc lists, trying to make sure
# each process has an equal number of observations to simulate
htmid_list = []
n_htmid_list = []
if not os.path.exists(args.out_dir):
os.mkdir(args.out_dir)
t_start = time.time()
# get list of ObservationMetaData to simulate
obs_gen = ObservationMetaDataGenerator(args.opsim_db, driver='sqlite')
obs_list = obs_gen.getObservationMetaData(night=15)
del obs_gen
sims_clean_up()
print('%d obs' % len(obs_list))
# use AlertDataGenerator to separate the ObservationMetaData
# by htmid
alert_gen = AlertDataGenerator()
alert_gen.subdivide_obs(obs_list, htmid_level=6)
# Create a dict that maps an obsHistID to the list of htmids
# of the trixels overlapping that observation
#
# Also separate list of obsHistIDs into n_proc
# lists of roughly balanced loads.
obshistid_to_htmid = {}
obshistid_list = []
for i_proc in range(args.n_proc):
obshistid_list.append([])
i_proc = 0
for htmid in alert_gen.htmid_list:
for obs in alert_gen.obs_from_htmid(htmid):
def test_avro_alert_generation_diff_dmag(self):
"""
Make sure everything works properly when the AlertDataGenerator
and the AvroAlertGenerator have different dmag thresholds
"""
dmag_cutoff_sqlite = 0.005
dmag_cutoff_avro = 0.2
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
star_db = StarAlertTestDBObj_avro(database=self.star_db_name,
driver='sqlite')
# assemble a dict of all of the alerts that need to be generated
obshistid_list = []
for obs in self.obs_list:
obshistid_list.append(obs.OpsimMetaData['obsHistID'])
obshistid_max = max(obshistid_list)
obshistid_bits = int(np.ceil(np.log(obshistid_max) / np.log(2.0)))
true_alert_dict = {}
obs_dict = {}
ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs)
cat.camera = obs_lsst_phosim.PhosimMapper().camera
for line in cat.iter_catalog():
if line[1] is None:
continue
dmag = line[2]
mag = line[3]
if (np.abs(dmag) > dmag_cutoff_avro and mag <=
self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]):
alertId = (line[0] << obshistid_bits) + obshistid
self.assertNotIn(alertId, true_alert_dict)
true_alert_dict[alertId] = {}
true_alert_dict[alertId]['chipName'] = line[1]
true_alert_dict[alertId]['dmag'] = dmag
true_alert_dict[alertId]['mag'] = mag
true_alert_dict[alertId]['ra'] = np.degrees(line[4])
true_alert_dict[alertId]['decl'] = np.degrees(line[5])
true_alert_dict[alertId]['xPix'] = line[6]
true_alert_dict[alertId]['yPix'] = line[7]
elif np.abs(dmag) > dmag_cutoff_sqlite:
ignored_sqlite += 1
self.assertGreater(len(true_alert_dict), 10)
self.assertGreater(ignored_sqlite,
50) # just make sure that some sqlite
# alerts were ignored by the more
# stringent avro cut
log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir,
suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(
htmid,
star_db,
photometry_class=TestAlertsVarCat_avro,
output_prefix='alert_test',
output_dir=self.alert_data_output_dir,
dmag_cutoff=dmag_cutoff_sqlite,
log_file_name=log_file_name)
obshistid_to_htmid = {}
for htmid in alert_gen.htmid_list:
for obs in alert_gen.obs_from_htmid(htmid):
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid not in obshistid_to_htmid:
obshistid_to_htmid[obshistid] = []
obshistid_to_htmid[obshistid].append(htmid)
avro_gen = AvroAlertGenerator()
avro_gen.load_schema(
os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData',
'avroSchema'))
sql_prefix_list = ['alert_test']
out_prefix = 'test_avro'
log_file_name = tempfile.mktemp(dir=self.avro_out_dir,
prefix='test_avro',
suffix='log.txt')
for obshistid in obshistid_list:
avro_gen.write_alerts(obshistid,
self.alert_data_output_dir,
sql_prefix_list,
obshistid_to_htmid[obshistid],
self.avro_out_dir,
out_prefix,
dmag_cutoff_avro,
lock=None,
log_file_name=log_file_name)
list_of_avro_files = os.listdir(self.avro_out_dir)
self.assertGreater(len(list_of_avro_files), 2)
alert_ct = 0
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
photParams = PhotometricParameters()
diasourceId_set = set()
for avro_file_name in list_of_avro_files:
if avro_file_name.endswith('log.txt'):
continue
full_name = os.path.join(self.avro_out_dir, avro_file_name)
with DataFileReader(open(full_name, 'rb'),
DatumReader()) as data_reader:
for alert in data_reader:
alert_ct += 1
obshistid = alert['alertId'] >> 20
obs = obs_dict[obshistid]
uniqueId = alert['diaObject']['diaObjectId']
true_alert_id = (uniqueId << obshistid_bits) + obshistid
self.assertIn(true_alert_id, true_alert_dict)
self.assertEqual(alert['l1dbId'], uniqueId)
true_alert = true_alert_dict[true_alert_id]
diaSource = alert['diaSource']
self.assertAlmostEqual(diaSource['ra'], true_alert['ra'],
10)
self.assertAlmostEqual(diaSource['decl'],
true_alert['decl'], 10)
self.assertAlmostEqual(diaSource['x'], true_alert['xPix'],
3)
self.assertAlmostEqual(diaSource['y'], true_alert['yPix'],
3)
self.assertAlmostEqual(diaSource['midPointTai'],
obs.mjd.TAI, 4)
true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag'])
true_q_mag = true_alert['mag'] - true_alert['dmag']
true_q_flux = dummy_sed.fluxFromMag(true_q_mag)
true_dflux = true_tot_flux - true_q_flux
self.assertAlmostEqual(diaSource['psFlux'] / true_dflux,
1.0, 6)
self.assertAlmostEqual(
diaSource['totFlux'] / true_tot_flux, 1.0, 6)
self.assertAlmostEqual(diaSource['diffFlux'] / true_dflux,
1.0, 6)
true_tot_snr, gamma = calcSNR_m5(true_alert['mag'],
bp_dict[obs.bandpass],
obs.m5[obs.bandpass],
photParams)
true_q_snr, gamma = calcSNR_m5(
true_q_mag, bp_dict[obs.bandpass],
self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]],
photParams)
true_tot_err = true_tot_flux / true_tot_snr
true_q_err = true_q_flux / true_q_snr
true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2)
self.assertAlmostEqual(
diaSource['snr'] / np.abs(true_dflux / true_diff_err),
1.0, 6)
self.assertAlmostEqual(
diaSource['totFluxErr'] / true_tot_err, 1.0, 6)
self.assertAlmostEqual(
diaSource['diffFluxErr'] / true_diff_err, 1.0, 6)
chipnum = int(true_alert['chipName'].replace(
'R', '').replace('S', '').replace(',', '').replace(
':', '').replace(' ', ''))
true_ccdid = (chipnum * 10**7) + obshistid
self.assertEqual(true_ccdid, diaSource['ccdVisitId'])
self.assertEqual(uniqueId, diaSource['diaObjectId'])
self.assertNotIn(diaSource['diaSourceId'], diasourceId_set)
diasourceId_set.add(diaSource['diaSourceId'])
diaObject = alert['diaObject']
obj_dex = (uniqueId // 1024) - 1
self.assertAlmostEqual(
0.001 * diaObject['pmRa'] / self.pmra_truth[obj_dex],
1.0, 5)
self.assertAlmostEqual(
0.001 * diaObject['pmDecl'] /
self.pmdec_truth[obj_dex], 1.0, 5)
self.assertAlmostEqual(
0.001 * diaObject['parallax'] / self.px_truth[obj_dex],
1.0, 5)
(true_ra_base, true_dec_base) = applyProperMotion(
self.ra_truth[obj_dex],
self.dec_truth[obj_dex],
self.pmra_truth[obj_dex],
self.pmdec_truth[obj_dex],
self.px_truth[obj_dex],
self.vrad_truth[obj_dex],
mjd=ModifiedJulianDate(TAI=diaObject['radecTai']))
self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7)
self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7)
self.assertEqual(alert_ct, len(true_alert_dict))
if not os.path.exists(args.out_dir):
os.mkdir(args.out_dir)
t_start = time.time()
# get list of ObservationMetaData to simulate
obs_gen = ObservationMetaDataGenerator(args.opsim_db, driver='sqlite')
obs_list = obs_gen.getObservationMetaData(night=15)
del obs_gen
sims_clean_up()
print('%d obs' % len(obs_list))
# use AlertDataGenerator to separate the ObservationMetaData
# by htmid
alert_gen = AlertDataGenerator()
alert_gen.subdivide_obs(obs_list, htmid_level=6)
# Create a dict that maps an obsHistID to the list of htmids
# of the trixels overlapping that observation
#
# Also separate list of obsHistIDs into n_proc
# lists of roughly balanced loads.
obshistid_to_htmid = {}
obshistid_list = []
for i_proc in range(args.n_proc):
obshistid_list.append([])
i_proc = 0
for htmid in alert_gen.htmid_list:
for obs in alert_gen.obs_from_htmid(htmid):
def test_avro_alert_generation_diff_dmag(self):
"""
Make sure everything works properly when the AlertDataGenerator
and the AvroAlertGenerator have different dmag thresholds
"""
dmag_cutoff_sqlite = 0.005
dmag_cutoff_avro = 0.2
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite')
# assemble a dict of all of the alerts that need to be generated
obshistid_list = []
for obs in self.obs_list:
obshistid_list.append(obs.OpsimMetaData['obsHistID'])
obshistid_max = max(obshistid_list)
obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0)))
true_alert_dict = {}
obs_dict = {}
ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs)
cat.camera = lsst_camera()
for line in cat.iter_catalog():
if line[1] is None:
continue
dmag = line[2]
mag = line[3]
if (np.abs(dmag) > dmag_cutoff_avro and
mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]):
alertId = (line[0] << obshistid_bits) + obshistid
self.assertNotIn(alertId, true_alert_dict)
true_alert_dict[alertId] = {}
true_alert_dict[alertId]['chipName'] = line[1]
true_alert_dict[alertId]['dmag'] = dmag
true_alert_dict[alertId]['mag'] = mag
true_alert_dict[alertId]['ra'] = np.degrees(line[4])
true_alert_dict[alertId]['decl'] = np.degrees(line[5])
true_alert_dict[alertId]['xPix'] = line[6]
true_alert_dict[alertId]['yPix'] = line[7]
elif np.abs(dmag) > dmag_cutoff_sqlite:
ignored_sqlite += 1
self.assertGreater(len(true_alert_dict), 10)
self.assertGreater(ignored_sqlite, 50) # just make sure that some sqlite
# alerts were ignored by the more
# stringent avro cut
log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat_avro,
output_prefix='alert_test',
output_dir=self.alert_data_output_dir,
dmag_cutoff=dmag_cutoff_sqlite,
log_file_name=log_file_name)
obshistid_to_htmid = {}
for htmid in alert_gen.htmid_list:
for obs in alert_gen.obs_from_htmid(htmid):
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid not in obshistid_to_htmid:
obshistid_to_htmid[obshistid] = []
obshistid_to_htmid[obshistid].append(htmid)
avro_gen = AvroAlertGenerator()
avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema'))
sql_prefix_list = ['alert_test']
out_prefix = 'test_avro'
log_file_name = tempfile.mktemp(dir=self.avro_out_dir,
prefix='test_avro',
suffix='log.txt')
for obshistid in obshistid_list:
avro_gen.write_alerts(obshistid, self.alert_data_output_dir,
sql_prefix_list,
obshistid_to_htmid[obshistid],
self.avro_out_dir, out_prefix,
dmag_cutoff_avro, lock=None,
log_file_name=log_file_name)
list_of_avro_files = os.listdir(self.avro_out_dir)
self.assertGreater(len(list_of_avro_files), 2)
alert_ct = 0
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
photParams = PhotometricParameters()
diasourceId_set = set()
for avro_file_name in list_of_avro_files:
if avro_file_name.endswith('log.txt'):
continue
full_name = os.path.join(self.avro_out_dir, avro_file_name)
with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader:
for alert in data_reader:
alert_ct += 1
obshistid = alert['alertId'] >> 20
obs = obs_dict[obshistid]
uniqueId = alert['diaObject']['diaObjectId']
true_alert_id = (uniqueId << obshistid_bits) + obshistid
self.assertIn(true_alert_id, true_alert_dict)
self.assertEqual(alert['l1dbId'], uniqueId)
true_alert = true_alert_dict[true_alert_id]
diaSource = alert['diaSource']
self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10)
self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10)
self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3)
self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3)
self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4)
true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag'])
true_q_mag = true_alert['mag'] - true_alert['dmag']
true_q_flux = dummy_sed.fluxFromMag(true_q_mag)
true_dflux = true_tot_flux - true_q_flux
self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6)
self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6)
self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6)
true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass],
obs.m5[obs.bandpass], photParams)
true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass],
self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]],
photParams)
true_tot_err = true_tot_flux/true_tot_snr
true_q_err = true_q_flux/true_q_snr
true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2)
self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err),
1.0, 6)
self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6)
self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6)
chipnum = int(true_alert['chipName'].replace('R', '').replace('S', '').
replace(',', '').replace(':', '').replace(' ', ''))
true_ccdid = (chipnum*10**7)+obshistid
self.assertEqual(true_ccdid, diaSource['ccdVisitId'])
self.assertEqual(uniqueId, diaSource['diaObjectId'])
self.assertNotIn(diaSource['diaSourceId'], diasourceId_set)
diasourceId_set.add(diaSource['diaSourceId'])
diaObject = alert['diaObject']
obj_dex = (uniqueId//1024) - 1
self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5)
self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5)
self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5)
(true_ra_base,
true_dec_base) = applyProperMotion(self.ra_truth[obj_dex],
self.dec_truth[obj_dex],
self.pmra_truth[obj_dex],
self.pmdec_truth[obj_dex],
self.px_truth[obj_dex],
self.vrad_truth[obj_dex],
mjd=ModifiedJulianDate(TAI=diaObject['radecTai']))
self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7)
self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7)
self.assertEqual(alert_ct, len(true_alert_dict))