def testApplyProperMotion(self): for mjd in self.mjdList: raRad, decRad = utils._applyProperMotion( self.raList, self.decList, self.pm_raList, self.pm_decList, self.pxList, self.v_radList, mjd=ModifiedJulianDate(TAI=mjd)) raDeg, decDeg = utils.applyProperMotion( np.degrees(self.raList), np.degrees(self.decList), utils.arcsecFromRadians(self.pm_raList), utils.arcsecFromRadians(self.pm_decList), utils.arcsecFromRadians(self.pxList), self.v_radList, mjd=ModifiedJulianDate(TAI=mjd)) dRa = utils.arcsecFromRadians(raRad - np.radians(raDeg)) np.testing.assert_array_almost_equal(dRa, np.zeros(self.nStars), 9) dDec = utils.arcsecFromRadians(raRad - np.radians(raDeg)) np.testing.assert_array_almost_equal(dDec, np.zeros(self.nStars), 9) for ra, dec, pm_ra, pm_dec, px, v_rad in \ zip(self.raList, self.decList, self.pm_raList, self.pm_decList, self.pxList, self.v_radList): raRad, decRad = utils._applyProperMotion( ra, dec, pm_ra, pm_dec, px, v_rad, mjd=ModifiedJulianDate(TAI=self.mjdList[0])) raDeg, decDeg = utils.applyProperMotion( np.degrees(ra), np.degrees(dec), utils.arcsecFromRadians(pm_ra), utils.arcsecFromRadians(pm_dec), utils.arcsecFromRadians(px), v_rad, mjd=ModifiedJulianDate(TAI=self.mjdList[0])) self.assertAlmostEqual( utils.arcsecFromRadians(raRad - np.radians(raDeg)), 0.0, 9) self.assertAlmostEqual( utils.arcsecFromRadians(decRad - np.radians(decDeg)), 0.0, 9)
def testApplyProperMotion(self): for mjd in self.mjdList: raRad, decRad = utils._applyProperMotion(self.raList, self.decList, self.pm_raList, self.pm_decList, self.pxList, self.v_radList, mjd=ModifiedJulianDate(TAI=mjd)) raDeg, decDeg = utils.applyProperMotion(np.degrees(self.raList), np.degrees(self.decList), utils.arcsecFromRadians( self.pm_raList), utils.arcsecFromRadians( self.pm_decList), utils.arcsecFromRadians( self.pxList), self.v_radList, mjd=ModifiedJulianDate(TAI=mjd)) dRa = utils.arcsecFromRadians(raRad - np.radians(raDeg)) np.testing.assert_array_almost_equal(dRa, np.zeros(self.nStars), 9) dDec = utils.arcsecFromRadians(raRad - np.radians(raDeg)) np.testing.assert_array_almost_equal( dDec, np.zeros(self.nStars), 9) for ra, dec, pm_ra, pm_dec, px, v_rad in \ zip(self.raList, self.decList, self.pm_raList, self.pm_decList, self.pxList, self.v_radList): raRad, decRad = utils._applyProperMotion(ra, dec, pm_ra, pm_dec, px, v_rad, mjd=ModifiedJulianDate(TAI=self.mjdList[0])) raDeg, decDeg = utils.applyProperMotion(np.degrees(ra), np.degrees(dec), utils.arcsecFromRadians(pm_ra), utils.arcsecFromRadians(pm_dec), utils.arcsecFromRadians(px), v_rad, mjd=ModifiedJulianDate(TAI=self.mjdList[0])) self.assertAlmostEqual(utils.arcsecFromRadians( raRad - np.radians(raDeg)), 0.0, 9) self.assertAlmostEqual(utils.arcsecFromRadians( decRad - np.radians(decDeg)), 0.0, 9)
def test_alert_data_generation(self): dmag_cutoff = 0.005 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5} _max_var_param_str = self.max_str_len class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject): objid = 'star_alert' tableid = 'stars' idColKey = 'simobjid' raColName = 'ra' decColName = 'dec' objectTypeId = 0 columns = [('raJ2000', 'ra*0.01745329252'), ('decJ2000', 'dec*0.01745329252'), ('parallax', 'px*0.01745329252/3600.0'), ('properMotionRa', 'pmra*0.01745329252/3600.0'), ('properMotionDec', 'pmdec*0.01745329252/3600.0'), ('radialVelocity', 'vrad'), ('variabilityParameters', 'varParamStr', str, _max_var_param_str)] class TestAlertsVarCatMixin(object): @register_method('alert_test') def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None): if len(params) == 0: return np.array([[], [], [], [], [], []]) if isinstance(expmjd, numbers.Number): dmags_out = np.zeros((6, self.num_variable_obj(params))) else: dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd))) for i_star in range(self.num_variable_obj(params)): if params['amp'][i_star] is not None: dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd) for i_filter in range(6): dmags_out[i_filter][i_star] = dmags return dmags_out class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog): pass class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoords, AstrometryStars, Variability, InstanceCatalog): column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert'] camera = obs_lsst_phosim.PhosimMapper().camera @compound('delta_umag', 'delta_gmag', 'delta_rmag', 'delta_imag', 'delta_zmag', 'delta_ymag') def get_TruthVariability(self): return self.applyVariability(self.column_by_name('varParamStr')) @cached def get_dmagAlert(self): return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass) @cached def get_magAlert(self): return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \ self.column_by_name('dmagAlert') star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite') # assemble the true light curves for each object; we need to figure out # if their np.max(dMag) ever goes over dmag_cutoff; then we will know if # we are supposed to simulate them true_lc_dict = {} true_lc_obshistid_dict = {} is_visible_dict = {} obs_dict = {} max_obshistid = -1 n_total_observations = 0 for obs in self.obs_list: obs_dict[obs.OpsimMetaData['obsHistID']] = obs obshistid = obs.OpsimMetaData['obsHistID'] if obshistid > max_obshistid: max_obshistid = obshistid cat = TestAlertsTruthCat(star_db, obs_metadata=obs) for line in cat.iter_catalog(): if line[1] is None: continue n_total_observations += 1 if line[0] not in true_lc_dict: true_lc_dict[line[0]] = {} true_lc_obshistid_dict[line[0]] = [] true_lc_dict[line[0]][obshistid] = line[2] true_lc_obshistid_dict[line[0]].append(obshistid) if line[0] not in is_visible_dict: is_visible_dict[line[0]] = False if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]: is_visible_dict[line[0]] = True obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2))) skipped_due_to_mag = 0 objects_to_simulate = [] obshistid_unqid_set = set() for obj_id in true_lc_dict: dmag_max = -1.0 for obshistid in true_lc_dict[obj_id]: if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max: dmag_max = np.abs(true_lc_dict[obj_id][obshistid]) if dmag_max >= dmag_cutoff: if not is_visible_dict[obj_id]: skipped_due_to_mag += 1 continue objects_to_simulate.append(obj_id) for obshistid in true_lc_obshistid_dict[obj_id]: obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid) self.assertGreater(len(objects_to_simulate), 10) self.assertGreater(skipped_due_to_mag, 0) log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt') alert_gen = AlertDataGenerator(testing=True) alert_gen.subdivide_obs(self.obs_list, htmid_level=6) for htmid in alert_gen.htmid_list: alert_gen.alert_data_from_htmid(htmid, star_db, photometry_class=TestAlertsVarCat, output_prefix='alert_test', output_dir=self.output_dir, dmag_cutoff=dmag_cutoff, log_file_name=log_file_name) dummy_sed = Sed() bp_dict = BandpassDict.loadTotalBandpassesFromFiles() phot_params = PhotometricParameters() # First, verify that the contents of the sqlite files are all correct n_tot_simulated = 0 alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, ' alert_query += 'meta.band, quiescent.flux, alert.dflux, ' alert_query += 'quiescent.snr, alert.snr, ' alert_query += 'alert.ra, alert.dec, alert.chipNum, ' alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, ' alert_query += 'ast.parallax ' alert_query += 'FROM alert_data AS alert ' alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId ' alert_query += 'INNER JOIN quiescent_flux AS quiescent ' alert_query += 'ON quiescent.uniqueId=alert.uniqueId ' alert_query += 'AND quiescent.band=meta.band ' alert_query += 'INNER JOIN baseline_astrometry AS ast ' alert_query += 'ON ast.uniqueId=alert.uniqueId' alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int), ('TAI', float), ('band', int), ('q_flux', float), ('dflux', float), ('q_snr', float), ('tot_snr', float), ('ra', float), ('dec', float), ('chipNum', int), ('xPix', float), ('yPix', float), ('pmRA', float), ('pmDec', float), ('parallax', float)]) sqlite_file_list = os.listdir(self.output_dir) n_tot_simulated = 0 obshistid_unqid_simulated_set = set() for file_name in sqlite_file_list: if not file_name.endswith('db'): continue full_name = os.path.join(self.output_dir, file_name) self.assertTrue(os.path.exists(full_name)) alert_db = DBObject(full_name, driver='sqlite') alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype) if len(alert_data) == 0: continue mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI']) for i_obj in range(len(alert_data)): n_tot_simulated += 1 obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) + alert_data['obshistId'][i_obj]) unq = alert_data['uniqueId'][i_obj] obj_dex = (unq//1024)-1 self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4) self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4) self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4) ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=mjd_list[i_obj]) distance = angularSeparation(ra_truth, dec_truth, alert_data['ra'][i_obj], alert_data['dec'][i_obj]) distance_arcsec = 3600.0*distance msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth, alert_data['ra'][i_obj], alert_data['dec'][i_obj]) self.assertLess(distance_arcsec, 0.0005, msg=msg) obs = obs_dict[alert_data['obshistId'][i_obj]] chipname = chipNameFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex], pm_ra=self.pmra_truth[obj_dex], pm_dec=self.pmdec_truth[obj_dex], parallax=self.px_truth[obj_dex], v_rad=self.vrad_truth[obj_dex], obs_metadata=obs, camera=self.camera) chipnum = int(chipname.replace('R', '').replace('S', ''). replace(' ', '').replace(';', '').replace(',', ''). replace(':', '')) self.assertEqual(chipnum, alert_data['chipNum'][i_obj]) xpix, ypix = pixelCoordsFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex], pm_ra=self.pmra_truth[obj_dex], pm_dec=self.pmdec_truth[obj_dex], parallax=self.px_truth[obj_dex], v_rad=self.vrad_truth[obj_dex], obs_metadata=obs, camera=self.camera) self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4) self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4) dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj]) self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]], dmag_sim, 3) mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]] m5 = obs.m5[mag_name] q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj]) self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex], q_mag, 4) snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex], bp_dict[mag_name], self.obs_mag_cutoff[alert_data['band'][i_obj]], phot_params) self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4) tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \ true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]] snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name], m5, phot_params) self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4) for val in obshistid_unqid_set: self.assertIn(val, obshistid_unqid_simulated_set) self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set)) astrometry_query = 'SELECT uniqueId, ra, dec, TAI ' astrometry_query += 'FROM baseline_astrometry' astrometry_dtype = np.dtype([('uniqueId', int), ('ra', float), ('dec', float), ('TAI', float)]) tai_list = [] for obs in self.obs_list: tai_list.append(obs.mjd.TAI) tai_list = np.array(tai_list) n_tot_ast_simulated = 0 for file_name in sqlite_file_list: if not file_name.endswith('db'): continue full_name = os.path.join(self.output_dir, file_name) self.assertTrue(os.path.exists(full_name)) alert_db = DBObject(full_name, driver='sqlite') astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype) if len(astrometry_data) == 0: continue mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI']) for i_obj in range(len(astrometry_data)): n_tot_ast_simulated += 1 obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1 ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=mjd_list[i_obj]) distance = angularSeparation(ra_truth, dec_truth, astrometry_data['ra'][i_obj], astrometry_data['dec'][i_obj]) self.assertLess(3600.0*distance, 0.0005) del alert_gen gc.collect() self.assertGreater(n_tot_simulated, 10) self.assertGreater(len(obshistid_unqid_simulated_set), 10) self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations) self.assertGreater(n_tot_ast_simulated, 0)
def extract_objects(df, header): """ Extract the object information needed by the sims code and pack into a new dataframe. Parameters ---------- df : pandas.DataFrame DataFrame containing the instance catalog object data. header : dictionary dictionary containing the PhoSim header cards as output by extract_commands() (necessary for correctly applying proper motion to stars) Returns ------- pandas.DataFrame A DataFrame with the columns expected by the sims code. """ # Check for unhandled source types and emit warning if any are present. valid_types = dict(point='pointSource', sersic2d='sersic') invalid_types = set(df['SOURCE_TYPE']) - set(valid_types) if invalid_types: warnings.warn("Instance catalog contains unhandled source types:\n%s\nSkipping these." % '\n'.join(invalid_types)) columns = ('uniqueId', 'galSimType', 'magNorm', 'sedFilepath', 'redshift', 'raJ2000', 'decJ2000', 'halfLightRadius', 'minorAxis', 'majorAxis', 'positionAngle', 'sindex', 'properMotionRa', 'properMotionDec', 'parallax', 'radialVelocity') # Process point sources and galaxies separately. source_type = 'point' stars = df.query("SOURCE_TYPE=='%s'" % source_type) phosim_stars = pd.DataFrame(np.zeros((len(stars), len(columns))), index=stars.index, columns=columns) phosim_stars['uniqueId'] = pd.to_numeric(stars['VALUE']).tolist() phosim_stars['galSimType'] = valid_types[source_type] phosim_stars['magNorm'] = pd.to_numeric(stars['MAG_NORM']).tolist() phosim_stars['sedFilepath'] = stars['SED_NAME'].tolist() phosim_stars['redshift'] = pd.to_numeric(stars['REDSHIFT']).tolist() phosim_stars['raJ2000'] = pd.to_numeric(stars['RA']).tolist() phosim_stars['decJ2000'] = pd.to_numeric(stars['DEC']).tolist() phosim_stars['properMotionRa'] = pd.to_numeric(stars['PAR5']).tolist() phosim_stars['properMotionDec'] = pd.to_numeric(stars['PAR6']).tolist() phosim_stars['parallax'] = pd.to_numeric(stars['PAR7']).tolist() phosim_stars['radialVelocity'] = pd.to_numeric(stars['PAR8']).tolist() if len(phosim_stars) > 0: phosim_stars = extract_extinction(stars, phosim_stars, 1) mjd = ModifiedJulianDate(TAI=header['mjd']) raICRS, decICRS = applyProperMotion(phosim_stars.raJ2000.values, phosim_stars.decJ2000.values, phosim_stars.properMotionRa.values, phosim_stars.properMotionDec.values, phosim_stars.parallax.values, phosim_stars.radialVelocity.values, mjd=mjd) phosim_stars = phosim_stars.assign(raICRS=raICRS, decICRS=decICRS) source_type = 'sersic2d' galaxies = df.query("SOURCE_TYPE == '%s'" % source_type) phosim_galaxies = pd.DataFrame(np.zeros((len(galaxies), len(columns))), index=galaxies.index, columns=columns) phosim_galaxies['uniqueId'] = pd.to_numeric(galaxies['VALUE']).tolist() phosim_galaxies['galSimType'] = valid_types[source_type] phosim_galaxies['magNorm'] = pd.to_numeric(galaxies['MAG_NORM']).tolist() phosim_galaxies['sedFilepath'] = galaxies['SED_NAME'].tolist() phosim_galaxies['redshift'] = pd.to_numeric(galaxies['REDSHIFT']).tolist() phosim_galaxies['raJ2000'] = pd.to_numeric(galaxies['RA']).tolist() phosim_galaxies['decJ2000'] = pd.to_numeric(galaxies['DEC']).tolist() phosim_galaxies['majorAxis'] = \ radiansFromArcsec(pd.to_numeric(galaxies['PAR1'])).tolist() phosim_galaxies['minorAxis'] = \ radiansFromArcsec(pd.to_numeric(galaxies['PAR2'])).tolist() phosim_galaxies['halfLightRadius'] = phosim_galaxies['majorAxis'] phosim_galaxies['positionAngle'] = \ (np.pi/180.*pd.to_numeric(galaxies['PAR3'])).tolist() phosim_galaxies['sindex'] = pd.to_numeric(galaxies['PAR4']).tolist() n_gal = len(phosim_galaxies.raJ2000.values) phosim_galaxies = phosim_galaxies.assign(raICRS=phosim_galaxies.raJ2000, decICRS=phosim_galaxies.decJ2000, properMotionRa=np.zeros(n_gal), properMotionDec=np.zeros(n_gal), parallax=np.zeros(n_gal), radialVelocity=np.zeros(n_gal)) if len(phosim_galaxies) > 0: phosim_galaxies = extract_extinction(galaxies, phosim_galaxies, 5) return pd.concat((phosim_stars, phosim_galaxies), ignore_index=True)
def test_alert_data_generation(self): dmag_cutoff = 0.005 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5} _max_var_param_str = self.max_str_len class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject): objid = 'star_alert' tableid = 'stars' idColKey = 'simobjid' raColName = 'ra' decColName = 'dec' objectTypeId = 0 columns = [('raJ2000', 'ra*0.01745329252'), ('decJ2000', 'dec*0.01745329252'), ('parallax', 'px*0.01745329252/3600.0'), ('properMotionRa', 'pmra*0.01745329252/3600.0'), ('properMotionDec', 'pmdec*0.01745329252/3600.0'), ('radialVelocity', 'vrad'), ('variabilityParameters', 'varParamStr', str, _max_var_param_str)] class TestAlertsVarCatMixin(object): @register_method('alert_test') def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None): if len(params) == 0: return np.array([[], [], [], [], [], []]) if isinstance(expmjd, numbers.Number): dmags_out = np.zeros((6, self.num_variable_obj(params))) else: dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd))) for i_star in range(self.num_variable_obj(params)): if params['amp'][i_star] is not None: dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd) for i_filter in range(6): dmags_out[i_filter][i_star] = dmags return dmags_out class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog): pass class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoordsLSST, AstrometryStars, Variability, InstanceCatalog): column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert'] @compound('delta_umag', 'delta_gmag', 'delta_rmag', 'delta_imag', 'delta_zmag', 'delta_ymag') def get_TruthVariability(self): return self.applyVariability(self.column_by_name('varParamStr')) @cached def get_dmagAlert(self): return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass) @cached def get_magAlert(self): return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \ self.column_by_name('dmagAlert') star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite') # assemble the true light curves for each object; we need to figure out # if their np.max(dMag) ever goes over dmag_cutoff; then we will know if # we are supposed to simulate them true_lc_dict = {} true_lc_obshistid_dict = {} is_visible_dict = {} obs_dict = {} max_obshistid = -1 n_total_observations = 0 for obs in self.obs_list: obs_dict[obs.OpsimMetaData['obsHistID']] = obs obshistid = obs.OpsimMetaData['obsHistID'] if obshistid > max_obshistid: max_obshistid = obshistid cat = TestAlertsTruthCat(star_db, obs_metadata=obs) for line in cat.iter_catalog(): if line[1] is None: continue n_total_observations += 1 if line[0] not in true_lc_dict: true_lc_dict[line[0]] = {} true_lc_obshistid_dict[line[0]] = [] true_lc_dict[line[0]][obshistid] = line[2] true_lc_obshistid_dict[line[0]].append(obshistid) if line[0] not in is_visible_dict: is_visible_dict[line[0]] = False if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]: is_visible_dict[line[0]] = True obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2))) skipped_due_to_mag = 0 objects_to_simulate = [] obshistid_unqid_set = set() for obj_id in true_lc_dict: dmag_max = -1.0 for obshistid in true_lc_dict[obj_id]: if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max: dmag_max = np.abs(true_lc_dict[obj_id][obshistid]) if dmag_max >= dmag_cutoff: if not is_visible_dict[obj_id]: skipped_due_to_mag += 1 continue objects_to_simulate.append(obj_id) for obshistid in true_lc_obshistid_dict[obj_id]: obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid) self.assertGreater(len(objects_to_simulate), 10) self.assertGreater(skipped_due_to_mag, 0) log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt') alert_gen = AlertDataGenerator(testing=True) alert_gen.subdivide_obs(self.obs_list, htmid_level=6) for htmid in alert_gen.htmid_list: alert_gen.alert_data_from_htmid(htmid, star_db, photometry_class=TestAlertsVarCat, output_prefix='alert_test', output_dir=self.output_dir, dmag_cutoff=dmag_cutoff, log_file_name=log_file_name) dummy_sed = Sed() bp_dict = BandpassDict.loadTotalBandpassesFromFiles() phot_params = PhotometricParameters() # First, verify that the contents of the sqlite files are all correct n_tot_simulated = 0 alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, ' alert_query += 'meta.band, quiescent.flux, alert.dflux, ' alert_query += 'quiescent.snr, alert.snr, ' alert_query += 'alert.ra, alert.dec, alert.chipNum, ' alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, ' alert_query += 'ast.parallax ' alert_query += 'FROM alert_data AS alert ' alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId ' alert_query += 'INNER JOIN quiescent_flux AS quiescent ' alert_query += 'ON quiescent.uniqueId=alert.uniqueId ' alert_query += 'AND quiescent.band=meta.band ' alert_query += 'INNER JOIN baseline_astrometry AS ast ' alert_query += 'ON ast.uniqueId=alert.uniqueId' alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int), ('TAI', float), ('band', int), ('q_flux', float), ('dflux', float), ('q_snr', float), ('tot_snr', float), ('ra', float), ('dec', float), ('chipNum', int), ('xPix', float), ('yPix', float), ('pmRA', float), ('pmDec', float), ('parallax', float)]) sqlite_file_list = os.listdir(self.output_dir) n_tot_simulated = 0 obshistid_unqid_simulated_set = set() for file_name in sqlite_file_list: if not file_name.endswith('db'): continue full_name = os.path.join(self.output_dir, file_name) self.assertTrue(os.path.exists(full_name)) alert_db = DBObject(full_name, driver='sqlite') alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype) if len(alert_data) == 0: continue mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI']) for i_obj in range(len(alert_data)): n_tot_simulated += 1 obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) + alert_data['obshistId'][i_obj]) unq = alert_data['uniqueId'][i_obj] obj_dex = (unq//1024)-1 self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4) self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4) self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4) ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=mjd_list[i_obj]) distance = angularSeparation(ra_truth, dec_truth, alert_data['ra'][i_obj], alert_data['dec'][i_obj]) distance_arcsec = 3600.0*distance msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth, alert_data['ra'][i_obj], alert_data['dec'][i_obj]) self.assertLess(distance_arcsec, 0.0005, msg=msg) obs = obs_dict[alert_data['obshistId'][i_obj]] chipname = chipNameFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex], pm_ra=self.pmra_truth[obj_dex], pm_dec=self.pmdec_truth[obj_dex], parallax=self.px_truth[obj_dex], v_rad=self.vrad_truth[obj_dex], obs_metadata=obs, band=obs.bandpass) chipnum = int(chipname.replace('R', '').replace('S', ''). replace(' ', '').replace(';', '').replace(',', ''). replace(':', '')) self.assertEqual(chipnum, alert_data['chipNum'][i_obj]) xpix, ypix = pixelCoordsFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex], pm_ra=self.pmra_truth[obj_dex], pm_dec=self.pmdec_truth[obj_dex], parallax=self.px_truth[obj_dex], v_rad=self.vrad_truth[obj_dex], obs_metadata=obs, band=obs.bandpass) self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4) self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4) dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj]) self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]], dmag_sim, 3) mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]] m5 = obs.m5[mag_name] q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj]) self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex], q_mag, 4) snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex], bp_dict[mag_name], self.obs_mag_cutoff[alert_data['band'][i_obj]], phot_params) self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4) tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \ true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]] snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name], m5, phot_params) self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4) for val in obshistid_unqid_set: self.assertIn(val, obshistid_unqid_simulated_set) self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set)) astrometry_query = 'SELECT uniqueId, ra, dec, TAI ' astrometry_query += 'FROM baseline_astrometry' astrometry_dtype = np.dtype([('uniqueId', int), ('ra', float), ('dec', float), ('TAI', float)]) tai_list = [] for obs in self.obs_list: tai_list.append(obs.mjd.TAI) tai_list = np.array(tai_list) n_tot_ast_simulated = 0 for file_name in sqlite_file_list: if not file_name.endswith('db'): continue full_name = os.path.join(self.output_dir, file_name) self.assertTrue(os.path.exists(full_name)) alert_db = DBObject(full_name, driver='sqlite') astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype) if len(astrometry_data) == 0: continue mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI']) for i_obj in range(len(astrometry_data)): n_tot_ast_simulated += 1 obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1 ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=mjd_list[i_obj]) distance = angularSeparation(ra_truth, dec_truth, astrometry_data['ra'][i_obj], astrometry_data['dec'][i_obj]) self.assertLess(3600.0*distance, 0.0005) del alert_gen gc.collect() self.assertGreater(n_tot_simulated, 10) self.assertGreater(len(obshistid_unqid_simulated_set), 10) self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations) self.assertGreater(n_tot_ast_simulated, 0)
def test_avro_alert_generation_diff_dmag(self): """ Make sure everything works properly when the AlertDataGenerator and the AvroAlertGenerator have different dmag thresholds """ dmag_cutoff_sqlite = 0.005 dmag_cutoff_avro = 0.2 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5} star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite') # assemble a dict of all of the alerts that need to be generated obshistid_list = [] for obs in self.obs_list: obshistid_list.append(obs.OpsimMetaData['obsHistID']) obshistid_max = max(obshistid_list) obshistid_bits = int(np.ceil(np.log(obshistid_max) / np.log(2.0))) true_alert_dict = {} obs_dict = {} ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro for obs in self.obs_list: obs_dict[obs.OpsimMetaData['obsHistID']] = obs obshistid = obs.OpsimMetaData['obsHistID'] cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs) cat.camera = obs_lsst_phosim.PhosimMapper().camera for line in cat.iter_catalog(): if line[1] is None: continue dmag = line[2] mag = line[3] if (np.abs(dmag) > dmag_cutoff_avro and mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]): alertId = (line[0] << obshistid_bits) + obshistid self.assertNotIn(alertId, true_alert_dict) true_alert_dict[alertId] = {} true_alert_dict[alertId]['chipName'] = line[1] true_alert_dict[alertId]['dmag'] = dmag true_alert_dict[alertId]['mag'] = mag true_alert_dict[alertId]['ra'] = np.degrees(line[4]) true_alert_dict[alertId]['decl'] = np.degrees(line[5]) true_alert_dict[alertId]['xPix'] = line[6] true_alert_dict[alertId]['yPix'] = line[7] elif np.abs(dmag) > dmag_cutoff_sqlite: ignored_sqlite += 1 self.assertGreater(len(true_alert_dict), 10) self.assertGreater(ignored_sqlite, 50) # just make sure that some sqlite # alerts were ignored by the more # stringent avro cut log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt') alert_gen = AlertDataGenerator(testing=True) alert_gen.subdivide_obs(self.obs_list, htmid_level=6) for htmid in alert_gen.htmid_list: alert_gen.alert_data_from_htmid( htmid, star_db, photometry_class=TestAlertsVarCat_avro, output_prefix='alert_test', output_dir=self.alert_data_output_dir, dmag_cutoff=dmag_cutoff_sqlite, log_file_name=log_file_name) obshistid_to_htmid = {} for htmid in alert_gen.htmid_list: for obs in alert_gen.obs_from_htmid(htmid): obshistid = obs.OpsimMetaData['obsHistID'] if obshistid not in obshistid_to_htmid: obshistid_to_htmid[obshistid] = [] obshistid_to_htmid[obshistid].append(htmid) avro_gen = AvroAlertGenerator() avro_gen.load_schema( os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema')) sql_prefix_list = ['alert_test'] out_prefix = 'test_avro' log_file_name = tempfile.mktemp(dir=self.avro_out_dir, prefix='test_avro', suffix='log.txt') for obshistid in obshistid_list: avro_gen.write_alerts(obshistid, self.alert_data_output_dir, sql_prefix_list, obshistid_to_htmid[obshistid], self.avro_out_dir, out_prefix, dmag_cutoff_avro, lock=None, log_file_name=log_file_name) list_of_avro_files = os.listdir(self.avro_out_dir) self.assertGreater(len(list_of_avro_files), 2) alert_ct = 0 dummy_sed = Sed() bp_dict = BandpassDict.loadTotalBandpassesFromFiles() photParams = PhotometricParameters() diasourceId_set = set() for avro_file_name in list_of_avro_files: if avro_file_name.endswith('log.txt'): continue full_name = os.path.join(self.avro_out_dir, avro_file_name) with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader: for alert in data_reader: alert_ct += 1 obshistid = alert['alertId'] >> 20 obs = obs_dict[obshistid] uniqueId = alert['diaObject']['diaObjectId'] true_alert_id = (uniqueId << obshistid_bits) + obshistid self.assertIn(true_alert_id, true_alert_dict) self.assertEqual(alert['l1dbId'], uniqueId) true_alert = true_alert_dict[true_alert_id] diaSource = alert['diaSource'] self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10) self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10) self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3) self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3) self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4) true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag']) true_q_mag = true_alert['mag'] - true_alert['dmag'] true_q_flux = dummy_sed.fluxFromMag(true_q_mag) true_dflux = true_tot_flux - true_q_flux self.assertAlmostEqual(diaSource['psFlux'] / true_dflux, 1.0, 6) self.assertAlmostEqual( diaSource['totFlux'] / true_tot_flux, 1.0, 6) self.assertAlmostEqual(diaSource['diffFlux'] / true_dflux, 1.0, 6) true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass], obs.m5[obs.bandpass], photParams) true_q_snr, gamma = calcSNR_m5( true_q_mag, bp_dict[obs.bandpass], self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]], photParams) true_tot_err = true_tot_flux / true_tot_snr true_q_err = true_q_flux / true_q_snr true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2) self.assertAlmostEqual( diaSource['snr'] / np.abs(true_dflux / true_diff_err), 1.0, 6) self.assertAlmostEqual( diaSource['totFluxErr'] / true_tot_err, 1.0, 6) self.assertAlmostEqual( diaSource['diffFluxErr'] / true_diff_err, 1.0, 6) chipnum = int(true_alert['chipName'].replace( 'R', '').replace('S', '').replace(',', '').replace( ':', '').replace(' ', '')) true_ccdid = (chipnum * 10**7) + obshistid self.assertEqual(true_ccdid, diaSource['ccdVisitId']) self.assertEqual(uniqueId, diaSource['diaObjectId']) self.assertNotIn(diaSource['diaSourceId'], diasourceId_set) diasourceId_set.add(diaSource['diaSourceId']) diaObject = alert['diaObject'] obj_dex = (uniqueId // 1024) - 1 self.assertAlmostEqual( 0.001 * diaObject['pmRa'] / self.pmra_truth[obj_dex], 1.0, 5) self.assertAlmostEqual( 0.001 * diaObject['pmDecl'] / self.pmdec_truth[obj_dex], 1.0, 5) self.assertAlmostEqual( 0.001 * diaObject['parallax'] / self.px_truth[obj_dex], 1.0, 5) (true_ra_base, true_dec_base) = applyProperMotion( self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=ModifiedJulianDate(TAI=diaObject['radecTai'])) self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7) self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7) self.assertEqual(alert_ct, len(true_alert_dict))
def test_avro_alert_generation_diff_dmag(self): """ Make sure everything works properly when the AlertDataGenerator and the AvroAlertGenerator have different dmag thresholds """ dmag_cutoff_sqlite = 0.005 dmag_cutoff_avro = 0.2 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5} star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite') # assemble a dict of all of the alerts that need to be generated obshistid_list = [] for obs in self.obs_list: obshistid_list.append(obs.OpsimMetaData['obsHistID']) obshistid_max = max(obshistid_list) obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0))) true_alert_dict = {} obs_dict = {} ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro for obs in self.obs_list: obs_dict[obs.OpsimMetaData['obsHistID']] = obs obshistid = obs.OpsimMetaData['obsHistID'] cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs) cat.camera = lsst_camera() for line in cat.iter_catalog(): if line[1] is None: continue dmag = line[2] mag = line[3] if (np.abs(dmag) > dmag_cutoff_avro and mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]): alertId = (line[0] << obshistid_bits) + obshistid self.assertNotIn(alertId, true_alert_dict) true_alert_dict[alertId] = {} true_alert_dict[alertId]['chipName'] = line[1] true_alert_dict[alertId]['dmag'] = dmag true_alert_dict[alertId]['mag'] = mag true_alert_dict[alertId]['ra'] = np.degrees(line[4]) true_alert_dict[alertId]['decl'] = np.degrees(line[5]) true_alert_dict[alertId]['xPix'] = line[6] true_alert_dict[alertId]['yPix'] = line[7] elif np.abs(dmag) > dmag_cutoff_sqlite: ignored_sqlite += 1 self.assertGreater(len(true_alert_dict), 10) self.assertGreater(ignored_sqlite, 50) # just make sure that some sqlite # alerts were ignored by the more # stringent avro cut log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt') alert_gen = AlertDataGenerator(testing=True) alert_gen.subdivide_obs(self.obs_list, htmid_level=6) for htmid in alert_gen.htmid_list: alert_gen.alert_data_from_htmid(htmid, star_db, photometry_class=TestAlertsVarCat_avro, output_prefix='alert_test', output_dir=self.alert_data_output_dir, dmag_cutoff=dmag_cutoff_sqlite, log_file_name=log_file_name) obshistid_to_htmid = {} for htmid in alert_gen.htmid_list: for obs in alert_gen.obs_from_htmid(htmid): obshistid = obs.OpsimMetaData['obsHistID'] if obshistid not in obshistid_to_htmid: obshistid_to_htmid[obshistid] = [] obshistid_to_htmid[obshistid].append(htmid) avro_gen = AvroAlertGenerator() avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema')) sql_prefix_list = ['alert_test'] out_prefix = 'test_avro' log_file_name = tempfile.mktemp(dir=self.avro_out_dir, prefix='test_avro', suffix='log.txt') for obshistid in obshistid_list: avro_gen.write_alerts(obshistid, self.alert_data_output_dir, sql_prefix_list, obshistid_to_htmid[obshistid], self.avro_out_dir, out_prefix, dmag_cutoff_avro, lock=None, log_file_name=log_file_name) list_of_avro_files = os.listdir(self.avro_out_dir) self.assertGreater(len(list_of_avro_files), 2) alert_ct = 0 dummy_sed = Sed() bp_dict = BandpassDict.loadTotalBandpassesFromFiles() photParams = PhotometricParameters() diasourceId_set = set() for avro_file_name in list_of_avro_files: if avro_file_name.endswith('log.txt'): continue full_name = os.path.join(self.avro_out_dir, avro_file_name) with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader: for alert in data_reader: alert_ct += 1 obshistid = alert['alertId'] >> 20 obs = obs_dict[obshistid] uniqueId = alert['diaObject']['diaObjectId'] true_alert_id = (uniqueId << obshistid_bits) + obshistid self.assertIn(true_alert_id, true_alert_dict) self.assertEqual(alert['l1dbId'], uniqueId) true_alert = true_alert_dict[true_alert_id] diaSource = alert['diaSource'] self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10) self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10) self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3) self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3) self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4) true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag']) true_q_mag = true_alert['mag'] - true_alert['dmag'] true_q_flux = dummy_sed.fluxFromMag(true_q_mag) true_dflux = true_tot_flux - true_q_flux self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6) self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6) self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6) true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass], obs.m5[obs.bandpass], photParams) true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass], self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]], photParams) true_tot_err = true_tot_flux/true_tot_snr true_q_err = true_q_flux/true_q_snr true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2) self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err), 1.0, 6) self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6) self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6) chipnum = int(true_alert['chipName'].replace('R', '').replace('S', ''). replace(',', '').replace(':', '').replace(' ', '')) true_ccdid = (chipnum*10**7)+obshistid self.assertEqual(true_ccdid, diaSource['ccdVisitId']) self.assertEqual(uniqueId, diaSource['diaObjectId']) self.assertNotIn(diaSource['diaSourceId'], diasourceId_set) diasourceId_set.add(diaSource['diaSourceId']) diaObject = alert['diaObject'] obj_dex = (uniqueId//1024) - 1 self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5) self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5) self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5) (true_ra_base, true_dec_base) = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=ModifiedJulianDate(TAI=diaObject['radecTai'])) self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7) self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7) self.assertEqual(alert_ct, len(true_alert_dict))