def check_pt(self, pt, answer):
"""
Take a Cartesian point (pt) and a known
htmid for that point (answer). Find the htmid
for the point using findHtmid and verify that
we get the expected answer.
"""
ra, dec = sphericalFromCartesian(pt)
ii = findHtmid(np.degrees(ra), np.degrees(dec), 3)
binary = '{0:b}'.format(ii)
self.assertEqual(binary, answer)
def check_pt(self, pt, answer):
"""
Take a Cartesian point (pt) and a known
htmid for that point (answer). Find the htmid
for the point using findHtmid and verify that
we get the expected answer.
"""
ra, dec = sphericalFromCartesian(pt)
ii = findHtmid(np.degrees(ra), np.degrees(dec), 3)
binary = '{0:b}'.format(ii)
self.assertEqual(binary, answer)
def test_findHtmid_vectorized(self):
"""
Test that findHtmid works correctly on vectors
"""
rng = np.random.RandomState(81723122)
n_samples = 1000
ra = rng.random_sample(n_samples)*360.0
dec = rng.random_sample(n_samples)*180.0-90.0
level = 7
htmid_vec = findHtmid(ra, dec, level)
self.assertIsInstance(htmid_vec, np.ndarray)
htmid_fast = _findHtmid_fast(ra, dec, level)
self.assertIsInstance(htmid_fast, np.ndarray)
np.testing.assert_array_equal(htmid_vec, htmid_fast)
for ii in range(n_samples):
htmid_slow = _findHtmid_slow(ra[ii], dec[ii], level)
self.assertIsInstance(htmid_slow, numbers.Number)
self.assertEqual(htmid_slow, htmid_vec[ii])
htmid_single = findHtmid(ra[ii], dec[ii], level)
self.assertIsInstance(htmid_single, numbers.Number)
self.assertEqual(htmid_single, htmid_vec[ii])
def test_findHtmid_vectorized(self):
"""
Test that findHtmid works correctly on vectors
"""
rng = np.random.RandomState(81723122)
n_samples = 1000
ra = rng.random_sample(n_samples) * 360.0
dec = rng.random_sample(n_samples) * 180.0 - 90.0
level = 7
htmid_vec = findHtmid(ra, dec, level)
self.assertIsInstance(htmid_vec, np.ndarray)
htmid_fast = _findHtmid_fast(ra, dec, level)
self.assertIsInstance(htmid_fast, np.ndarray)
np.testing.assert_array_equal(htmid_vec, htmid_fast)
for ii in range(n_samples):
htmid_slow = _findHtmid_slow(ra[ii], dec[ii], level)
self.assertIsInstance(htmid_slow, numbers.Number)
self.assertEqual(htmid_slow, htmid_vec[ii])
htmid_single = findHtmid(ra[ii], dec[ii], level)
self.assertIsInstance(htmid_single, numbers.Number)
self.assertEqual(htmid_single, htmid_vec[ii])
def test_against_fatboy(self):
"""
Test findHtmid against a random selection of stars from fatboy
"""
dtype = np.dtype([('htmid', int), ('ra', float), ('dec', float)])
data = np.genfromtxt(os.path.join(getPackageDir('sims_utils'), 'tests',
'testData', 'htmid_test_data.txt'),
dtype=dtype)
self.assertGreater(len(data), 20)
for i_pt in range(len(data)):
htmid_test = findHtmid(data['ra'][i_pt], data['dec'][i_pt], 21)
self.assertEqual(htmid_test, data['htmid'][i_pt])
level_test = levelFromHtmid(htmid_test)
self.assertEqual(level_test, 21)
def test_against_fatboy(self):
"""
Test findHtmid against a random selection of stars from fatboy
"""
dtype = np.dtype([('htmid', int), ('ra', float), ('dec', float)])
data = np.genfromtxt(os.path.join(getPackageDir('sims_utils'), 'tests',
'testData', 'htmid_test_data.txt'),
dtype=dtype)
self.assertGreater(len(data), 20)
for i_pt in range(len(data)):
htmid_test = findHtmid(data['ra'][i_pt], data['dec'][i_pt], 21)
self.assertEqual(htmid_test, data['htmid'][i_pt])
level_test = levelFromHtmid(htmid_test)
self.assertEqual(level_test, 21)
def test_trixel_from_htmid(self):
"""
Check that trixelFromHtmid works by
finding the htmid from an RA, Dec pair,
instantiating the Trixel corresponding
to that htmid, and verifying that that
Trixel (and not its neighbors) contains
the RA, Dec pair.
"""
rng = np.random.RandomState(88)
n_tests = 100
for i_test in range(n_tests):
pt = rng.normal(0.0, 1.0, 3)
ra, dec = sphericalFromCartesian(pt)
ra = np.degrees(ra)
dec = np.degrees(dec)
ii = findHtmid(ra, dec, 5)
tt = trixelFromHtmid(ii)
self.assertTrue(tt.contains(ra, dec))
tt1 = trixelFromHtmid(ii-1)
self.assertFalse(tt1.contains(ra, dec))
tt2 = trixelFromHtmid(ii+1)
self.assertFalse(tt2.contains(ra, dec))
def test_trixel_from_htmid(self):
"""
Check that trixelFromHtmid works by
finding the htmid from an RA, Dec pair,
instantiating the Trixel corresponding
to that htmid, and verifying that that
Trixel (and not its neighbors) contains
the RA, Dec pair.
"""
rng = np.random.RandomState(88)
n_tests = 100
for i_test in range(n_tests):
pt = rng.normal(0.0, 1.0, 3)
ra, dec = sphericalFromCartesian(pt)
ra = np.degrees(ra)
dec = np.degrees(dec)
ii = findHtmid(ra, dec, 5)
tt = trixelFromHtmid(ii)
self.assertTrue(tt.contains(ra, dec))
tt1 = trixelFromHtmid(ii - 1)
self.assertFalse(tt1.contains(ra, dec))
tt2 = trixelFromHtmid(ii + 1)
self.assertFalse(tt2.contains(ra, dec))
def setUpClass(cls):
print('setting up %s' % sims_clean_up.targets)
cls.camera = obs_lsst_phosim.PhosimMapper().camera
# These represent the dimmest magnitudes at which objects
# are considered visible in each of the LSST filters
# (taken from Table 2 of the overview paper)
cls.obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60)
cls.opsim_db = os.path.join(getPackageDir('sims_data'),
'OpSimData',
'opsimblitz1_1133_sqlite.db')
rng = np.random.RandomState(8123)
obs_gen = ObservationMetaDataGenerator(database=cls.opsim_db)
cls.obs_list = obs_gen.getObservationMetaData(night=(0, 2))
cls.obs_list = rng.choice(cls.obs_list, 10, replace=False)
fieldid_list = []
for obs in cls.obs_list:
fieldid_list.append(obs.OpsimMetaData['fieldID'])
# make sure we have selected observations such that the
# same field is revisited more than once
assert len(np.unique(fieldid_list)) < len(fieldid_list)
cls.input_dir = tempfile.mkdtemp(prefix='alertDataGen',
dir=ROOT)
cls.star_db_name = tempfile.mktemp(prefix='alertDataGen_star_db',
dir=cls.input_dir,
suffix='.db')
conn = sqlite3.connect(cls.star_db_name)
cursor = conn.cursor()
cursor.execute('''CREATE TABLE stars
(simobjid int, htmid int, ra real, dec real,
umag real, gmag real, rmag real,
imag real, zmag real, ymag real,
px real, pmra real, pmdec real,
vrad real, varParamStr text)''')
conn.commit()
n_stars = 10
cls.ra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.dec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
u_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
g_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
r_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
i_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
z_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
y_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.px_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.pmra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.pmdec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.vrad_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.amp_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.period_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
id_offset = -n_stars
for obs in cls.obs_list:
id_offset += n_stars
ra_0 = obs.pointingRA
dec_0 = obs.pointingDec
rr = rng.random_sample(n_stars)
theta = rng.random_sample(n_stars)*2.0*np.pi
ra = ra_0 + rr*np.cos(theta)
dec = dec_0 + rr*np.sin(theta)
var_period = rng.random_sample(n_stars)*0.25
var_amp = rng.random_sample(n_stars)*1.0 + 0.01
subset = rng.randint(0, high=len(var_amp)-1, size=3)
var_amp[subset[:2]] = 0.0
var_amp[subset[-1]] = -1.0
umag = rng.random_sample(n_stars)*5.0 + 15.0
gmag = rng.random_sample(n_stars)*5.0 + 15.0
rmag = rng.random_sample(n_stars)*5.0 + 15.0
imag = rng.random_sample(n_stars)*5.0 + 15.0
zmag = rng.random_sample(n_stars)*5.0 + 15.0
ymag = rng.random_sample(n_stars)*5.0 + 15.0
px = rng.random_sample(n_stars)*0.1 # say it is arcsec
pmra = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr
pmdec = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr
vrad = rng.random_sample(n_stars)*600.0 - 300.0
subset = rng.randint(0, high=n_stars-1, size=3)
umag[subset] = 40.0
gmag[subset] = 40.0
rmag[subset] = 40.0
imag[subset] = 40.0
zmag[subset] = 40.0
ymag[subset] = 40.0
cls.ra_truth[id_offset:id_offset+n_stars] = np.round(ra, decimals=6)
cls.dec_truth[id_offset:id_offset+n_stars] = np.round(dec, decimals=6)
u_truth[id_offset:id_offset+n_stars] = np.round(umag, decimals=4)
g_truth[id_offset:id_offset+n_stars] = np.round(gmag, decimals=4)
r_truth[id_offset:id_offset+n_stars] = np.round(rmag, decimals=4)
i_truth[id_offset:id_offset+n_stars] = np.round(imag, decimals=4)
z_truth[id_offset:id_offset+n_stars] = np.round(zmag, decimals=4)
y_truth[id_offset:id_offset+n_stars] = np.round(ymag, decimals=4)
cls.px_truth[id_offset:id_offset+n_stars] = np.round(px, decimals=4)
cls.pmra_truth[id_offset:id_offset+n_stars] = np.round(pmra, decimals=4)
cls.pmdec_truth[id_offset:id_offset+n_stars] = np.round(pmdec, decimals=4)
cls.vrad_truth[id_offset:id_offset+n_stars] = np.round(vrad, decimals=4)
cls.amp_truth[id_offset:id_offset+n_stars] = np.round(var_amp, decimals=4)
cls.period_truth[id_offset:id_offset+n_stars] = np.round(var_period, decimals=4)
cls.max_str_len = -1
for i_star in range(n_stars):
if var_amp[i_star] >= -0.1:
varParamStr = ('{"m":"alert_test", "p":{"amp":%.4f, "per": %.4f}}'
% (var_amp[i_star], var_period[i_star]))
else:
varParamStr = 'None'
if len(varParamStr) > cls.max_str_len:
cls.max_str_len = len(varParamStr)
htmid = findHtmid(ra[i_star], dec[i_star], 21)
query = ('''INSERT INTO stars VALUES(%d, %d, %.6f, %.6f,
%.4f, %.4f, %.4f, %.4f, %.4f, %.4f,
%.4f, %.4f, %.4f, %.4f, '%s')'''
% (i_star+id_offset+1, htmid, ra[i_star], dec[i_star],
umag[i_star], gmag[i_star], rmag[i_star],
imag[i_star], zmag[i_star], ymag[i_star],
px[i_star], pmra[i_star], pmdec[i_star],
vrad[i_star], varParamStr))
cursor.execute(query)
conn.commit()
conn.close()
cls.output_dir = tempfile.mkdtemp(dir=ROOT, prefix='alert_gen_output')
cls.mag0_truth_dict = {}
cls.mag0_truth_dict[0] = u_truth
cls.mag0_truth_dict[1] = g_truth
cls.mag0_truth_dict[2] = r_truth
cls.mag0_truth_dict[3] = i_truth
cls.mag0_truth_dict[4] = z_truth
cls.mag0_truth_dict[5] = y_truth
def setUpClass(cls):
print('setting up %s' % sims_clean_up.targets)
# These represent the dimmest magnitudes at which objects
# are considered visible in each of the LSST filters
# (taken from Table 2 of the overview paper)
cls.obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60)
cls.opsim_db = os.path.join(getPackageDir('sims_data'),
'OpSimData',
'opsimblitz1_1133_sqlite.db')
rng = np.random.RandomState(8123)
obs_gen = ObservationMetaDataGenerator(database=cls.opsim_db)
cls.obs_list = obs_gen.getObservationMetaData(night=(0, 2))
cls.obs_list = rng.choice(cls.obs_list, 10, replace=False)
fieldid_list = []
for obs in cls.obs_list:
fieldid_list.append(obs.OpsimMetaData['fieldID'])
# make sure we have selected observations such that the
# same field is revisited more than once
assert len(np.unique(fieldid_list)) < len(fieldid_list)
cls.input_dir = tempfile.mkdtemp(prefix='alertDataGen',
dir=ROOT)
cls.star_db_name = tempfile.mktemp(prefix='alertDataGen_star_db',
dir=cls.input_dir,
suffix='.db')
conn = sqlite3.connect(cls.star_db_name)
cursor = conn.cursor()
cursor.execute('''CREATE TABLE stars
(simobjid int, htmid int, ra real, dec real,
umag real, gmag real, rmag real,
imag real, zmag real, ymag real,
px real, pmra real, pmdec real,
vrad real, varParamStr text)''')
conn.commit()
n_stars = 10
cls.ra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.dec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
u_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
g_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
r_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
i_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
z_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
y_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.px_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.pmra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.pmdec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.vrad_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.amp_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
cls.period_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
id_offset = -n_stars
for obs in cls.obs_list:
id_offset += n_stars
ra_0 = obs.pointingRA
dec_0 = obs.pointingDec
rr = rng.random_sample(n_stars)
theta = rng.random_sample(n_stars)*2.0*np.pi
ra = ra_0 + rr*np.cos(theta)
dec = dec_0 + rr*np.sin(theta)
var_period = rng.random_sample(n_stars)*0.25
var_amp = rng.random_sample(n_stars)*1.0 + 0.01
subset = rng.randint(0, high=len(var_amp)-1, size=3)
var_amp[subset[:2]] = 0.0
var_amp[subset[-1]] = -1.0
umag = rng.random_sample(n_stars)*5.0 + 15.0
gmag = rng.random_sample(n_stars)*5.0 + 15.0
rmag = rng.random_sample(n_stars)*5.0 + 15.0
imag = rng.random_sample(n_stars)*5.0 + 15.0
zmag = rng.random_sample(n_stars)*5.0 + 15.0
ymag = rng.random_sample(n_stars)*5.0 + 15.0
px = rng.random_sample(n_stars)*0.1 # say it is arcsec
pmra = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr
pmdec = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr
vrad = rng.random_sample(n_stars)*600.0 - 300.0
subset = rng.randint(0, high=n_stars-1, size=3)
umag[subset] = 40.0
gmag[subset] = 40.0
rmag[subset] = 40.0
imag[subset] = 40.0
zmag[subset] = 40.0
ymag[subset] = 40.0
cls.ra_truth[id_offset:id_offset+n_stars] = np.round(ra, decimals=6)
cls.dec_truth[id_offset:id_offset+n_stars] = np.round(dec, decimals=6)
u_truth[id_offset:id_offset+n_stars] = np.round(umag, decimals=4)
g_truth[id_offset:id_offset+n_stars] = np.round(gmag, decimals=4)
r_truth[id_offset:id_offset+n_stars] = np.round(rmag, decimals=4)
i_truth[id_offset:id_offset+n_stars] = np.round(imag, decimals=4)
z_truth[id_offset:id_offset+n_stars] = np.round(zmag, decimals=4)
y_truth[id_offset:id_offset+n_stars] = np.round(ymag, decimals=4)
cls.px_truth[id_offset:id_offset+n_stars] = np.round(px, decimals=4)
cls.pmra_truth[id_offset:id_offset+n_stars] = np.round(pmra, decimals=4)
cls.pmdec_truth[id_offset:id_offset+n_stars] = np.round(pmdec, decimals=4)
cls.vrad_truth[id_offset:id_offset+n_stars] = np.round(vrad, decimals=4)
cls.amp_truth[id_offset:id_offset+n_stars] = np.round(var_amp, decimals=4)
cls.period_truth[id_offset:id_offset+n_stars] = np.round(var_period, decimals=4)
cls.max_str_len = -1
for i_star in range(n_stars):
if var_amp[i_star] >= -0.1:
varParamStr = ('{"m":"alert_test", "p":{"amp":%.4f, "per": %.4f}}'
% (var_amp[i_star], var_period[i_star]))
else:
varParamStr = 'None'
if len(varParamStr) > cls.max_str_len:
cls.max_str_len = len(varParamStr)
htmid = findHtmid(ra[i_star], dec[i_star], 21)
query = ('''INSERT INTO stars VALUES(%d, %d, %.6f, %.6f,
%.4f, %.4f, %.4f, %.4f, %.4f, %.4f,
%.4f, %.4f, %.4f, %.4f, '%s')'''
% (i_star+id_offset+1, htmid, ra[i_star], dec[i_star],
umag[i_star], gmag[i_star], rmag[i_star],
imag[i_star], zmag[i_star], ymag[i_star],
px[i_star], pmra[i_star], pmdec[i_star],
vrad[i_star], varParamStr))
cursor.execute(query)
conn.commit()
conn.close()
cls.output_dir = tempfile.mkdtemp(dir=ROOT, prefix='alert_gen_output')
cls.mag0_truth_dict = {}
cls.mag0_truth_dict[0] = u_truth
cls.mag0_truth_dict[1] = g_truth
cls.mag0_truth_dict[2] = r_truth
cls.mag0_truth_dict[3] = i_truth
cls.mag0_truth_dict[4] = z_truth
cls.mag0_truth_dict[5] = y_truth
per = duration / (i_file + 1)
predict = per * len(data_dir_file_names)
full_name = os.path.join(args.in_dir, file_name)
print('reading %s; elapsed %.2e; predict %.2e' %
(file_name, duration, predict))
with open(full_name, 'r') as input_file:
input_lines = input_file.readlines()[1:]
params_list = []
for line in input_lines:
params = line.strip().split(',')
if len(params) == 23:
htmid = findHtmid(float(params[21]),
float(params[22]),
max_level=htmid_level)
pv = [htmid, int(params[0])]
pv += [float(pp) for pp in params[1:6]]
pv += [float(params[11])]
pv += [params[20]]
pv += [float(params[21]), float(params[22])]
elif len(params) == 10:
htmid = findHtmid(float(params[8]),
float(params[9]),
max_level=htmid_level)
pv = [htmid, int(params[0])]
pv += [float(params[1]), float(params[2])]
obs_mag_i = obs_mag_i[valid]
bhm = bhm[valid]
galaxy_id = galaxy_id[valid]
ra = ra[valid]
dec = dec[valid]
for other_bp in bands:
sf_dict[other_bp] = sf_dict[other_bp][valid]
tau_dict[other_bp] = tau_dict[other_bp][valid]
interpolated_m_i = np.interp(redshift, z_grid, m_i_grid)
interpolated_mag_norm = np.interp(redshift, z_grid, mag_norm_grid)
mag_norm = interpolated_mag_norm + (obs_mag_i - interpolated_m_i)
seed_arr = rng.randint(1, high=10000000, size=len(redshift))
htmid = findHtmid(ra, dec, htmid_level)
varParamStr_format \
= ('{"m": "applyAgn", "p": {"seed": %d, '
'"agn_sf_u": %.3e, "agn_sf_g": %.3e, "agn_sf_r": %.3e, '
'"agn_sf_i": %.3e, "agn_sf_z": %.3e, "agn_sf_y": %.3e, '
'"agn_tau_u" : %.3e, "agn_tau_g" : %.3e, "agn_tau_r" : %.3e, '
'"agn_tau_i" : %.3e, "agn_tau_z" : %.3e, "agn_tau_y" : %.3e}}')
row_by_row = zip(galaxy_id, htmid, mag_norm, redshift, abs_mag_i,
ra, dec, seed_arr, sf_dict['u'], sf_dict['g'],
sf_dict['r'], sf_dict['i'], sf_dict['z'],
sf_dict['y'], tau_dict['u'], tau_dict['g'],
tau_dict['r'], tau_dict['i'], tau_dict['z'],
tau_dict['y'])
def get_htmid(self):
ra = np.degrees(self.column_by_name('raJ2000'))
dec = np.degrees(self.column_by_name('decJ2000'))
return findHtmid(ra, dec, max_level=_truth_trixel_level)
get_physical_characteristics.teff_dict[sed_name] = tt
get_physical_characteristics.metal_dict[sed_name] = mm
get_physical_characteristics.logg_dict[sed_name] = gg
return tt, mm, gg
if __name__ == "__main__":
trixel_dict = getAllTrixels(6)
hs1 = halfSpaceFromRaDec(0.0, 90.0, 91.3)
hs2 = halfSpaceFromRaDec(0.0, -90.0, 91.3)
for ra in range(0, 360, 20):
htmid = findHtmid(ra, 0.0, max_level=6)
tx = trixelFromHtmid(htmid)
assert hs1.contains_trixel(tx) != 'outside'
assert hs2.contains_trixel(tx) != 'outside'
valid_htmid = []
n_6 = 0
for htmid in trixel_dict:
if levelFromHtmid(htmid) != 6:
continue
n_6 += 1
tx = trixel_dict[htmid]
if hs1.contains_trixel(tx) != 'outside':
if hs2.contains_trixel(tx) != 'outside':
valid_htmid.append(htmid)
print(n_6, len(valid_htmid))
