def setUpClass(cls):
cls._tile_radius = angularSeparation(0.0, 0.0, 2.0, 2.0)
dir_name = os.path.join(getPackageDir('sims_catUtils'), 'tests')
cls._tmpdir = tempfile.mkdtemp(dir=dir_name)
cls._temp_gal_db = os.path.join(cls._tmpdir, 'test_galaxies.db')
ra_min = -2.25
cls._ra_min = ra_min
ra_max = 2.251
cls._d_ra = 0.05
ra_grid = np.arange(ra_min, ra_max, cls._d_ra)
dec_grid = np.arange(ra_min, ra_max, cls._d_ra)
print('raw grid %d' % len(ra_grid))
ra_dec_mesh = np.meshgrid(ra_grid, dec_grid)
ra_grid = ra_dec_mesh[0].flatten()
dec_grid = ra_dec_mesh[1].flatten()
# add a very small offset so that numerical precision
# does not foul things up on the tile boundaries
rng = np.random.RandomState(7163)
ra_grid += 1.0e-5 * (rng.random_sample(len(ra_grid)) - 0.5)
dec_grid += 1.0e-5 * (rng.random_sample(len(dec_grid)) - 0.5)
galtag = (100 * np.round(45 + ra_grid / cls._d_ra) +
np.round(45 + dec_grid / cls._d_ra)).astype(int)
assert len(galtag) == len(np.unique(galtag))
htmid_grid = htm.findHtmid(ra_grid, dec_grid, 21)
print('got htmid %d' % len(htmid_grid))
print(htm.levelFromHtmid(htmid_grid.min()))
print(htm.levelFromHtmid(htmid_grid.max()))
assert htm.levelFromHtmid(htmid_grid.min()) == 21
assert htm.levelFromHtmid(htmid_grid.max()) == 21
gid = np.arange(len(ra_grid), dtype=int)
assert len(galtag) == len(np.unique(galtag))
print(ra_grid.max(), ra_grid.min())
with sqlite3.connect(cls._temp_gal_db) as conn:
c = conn.cursor()
query = '''CREATE TABLE galaxy(htmid int, id int,
galid text, ra real, dec real, galtag int, redshift)'''
c.execute(query).fetchall()
values = ((int(hh), int(ii), str(ii), r, d, int(g), 0.5)
for hh, ii, r, d, g in zip(htmid_grid, gid, ra_grid,
dec_grid, galtag))
c.executemany('INSERT INTO galaxy VALUES (?,?,?,?,?,?,?)', values)
constraint = 'isvar=1 '
htmid_21_min = htmid_query<<2*(21-query_level)
htmid_21_max = (htmid_query+1)<<2*(21-query_level)
constraint += 'AND htmid>=%d AND htmid<%d' % (htmid_21_min, htmid_21_max)
data_iter = star_db.query_columns(col_names, obs_metadata=obs_query,
chunk_size=args.q_chunk_size,
constraint=constraint)
p_list = []
i_chunk = 0
to_concatenate = []
for chunk in data_iter:
htmid_found = htm.findHtmid(chunk['ra'],
chunk['decl'],
query_level)
valid = np.where(htmid_found==htmid_query)
if len(valid[0]) == 0:
continue
chunk = chunk[valid]
n_tot += len(chunk)
# multiprocessing code
if len(chunk)<args.p_chunk_size:
to_concatenate.append(chunk)
tot_sub = 0
for sub_chunk in to_concatenate:
tot_sub += len(sub_chunk)
zmag real, ymag real)'''
out_c.execute(creation_query)
out_conn.commit()
colnames = ['simobjid', 'ra', 'decl', 'gal_l', 'gal_b',
'magNorm', 'mura', 'mudecl', 'parallax', 'ebv',
'radialVelocity', 'varParamStr', 'sedFilename',
'umag', 'gmag', 'rmag', 'imag', 'zmag', 'ymag']
data_iter = in_db.query_columns(colnames=colnames, obs_metadata=dc2_obs,
chunk_size=100000)
for data_chunk in data_iter:
# spatial indexing to help with queries
htmid = htm.findHtmid(data_chunk['ra'], data_chunk['decl'], 6)
vals = ((int(d['simobjid']), int(hh),
d['ra'], d['decl'], d['gal_l'], d['gal_b'],
d['magNorm'], d['mura'], d['mudecl'], d['parallax'],
d['ebv'], d['radialVelocity'], d['varParamStr'],
d['sedFilename'], d['umag'], d['gmag'],
d['rmag'], d['imag'], d['zmag'], d['ymag'])
for d, hh in zip(data_chunk, htmid))
out_c.executemany('''INSERT INTO stars VALUES
(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', vals)
out_conn.commit()
out_c.execute('''CREATE INDEX htmid_6_idx ON stars (htmid_6)''')
out_conn.commit()