def _run(self, simData, cols_present=False):
if cols_present:
# Column already present in data; assume it is correct and does not need recalculating.
return simData
if self.degrees:
coord_x, coord_y, coord_z = xyz_from_ra_dec(simData[self.raCol],
simData[self.decCol])
field_ids = self.tree.query_ball_point(list(zip(coord_x, coord_y, coord_z)),
xyz_angular_radius())
else:
# use _xyz private method (sending radians)
coord_x, coord_y, coord_z = _xyz_from_ra_dec(simData[self.raCol],
simData[self.decCol])
field_ids = self.tree.query_ball_point(list(zip(coord_x, coord_y, coord_z)),
xyz_angular_radius())
simData['opsimFieldId'] = np.array([ids[0] for ids in field_ids]) + 1
return simData
def do_photometry(chunk, obs_lock, obs_metadata_dict, star_lock,
star_data_dict, job_lock, job_dict, out_dir):
"""
-Take a chunk from the stellar database
-Calculate the light curves for all of the stars in the chunk
-Write the light curves to disk
-Save the metadata and static data to dicts to be output later
Parameters
----------
chunk -- a set of stars returned by sqlite3.fetchmany()
the columns in chunk are:
simobjid, hpid, sedFilename, magNorm, ebv, varParamStr,
parallax, ra, decl
Note: chunk must be from one hpid (healpix pixel ID)
obs_lock -- the lock corresponding to the observation metadata
for this healpixel
obs_metadata_dict -- the dict where observation metadata is stored
star_lock -- the lock corresponding to the stellar metadata for this
healpixel
star_data_dict -- the dict where stellar metadata (i.e. static data) is
stored
job_lock -- the lock used to prevent too many processes from entering
light curve generation at once
job_dict -- the dict keeping track of how many jobs are doing light curve
generation
out_dir -- directory where light curve files are to be stored
Returns
-------
None
"""
dummy_sed = sims_photUtils.Sed()
# find the lookup file associating healpixel with obsHistID;
# this should probably actually be passed into the method
data_dir = '/astro/store/pogo4/danielsf/desc_dc2_truth'
assert os.path.isdir(data_dir)
hpid_lookup_name = os.path.join(data_dir, 'hpid_to_obsHistID_lookup.h5')
assert os.path.isfile(hpid_lookup_name)
metadata_dict = {}
metadata_keys = ['obsHistID', 'ra', 'dec', 'rotTelPos', 'mjd', 'filter']
hpid = int(chunk[0][1])
with h5py.File(hpid_lookup_name, 'r') as in_file:
valid_obsid = in_file['%d' % hpid][()]
for k in metadata_keys:
metadata_dict[k] = in_file[k][()]
valid_obsid = np.sort(valid_obsid)
valid_dex = np.searchsorted(metadata_dict['obsHistID'], valid_obsid)
np.testing.assert_array_equal(metadata_dict['obsHistID'][valid_dex],
valid_obsid)
for k in metadata_dict.keys():
metadata_dict[k] = metadata_dict[k][valid_dex]
# generate delta_magnitude light curves
has_dmag = False
while not has_dmag:
# make sure no more than 5 processes are doing this at once
with job_lock:
if job_dict['running_dmag'] >= 5:
continue
else:
job_dict['running_dmag'] += 1
#print('running dmag %d' % job_dict['running_dmag'])
t_start = time.time()
var_gen = VariabilityGenerator(chunk)
dmag_raw = var_gen.applyVariability(var_gen.varParamStr,
expmjd=metadata_dict['mjd'])
dmag_raw = dmag_raw.transpose([1, 2, 0])
# transpose so the columns are (star, mjd, filter)
assert dmag_raw.shape == (len(chunk), len(metadata_dict['mjd']), 6)
dmag = np.zeros((len(chunk), len(metadata_dict['mjd'])), dtype=float)
for i_mjd in range(len(metadata_dict['mjd'])):
dmag[:, i_mjd] = dmag_raw[:, i_mjd, metadata_dict['filter'][i_mjd]]
del dmag_raw
has_dmag = True
with job_lock:
job_dict['running_dmag'] -= 1
#print('running dmag %d' % job_dict['running_dmag'])
quiescent_fluxes = np.zeros((len(chunk), 6), dtype=float)
for i_bp, bp in enumerate('ugrizy'):
quiescent_fluxes[:, i_bp] = dummy_sed.fluxFromMag(
var_gen.column_by_name('quiescent_%s' % bp))
t_dmag = time.time() - t_start
star_ra = np.array([c[7] for c in chunk])
star_dec = np.array([c[8] for c in chunk])
# Now figure out which obsHistID are observing which stars
obs_mask = np.zeros((len(chunk), len(metadata_dict['mjd'])), dtype=bool)
fov_radius = 2.1 # in degrees
t_start = time.time()
for i_obs in range(len(metadata_dict['mjd'])):
ra = np.degrees(metadata_dict['ra'][i_obs])
dec = np.degrees(metadata_dict['dec'][i_obs])
## back-of-the-envelope implementation
## (does not use focal plane model with chip gaps, etc.)
#ang_dist = angularSeparation(ra, dec, star_ra, star_dec)
#valid = ang_dist<fov_radius
rotTel = np.degrees(metadata_dict['rotTelPos'][i_obs])
obs_md = ObservationMetaData(pointingRA=ra,
pointingDec=dec,
mjd=metadata_dict['mjd'][i_obs])
rotSky = getRotSkyPos(ra, dec, obs_md, rotTel)
obs_md.rotSkyPos = rotSky
chip_name = chipNameFromRaDecLSST(star_ra,
star_dec,
obs_metadata=obs_md).astype(str)
valid = np.char.find(chip_name, 'None') < 0
obs_mask[:, i_obs] = valid
# verify that any stars with empty light curves are, in fact
# outside of DC2
region_of_interest = DC2_bounds()
xyz_offenders = []
for i_star in range(len(chunk)):
if obs_mask[i_star].sum() == 0:
xyz_offenders.append(
xyz_from_ra_dec(star_ra[i_star], star_dec[i_star]))
if len(xyz_offenders) > 0:
xyz_offenders = np.array(xyz_offenders)
v0 = region_of_interest.hs_list[0].contains_many_pts(xyz_offenders)
v1 = region_of_interest.hs_list[1].contains_many_pts(xyz_offenders)
v2 = region_of_interest.hs_list[2].contains_many_pts(xyz_offenders)
v3 = region_of_interest.hs_list[3].contains_many_pts(xyz_offenders)
if (v0 & v1 & v2 & v3).sum() > 0:
msg = "\n\nsome stars in healpixel %d unobserved\n\n" % hpid
raise RuntimeError(msg)
t_flux = time.time()
dflux_arr = []
for i_star in range(len(chunk)):
star_filter = metadata_dict['filter'][obs_mask[i_star]]
q_flux = quiescent_fluxes[i_star][star_filter]
assert len(q_flux) == obs_mask[i_star].sum()
q_mag = dummy_sed.magFromFlux(q_flux)
tot_mag = q_mag + dmag[i_star, obs_mask[i_star]]
tot_flux = dummy_sed.fluxFromMag(tot_mag)
dflux = tot_flux - q_flux
dflux_arr.append(dflux)
assert len(dflux) == obs_mask[i_star].sum()
# store metadata in the output dicts
with obs_lock:
obs_metadata_dict['mjd'].append(metadata_dict['mjd'])
obs_metadata_dict['obsHistID'].append(metadata_dict['obsHistID'])
obs_metadata_dict['filter'].append(metadata_dict['filter'])
with star_lock:
local_simobjid = var_gen.column_by_name('simobjid')
out_file_name = create_out_name(out_dir, hpid)
# write the light curves to disk now; otherwise, the memory
# footprint of the job becomes too large
with h5py.File(out_file_name, 'a') as out_file:
for i_star in range(len(local_simobjid)):
if len(dflux_arr[i_star]) == 0:
continue
simobjid = local_simobjid[i_star]
out_file.create_dataset('%d_flux' % simobjid,
data=dflux_arr[i_star])
out_file.create_dataset(
'%d_obsHistID' % simobjid,
data=metadata_dict['obsHistID'][obs_mask[i_star]])
star_data_dict['simobjid'].append(local_simobjid)
star_data_dict['ra'].append(star_ra)
star_data_dict['dec'].append(star_dec)
for i_bp, bp in enumerate('ugrizy'):
star_data_dict['quiescent_%s' % bp].append(quiescent_fluxes[:,
i_bp])
def make_lookup(chunk, hs_list, my_lock, output_dict, mgr):
"""
chunk will be the result of sqlite3.fetchmany() on the OpSim database;
each row will be (obsHistID, descDitheredRA, descDitheredDec,
descDitheredRotTelPos, expMJD, filter)
hs_list is a list of HalfSpaces defining DC2
output_dict will need to be initialized with the valid values of htmid
"""
print('looking up')
bp_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
radius_deg = 2.1
radius_rad = np.radians(radius_deg)
local_lookup = {}
local_dex = {}
obsHistID_arr = []
mjd_arr = []
ra_arr = []
dec_arr = []
rotTelPos_arr = []
filter_arr = []
for pointing in chunk:
obs_id = int(pointing[0])
ra = float(pointing[1])
dec = float(pointing[2])
rotTelPos = float(pointing[3])
mjd = float(pointing[4])
bp = bp_to_int[pointing[5]]
obs_xyz = xyz_from_ra_dec(np.degrees(ra), np.degrees(dec))
obs_hs = htmModule.halfSpaceFromRaDec(np.degrees(ra), np.degrees(dec),
radius_deg)
is_valid_pointing = True
for hs in hs_list:
if not hs.intersects_circle(obs_hs.vector, obs_hs.phi):
is_valid_pointing = False
break
if not is_valid_pointing:
continue
logged_metadata = False
healpixel_list = healpy.query_disc(32,
obs_xyz,
radius_rad,
nest=False,
inclusive=True)
for hpid in healpixel_list:
if hpid not in local_lookup:
local_lookup[hpid] = -1 * np.ones(len(chunk), dtype=int)
local_dex[hpid] = 0
local_lookup[hpid][local_dex[hpid]] = obs_id
local_dex[hpid] += 1
if not logged_metadata:
obsHistID_arr.append(obs_id)
ra_arr.append(ra)
dec_arr.append(dec)
mjd_arr.append(mjd)
rotTelPos_arr.append(rotTelPos)
filter_arr.append(bp)
logged_metadata = True
obsHistID_arr = np.array(obsHistID_arr)
ra_arr = np.array(ra_arr)
dec_arr = np.array(dec_arr)
mjd_arr = np.array(mjd_arr)
rotTelPos_arr = np.array(rotTelPos_arr)
filter_arr = np.array(filter_arr)
with my_lock:
for hpid in local_lookup:
valid = np.where(local_lookup[hpid] >= 0)
if hpid not in output_dict:
output_dict[hpid] = mgr.list()
output_dict[hpid].append(local_lookup[hpid][valid])
if len(output_dict['obsHistID']) > 0:
proto_obs = np.concatenate(output_dict['obsHistID'])
valid = ~np.isin(obsHistID_arr, proto_obs)
else:
valid = np.array([True] * len(obsHistID_arr))
output_dict['obsHistID'].append(obsHistID_arr[valid])
output_dict['ra'].append(ra_arr[valid])
output_dict['dec'].append(dec_arr[valid])
output_dict['mjd'].append(mjd_arr[valid])
output_dict['rotTelPos'].append(rotTelPos_arr[valid])
output_dict['filter'].append(filter_arr[valid])
print('leaving')
def test_four_corners(self):
"""
Test field of view centered at a corner between four tiles
(not on the equator)
"""
ra_obs = 44.0
dec_obs = 34.0
radius = 2.8 # this should be enough to get galaxies to appear
# in more than one tile
# construct bounding half spaces for the four quadrants
quadrant_half_spaces = []
quadrant_centers = [] # prime tile coords of tile corners
for ra_q, dec_q in zip([46.0, 42.0, 42.0, 46.0],
[36.0, 36.0, 32.0, 32.0]):
cos_ra = np.cos(np.radians(ra_q))
sin_ra = np.sin(np.radians(ra_q))
cos_dec = np.cos(np.radians(dec_q))
sin_dec = np.sin(np.radians(dec_q))
m_ra = np.array([[cos_ra, sin_ra, 0.0], [-sin_ra, cos_ra, 0.0],
[0.0, 0.0, 1.0]])
m_dec = np.array([[cos_dec, 0.0, sin_dec], [0.0, 1.0, 0.0],
[-sin_dec, 0.0, cos_dec]])
upper_hs_vv = np.array([0.0, 0.0, -1.0])
upper_hs_vv = np.dot(m_dec, np.dot(m_ra, upper_hs_vv))
rr, dd = ra_dec_from_xyz(upper_hs_vv[0], upper_hs_vv[1],
upper_hs_vv[2])
upper_hs = htm.halfSpaceFromRaDec(rr, dd, 90.0 + dec_q + 2.0)
lower_hs_vv = np.array([0.0, 0.0, 1.0])
lower_hs_vv = np.dot(m_dec, np.dot(m_ra, lower_hs_vv))
rr, dd = ra_dec_from_xyz(lower_hs_vv[0], lower_hs_vv[1],
lower_hs_vv[2])
lower_hs = htm.halfSpaceFromRaDec(rr, dd, 92.0 - dec_q)
left_hs_vv = xyz_from_ra_dec(ra_q + 88.0, 0.0)
left_hs_vv = np.dot(m_dec, np.dot(m_ra, left_hs_vv))
rr, dd = ra_dec_from_xyz(left_hs_vv[0], left_hs_vv[1],
left_hs_vv[2])
left_hs = htm.halfSpaceFromRaDec(rr, dd, 90.0)
right_hs_vv = xyz_from_ra_dec(ra_q - 88.0, 0.0)
right_hs_vv = np.dot(m_dec, np.dot(m_ra, right_hs_vv))
rr, dd = ra_dec_from_xyz(right_hs_vv[0], right_hs_vv[1],
right_hs_vv[2])
right_hs = htm.halfSpaceFromRaDec(rr, dd, 90.0)
quadrant_half_spaces.append(
(upper_hs, lower_hs, left_hs, right_hs))
corner = xyz_from_ra_dec(44.0, 34.0)
rot_corner = np.dot(m_dec, np.dot(m_ra, corner))
quadrant_centers.append(
ra_dec_from_xyz(rot_corner[0], rot_corner[1], rot_corner[2]))
gal_tag_1st_quad = set()
gal_tag_2nd_quad = set()
gal_tag_3rd_quad = set()
gal_tag_4th_quad = set()
n_multiple_quad = 0
with sqlite3.connect(self._temp_gal_db) as db_conn:
c = db_conn.cursor()
query = "SELECT ra, dec, galtag FROM galaxy "
results = c.execute(query).fetchall()
for gal in results:
n_quad = 0
vv = xyz_from_ra_dec(gal[0], gal[1])
dd = list([
angularSeparation(c[0], c[1], gal[0], gal[1])
for c in quadrant_centers
])
if gal[2] == 6785:
for hs in quadrant_half_spaces[3]:
print('6785 contained ', hs.contains_pt(vv))
if (dd[0] <= radius
and quadrant_half_spaces[0][0].contains_pt(vv)
and quadrant_half_spaces[0][1].contains_pt(vv)
and quadrant_half_spaces[0][2].contains_pt(vv)
and quadrant_half_spaces[0][3].contains_pt(vv)):
n_quad += 1
gal_tag_1st_quad.add(gal[2])
if (dd[1] <= radius
and quadrant_half_spaces[1][0].contains_pt(vv)
and quadrant_half_spaces[1][1].contains_pt(vv)
and quadrant_half_spaces[1][2].contains_pt(vv)
and quadrant_half_spaces[1][3].contains_pt(vv)):
n_quad += 1
gal_tag_2nd_quad.add(gal[2])
if (dd[2] <= radius
and quadrant_half_spaces[2][0].contains_pt(vv)
and quadrant_half_spaces[2][1].contains_pt(vv)
and quadrant_half_spaces[2][2].contains_pt(vv)
and quadrant_half_spaces[2][3].contains_pt(vv)):
n_quad += 1
gal_tag_3rd_quad.add(gal[2])
if (dd[3] <= radius
and quadrant_half_spaces[3][0].contains_pt(vv)
and quadrant_half_spaces[3][1].contains_pt(vv)
and quadrant_half_spaces[3][2].contains_pt(vv)
and quadrant_half_spaces[3][3].contains_pt(vv)):
n_quad += 1
gal_tag_4th_quad.add(gal[2])
if n_quad > 1:
n_multiple_quad += 1
self.assertGreater(n_multiple_quad, 100)
gal_found_1st_quad = set()
gal_found_2nd_quad = set()
gal_found_3rd_quad = set()
ra_dec_3 = {}
gal_found_4th_quad = set()
ra_dec_4 = {}
obs = ObservationMetaData(pointingRA=ra_obs,
pointingDec=dec_obs,
boundType='circle',
boundLength=radius)
dbobj = UWGal.UWGalaxyTileObj(database=self._temp_gal_db,
driver='sqlite')
data_iter = dbobj.query_columns(['galtileid', 'ra', 'dec', 'galtag'],
obs_metadata=obs)
for chunk in data_iter:
for gal in chunk:
if gal['ra'] >= ra_obs and gal['dec'] >= dec_obs:
quad_set = gal_found_1st_quad
elif gal['ra'] < ra_obs and gal['dec'] >= dec_obs:
quad_set = gal_found_2nd_quad
elif gal['ra'] < ra_obs and gal['dec'] < dec_obs:
quad_set = gal_found_3rd_quad
ra_dec_3[gal['galtag']] = (gal['ra'], gal['dec'])
elif gal['ra'] >= ra_obs and gal['dec'] < dec_obs:
quad_set = gal_found_4th_quad
ra_dec_4[gal['galtag']] = (gal['ra'], gal['dec'])
else:
raise RuntimeError(
"Unsure what quadrant galaxy belongs in")
assert gal['galtag'] not in quad_set
quad_set.add(gal['galtag'])
test_sum = 0
control_sum = 0
for test, control, ra_dec in zip([
gal_found_1st_quad, gal_found_2nd_quad, gal_found_3rd_quad,
gal_found_4th_quad
], [
gal_tag_1st_quad, gal_tag_2nd_quad, gal_tag_3rd_quad,
gal_tag_4th_quad
], [None, None, ra_dec_3, ra_dec_4]):
n_erroneous_test = 0
for tag in test:
if tag not in control:
n_erroneous_test += 1
dd = angularSeparation(ra_obs, dec_obs, ra_dec[tag][0],
ra_dec[tag][1])
print(' bad test %d %e -- %.4f %.4f' %
(tag, dd, ra_dec[tag][0], ra_dec[tag][1]))
n_erroneous_control = 0
for tag in control:
if tag not in test:
n_erroneous_control += 1
print('n_test %d bad %d' % (len(test), n_erroneous_test))
print('n_control %d bad %d\n' %
(len(control), n_erroneous_control))
test_sum += len(test)
control_sum += len(control)
print('test_sum %d' % test_sum)
print('control_sum %d' % control_sum)
self.assertEqual(len(gal_found_1st_quad), len(gal_tag_1st_quad))
self.assertEqual(len(gal_found_2nd_quad), len(gal_tag_2nd_quad))
self.assertEqual(len(gal_found_3rd_quad), len(gal_tag_3rd_quad))
self.assertEqual(len(gal_found_4th_quad), len(gal_tag_4th_quad))
for tag in gal_found_1st_quad:
self.assertIn(tag, gal_tag_1st_quad)
for tag in gal_found_2nd_quad:
self.assertIn(tag, gal_tag_2nd_quad)
for tag in gal_found_3rd_quad:
self.assertIn(tag, gal_tag_3rd_quad)
for tag in gal_found_4th_quad:
self.assertIn(tag, gal_tag_4th_quad)