def _run(self, simData, cols_present=False): if cols_present: # Column already present in data; assume it is correct and does not need recalculating. return simData if self.degrees: coord_x, coord_y, coord_z = xyz_from_ra_dec(simData[self.raCol], simData[self.decCol]) field_ids = self.tree.query_ball_point(list(zip(coord_x, coord_y, coord_z)), xyz_angular_radius()) else: # use _xyz private method (sending radians) coord_x, coord_y, coord_z = _xyz_from_ra_dec(simData[self.raCol], simData[self.decCol]) field_ids = self.tree.query_ball_point(list(zip(coord_x, coord_y, coord_z)), xyz_angular_radius()) simData['opsimFieldId'] = np.array([ids[0] for ids in field_ids]) + 1 return simData
def do_photometry(chunk, obs_lock, obs_metadata_dict, star_lock, star_data_dict, job_lock, job_dict, out_dir): """ -Take a chunk from the stellar database -Calculate the light curves for all of the stars in the chunk -Write the light curves to disk -Save the metadata and static data to dicts to be output later Parameters ---------- chunk -- a set of stars returned by sqlite3.fetchmany() the columns in chunk are: simobjid, hpid, sedFilename, magNorm, ebv, varParamStr, parallax, ra, decl Note: chunk must be from one hpid (healpix pixel ID) obs_lock -- the lock corresponding to the observation metadata for this healpixel obs_metadata_dict -- the dict where observation metadata is stored star_lock -- the lock corresponding to the stellar metadata for this healpixel star_data_dict -- the dict where stellar metadata (i.e. static data) is stored job_lock -- the lock used to prevent too many processes from entering light curve generation at once job_dict -- the dict keeping track of how many jobs are doing light curve generation out_dir -- directory where light curve files are to be stored Returns ------- None """ dummy_sed = sims_photUtils.Sed() # find the lookup file associating healpixel with obsHistID; # this should probably actually be passed into the method data_dir = '/astro/store/pogo4/danielsf/desc_dc2_truth' assert os.path.isdir(data_dir) hpid_lookup_name = os.path.join(data_dir, 'hpid_to_obsHistID_lookup.h5') assert os.path.isfile(hpid_lookup_name) metadata_dict = {} metadata_keys = ['obsHistID', 'ra', 'dec', 'rotTelPos', 'mjd', 'filter'] hpid = int(chunk[0][1]) with h5py.File(hpid_lookup_name, 'r') as in_file: valid_obsid = in_file['%d' % hpid][()] for k in metadata_keys: metadata_dict[k] = in_file[k][()] valid_obsid = np.sort(valid_obsid) valid_dex = np.searchsorted(metadata_dict['obsHistID'], valid_obsid) np.testing.assert_array_equal(metadata_dict['obsHistID'][valid_dex], valid_obsid) for k in metadata_dict.keys(): metadata_dict[k] = metadata_dict[k][valid_dex] # generate delta_magnitude light curves has_dmag = False while not has_dmag: # make sure no more than 5 processes are doing this at once with job_lock: if job_dict['running_dmag'] >= 5: continue else: job_dict['running_dmag'] += 1 #print('running dmag %d' % job_dict['running_dmag']) t_start = time.time() var_gen = VariabilityGenerator(chunk) dmag_raw = var_gen.applyVariability(var_gen.varParamStr, expmjd=metadata_dict['mjd']) dmag_raw = dmag_raw.transpose([1, 2, 0]) # transpose so the columns are (star, mjd, filter) assert dmag_raw.shape == (len(chunk), len(metadata_dict['mjd']), 6) dmag = np.zeros((len(chunk), len(metadata_dict['mjd'])), dtype=float) for i_mjd in range(len(metadata_dict['mjd'])): dmag[:, i_mjd] = dmag_raw[:, i_mjd, metadata_dict['filter'][i_mjd]] del dmag_raw has_dmag = True with job_lock: job_dict['running_dmag'] -= 1 #print('running dmag %d' % job_dict['running_dmag']) quiescent_fluxes = np.zeros((len(chunk), 6), dtype=float) for i_bp, bp in enumerate('ugrizy'): quiescent_fluxes[:, i_bp] = dummy_sed.fluxFromMag( var_gen.column_by_name('quiescent_%s' % bp)) t_dmag = time.time() - t_start star_ra = np.array([c[7] for c in chunk]) star_dec = np.array([c[8] for c in chunk]) # Now figure out which obsHistID are observing which stars obs_mask = np.zeros((len(chunk), len(metadata_dict['mjd'])), dtype=bool) fov_radius = 2.1 # in degrees t_start = time.time() for i_obs in range(len(metadata_dict['mjd'])): ra = np.degrees(metadata_dict['ra'][i_obs]) dec = np.degrees(metadata_dict['dec'][i_obs]) ## back-of-the-envelope implementation ## (does not use focal plane model with chip gaps, etc.) #ang_dist = angularSeparation(ra, dec, star_ra, star_dec) #valid = ang_dist<fov_radius rotTel = np.degrees(metadata_dict['rotTelPos'][i_obs]) obs_md = ObservationMetaData(pointingRA=ra, pointingDec=dec, mjd=metadata_dict['mjd'][i_obs]) rotSky = getRotSkyPos(ra, dec, obs_md, rotTel) obs_md.rotSkyPos = rotSky chip_name = chipNameFromRaDecLSST(star_ra, star_dec, obs_metadata=obs_md).astype(str) valid = np.char.find(chip_name, 'None') < 0 obs_mask[:, i_obs] = valid # verify that any stars with empty light curves are, in fact # outside of DC2 region_of_interest = DC2_bounds() xyz_offenders = [] for i_star in range(len(chunk)): if obs_mask[i_star].sum() == 0: xyz_offenders.append( xyz_from_ra_dec(star_ra[i_star], star_dec[i_star])) if len(xyz_offenders) > 0: xyz_offenders = np.array(xyz_offenders) v0 = region_of_interest.hs_list[0].contains_many_pts(xyz_offenders) v1 = region_of_interest.hs_list[1].contains_many_pts(xyz_offenders) v2 = region_of_interest.hs_list[2].contains_many_pts(xyz_offenders) v3 = region_of_interest.hs_list[3].contains_many_pts(xyz_offenders) if (v0 & v1 & v2 & v3).sum() > 0: msg = "\n\nsome stars in healpixel %d unobserved\n\n" % hpid raise RuntimeError(msg) t_flux = time.time() dflux_arr = [] for i_star in range(len(chunk)): star_filter = metadata_dict['filter'][obs_mask[i_star]] q_flux = quiescent_fluxes[i_star][star_filter] assert len(q_flux) == obs_mask[i_star].sum() q_mag = dummy_sed.magFromFlux(q_flux) tot_mag = q_mag + dmag[i_star, obs_mask[i_star]] tot_flux = dummy_sed.fluxFromMag(tot_mag) dflux = tot_flux - q_flux dflux_arr.append(dflux) assert len(dflux) == obs_mask[i_star].sum() # store metadata in the output dicts with obs_lock: obs_metadata_dict['mjd'].append(metadata_dict['mjd']) obs_metadata_dict['obsHistID'].append(metadata_dict['obsHistID']) obs_metadata_dict['filter'].append(metadata_dict['filter']) with star_lock: local_simobjid = var_gen.column_by_name('simobjid') out_file_name = create_out_name(out_dir, hpid) # write the light curves to disk now; otherwise, the memory # footprint of the job becomes too large with h5py.File(out_file_name, 'a') as out_file: for i_star in range(len(local_simobjid)): if len(dflux_arr[i_star]) == 0: continue simobjid = local_simobjid[i_star] out_file.create_dataset('%d_flux' % simobjid, data=dflux_arr[i_star]) out_file.create_dataset( '%d_obsHistID' % simobjid, data=metadata_dict['obsHistID'][obs_mask[i_star]]) star_data_dict['simobjid'].append(local_simobjid) star_data_dict['ra'].append(star_ra) star_data_dict['dec'].append(star_dec) for i_bp, bp in enumerate('ugrizy'): star_data_dict['quiescent_%s' % bp].append(quiescent_fluxes[:, i_bp])
def make_lookup(chunk, hs_list, my_lock, output_dict, mgr): """ chunk will be the result of sqlite3.fetchmany() on the OpSim database; each row will be (obsHistID, descDitheredRA, descDitheredDec, descDitheredRotTelPos, expMJD, filter) hs_list is a list of HalfSpaces defining DC2 output_dict will need to be initialized with the valid values of htmid """ print('looking up') bp_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5} radius_deg = 2.1 radius_rad = np.radians(radius_deg) local_lookup = {} local_dex = {} obsHistID_arr = [] mjd_arr = [] ra_arr = [] dec_arr = [] rotTelPos_arr = [] filter_arr = [] for pointing in chunk: obs_id = int(pointing[0]) ra = float(pointing[1]) dec = float(pointing[2]) rotTelPos = float(pointing[3]) mjd = float(pointing[4]) bp = bp_to_int[pointing[5]] obs_xyz = xyz_from_ra_dec(np.degrees(ra), np.degrees(dec)) obs_hs = htmModule.halfSpaceFromRaDec(np.degrees(ra), np.degrees(dec), radius_deg) is_valid_pointing = True for hs in hs_list: if not hs.intersects_circle(obs_hs.vector, obs_hs.phi): is_valid_pointing = False break if not is_valid_pointing: continue logged_metadata = False healpixel_list = healpy.query_disc(32, obs_xyz, radius_rad, nest=False, inclusive=True) for hpid in healpixel_list: if hpid not in local_lookup: local_lookup[hpid] = -1 * np.ones(len(chunk), dtype=int) local_dex[hpid] = 0 local_lookup[hpid][local_dex[hpid]] = obs_id local_dex[hpid] += 1 if not logged_metadata: obsHistID_arr.append(obs_id) ra_arr.append(ra) dec_arr.append(dec) mjd_arr.append(mjd) rotTelPos_arr.append(rotTelPos) filter_arr.append(bp) logged_metadata = True obsHistID_arr = np.array(obsHistID_arr) ra_arr = np.array(ra_arr) dec_arr = np.array(dec_arr) mjd_arr = np.array(mjd_arr) rotTelPos_arr = np.array(rotTelPos_arr) filter_arr = np.array(filter_arr) with my_lock: for hpid in local_lookup: valid = np.where(local_lookup[hpid] >= 0) if hpid not in output_dict: output_dict[hpid] = mgr.list() output_dict[hpid].append(local_lookup[hpid][valid]) if len(output_dict['obsHistID']) > 0: proto_obs = np.concatenate(output_dict['obsHistID']) valid = ~np.isin(obsHistID_arr, proto_obs) else: valid = np.array([True] * len(obsHistID_arr)) output_dict['obsHistID'].append(obsHistID_arr[valid]) output_dict['ra'].append(ra_arr[valid]) output_dict['dec'].append(dec_arr[valid]) output_dict['mjd'].append(mjd_arr[valid]) output_dict['rotTelPos'].append(rotTelPos_arr[valid]) output_dict['filter'].append(filter_arr[valid]) print('leaving')
def test_four_corners(self): """ Test field of view centered at a corner between four tiles (not on the equator) """ ra_obs = 44.0 dec_obs = 34.0 radius = 2.8 # this should be enough to get galaxies to appear # in more than one tile # construct bounding half spaces for the four quadrants quadrant_half_spaces = [] quadrant_centers = [] # prime tile coords of tile corners for ra_q, dec_q in zip([46.0, 42.0, 42.0, 46.0], [36.0, 36.0, 32.0, 32.0]): cos_ra = np.cos(np.radians(ra_q)) sin_ra = np.sin(np.radians(ra_q)) cos_dec = np.cos(np.radians(dec_q)) sin_dec = np.sin(np.radians(dec_q)) m_ra = np.array([[cos_ra, sin_ra, 0.0], [-sin_ra, cos_ra, 0.0], [0.0, 0.0, 1.0]]) m_dec = np.array([[cos_dec, 0.0, sin_dec], [0.0, 1.0, 0.0], [-sin_dec, 0.0, cos_dec]]) upper_hs_vv = np.array([0.0, 0.0, -1.0]) upper_hs_vv = np.dot(m_dec, np.dot(m_ra, upper_hs_vv)) rr, dd = ra_dec_from_xyz(upper_hs_vv[0], upper_hs_vv[1], upper_hs_vv[2]) upper_hs = htm.halfSpaceFromRaDec(rr, dd, 90.0 + dec_q + 2.0) lower_hs_vv = np.array([0.0, 0.0, 1.0]) lower_hs_vv = np.dot(m_dec, np.dot(m_ra, lower_hs_vv)) rr, dd = ra_dec_from_xyz(lower_hs_vv[0], lower_hs_vv[1], lower_hs_vv[2]) lower_hs = htm.halfSpaceFromRaDec(rr, dd, 92.0 - dec_q) left_hs_vv = xyz_from_ra_dec(ra_q + 88.0, 0.0) left_hs_vv = np.dot(m_dec, np.dot(m_ra, left_hs_vv)) rr, dd = ra_dec_from_xyz(left_hs_vv[0], left_hs_vv[1], left_hs_vv[2]) left_hs = htm.halfSpaceFromRaDec(rr, dd, 90.0) right_hs_vv = xyz_from_ra_dec(ra_q - 88.0, 0.0) right_hs_vv = np.dot(m_dec, np.dot(m_ra, right_hs_vv)) rr, dd = ra_dec_from_xyz(right_hs_vv[0], right_hs_vv[1], right_hs_vv[2]) right_hs = htm.halfSpaceFromRaDec(rr, dd, 90.0) quadrant_half_spaces.append( (upper_hs, lower_hs, left_hs, right_hs)) corner = xyz_from_ra_dec(44.0, 34.0) rot_corner = np.dot(m_dec, np.dot(m_ra, corner)) quadrant_centers.append( ra_dec_from_xyz(rot_corner[0], rot_corner[1], rot_corner[2])) gal_tag_1st_quad = set() gal_tag_2nd_quad = set() gal_tag_3rd_quad = set() gal_tag_4th_quad = set() n_multiple_quad = 0 with sqlite3.connect(self._temp_gal_db) as db_conn: c = db_conn.cursor() query = "SELECT ra, dec, galtag FROM galaxy " results = c.execute(query).fetchall() for gal in results: n_quad = 0 vv = xyz_from_ra_dec(gal[0], gal[1]) dd = list([ angularSeparation(c[0], c[1], gal[0], gal[1]) for c in quadrant_centers ]) if gal[2] == 6785: for hs in quadrant_half_spaces[3]: print('6785 contained ', hs.contains_pt(vv)) if (dd[0] <= radius and quadrant_half_spaces[0][0].contains_pt(vv) and quadrant_half_spaces[0][1].contains_pt(vv) and quadrant_half_spaces[0][2].contains_pt(vv) and quadrant_half_spaces[0][3].contains_pt(vv)): n_quad += 1 gal_tag_1st_quad.add(gal[2]) if (dd[1] <= radius and quadrant_half_spaces[1][0].contains_pt(vv) and quadrant_half_spaces[1][1].contains_pt(vv) and quadrant_half_spaces[1][2].contains_pt(vv) and quadrant_half_spaces[1][3].contains_pt(vv)): n_quad += 1 gal_tag_2nd_quad.add(gal[2]) if (dd[2] <= radius and quadrant_half_spaces[2][0].contains_pt(vv) and quadrant_half_spaces[2][1].contains_pt(vv) and quadrant_half_spaces[2][2].contains_pt(vv) and quadrant_half_spaces[2][3].contains_pt(vv)): n_quad += 1 gal_tag_3rd_quad.add(gal[2]) if (dd[3] <= radius and quadrant_half_spaces[3][0].contains_pt(vv) and quadrant_half_spaces[3][1].contains_pt(vv) and quadrant_half_spaces[3][2].contains_pt(vv) and quadrant_half_spaces[3][3].contains_pt(vv)): n_quad += 1 gal_tag_4th_quad.add(gal[2]) if n_quad > 1: n_multiple_quad += 1 self.assertGreater(n_multiple_quad, 100) gal_found_1st_quad = set() gal_found_2nd_quad = set() gal_found_3rd_quad = set() ra_dec_3 = {} gal_found_4th_quad = set() ra_dec_4 = {} obs = ObservationMetaData(pointingRA=ra_obs, pointingDec=dec_obs, boundType='circle', boundLength=radius) dbobj = UWGal.UWGalaxyTileObj(database=self._temp_gal_db, driver='sqlite') data_iter = dbobj.query_columns(['galtileid', 'ra', 'dec', 'galtag'], obs_metadata=obs) for chunk in data_iter: for gal in chunk: if gal['ra'] >= ra_obs and gal['dec'] >= dec_obs: quad_set = gal_found_1st_quad elif gal['ra'] < ra_obs and gal['dec'] >= dec_obs: quad_set = gal_found_2nd_quad elif gal['ra'] < ra_obs and gal['dec'] < dec_obs: quad_set = gal_found_3rd_quad ra_dec_3[gal['galtag']] = (gal['ra'], gal['dec']) elif gal['ra'] >= ra_obs and gal['dec'] < dec_obs: quad_set = gal_found_4th_quad ra_dec_4[gal['galtag']] = (gal['ra'], gal['dec']) else: raise RuntimeError( "Unsure what quadrant galaxy belongs in") assert gal['galtag'] not in quad_set quad_set.add(gal['galtag']) test_sum = 0 control_sum = 0 for test, control, ra_dec in zip([ gal_found_1st_quad, gal_found_2nd_quad, gal_found_3rd_quad, gal_found_4th_quad ], [ gal_tag_1st_quad, gal_tag_2nd_quad, gal_tag_3rd_quad, gal_tag_4th_quad ], [None, None, ra_dec_3, ra_dec_4]): n_erroneous_test = 0 for tag in test: if tag not in control: n_erroneous_test += 1 dd = angularSeparation(ra_obs, dec_obs, ra_dec[tag][0], ra_dec[tag][1]) print(' bad test %d %e -- %.4f %.4f' % (tag, dd, ra_dec[tag][0], ra_dec[tag][1])) n_erroneous_control = 0 for tag in control: if tag not in test: n_erroneous_control += 1 print('n_test %d bad %d' % (len(test), n_erroneous_test)) print('n_control %d bad %d\n' % (len(control), n_erroneous_control)) test_sum += len(test) control_sum += len(control) print('test_sum %d' % test_sum) print('control_sum %d' % control_sum) self.assertEqual(len(gal_found_1st_quad), len(gal_tag_1st_quad)) self.assertEqual(len(gal_found_2nd_quad), len(gal_tag_2nd_quad)) self.assertEqual(len(gal_found_3rd_quad), len(gal_tag_3rd_quad)) self.assertEqual(len(gal_found_4th_quad), len(gal_tag_4th_quad)) for tag in gal_found_1st_quad: self.assertIn(tag, gal_tag_1st_quad) for tag in gal_found_2nd_quad: self.assertIn(tag, gal_tag_2nd_quad) for tag in gal_found_3rd_quad: self.assertIn(tag, gal_tag_3rd_quad) for tag in gal_found_4th_quad: self.assertIn(tag, gal_tag_4th_quad)