def test_findAllTrixels_radius(self): """ Test the method that attempts to find all of the trixels inside a given half space by approximating the angular scale of the trixels and verifying that all returned trixels are within radius+angular scale of the center of the half space. """ level = 5 # approximate the linear angular scale (in degrees) # of a trixel grid using the fact that there are # 8*4**(level-1) trixels in the grid as per equation 2.5 of # # https://www.microsoft.com/en-us/research/wp-content/uploads/2005/09/tr-2005-123.pdf angular_scale = np.sqrt(4.0*np.pi*(180.0/np.pi)**2/(8.0*4.0**(level-1))) ra = 43.0 dec = 22.0 radius = 20.0 half_space = halfSpaceFromRaDec(ra, dec, radius) trixel_list = half_space.findAllTrixels(level) self.assertGreater(len(trixel_list), 2) # first, check that all of the returned trixels are # inside the HalfSpace good_htmid_list = [] for i_limit, limits in enumerate(trixel_list): # verify that the tuples have been sorted by # htmid_min if i_limit > 0: self.assertGreater(limits[0], trixel_list[i_limit-1][1]) for htmid in range(limits[0], limits[1]+1): test_trixel = trixelFromHtmid(htmid) ra_trix, dec_trix = test_trixel.get_center() good_htmid_list.append(htmid) self.assertNotEqual(half_space.contains_trixel(test_trixel), 'outside') # check that the returned trixels are within # radius+angular_scale of the center of the HalfSpace self.assertLess(angularSeparation(ra, dec, ra_trix, dec_trix), radius+angular_scale) # next, verify that all of the possible trixels that # were not returned are outside the HalfSpace for base_htmid in range(8, 16): htmid_0 = base_htmid << 2*(level-1) self.assertEqual(levelFromHtmid(htmid_0), level) for ii in range(2**(2*level-2)): htmid = htmid_0 + ii self.assertEqual(levelFromHtmid(htmid), level) if htmid not in good_htmid_list: test_trixel = trixelFromHtmid(htmid) self.assertEqual(half_space.contains_trixel(test_trixel), 'outside') ra_trix, dec_trix = test_trixel.get_center() self.assertGreater(angularSeparation(ra, dec, ra_trix, dec_trix), radius)
def _init_from_corners(self, box_corners): ra_range = [c[0] for c in box_corners] ra_min = min(ra_range) ra_max = max(ra_range) dec_range = [c[1] for c in box_corners] dec_min = min(dec_range) dec_max = max(dec_range) tol = 1.0e-10 for i_c1 in range(len(box_corners)): c1 = box_corners[i_c1] pt1 = cartesianFromSpherical(np.degrees(c1[0]), np.degrees(c1[1])) for i_c2 in range(i_c1+1, len(box_corners), 1): hs = None c2 = box_corners[i_c2] pt2 = cartesianFromSpherical(np.degrees(c2[0]), np.degrees(c2[1])) if np.abs(1.0-np.dot(pt1, pt2))<tol: continue dra = np.abs(c1[0]-c2[0]) ddec = np.abs(c1[1]-c2[1]) if dra<tol and ddec>tol: # The RAs of the two corners is identical, but the Decs are # different; this Half Space is defined by a Great Circle if np.abs(c1[0]-ra_min)<tol: inner_pt = (ra_min+0.001, dec_min+0.001) else: inner_pt = (ra_max-0.001, dec_min+0.001) hs = halfSpaceFromPoints(c1, c2, inner_pt) elif ddec<tol and dra>tol: # The Decs of the two corners is identical, bu the RAs are # different; this Half Space is defined by a line of constant # Dec and should be centered at one of the poles if np.abs(c1[1]-dec_min)<tol: hs = halfSpaceFromRaDec(0.0, 90.0, 90.0-dec_min) else: hs = halfSpaceFromRaDec(0.0, -90.0, 90.0+dec_max) else: continue if hs is None: raise RuntimeError("Somehow Half Space == None") self._hs_list.append(hs)
def test_HalfSpaceIntersection(self): # Test that the two roots of an intersection are the # correct angular distance from the centers of the # half spaces ra1 = 22.0 dec1 = 45.0 rad1 = 10.0 ra2 = 23.5 dec2 = 37.9 rad2 = 9.2 hs1 = halfSpaceFromRaDec(ra1, dec1, rad1) hs2 = halfSpaceFromRaDec(ra2, dec2, rad2) roots = intersectHalfSpaces(hs1, hs2) self.assertEqual(len(roots), 2) self.assertAlmostEqual(np.sqrt(np.sum(roots[0]**2)), 1.0, 10) self.assertAlmostEqual(np.sqrt(np.sum(roots[1]**2)), 1.0, 10) ra_r1, dec_r1 = np.degrees(sphericalFromCartesian(roots[0])) ra_r2, dec_r2 = np.degrees(sphericalFromCartesian(roots[1])) dd = angularSeparation(ra1, dec1, ra_r1, dec_r1) self.assertAlmostEqual(dd, rad1, 10) dd = angularSeparation(ra1, dec1, ra_r2, dec_r2) self.assertAlmostEqual(dd, rad1, 10) dd = angularSeparation(ra2, dec2, ra_r1, dec_r1) self.assertAlmostEqual(dd, rad2, 10) dd = angularSeparation(ra2, dec2, ra_r2, dec_r2) self.assertAlmostEqual(dd, rad2, 10) # test that two non-intersecting HalfSpaces return no roots hs1 = halfSpaceFromRaDec(0.0, 90.0, 1.0) hs2 = halfSpaceFromRaDec(20.0, -75.0, 5.0) roots = intersectHalfSpaces(hs1, hs2) self.assertEqual(len(roots), 0) # test that two half spaces that are inside each other # return no roots hs1 = halfSpaceFromRaDec(77.0, 10.0, 20.0) hs2 = halfSpaceFromRaDec(75.0, 8.0, 0.2) roots = intersectHalfSpaces(hs1, hs2) self.assertEqual(len(roots), 0) # test that two half spaces with identical centers # return no roots hs1 = halfSpaceFromRaDec(11.0, -23.0, 1.0) hs2 = halfSpaceFromRaDec(11.0, -23.0, 0.2) roots = intersectHalfSpaces(hs1, hs2) self.assertEqual(len(roots), 0) roots = intersectHalfSpaces(hs1, hs1) self.assertEqual(len(roots), 0)
def test_findAllTrixels_brute(self): """ Use the method trixel_intersects_half_space defined at the top of this script to verify that HalfSpace.findAllTrixels works """ level = 7 trixel_dict = getAllTrixels(level) all_htmid = [] for htmid in trixel_dict.keys(): if levelFromHtmid(htmid) == level: all_htmid.append(htmid) hspace = halfSpaceFromRaDec(36.0, 22.1, 2.0) # make sure that the two methods of determining if # a HalfSpace contains a trixel (HalfSpace.contains_trixel # and trixel_interects_half_space) agree for htmid in all_htmid: trix = trixel_dict[htmid] msg = 'offending htmid %d' % htmid if trixel_intersects_half_space(trix, hspace): self.assertNotEqual(hspace.contains_trixel(trix), 'outside', msg=msg) else: self.assertEqual(hspace.contains_trixel(trix), 'outside', msg=msg) trixel_limits = hspace.findAllTrixels(level) intersecting_htmid = set() # check that all of the trixels included in the limits # do, in fact, intersect or exist in the HalfSpace for lim in trixel_limits: for htmid in range(lim[0], lim[1] + 1): trix = trixel_dict[htmid] self.assertTrue(trixel_intersects_half_space(trix, hspace)) intersecting_htmid.add(htmid) # check that all of the trixels not included in the limits # are, in fact, outside of the HalfSpace self.assertLess(len(intersecting_htmid), len(all_htmid)) self.assertGreater(len(intersecting_htmid), 0) for htmid in all_htmid: if htmid in intersecting_htmid: continue trix = trixel_dict[htmid] self.assertFalse(trixel_intersects_half_space(trix, hspace))
def test_findAllTrixels_brute(self): """ Use the method trixel_intersects_half_space defined at the top of this script to verify that HalfSpace.findAllTrixels works """ level = 7 trixel_dict = getAllTrixels(level) all_htmid = [] for htmid in trixel_dict.keys(): if levelFromHtmid(htmid) == level: all_htmid.append(htmid) hspace = halfSpaceFromRaDec(36.0, 22.1, 2.0) # make sure that the two methods of determining if # a HalfSpace contains a trixel (HalfSpace.contains_trixel # and trixel_interects_half_space) agree for htmid in all_htmid: trix = trixel_dict[htmid] msg = 'offending htmid %d' % htmid if trixel_intersects_half_space(trix, hspace): self.assertNotEqual(hspace.contains_trixel(trix), 'outside', msg=msg) else: self.assertEqual(hspace.contains_trixel(trix), 'outside', msg=msg) trixel_limits = hspace.findAllTrixels(level) intersecting_htmid = set() # check that all of the trixels included in the limits # do, in fact, intersect or exist in the HalfSpace for lim in trixel_limits: for htmid in range(lim[0], lim[1]+1): trix = trixel_dict[htmid] self.assertTrue(trixel_intersects_half_space(trix, hspace)) intersecting_htmid.add(htmid) # check that all of the trixels not included in the limits # are, in fact, outside of the HalfSpace self.assertLess(len(intersecting_htmid), len(all_htmid)) self.assertGreater(len(intersecting_htmid), 0) for htmid in all_htmid: if htmid in intersecting_htmid: continue trix = trixel_dict[htmid] self.assertFalse(trixel_intersects_half_space(trix, hspace))
def _prefilter_galaxy_id(self, obs_metadata): """ Accept an ObservationMetaData characterizing the current pointing. Return a numpy array of galaxy_ids that are in the field of view and actually contain an AGN. """ half_space = halfSpaceFromRaDec(obs_metadata.pointingRA, obs_metadata.pointingDec, obs_metadata.boundLength) if (not hasattr(self, "_agn_query_results") or half_space != self._cached_half_space): self._do_agn_query(half_space) return np.sort(self._agn_query_results['galaxy_id'])
def __init__(self, dbobj, colnames, obs_metadata, chunk_size, constraint): """ Parameters ---------- dbobj -- a CatalogDBObject connected to the 'galaxies' table on the UW CatSim server colnames -- a list of the columns to query chunk_size -- size of chunks to return constraint -- a string specifying a SQL 'WHERE' clause """ self.arbitrarySQL = False self.dbobj = dbobj if 'ra' not in colnames: query_colnames = ['htmid', 'galid', 'ra', 'dec'] + colnames else: query_colnames = ['htmid', 'galid'] + colnames self._column_query = dbobj._get_column_query(query_colnames) self.chunk_size = chunk_size tile_idx_list = np.sort(self._find_tiles(obs_metadata)) self._trixel_search_level = 9 self.obs_metadata = obs_metadata total_trixel_bounds = [] self._00_bounds = [] self._rotate_to_sky = [] self._sky_tile = [] self._tile_idx = [] # construct a HalfSpace based on obs_metadata self.obs_hs = halfSpaceFromRaDec(obs_metadata.pointingRA, obs_metadata.pointingDec, obs_metadata.boundLength) obs_where_clause = "(" for tile_idx in tile_idx_list: rotate_to_00 = self.uwgalaxy_tiles.rotation_matrix(tile_idx) # find the bounds for trixels contained by the field of view # when rotated from the current tile to RA=Dec=0 new_vv = np.dot(rotate_to_00, self.obs_hs.vector) new_ra, new_dec = sphericalFromCartesian(new_vv) new_obs = ObservationMetaData(pointingRA=np.degrees(new_ra), pointingDec=np.degrees(new_dec), boundType='circle', boundLength=self.obs_metadata.boundLength) if obs_where_clause != "(": obs_where_clause += " OR (" else: obs_where_clause += "(" obs_where_clause += new_obs.bounds.to_SQL('ra', 'dec') obs_where_clause += ")" obs_hs_00 = HalfSpace(new_vv, self.obs_hs.dd) obs_hs_00_trixels = obs_hs_00.findAllTrixels(self._trixel_search_level) # find the trixels in the current tile when it is rotated # to RA=Dec=0 sky_tile = self.uwgalaxy_tiles.tile(tile_idx) single_tile = sky_tile.rotate(rotate_to_00) local_bounds = single_tile.find_all_trixels(self._trixel_search_level) local_bounds = HalfSpace.join_trixel_bound_sets(local_bounds, obs_hs_00_trixels) total_trixel_bounds += local_bounds self._sky_tile.append(sky_tile) self._00_bounds.append(local_bounds) self._rotate_to_sky.append(np.linalg.inv(rotate_to_00)) self._tile_idx.append(tile_idx) obs_where_clause += ")" total_trixel_bounds = HalfSpace.merge_trixel_bounds(total_trixel_bounds) where_clause = "(" for i_bound, bound in enumerate(total_trixel_bounds): if i_bound>0: where_clause += " OR " htmid_min = bound[0] << 2*(21-self._trixel_search_level) htmid_max = (bound[1]+1) << 2*(21-self._trixel_search_level) assert levelFromHtmid(htmid_min) == 21 assert levelFromHtmid(htmid_max) == 21 assert htmid_min<htmid_max where_clause += "(htmid>=%d AND htmid<=%d)" % (htmid_min, htmid_max) where_clause += ")" where_clause += " AND " where_clause += obs_where_clause if constraint is not None: where_clause += " AND (%s)" % text(constraint) query = self._column_query query = query.filter(text(where_clause)) query = query.order_by('redshift') self._galaxy_query = dbobj.connection.session.execute(query) self._tile_to_do = 0 self._has_J2000 = False if 'raJ2000' in colnames: self._has_J2000 = True self._valid_tiles = 0 self._n_chunks = 0 self._n_rows = 0 self._rows_kept = 0
def _postprocess_results(self, master_chunk, obs_metadata): """ query the database specified by agn_params_db to find the AGN varParamStr associated with each AGN """ if self.agn_params_db is None: return(master_chunk) if self.agn_objid is None: gid_name = 'galaxy_id' varpar_name = 'varParamStr' else: gid_name = self.agn_objid + '_' + 'galaxy_id' if not gid_name in master_chunk.dtype.names: gid_name = 'galaxy_id' varpar_name = self.agn_objid + '_' + 'varParamStr' if not varpar_name in master_chunk.dtype.names: varpar_name = 'varParamStr' magnorm_name = 'agnMagNorm' half_space = halfSpaceFromRaDec(obs_metadata.pointingRA, obs_metadata.pointingDec, obs_metadata.boundLength) if (not hasattr(self, "_agn_query_results") or half_space != self._cached_half_space): self._do_agn_query(half_space) gid_arr = master_chunk[gid_name] m_sorted_dex = np.argsort(gid_arr) m_sorted_id = gid_arr[m_sorted_dex] valid_agn_dex = np.where(np.logical_and(self._agn_query_results['galaxy_id']>=gid_arr.min(), self._agn_query_results['galaxy_id']<=gid_arr.max())) valid_agn = {} for k in self._agn_query_results: valid_agn[k] = self._agn_query_results[k][valid_agn_dex] # find the indices of the elements in master_chunk # that correspond to elements in agn_chunk m_elements = np.in1d(m_sorted_id, valid_agn['galaxy_id'], assume_unique=True) m_dex = m_sorted_dex[m_elements] # find the indices of the elements in agn_chunk # that correspond to elements in master_chunk a_dex = np.in1d(valid_agn['galaxy_id'], m_sorted_id, assume_unique=True) # make sure we have matched elements correctly np.testing.assert_array_equal(valid_agn['galaxy_id'][a_dex], master_chunk[gid_name][m_dex]) if varpar_name in master_chunk.dtype.names: master_chunk[varpar_name][m_dex] = valid_agn['varParamStr'][a_dex] if magnorm_name in master_chunk.dtype.names: master_chunk[magnorm_name][m_dex] = valid_agn['magNorm'][a_dex] return self._final_pass(master_chunk)
def get_pointing_htmid(pointing_dir, opsim_db_name, ra_colname='descDitheredRA', dec_colname='descDitheredDec', rottel_colname='descDitheredRotTelPos'): """ For a list of OpSim pointings, find dicts mapping those pointings to: - The trixels filling the pointings - The MJDs of the pointings - The telescope filters of the pointings Parameters ---------- pointing_dir contains a series of files that are two columns: obshistid, mjd. The files must each have 'visits' in their name. These specify the pointings for which we are assembling data. See: https://github.com/LSSTDESC/DC2_Repo/tree/master/data/Run1.1 for an example. opsim_db_name is the path to the OpSim database from which to take those pointings ra_colname is the column used for RA of the pointing (default: descDitheredRA) dec_colname is the column used for the Dec of the pointing (default: descDitheredDec) rottel_colname is the column used for the rotTelPos of the pointing (default: desckDitheredRotTelPos') Returns ------- htmid_bound_dict -- a dict keyed on ObsHistID. Values are the list of trixels filling the OpSim pointing, as returned by lsst.sims.utils.HalfSpace.findAllTrixels mjd_dict -- a dict keyed on ObsHistID. Values are the MJD(TAI) of the OpSim pointings. filter_dict -- a dict keyed on ObsHistID. Values are the 'ugrizy' filter of the OpSim pointings. obsmd_dict -- a dict keyed on ObsHistID. The values are ObservationMetaData with the RA, Dec, MJD, and rotSkyPos of the pointings (for use in focal plane geometry calculations) """ radius = 2.0 # field of view of a pointing in degrees if not os.path.isfile(opsim_db_name): raise RuntimeError("%s is not a valid file name" % opsim_db_name) if not os.path.isdir(pointing_dir): raise RuntimeError("%s is not a valid dir name" % pointing_dir) dtype = np.dtype([('obshistid', int), ('mjd', float)]) obs_data = None for file_name in os.listdir(pointing_dir): if 'visits' in file_name: full_name = os.path.join(pointing_dir, file_name) data = np.genfromtxt(full_name, dtype=dtype) if obs_data is None: obs_data = data['obshistid'] else: obs_data = np.concatenate((obs_data, data['obshistid']), axis=0) obs_data = np.sort(obs_data) db = DBObject(opsim_db_name, driver='sqlite') dtype = np.dtype([('obshistid', int), ('mjd', float), ('band', str, 1), ('ra', float), ('dec', float), ('rotTelPos', float)]) htmid_bound_dict = {} mjd_dict = {} filter_dict = {} obsmd_dict = {} d_obs = len(obs_data) // 5 for i_start in range(0, len(obs_data), d_obs): i_end = i_start + d_obs if len(obs_data) - i_start < d_obs: i_end = len(obs_data) subset = obs_data[i_start:i_end] query = 'SELECT obsHistId, expMJD, filter,' query += ' %s, %s, %s FROM Summary' % (ra_colname, dec_colname, rottel_colname) query += ' WHERE obsHistID BETWEEN %d and %e' % (subset.min(), subset.max()) query += ' GROUP BY obsHistID' results = db.execute_arbitrary(query, dtype=dtype) for ii in range(len(results)): obshistid = results['obshistid'][ii] if obshistid not in obs_data: continue hs = halfSpaceFromRaDec(np.degrees(results['ra'][ii]), np.degrees(results['dec'][ii]), radius) trixel_bounds = hs.findAllTrixels(_truth_trixel_level) htmid_bound_dict[obshistid] = trixel_bounds mjd_dict[obshistid] = results['mjd'][ii] filter_dict[obshistid] = results['band'][ii] obs_md = ObservationMetaData( pointingRA=np.degrees(results['ra'][ii]), pointingDec=np.degrees(results['dec'][ii]), mjd=results['mjd'][ii]) rotsky_rad = _getRotSkyPos(results['ra'][ii], results['dec'][ii], obs_md, results['rotTelPos'][ii]) obsmd_dict[obshistid] = ObservationMetaData( pointingRA=np.degrees(results['ra'][ii]), pointingDec=np.degrees(results['dec'][ii]), mjd=results['mjd'][ii], rotSkyPos=np.degrees(rotsky_rad)) assert len(obs_data) == len(htmid_bound_dict) return htmid_bound_dict, mjd_dict, filter_dict, obsmd_dict
raise RuntimeError('Do not know how to get ' 'physical characteristics for ' 'sub_dir %s' % sub_dir) get_physical_characteristics.teff_dict[sed_name] = tt get_physical_characteristics.metal_dict[sed_name] = mm get_physical_characteristics.logg_dict[sed_name] = gg return tt, mm, gg if __name__ == "__main__": trixel_dict = getAllTrixels(6) hs1 = halfSpaceFromRaDec(0.0, 90.0, 91.3) hs2 = halfSpaceFromRaDec(0.0, -90.0, 91.3) for ra in range(0, 360, 20): htmid = findHtmid(ra, 0.0, max_level=6) tx = trixelFromHtmid(htmid) assert hs1.contains_trixel(tx) != 'outside' assert hs2.contains_trixel(tx) != 'outside' valid_htmid = [] n_6 = 0 for htmid in trixel_dict: if levelFromHtmid(htmid) != 6: continue n_6 += 1 tx = trixel_dict[htmid] if hs1.contains_trixel(tx) != 'outside':
def test_findAllTrixels_radius(self): """ Test the method that attempts to find all of the trixels inside a given half space by approximating the angular scale of the trixels and verifying that all returned trixels are within radius+angular scale of the center of the half space. """ level = 5 # approximate the linear angular scale (in degrees) # of a trixel grid using the fact that there are # 8*4**(level-1) trixels in the grid as per equation 2.5 of # # https://www.microsoft.com/en-us/research/wp-content/uploads/2005/09/tr-2005-123.pdf angular_scale = np.sqrt(4.0 * np.pi * (180.0 / np.pi)**2 / (8.0 * 4.0**(level - 1))) ra = 43.0 dec = 22.0 radius = 20.0 half_space = halfSpaceFromRaDec(ra, dec, radius) trixel_list = half_space.findAllTrixels(level) self.assertGreater(len(trixel_list), 2) # first, check that all of the returned trixels are # inside the HalfSpace good_htmid_list = [] for i_limit, limits in enumerate(trixel_list): # verify that the tuples have been sorted by # htmid_min if i_limit > 0: self.assertGreater(limits[0], trixel_list[i_limit - 1][1]) for htmid in range(limits[0], limits[1] + 1): test_trixel = trixelFromHtmid(htmid) ra_trix, dec_trix = test_trixel.get_center() good_htmid_list.append(htmid) self.assertNotEqual(half_space.contains_trixel(test_trixel), 'outside') # check that the returned trixels are within # radius+angular_scale of the center of the HalfSpace self.assertLess(angularSeparation(ra, dec, ra_trix, dec_trix), radius + angular_scale) # next, verify that all of the possible trixels that # were not returned are outside the HalfSpace for base_htmid in range(8, 16): htmid_0 = base_htmid << 2 * (level - 1) self.assertEqual(levelFromHtmid(htmid_0), level) for ii in range(2**(2 * level - 2)): htmid = htmid_0 + ii self.assertEqual(levelFromHtmid(htmid), level) if htmid not in good_htmid_list: test_trixel = trixelFromHtmid(htmid) self.assertEqual(half_space.contains_trixel(test_trixel), 'outside') ra_trix, dec_trix = test_trixel.get_center() self.assertGreater( angularSeparation(ra, dec, ra_trix, dec_trix), radius)
npix = float(hp.nside2npix(nside)) # If the area has more than this number of objects, flag it as a max breakLimit = 1e6 chunk_size = 10000 for i in np.arange(indxMin, int(npix)): lastCP = '' # wonder what the units of boundLength are...degrees! And it's a radius # The newer interface: #obs_metadata = ObservationMetaData(boundType='circle', ## pointingRA=np.degrees(ra[i]), # pointingDec=np.degrees(dec[i]), # boundLength=boundLength, mjd=5700) #t = dbobj.getCatalog('ref_catalog_star', obs_metadata=obs_metadata) hs = halfSpaceFromRaDec(ra[i], dec[i], radius) current_level = 7 n_bits_off = 2*(21-current_level) tx_list = hs.findAllTrixels(current_level) # actually construct the query query = 'SELECT ra, dec, phot_g_mean_mag ' query += 'FROM gaia_2016 ' query += 'WHERE ' for i_pair, pair in enumerate(tx_list): min_tx = int(pair[0]<<n_bits_off) max_tx = int((pair[1]+1)<<n_bits_off) query += '(htmid>=%d AND htmid<=%d)' % (min_tx, max_tx) if i_pair<len(tx_list)-1: