def match_stars_by_position(star_catalog, vphas_cat, log): """Function to cross-match stars by position. Returns: :param dict match_index: { Index in vphas_cat: index in star_cat } """ ddx = np.where(star_catalog['clean'] == 1.0)[0] det_stars = SkyCoord(star_catalog['RA'][ddx], star_catalog['DEC'][ddx], unit="deg") vdx = np.where(vphas_cat['clean'] == 1.0)[0] cat_stars = SkyCoord(vphas_cat['_RAJ2000'][vdx], vphas_cat['_DEJ2000'][vdx], unit="deg") tolerance = 2.0 * u.arcsec match_data = matching.search_around_sky(det_stars, cat_stars, seplimit=tolerance) idx = np.argsort(match_data[2].value) det_index = match_data[0][idx] cat_index = match_data[1][idx] match_index = np.array(zip(ddx[det_index], vdx[cat_index])) log.info('Matched ' + str(len(match_index))) return match_index
def calc_offsets_from_red(self): """Method to calculate the offset of the blue and green from the red, by matching stars between the star catalogs""" red_stars = SkyCoord(self.r_cat['RA'], self.r_cat['DEC'], unit="deg") for key, par in {'b_offset': 'b_cat', 'g_offset': 'g_cat'}.items(): f = par.split('_')[0] cat = getattr(self, par) cat_stars = SkyCoord(cat['RA'], cat['DEC'], unit="deg") match_data = matching.search_around_sky(red_stars, cat_stars, seplimit=self.tolerance) dx = self.r_cat['x'][match_data[0]] - cat['x'][match_data[1]] dy = self.r_cat['y'][match_data[0]] - cat['y'][match_data[1]] (hist_dx, binsx) = np.histogram(dx, bins=50) (hist_dy, binsy) = np.histogram(dy, bins=50) idx = np.where(hist_dx == max(hist_dx))[0][0] idy = np.where(hist_dy == max(hist_dy))[0][0] xmin = binsx[idx - 2] xmax = binsx[idx + 2] ymin = binsy[idy - 2] ymax = binsy[idy + 2] idx = np.where(dx >= xmin)[0] jdx = np.where(dx <= xmax)[0] kdx = list(set(idx).intersection(set(jdx))) deltax = np.median(dx[kdx]) + getattr(self, f + '_offset_x') idy = np.where(dy >= ymin)[0] jdy = np.where(dy <= ymax)[0] kdy = list(set(idy).intersection(set(jdy))) deltay = np.median(dy[kdy]) + getattr(self, f + '_offset_y') if par == 'b_cat': setattr(self, key, (-deltax, -deltay)) else: setattr(self, key, (deltax, deltay)) plot_offsets(par, dx, dy)
def xmatch_catalogs(catalog1, catalog2, log): """Function to cross-match objects between two catalogs in astropy.Table format. Based on code by Y. Tsapras. """ stars1 = SkyCoord(catalog1['RA_J2000'], catalog1['DEC_J2000'], unit=u.deg) stars2 = SkyCoord(catalog2['RA_J2000'], catalog2['DEC_J2000'], unit=u.deg) match_table = matching.search_around_sky(stars1, stars2, seplimit=0.5 * u.arcsec) matches1 = match_table[0].tolist() blends = {} blend_idx = set([x for x in matches1 if matches1.count(x) > 1]) for b in blend_idx: idx = np.where(match_table[0] == b)[0] for i in idx: c = match_table[1][i] if b not in blends.keys(): blends[b] = [ (catalog1['RA_J2000'][b], catalog1['DEC_J2000'][b]), (c, catalog2['RA_J2000'][c], catalog2['DEC_J2000'][c]) ] else: blends[b].append( (c, catalog2['RA_J2000'][c], catalog2['DEC_J2000'][c])) log.info('Match '+str(b)+' '+\ str(catalog1['RA_J2000'][b])+' '+str(catalog1['DEC_J2000'][b])+' -> '+\ str(c)+' '+str(catalog2['RA_J2000'][b])+' '+str(catalog2['DEC_J2000'][b])) return match_table, blends
def find_target_data(params,star_catalog): """Function to identify the photometry for a given target, if present in the star catalogue""" target = {} if params['target_ra'] != None: target_location = SkyCoord([params['target_ra']], [params['target_dec']], unit=(u.hourangle, u.deg)) stars = SkyCoord(star_catalog['RA'], star_catalog['DEC'], unit="deg") tolerance = 5.0 * u.arcsec match_data = matching.search_around_sky(target_location, stars, seplimit=tolerance) if len(match_data[0]) > 0: idx = np.argsort(match_data[2].value) print str(len(match_data[1]))+' matching objects in the catalog' target = {'star_index': star_catalog['star_index'][match_data[1][idx[0]]], 'x': star_catalog['x_pixel'][match_data[1][idx[0]]], 'y': star_catalog['y_pixel'][match_data[1][idx[0]]], 'ra': star_catalog['RA'][match_data[1][idx[0]]], 'dec': star_catalog['DEC'][match_data[1][idx[0]]], 'mag': star_catalog['mag'][match_data[1][idx[0]]], 'mag_err': star_catalog['mag_err'][match_data[1][idx[0]]]} print 'Target identified as '+repr(target) print 'List of matching objects: ' for i in range(len(match_data[1])): print str(star_catalog['star_index'][match_data[1][idx[i]]])+' '+\ str(star_catalog['x_pixel'][match_data[1][idx[i]]])+' '+\ str(star_catalog['y_pixel'][match_data[1][idx[i]]])+' '+\ str(star_catalog['RA'][match_data[1][idx[i]]])+' '+\ str(star_catalog['DEC'][match_data[1][idx[i]]]),match_data[2][idx[i]] return target
def cross_match_stars(f1,dataset1,f2,dataset2,log): """Function to cross-match stars by position. Returns: :param dict match_index: { Index in vphas_cat: index in star_cat } """ stars1 = SkyCoord(dataset1['RA'], dataset1['DEC'], unit="deg") stars2 = SkyCoord(dataset2['RA'], dataset2['DEC'], unit="deg") tolerance = 1.0 * u.arcsec match_data = matching.search_around_sky(stars1, stars2, seplimit=tolerance) idx = np.argsort(match_data[2].value) match_index = np.array(zip(match_data[0][idx],match_data[1][idx])) log.info('Matched '+str(len(match_index))+' stars between '+f1+' and '+f2) return match_index
def test_search_around(): from astropy.coordinates import ICRS, SkyCoord from astropy.coordinates.matching import search_around_sky, search_around_3d coo1 = ICRS([4, 2.1] * u.degree, [0, 0] * u.degree, distance=[1, 5] * u.kpc) coo2 = ICRS([1, 2, 3, 4] * u.degree, [0, 0, 0, 0] * u.degree, distance=[1, 1, 1, 5] * u.kpc) idx1_1deg, idx2_1deg, d2d_1deg, d3d_1deg = search_around_sky( coo1, coo2, 1.01 * u.deg) idx1_0p05deg, idx2_0p05deg, d2d_0p05deg, d3d_0p05deg = search_around_sky( coo1, coo2, 0.05 * u.deg) assert list(zip(idx1_1deg, idx2_1deg)) == [(0, 2), (0, 3), (1, 1), (1, 2)] assert d2d_1deg[0] == 1.0 * u.deg assert_allclose(d2d_1deg, [1, 0, .1, .9] * u.deg) assert list(zip(idx1_0p05deg, idx2_0p05deg)) == [(0, 3)] idx1_1kpc, idx2_1kpc, d2d_1kpc, d3d_1kpc = search_around_3d( coo1, coo2, 1 * u.kpc) idx1_sm, idx2_sm, d2d_sm, d3d_sm = search_around_3d( coo1, coo2, 0.05 * u.kpc) assert list(zip(idx1_1kpc, idx2_1kpc)) == [(0, 0), (0, 1), (0, 2), (1, 3)] assert list(zip(idx1_sm, idx2_sm)) == [(0, 1), (0, 2)] assert_allclose(d2d_sm, [2, 1] * u.deg) # Test for the non-matches, #4877 coo1 = ICRS([4.1, 2.1] * u.degree, [0, 0] * u.degree, distance=[1, 5] * u.kpc) idx1, idx2, d2d, d3d = search_around_sky(coo1, coo2, 1 * u.arcsec) assert idx1.size == idx2.size == d2d.size == d3d.size == 0 assert idx1.dtype == idx2.dtype == np.int assert d2d.unit == u.deg assert d3d.unit == u.kpc idx1, idx2, d2d, d3d = search_around_3d(coo1, coo2, 1 * u.m) assert idx1.size == idx2.size == d2d.size == d3d.size == 0 assert idx1.dtype == idx2.dtype == np.int assert d2d.unit == u.deg assert d3d.unit == u.kpc # Test when one or both of the coordinate arrays is empty, #4875 empty = ICRS(ra=[] * u.degree, dec=[] * u.degree, distance=[] * u.kpc) idx1, idx2, d2d, d3d = search_around_sky(empty, coo2, 1 * u.arcsec) assert idx1.size == idx2.size == d2d.size == d3d.size == 0 assert idx1.dtype == idx2.dtype == np.int assert d2d.unit == u.deg assert d3d.unit == u.kpc idx1, idx2, d2d, d3d = search_around_sky(coo1, empty, 1 * u.arcsec) assert idx1.size == idx2.size == d2d.size == d3d.size == 0 assert idx1.dtype == idx2.dtype == np.int assert d2d.unit == u.deg assert d3d.unit == u.kpc empty = ICRS(ra=[] * u.degree, dec=[] * u.degree, distance=[] * u.kpc) idx1, idx2, d2d, d3d = search_around_sky(empty, empty[:], 1 * u.arcsec) assert idx1.size == idx2.size == d2d.size == d3d.size == 0 assert idx1.dtype == idx2.dtype == np.int assert d2d.unit == u.deg assert d3d.unit == u.kpc idx1, idx2, d2d, d3d = search_around_3d(empty, coo2, 1 * u.m) assert idx1.size == idx2.size == d2d.size == d3d.size == 0 assert idx1.dtype == idx2.dtype == np.int assert d2d.unit == u.deg assert d3d.unit == u.kpc idx1, idx2, d2d, d3d = search_around_3d(coo1, empty, 1 * u.m) assert idx1.size == idx2.size == d2d.size == d3d.size == 0 assert idx1.dtype == idx2.dtype == np.int assert d2d.unit == u.deg assert d3d.unit == u.kpc idx1, idx2, d2d, d3d = search_around_3d(empty, empty[:], 1 * u.m) assert idx1.size == idx2.size == d2d.size == d3d.size == 0 assert idx1.dtype == idx2.dtype == np.int assert d2d.unit == u.deg assert d3d.unit == u.kpc # Test that input without distance units results in a # 'dimensionless_unscaled' unit cempty = SkyCoord(ra=[], dec=[], unit=u.deg) idx1, idx2, d2d, d3d = search_around_3d(cempty, cempty[:], 1 * u.m) assert d2d.unit == u.deg assert d3d.unit == u.dimensionless_unscaled idx1, idx2, d2d, d3d = search_around_sky(cempty, cempty[:], 1 * u.m) assert d2d.unit == u.deg assert d3d.unit == u.dimensionless_unscaled
def generate_pairs(ar_dec, ar_ra, coord_permutation, coord_set, local_end_index, local_start_index, max_angular_separation, bundle_start_index=0): """ Generate QSO pairs, in bundles. Each time, a bundle of QSOs is matched against the full list :param ar_dec: declination array :param ar_ra: right ascension array :param coord_permutation: pseudo-random permutation of qso indices, for counting each pair only once :param coord_set: coordinate set of all QSOs :param local_end_index: last QSO index for this MPI node :param local_start_index: first QSO index for this MPI node :param max_angular_separation: maximum angular separation for sky search :param bundle_start_index: skip all bundles prior to bundle index :return: """ # each node matches a range of objects against the full list. qso_bundle_size = settings.get_qso_bundle_size() bundles = list( get_bundles(local_start_index, local_end_index, qso_bundle_size)) num_bundles = len(bundles) # if the number of bundles is not the same across all MPI nodes (which should be rare), # we are not allowed to use the synchronized version of print. num_bundles = comm.allgather(num_bundles) mpi_helper.r_print("number of QSO bundles per node:", num_bundles) is_num_bundles_equal = np.all(np.array(num_bundles) == num_bundles[0]) print_func = mpi_helper.l_print if is_num_bundles_equal else mpi_helper.l_print_no_barrier mpi_helper.r_print('using print function:', print_func.__name__) bundle_iterator = islice(enumerate(bundles), bundle_start_index, None) for bundle_index, (bundle_start, bundle_size) in bundle_iterator: print_func('matching ', bundle_size, ' objects, starting at :', bundle_start) print_func('node progress:{:.1f}%'.format( 100. * (bundle_start - local_start_index) / (local_end_index - local_start_index))) count = matching.search_around_sky( coord_set[bundle_start:bundle_start + bundle_size], coord_set, max_angular_separation) # search around sky returns indices in the input lists. # each node should add its offset to get the QSO index in the original list (only for x[0]). # qso2 contains the unmodified index to the full list of QSOs. # the third vector is a count so we can keep a reference to the angles vector. qso_index_1 = count[0] + bundle_start qso_index_2 = count[1] # find the mean ra,dec for each pair qso_ra_pairs = np.vstack((ar_ra[qso_index_1], ar_ra[qso_index_2])) qso_dec_pairs = np.vstack((ar_dec[qso_index_1], ar_dec[qso_index_2])) # we can safely assume that separations are small enough so we don't have catastrophic cancellation of the mean, # so checking the unit radius value is not required pair_means_ra, local_pair_means_dec, _ = find_spherical_mean_deg( qso_ra_pairs, qso_dec_pairs, axis=0) sky_groups = SkyGroups(nside=settings.get_healpix_nside()) group_id = sky_groups.get_group_ids(pair_means_ra, local_pair_means_dec) qso_pairs_with_unity = np.vstack( (qso_index_1, qso_index_2, group_id, np.arange(count[0].size))) qso_pair_angles = count[2].to(u.rad).value print_func('number of QSO pairs (including identity pairs):', count[0].size) print_func('angle vector size:', qso_pair_angles.size) # remove pairs of the same QSO, which have different [plate,mjd,fiber] # assume that QSOs within roughly 10 arc-second (5e-5 rads) are the same object. # also keep only 1 instance of each pair (keep only: qso1_index_hash < qso2_index_hash) qso_pairs = qso_pairs_with_unity.T[np.logical_and( qso_pair_angles > 5e-5, coord_permutation[qso_pairs_with_unity[0]] < coord_permutation[qso_pairs_with_unity[1]])] print_func('total number of redundant objects removed:', qso_pairs_with_unity.shape[1] - qso_pairs.shape[0]) print_func('number of QSO pairs:', qso_pairs.shape[0]) yield bundle_index, qso_pair_angles, qso_pairs
def profile_main(): # x = coord.SkyCoord(ra=10.68458*u.deg, dec=41.26917*u.deg, frame='icrs') # min_distance = cd.comoving_distance_transverse(2.1, **fidcosmo) # print('minimum distance', min_distance, 'Mpc/rad') # initialize data sources qso_record_table = table.Table(np.load(settings.get_qso_metadata_npy())) # prepare data for quicker access qso_record_list = [QSORecord.from_row(i) for i in qso_record_table] ar_ra = np.array([i.ra for i in qso_record_list]) ar_dec = np.array([i.dec for i in qso_record_list]) ar_z = np.array([i.z for i in qso_record_list]) ar_extinction = np.array([i.extinction_g for i in qso_record_list]) ar_distance = cd.fast_comoving_distance(ar_z) mpi_helper.r_print('QSO table size:', len(ar_distance)) # TODO: find a more precise value instead of z=1.9 # set maximum QSO angular separation to 200Mpc/h (in co-moving coordinates) # the article assumes h is measured in units of 100km/s/mpc radius_quantity = (200. * (100. * u.km / (u.Mpc * u.s)) / cd.H0 ) # type: u.Quantity radius = radius_quantity.value max_angular_separation = radius / (cd.comoving_distance(1.9) / u.radian) mpi_helper.r_print('maximum separation of QSOs:', Angle(max_angular_separation).to_string(unit=u.degree)) # print(ar_list) coord_set = coord.SkyCoord(ra=ar_ra * u.degree, dec=ar_dec * u.degree, distance=ar_distance * u.Mpc) # print(coord_set) # find all QSO pairs chunk_sizes, chunk_offsets = mpi_helper.get_chunks(len(coord_set), comm.size) local_start_index = chunk_offsets[comm.rank] local_end_index = local_start_index + chunk_sizes[comm.rank] mpi_helper.l_print('matching objects in range:', local_start_index, 'to', local_end_index) # each node matches a range of objects against the full list. count = matching.search_around_sky( coord_set[local_start_index:local_end_index], coord_set, max_angular_separation) # search around sky returns indices in the input lists. # each node should add its offset to get the QSO index in the original list (only for x[0]). # qso2 which contains the unmodified index to the full list of QSOs. # the third vector is a count so we can keep a reference to the angles vector. local_qso_index_1 = count[0] + local_start_index local_qso_index_2 = count[1] # find the mean ra,dec for each pair local_qso_ra_pairs = np.vstack( (ar_ra[local_qso_index_1], ar_ra[local_qso_index_2])) local_qso_dec_pairs = np.vstack( (ar_dec[local_qso_index_1], ar_dec[local_qso_index_2])) # we can safely assume that separations is small enough so we don't have catastrophic cancellation of the mean, # so checking the unit radius value is not required local_pair_means_ra, local_pair_means_dec, _ = find_spherical_mean_deg( local_qso_ra_pairs, local_qso_dec_pairs, axis=0) sky_groups = SkyGroups(nside=settings.get_healpix_nside()) group_id = sky_groups.get_group_ids(local_pair_means_ra, local_pair_means_dec) local_qso_pairs_with_unity = np.vstack( (local_qso_index_1, local_qso_index_2, group_id, np.arange(count[0].size))) local_qso_pair_angles = count[2].to(u.rad).value mpi_helper.l_print('number of QSO pairs (including identity pairs):', count[0].size) mpi_helper.l_print('angle vector size:', local_qso_pair_angles.size) # remove pairs of the same QSO. # local_qso_pairs = local_qso_pairs_with_unity.T[local_qso_pairs_with_unity[1] != local_qso_pairs_with_unity[0]] # remove pairs of the same QSO, which have different [plate,mjd,fiber] # assume that QSOs within roughly 10 arc-second (5e-5 rads) are the same object. local_qso_pairs = local_qso_pairs_with_unity.T[ local_qso_pair_angles > 5e-5] mpi_helper.l_print( 'total number of redundant objects removed:', local_qso_pairs_with_unity.shape[1] - local_qso_pairs.shape[0] - chunk_sizes[comm.rank]) # l_print(pairs) mpi_helper.l_print('number of QSO pairs:', local_qso_pairs.shape[0]) # l_print('angle vector:', x[2]) # divide the work into sub chunks # Warning: the number of sub chunks must be identical for all nodes because gather is called after each sub chunk. # divide by comm.size to make sub chunk size independent of number of nodes. num_sub_chunks_per_node = settings.get_mpi_num_sub_chunks() // comm.size pixel_pair_sub_chunks = mpi_helper.get_chunks(local_qso_pairs.shape[0], num_sub_chunks_per_node) sub_chunk_helper = SubChunkHelper(ar_extinction) for i, j, k in zip(pixel_pair_sub_chunks[0], pixel_pair_sub_chunks[1], itertools.count()): sub_chunk_start = j sub_chunk_end = j + i mpi_helper.l_print("sub_chunk: size", i, ", starting at", j, ",", k, "out of", len(pixel_pair_sub_chunks[0])) sub_chunk_helper.add_pairs_in_sub_chunk( local_qso_pair_angles, local_qso_pairs[sub_chunk_start:sub_chunk_end])