def match_stars_by_position(star_catalog, vphas_cat, log):
"""Function to cross-match stars by position.
Returns:
:param dict match_index: { Index in vphas_cat: index in star_cat }
"""
ddx = np.where(star_catalog['clean'] == 1.0)[0]
det_stars = SkyCoord(star_catalog['RA'][ddx],
star_catalog['DEC'][ddx],
unit="deg")
vdx = np.where(vphas_cat['clean'] == 1.0)[0]
cat_stars = SkyCoord(vphas_cat['_RAJ2000'][vdx],
vphas_cat['_DEJ2000'][vdx],
unit="deg")
tolerance = 2.0 * u.arcsec
match_data = matching.search_around_sky(det_stars,
cat_stars,
seplimit=tolerance)
idx = np.argsort(match_data[2].value)
det_index = match_data[0][idx]
cat_index = match_data[1][idx]
match_index = np.array(zip(ddx[det_index], vdx[cat_index]))
log.info('Matched ' + str(len(match_index)))
return match_index
def calc_offsets_from_red(self):
"""Method to calculate the offset of the blue and green from the red,
by matching stars between the star catalogs"""
red_stars = SkyCoord(self.r_cat['RA'], self.r_cat['DEC'], unit="deg")
for key, par in {'b_offset': 'b_cat', 'g_offset': 'g_cat'}.items():
f = par.split('_')[0]
cat = getattr(self, par)
cat_stars = SkyCoord(cat['RA'], cat['DEC'], unit="deg")
match_data = matching.search_around_sky(red_stars,
cat_stars,
seplimit=self.tolerance)
dx = self.r_cat['x'][match_data[0]] - cat['x'][match_data[1]]
dy = self.r_cat['y'][match_data[0]] - cat['y'][match_data[1]]
(hist_dx, binsx) = np.histogram(dx, bins=50)
(hist_dy, binsy) = np.histogram(dy, bins=50)
idx = np.where(hist_dx == max(hist_dx))[0][0]
idy = np.where(hist_dy == max(hist_dy))[0][0]
xmin = binsx[idx - 2]
xmax = binsx[idx + 2]
ymin = binsy[idy - 2]
ymax = binsy[idy + 2]
idx = np.where(dx >= xmin)[0]
jdx = np.where(dx <= xmax)[0]
kdx = list(set(idx).intersection(set(jdx)))
deltax = np.median(dx[kdx]) + getattr(self, f + '_offset_x')
idy = np.where(dy >= ymin)[0]
jdy = np.where(dy <= ymax)[0]
kdy = list(set(idy).intersection(set(jdy)))
deltay = np.median(dy[kdy]) + getattr(self, f + '_offset_y')
if par == 'b_cat':
setattr(self, key, (-deltax, -deltay))
else:
setattr(self, key, (deltax, deltay))
plot_offsets(par, dx, dy)
def xmatch_catalogs(catalog1, catalog2, log):
"""Function to cross-match objects between two catalogs in astropy.Table
format.
Based on code by Y. Tsapras.
"""
stars1 = SkyCoord(catalog1['RA_J2000'], catalog1['DEC_J2000'], unit=u.deg)
stars2 = SkyCoord(catalog2['RA_J2000'], catalog2['DEC_J2000'], unit=u.deg)
match_table = matching.search_around_sky(stars1,
stars2,
seplimit=0.5 * u.arcsec)
matches1 = match_table[0].tolist()
blends = {}
blend_idx = set([x for x in matches1 if matches1.count(x) > 1])
for b in blend_idx:
idx = np.where(match_table[0] == b)[0]
for i in idx:
c = match_table[1][i]
if b not in blends.keys():
blends[b] = [
(catalog1['RA_J2000'][b], catalog1['DEC_J2000'][b]),
(c, catalog2['RA_J2000'][c], catalog2['DEC_J2000'][c])
]
else:
blends[b].append(
(c, catalog2['RA_J2000'][c], catalog2['DEC_J2000'][c]))
log.info('Match '+str(b)+' '+\
str(catalog1['RA_J2000'][b])+' '+str(catalog1['DEC_J2000'][b])+' -> '+\
str(c)+' '+str(catalog2['RA_J2000'][b])+' '+str(catalog2['DEC_J2000'][b]))
return match_table, blends
def find_target_data(params,star_catalog):
"""Function to identify the photometry for a given target, if present
in the star catalogue"""
target = {}
if params['target_ra'] != None:
target_location = SkyCoord([params['target_ra']], [params['target_dec']], unit=(u.hourangle, u.deg))
stars = SkyCoord(star_catalog['RA'], star_catalog['DEC'], unit="deg")
tolerance = 5.0 * u.arcsec
match_data = matching.search_around_sky(target_location, stars,
seplimit=tolerance)
if len(match_data[0]) > 0:
idx = np.argsort(match_data[2].value)
print str(len(match_data[1]))+' matching objects in the catalog'
target = {'star_index': star_catalog['star_index'][match_data[1][idx[0]]],
'x': star_catalog['x_pixel'][match_data[1][idx[0]]],
'y': star_catalog['y_pixel'][match_data[1][idx[0]]],
'ra': star_catalog['RA'][match_data[1][idx[0]]],
'dec': star_catalog['DEC'][match_data[1][idx[0]]],
'mag': star_catalog['mag'][match_data[1][idx[0]]],
'mag_err': star_catalog['mag_err'][match_data[1][idx[0]]]}
print 'Target identified as '+repr(target)
print 'List of matching objects: '
for i in range(len(match_data[1])):
print str(star_catalog['star_index'][match_data[1][idx[i]]])+' '+\
str(star_catalog['x_pixel'][match_data[1][idx[i]]])+' '+\
str(star_catalog['y_pixel'][match_data[1][idx[i]]])+' '+\
str(star_catalog['RA'][match_data[1][idx[i]]])+' '+\
str(star_catalog['DEC'][match_data[1][idx[i]]]),match_data[2][idx[i]]
return target
def cross_match_stars(f1,dataset1,f2,dataset2,log):
"""Function to cross-match stars by position.
Returns:
:param dict match_index: { Index in vphas_cat: index in star_cat }
"""
stars1 = SkyCoord(dataset1['RA'], dataset1['DEC'], unit="deg")
stars2 = SkyCoord(dataset2['RA'], dataset2['DEC'], unit="deg")
tolerance = 1.0 * u.arcsec
match_data = matching.search_around_sky(stars1, stars2,
seplimit=tolerance)
idx = np.argsort(match_data[2].value)
match_index = np.array(zip(match_data[0][idx],match_data[1][idx]))
log.info('Matched '+str(len(match_index))+' stars between '+f1+' and '+f2)
return match_index
def test_search_around():
from astropy.coordinates import ICRS, SkyCoord
from astropy.coordinates.matching import search_around_sky, search_around_3d
coo1 = ICRS([4, 2.1] * u.degree, [0, 0] * u.degree,
distance=[1, 5] * u.kpc)
coo2 = ICRS([1, 2, 3, 4] * u.degree, [0, 0, 0, 0] * u.degree,
distance=[1, 1, 1, 5] * u.kpc)
idx1_1deg, idx2_1deg, d2d_1deg, d3d_1deg = search_around_sky(
coo1, coo2, 1.01 * u.deg)
idx1_0p05deg, idx2_0p05deg, d2d_0p05deg, d3d_0p05deg = search_around_sky(
coo1, coo2, 0.05 * u.deg)
assert list(zip(idx1_1deg, idx2_1deg)) == [(0, 2), (0, 3), (1, 1), (1, 2)]
assert d2d_1deg[0] == 1.0 * u.deg
assert_allclose(d2d_1deg, [1, 0, .1, .9] * u.deg)
assert list(zip(idx1_0p05deg, idx2_0p05deg)) == [(0, 3)]
idx1_1kpc, idx2_1kpc, d2d_1kpc, d3d_1kpc = search_around_3d(
coo1, coo2, 1 * u.kpc)
idx1_sm, idx2_sm, d2d_sm, d3d_sm = search_around_3d(
coo1, coo2, 0.05 * u.kpc)
assert list(zip(idx1_1kpc, idx2_1kpc)) == [(0, 0), (0, 1), (0, 2), (1, 3)]
assert list(zip(idx1_sm, idx2_sm)) == [(0, 1), (0, 2)]
assert_allclose(d2d_sm, [2, 1] * u.deg)
# Test for the non-matches, #4877
coo1 = ICRS([4.1, 2.1] * u.degree, [0, 0] * u.degree,
distance=[1, 5] * u.kpc)
idx1, idx2, d2d, d3d = search_around_sky(coo1, coo2, 1 * u.arcsec)
assert idx1.size == idx2.size == d2d.size == d3d.size == 0
assert idx1.dtype == idx2.dtype == np.int
assert d2d.unit == u.deg
assert d3d.unit == u.kpc
idx1, idx2, d2d, d3d = search_around_3d(coo1, coo2, 1 * u.m)
assert idx1.size == idx2.size == d2d.size == d3d.size == 0
assert idx1.dtype == idx2.dtype == np.int
assert d2d.unit == u.deg
assert d3d.unit == u.kpc
# Test when one or both of the coordinate arrays is empty, #4875
empty = ICRS(ra=[] * u.degree, dec=[] * u.degree, distance=[] * u.kpc)
idx1, idx2, d2d, d3d = search_around_sky(empty, coo2, 1 * u.arcsec)
assert idx1.size == idx2.size == d2d.size == d3d.size == 0
assert idx1.dtype == idx2.dtype == np.int
assert d2d.unit == u.deg
assert d3d.unit == u.kpc
idx1, idx2, d2d, d3d = search_around_sky(coo1, empty, 1 * u.arcsec)
assert idx1.size == idx2.size == d2d.size == d3d.size == 0
assert idx1.dtype == idx2.dtype == np.int
assert d2d.unit == u.deg
assert d3d.unit == u.kpc
empty = ICRS(ra=[] * u.degree, dec=[] * u.degree, distance=[] * u.kpc)
idx1, idx2, d2d, d3d = search_around_sky(empty, empty[:], 1 * u.arcsec)
assert idx1.size == idx2.size == d2d.size == d3d.size == 0
assert idx1.dtype == idx2.dtype == np.int
assert d2d.unit == u.deg
assert d3d.unit == u.kpc
idx1, idx2, d2d, d3d = search_around_3d(empty, coo2, 1 * u.m)
assert idx1.size == idx2.size == d2d.size == d3d.size == 0
assert idx1.dtype == idx2.dtype == np.int
assert d2d.unit == u.deg
assert d3d.unit == u.kpc
idx1, idx2, d2d, d3d = search_around_3d(coo1, empty, 1 * u.m)
assert idx1.size == idx2.size == d2d.size == d3d.size == 0
assert idx1.dtype == idx2.dtype == np.int
assert d2d.unit == u.deg
assert d3d.unit == u.kpc
idx1, idx2, d2d, d3d = search_around_3d(empty, empty[:], 1 * u.m)
assert idx1.size == idx2.size == d2d.size == d3d.size == 0
assert idx1.dtype == idx2.dtype == np.int
assert d2d.unit == u.deg
assert d3d.unit == u.kpc
# Test that input without distance units results in a
# 'dimensionless_unscaled' unit
cempty = SkyCoord(ra=[], dec=[], unit=u.deg)
idx1, idx2, d2d, d3d = search_around_3d(cempty, cempty[:], 1 * u.m)
assert d2d.unit == u.deg
assert d3d.unit == u.dimensionless_unscaled
idx1, idx2, d2d, d3d = search_around_sky(cempty, cempty[:], 1 * u.m)
assert d2d.unit == u.deg
assert d3d.unit == u.dimensionless_unscaled
def generate_pairs(ar_dec,
ar_ra,
coord_permutation,
coord_set,
local_end_index,
local_start_index,
max_angular_separation,
bundle_start_index=0):
"""
Generate QSO pairs, in bundles.
Each time, a bundle of QSOs is matched against the full list
:param ar_dec: declination array
:param ar_ra: right ascension array
:param coord_permutation: pseudo-random permutation of qso indices, for counting each pair only once
:param coord_set: coordinate set of all QSOs
:param local_end_index: last QSO index for this MPI node
:param local_start_index: first QSO index for this MPI node
:param max_angular_separation: maximum angular separation for sky search
:param bundle_start_index: skip all bundles prior to bundle index
:return:
"""
# each node matches a range of objects against the full list.
qso_bundle_size = settings.get_qso_bundle_size()
bundles = list(
get_bundles(local_start_index, local_end_index, qso_bundle_size))
num_bundles = len(bundles)
# if the number of bundles is not the same across all MPI nodes (which should be rare),
# we are not allowed to use the synchronized version of print.
num_bundles = comm.allgather(num_bundles)
mpi_helper.r_print("number of QSO bundles per node:", num_bundles)
is_num_bundles_equal = np.all(np.array(num_bundles) == num_bundles[0])
print_func = mpi_helper.l_print if is_num_bundles_equal else mpi_helper.l_print_no_barrier
mpi_helper.r_print('using print function:', print_func.__name__)
bundle_iterator = islice(enumerate(bundles), bundle_start_index, None)
for bundle_index, (bundle_start, bundle_size) in bundle_iterator:
print_func('matching ', bundle_size, ' objects, starting at :',
bundle_start)
print_func('node progress:{:.1f}%'.format(
100. * (bundle_start - local_start_index) /
(local_end_index - local_start_index)))
count = matching.search_around_sky(
coord_set[bundle_start:bundle_start + bundle_size], coord_set,
max_angular_separation)
# search around sky returns indices in the input lists.
# each node should add its offset to get the QSO index in the original list (only for x[0]).
# qso2 contains the unmodified index to the full list of QSOs.
# the third vector is a count so we can keep a reference to the angles vector.
qso_index_1 = count[0] + bundle_start
qso_index_2 = count[1]
# find the mean ra,dec for each pair
qso_ra_pairs = np.vstack((ar_ra[qso_index_1], ar_ra[qso_index_2]))
qso_dec_pairs = np.vstack((ar_dec[qso_index_1], ar_dec[qso_index_2]))
# we can safely assume that separations are small enough so we don't have catastrophic cancellation of the mean,
# so checking the unit radius value is not required
pair_means_ra, local_pair_means_dec, _ = find_spherical_mean_deg(
qso_ra_pairs, qso_dec_pairs, axis=0)
sky_groups = SkyGroups(nside=settings.get_healpix_nside())
group_id = sky_groups.get_group_ids(pair_means_ra,
local_pair_means_dec)
qso_pairs_with_unity = np.vstack(
(qso_index_1, qso_index_2, group_id, np.arange(count[0].size)))
qso_pair_angles = count[2].to(u.rad).value
print_func('number of QSO pairs (including identity pairs):',
count[0].size)
print_func('angle vector size:', qso_pair_angles.size)
# remove pairs of the same QSO, which have different [plate,mjd,fiber]
# assume that QSOs within roughly 10 arc-second (5e-5 rads) are the same object.
# also keep only 1 instance of each pair (keep only: qso1_index_hash < qso2_index_hash)
qso_pairs = qso_pairs_with_unity.T[np.logical_and(
qso_pair_angles > 5e-5, coord_permutation[qso_pairs_with_unity[0]]
< coord_permutation[qso_pairs_with_unity[1]])]
print_func('total number of redundant objects removed:',
qso_pairs_with_unity.shape[1] - qso_pairs.shape[0])
print_func('number of QSO pairs:', qso_pairs.shape[0])
yield bundle_index, qso_pair_angles, qso_pairs
def profile_main():
# x = coord.SkyCoord(ra=10.68458*u.deg, dec=41.26917*u.deg, frame='icrs')
# min_distance = cd.comoving_distance_transverse(2.1, **fidcosmo)
# print('minimum distance', min_distance, 'Mpc/rad')
# initialize data sources
qso_record_table = table.Table(np.load(settings.get_qso_metadata_npy()))
# prepare data for quicker access
qso_record_list = [QSORecord.from_row(i) for i in qso_record_table]
ar_ra = np.array([i.ra for i in qso_record_list])
ar_dec = np.array([i.dec for i in qso_record_list])
ar_z = np.array([i.z for i in qso_record_list])
ar_extinction = np.array([i.extinction_g for i in qso_record_list])
ar_distance = cd.fast_comoving_distance(ar_z)
mpi_helper.r_print('QSO table size:', len(ar_distance))
# TODO: find a more precise value instead of z=1.9
# set maximum QSO angular separation to 200Mpc/h (in co-moving coordinates)
# the article assumes h is measured in units of 100km/s/mpc
radius_quantity = (200. * (100. * u.km / (u.Mpc * u.s)) / cd.H0
) # type: u.Quantity
radius = radius_quantity.value
max_angular_separation = radius / (cd.comoving_distance(1.9) / u.radian)
mpi_helper.r_print('maximum separation of QSOs:',
Angle(max_angular_separation).to_string(unit=u.degree))
# print(ar_list)
coord_set = coord.SkyCoord(ra=ar_ra * u.degree,
dec=ar_dec * u.degree,
distance=ar_distance * u.Mpc)
# print(coord_set)
# find all QSO pairs
chunk_sizes, chunk_offsets = mpi_helper.get_chunks(len(coord_set),
comm.size)
local_start_index = chunk_offsets[comm.rank]
local_end_index = local_start_index + chunk_sizes[comm.rank]
mpi_helper.l_print('matching objects in range:', local_start_index, 'to',
local_end_index)
# each node matches a range of objects against the full list.
count = matching.search_around_sky(
coord_set[local_start_index:local_end_index], coord_set,
max_angular_separation)
# search around sky returns indices in the input lists.
# each node should add its offset to get the QSO index in the original list (only for x[0]).
# qso2 which contains the unmodified index to the full list of QSOs.
# the third vector is a count so we can keep a reference to the angles vector.
local_qso_index_1 = count[0] + local_start_index
local_qso_index_2 = count[1]
# find the mean ra,dec for each pair
local_qso_ra_pairs = np.vstack(
(ar_ra[local_qso_index_1], ar_ra[local_qso_index_2]))
local_qso_dec_pairs = np.vstack(
(ar_dec[local_qso_index_1], ar_dec[local_qso_index_2]))
# we can safely assume that separations is small enough so we don't have catastrophic cancellation of the mean,
# so checking the unit radius value is not required
local_pair_means_ra, local_pair_means_dec, _ = find_spherical_mean_deg(
local_qso_ra_pairs, local_qso_dec_pairs, axis=0)
sky_groups = SkyGroups(nside=settings.get_healpix_nside())
group_id = sky_groups.get_group_ids(local_pair_means_ra,
local_pair_means_dec)
local_qso_pairs_with_unity = np.vstack(
(local_qso_index_1, local_qso_index_2, group_id,
np.arange(count[0].size)))
local_qso_pair_angles = count[2].to(u.rad).value
mpi_helper.l_print('number of QSO pairs (including identity pairs):',
count[0].size)
mpi_helper.l_print('angle vector size:', local_qso_pair_angles.size)
# remove pairs of the same QSO.
# local_qso_pairs = local_qso_pairs_with_unity.T[local_qso_pairs_with_unity[1] != local_qso_pairs_with_unity[0]]
# remove pairs of the same QSO, which have different [plate,mjd,fiber]
# assume that QSOs within roughly 10 arc-second (5e-5 rads) are the same object.
local_qso_pairs = local_qso_pairs_with_unity.T[
local_qso_pair_angles > 5e-5]
mpi_helper.l_print(
'total number of redundant objects removed:',
local_qso_pairs_with_unity.shape[1] - local_qso_pairs.shape[0] -
chunk_sizes[comm.rank])
# l_print(pairs)
mpi_helper.l_print('number of QSO pairs:', local_qso_pairs.shape[0])
# l_print('angle vector:', x[2])
# divide the work into sub chunks
# Warning: the number of sub chunks must be identical for all nodes because gather is called after each sub chunk.
# divide by comm.size to make sub chunk size independent of number of nodes.
num_sub_chunks_per_node = settings.get_mpi_num_sub_chunks() // comm.size
pixel_pair_sub_chunks = mpi_helper.get_chunks(local_qso_pairs.shape[0],
num_sub_chunks_per_node)
sub_chunk_helper = SubChunkHelper(ar_extinction)
for i, j, k in zip(pixel_pair_sub_chunks[0], pixel_pair_sub_chunks[1],
itertools.count()):
sub_chunk_start = j
sub_chunk_end = j + i
mpi_helper.l_print("sub_chunk: size", i, ", starting at", j, ",", k,
"out of", len(pixel_pair_sub_chunks[0]))
sub_chunk_helper.add_pairs_in_sub_chunk(
local_qso_pair_angles,
local_qso_pairs[sub_chunk_start:sub_chunk_end])
