def crossmatch_cartesian(coords_image, coords_ref, tolerance=None):
"""
Crossmatch source coordinates with reference-star coordinates.
Parameters
----------
coords_image : array-like
The coordinates of points to match
coords_ref : array-like
The coordinates of reference points to match
tolerance : float
Crossmatch distance in pixels (default: 5)
"""
if tolerance is None:
tolerance = 5.
try:
kdt = KDT(coords_ref, balanced_tree=False, compact_nodes=False)
except TypeError:
kdt = KDT(coords_ref)
ds,ind_ref = kdt.query(coords_image, k=1, distance_upper_bound=tolerance)
mask_xmatch = ds < tolerance
ind_image = np.arange(len(coords_image))
return ind_image[mask_xmatch], ind_ref[mask_xmatch], ds[mask_xmatch]
def check_n_in_aper(radius_factor=1, k=100):
for catfile in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"):
print
print catfile
names = open(catfile).readline().split()[1:]
cat = np.recfromtxt(catfile, names=names)
xscfile = catfile.replace('combined_hdr_catalog.txt', '2mass_xsc.tbl')
print xscfile
names = open(xscfile).read().split('\n')[76].split('|')[1:-1]
xsc = np.recfromtxt(xscfile, skip_header=80, names=names)
n_in_aper = []
coords = radec_to_coords(cat.ra, cat.dec)
kdt = KDT(coords)
for i in range(xsc.size):
r_deg = xsc.r_ext[i] / 3600.
idx, ds = spherematch2(xsc.ra[i],
xsc.dec[i],
cat.ra,
cat.dec,
kdt,
tolerance=radius_factor * r_deg,
k=k)
n_in_aper.append(ds.size)
for i in [(i, n_in_aper.count(i)) for i in set(n_in_aper)]:
print i
def closest_pair(points):
# Brute force
# minDist = float('inf')
# out = None
# for pair in combinations(points, 2):
# d = dist(*pair)
# if d < minDist:
# minDist = d
# out = pair
# return out
# My KDTree
# tree = kdtree(list(points))
# minDist, pair = float('inf'), None
# for point in points:
# newDist, newPoint = nearest(tree, point, 0, float('inf'), None, 0)
# if newDist < minDist:
# minDist = newDist
# pair = (newPoint, point)
# return pair
# Use scipy
tree = KDT(points)
dists, idxs = tree.query(points, k=2)
dists, idxs = dists[:, 1], idxs[:, 1]
minIdx = dists.argmin()
return points[minIdx], points[idxs[minIdx]]
def CartMatch(coord1, coord2, tol=None, nnearest=1):
"""
Cartesian Coordinate mathcing
"""
# sanitize
coord1 = np.array(coord1, ndmin=1)
coord2 = np.array(coord2, ndmin=1)
# check the dimensions of the coordinate
npairs1 = len(coord1)
ndim1 = 1 if len( np.shape(coord1) ) == 1 else \
np.shape(coord1)[1]
npairs2 = len(coord2)
ndim2 = 1 if len( np.shape(coord2) ) == 1 else \
np.shape(coord2)[1]
# check whether the coord1 and coord2 have the same shape
if ndim1 != ndim2:
raise RuntimeError("The dims of coord1/2 are not the same.")
else:
ndim = ndim1
# make proper arrays if they are 1d arrays
if ndim == 1:
coord1 = np.array([coord1, np.zeros(len(coord1))]).T
coord2 = np.array([coord2, np.zeros(len(coord2))]).T
# kdtree the coord2
kdt = KDT(coord2)
if nnearest == 1:
idxs2 = kdt.query(coord1)[1]
elif nnearest > 1:
idxs2 = kdt.query(coord1, nnearest)[1][:, -1]
else:
raise ValueError('invalid nnearest ' + str(nnearest))
# distance - warning: this could be over float if the precision is not enough, we assume that case is beyond the distance of interest...
ds = np.sqrt(np.sum((coord1 - coord2[idxs2])**2, axis=1))
# index of coord1
idxs1 = np.arange(npairs1)
# distance filtering
if tol is not None:
msk = ds < tol
idxs1 = idxs1[msk]
idxs2 = idxs2[msk]
ds = ds[msk]
return idxs1, idxs2, ds
def build_sp_kdtree(ra, dec, degree=False):
"""
ra and dec are in radian, if not set degree to True
"""
x, y, z = radec2xyz(ra, dec, degree=degree)
coords = np.empty((x.size, 3))
coords[:, 0] = x
coords[:, 1] = y
coords[:, 2] = z
tree = KDT(coords, balanced_tree=False)
return tree
def measure_particleparticle(subhalostars,
emptybox,
boxx,
boxy,
boxz,
kneighbors=3,
maxNkernels=5,
pixelsizekpc=0.25,
distance=10.,
projection='None',
outtag=None):
if verbose:
print_verbose_string('Measuring particle-particle distances in 3D ...')
'Choose which key to use for projection'
if projection == 'None':
projkey = 'CoordinatesOrigin'
else:
projkey = 'Coordinates' + projection
'Put particle coordinates in physical units'
ptclx = subhalostars[projkey][:, 0] * (a / h)
ptcly = subhalostars[projkey][:, 1] * (a / h)
ptclz = subhalostars[projkey][:, 2] * (a / h)
ptcld2o = np.sqrt(ptclx**2 + ptcly**2 + ptclz**2)
'Calculate distance to the kth nearest neighbor (star particles), and the physical extent of the smoothing'
ptclcoords = zip(ptclx, ptcly, ptclz)
ptclkdt = KDT(ptclcoords)
ptcldistances_k, ids_k = ptclkdt.query(ptclcoords, k=kneighbors)
rstell = ptcldistances_k[:, -1] + pixelsizekpc
rstell[rstell > 30] = 30.
maxrstell = maxNkernels * rstell
'Plot kth nearest neighbor and smoothing extents for this galaxy'
plot_kneighbors(ptclkdt, ptclcoords, outtag, pixelsizekpc)
plot_smoothing_extent(maxrstell, outtag, pixelsizekpc)
'Place each particle in its pixel bin'
'NOTE: the xyz values in the box[xyz] arrays are *not* pixel centers -- they are the left edges of the pixels!'
'digitize returns indices such that boxx[i-1] <= ptclx < boxx[i] .. but need to subtract 1 otherwise we have too much shifting'
ptclx_indices = np.digitize(ptclx, boxx[:, 0, 0]) - 1
ptcly_indices = np.digitize(ptcly, boxy[0, :, 0]) - 1
ptclz_indices = np.digitize(ptclz, boxz[0, 0, :]) - 1
return ptclx_indices, ptcly_indices, ptclz_indices, rstell, maxrstell, ptcld2o, ptclx, ptcly, ptclz
def spherematch(ra1, dec1, ra2, dec2, tolerance=1 / 3600.):
"""
Uses a k-d tree to efficiently match two pairs of coordinates in spherical
geometry, with a tolerance in degrees.
"""
args = ra1, dec1, ra2, dec2
ra1, dec1, ra2, dec2 = map(partial(np.array, copy=False), args)
coords1 = radec_to_coords(ra1, dec1)
coords2 = radec_to_coords(ra2, dec2)
kdt = KDT(coords2)
idx2 = kdt.query(coords1)[1]
ds = great_circle_distance(ra1, dec1, ra2[idx2], dec2[idx2])
idx1 = np.arange(ra1.size)
msk = ds < tolerance
idx1 = idx1[msk]
idx2 = idx2[msk]
ds = ds[msk]
return idx1, idx2, ds
def comoving_neighbors(zcoord1, zcoord2, nthneighbor=1, maxscale=None):
"""
"""
ccoord1 = radecz_to_comoving(zcoord1)
ccoord2 = radecz_to_comoving(zcoord2)
kdt = KDT(ccoord2)
if nthneighbor == 1:
indx2 = kdt.query(ccoord1)[1]
elif nthneighbor > 1:
indx2 = kdt.query(ccoord1, nthneighbor)[1][:, -1]
else:
raise ValueError('{0}th neighbor cannot be found'.format(nthneighbor))
ds = np.linalg.norm(ccoord1-ccoord2[indx2,:], axis=1)
indx1 = np.arange(ds.size)
if maxscale != None:
iselect = ds < maxscale
indx1 = indx1[iselect]
indx2 = indx2[iselect]
ds = ds[iselect]
return indx1, indx2, ds
def pix_match(targname, iter, catDir='./', matchtol=2):
infileF606W = catDir + targname + '_F606W_cut_std_{0:d}.dat'.format(iter)
infileF814W = catDir + targname + '_F814W_cut_std_{0:d}.dat'.format(iter)
f606w = np.loadtxt(infileF606W, dtype='float')
f814w = np.loadtxt(infileF814W, dtype='float')
print('Length F606W', len(f606w))
print('Length F814W', len(f814w))
xt_f606w = np.vstack(
(f606w[:, xt1], f606w[:, xt2], f606w[:, xt3], f606w[:, xt4]))
xt_f814w = np.vstack(
(f814w[:, xt1], f814w[:, xt2], f814w[:, xt3], f814w[:, xt4]))
yt_f606w = np.vstack(
(f606w[:, yt1], f606w[:, yt2], f606w[:, yt3], f606w[:, yt4]))
yt_f814w = np.vstack(
(f814w[:, yt1], f814w[:, yt2], f814w[:, yt3], f814w[:, yt4]))
xm_606 = np.nanmean(xt_f606w, axis=0)
xm_814 = np.nanmean(xt_f814w, axis=0)
ym_606 = np.nanmean(yt_f606w, axis=0)
ym_814 = np.nanmean(yt_f814w, axis=0)
coords1 = np.empty((xm_606.size, 2))
coords2 = np.empty((xm_814.size, 2))
coords1[:, 0] = xm_606
coords1[:, 1] = ym_606
coords2[:, 0] = xm_814
coords2[:, 1] = ym_814
kdt = KDT(coords1)
idxs2 = kdt.query(coords2)[1]
ds = distArr(xm_814, ym_814, xm_606[idxs2], ym_606[idxs2])
idxs1 = np.arange(xm_814.size)
msk = ds < matchtol
idxs1 = idxs1[msk]
idxs2 = idxs2[msk]
ds = ds[msk]
# print(max(idxs1))
# print(max(idxs2))
outfile = catDir + targname + '-cut_F606W_match_{0:d}.dat'.format(iter)
np.savetxt(outfile, idxs2, fmt='%4i')
outfile = catDir + targname + '-cut_F814W_match_{0:d}.dat'.format(iter)
np.savetxt(outfile, idxs1, fmt='%4i')
outfile = catDir + targname + '-cut_ds_{0:d}.dat'.format(iter)
np.savetxt(outfile, ds, fmt='%1.4f')
return None
def match(x1, y1, m1, x2, y2, m2, dr_tol, dm_tol=None):
"""
Finds matches between two different catalogs. No transformations are done and it
is assumed that the two catalogs are already on the same coordinate system
and magnitude system.
For two stars to be matched, they must be within a specified radius (dr_tol) and
delta-magnitude (dm_tol). For stars with more than 1 neighbor (within the tolerances),
if one is found that is the best match in both brightness and positional offsets
(closest in both), then the match is made. Otherwise,
their is a conflict and no match is returned for the star.
Parameters
x1 : array-like
X coordinate in the first catalog
y1 : array-like
Y coordinate in the first catalog (shape of array must match `x1`)
m1 : array-like
Magnitude in the first catalog. Must have the same shape as x1.
x2 : array-like
X coordinate in the second catalog
y2 : array-like
Y coordinate in the second catalog (shape of array must match `x2`)
m2 : array-like
Magnitude in the second catalog. Must have the same shape as x2.
dr_tol : float
How close (in units of the first catalog) a match has to be to count as a match.
For stars with more than one nearest neighbor, the delta-magnitude is checked
and the closest in delta-mag is chosen.
dm_tol : float or None, optional
How close in delta-magnitude a match has to be to count as a match.
If None, then any delta-magnitude is allowed.
Returns
-------
idx1 : int array
Indicies into the first catalog of the matches. Will never be
larger than `x1`/`y1`.
idx2 : int array
Indicies into the second catalog of the matches. Will never be
larger than `x1`/`y1`.
dr : float array
Distance between the matches.
dm : float array
Delta-mag between the matches. (m1 - m2)
"""
x1 = np.array(x1, copy=False)
y1 = np.array(y1, copy=False)
m1 = np.array(m1, copy=False)
x2 = np.array(x2, copy=False)
y2 = np.array(y2, copy=False)
m2 = np.array(m2, copy=False)
if x1.shape != y1.shape:
raise ValueError('x1 and y1 do not match!')
if x2.shape != y2.shape:
raise ValueError('x2 and y2 do not match!')
# Setup coords1 pairs and coords 2 pairs
# this is equivalent to, but faster than just doing np.array([x1, y1])
coords1 = np.empty((x1.size, 2))
coords1[:, 0] = x1
coords1[:, 1] = y1
# this is equivalent to, but faster than just doing np.array([x1, y1])
coords2 = np.empty((x2.size, 2))
coords2[:, 0] = x2
coords2[:, 1] = y2
# Utimately we will generate arrays of indices.
# idxs1 is the indices for matches into catalog 1. This
# is just a place holder for which stars actually
# have matches.
idxs1 = np.ones(x1.size, dtype=int) * -1
idxs2 = np.ones(x1.size, dtype=int) * -1
# The matching will be done using a KDTree.
kdt = KDT(coords2)
# This returns the number of neighbors within the specified
# radius. We will use this to find those stars that have no or one
# match and deal with them easily. The more complicated conflict
# cases will be dealt with afterward.
i2_match = kdt.query_ball_point(coords1, dr_tol)
Nmatch = np.array([len(idxs) for idxs in i2_match])
# What is the largest number of matches we have for a given star?
Nmatch_max = Nmatch.max()
# Loop through and handle all the different numbers of matches.
# This turns out to be the most efficient so we can use numpy
# array operations. Remember, skip the Nmatch=0 objects... they
# already have indices set to -1.
for nn in range(1, Nmatch_max + 1):
i1_nn = np.where(Nmatch == nn)[0]
if len(i1_nn) == 0:
continue
if nn == 1:
i2_nn = np.array([i2_match[mm][0] for mm in i1_nn])
if dm_tol != None:
dm = np.abs(m1[i1_nn] - m2[i2_nn])
keep = dm < dm_tol
idxs1[i1_nn[keep]] = i1_nn[keep]
idxs2[i1_nn[keep]] = i2_nn[keep]
else:
idxs1[i1_nn] = i1_nn
idxs2[i1_nn] = i2_nn
else:
i2_tmp = np.array([i2_match[mm] for mm in i1_nn])
# Repeat star list 1 positions and magnitudes
# for nn times (tile then transpose)
x1_nn = np.tile(x1[i1_nn], (nn, 1)).T
y1_nn = np.tile(y1[i1_nn], (nn, 1)).T
m1_nn = np.tile(m1[i1_nn], (nn, 1)).T
# Get out star list 2 positions and magnitudes
x2_nn = x2[i2_tmp]
y2_nn = y2[i2_tmp]
m2_nn = m2[i2_tmp]
dr = np.abs(x1_nn - x2_nn, y1_nn - y2_nn)
dm = np.abs(m1_nn - m2_nn)
if dm_tol != None:
# Don't even consider stars that exceed our
# delta-mag threshold.
dr_msk = np.ma.masked_where(dm > dm_tol, dr)
dm_msk = np.ma.masked_where(dm > dm_tol, dm)
# Remember that argmin on masked arrays can find
# one of the masked array elements if ALL are masked.
# But our subsequent "keep" check should get rid of all
# of these.
dm_min = dm_msk.argmin(axis=1)
dr_min = dr_msk.argmin(axis=1)
# Double check that "min" choice is still within our
# detla-mag tolerence.
dm_tmp = np.choose(dm_min, dm.T)
keep = (dm_min == dr_min) & (dm_tmp < dm_tol)
else:
dm_min = dm.argmin(axis=1)
dr_min = dr.argmin(axis=1)
keep = (dm_min == dr_min)
i2_keep_2D = i2_tmp[keep]
dr_keep = dr_min[keep] # which i2 star for a given i1 star
ii_keep = np.arange(
len(dr_keep)) # a running index for the i2 keeper stars.
idxs1[i1_nn[keep]] = i1_nn[keep]
idxs2[i1_nn[keep]] = i2_keep_2D[ii_keep, dr_keep]
idxs1 = idxs1[idxs1 >= 0]
idxs2 = idxs2[idxs2 >= 0]
dr = np.hypot(x1[idxs1] - x2[idxs2], y1[idxs1] - y2[idxs2])
dm = m1[idxs1] - m2[idxs2]
# Deal with duplicates
duplicates = [
item for item, count in Counter(idxs2).iteritems() if count > 1
]
print 'Found {0:d} out of {1:d} duplicates'.format(len(duplicates),
len(dm))
# for dd in range(len(duplicates)):
# dups = np.where(idxs2 == duplicates[dd])[0]
# # Handle them in brightness order -- brightest first in the first starlist
# fsort = m1[dups].argsort()
# # For every duplicate, match to the star that is closest in space and
# # magnitude. HMMMM.... this doesn't seem like it will work optimally.
return idxs1, idxs2, dr, dm
def carte2d_and_z_match(x1, y1, z1, x2, y2, z2, ztol, stol):
"""
Finds matches in one catalog to another.
Parameters
x1 : array-like
Cartesian coordinate x of the first catalog
y1 : array-like
Cartesian coordinate y of the first catalog (shape of array must match `x1`)
z1 : array-like
Cartesian coordinate z of the first catalog (shape of array must match `x1`)
x2 : array-like
Cartesian coordinate x of the second catalog
y2 : array-like
Cartesian coordinate y of the second catalog (shape of array must match `x2`)
z2 : array-like
Cartesian coordinate z of the second catalog (shape of array must match `x2`)
ztol: float or array-like
The tolarance in z direction. Its shape must match to `x1` if it is an array.
stol: float or None, optional
How close (in the unit of the cartesian coordinate) a match has to be to count as a match. If None,
all nearest neighbors for the first catalog will be returned.
nnearest : int, optional
The nth neighbor to find. E.g., 1 for the nearest nearby, 2 for the
second nearest neighbor, etc. Particularly useful if you want to get
the nearest *non-self* neighbor of a catalog. To do this, use:
``carte2dmatch(x, y, x, y, nnearest=2)``
Returns
-------
idx1 : int array
Indecies into the first catalog of the matches. Will never be
larger than `x1`/`y1`.
idx2 : int array
Indecies into the second catalog of the matches. Will never be
larger than `x1`/`y1`.
ds : float array
Distance (in the unit of the cartesian coordinate) between the matches
dz : float array
Distance (in the unit of the cartesian coordinate) between the matches
"""
# sanitize
x1 = np.array(x1, copy=False)
y1 = np.array(y1, copy=False)
z1 = np.array(z1, copy=False)
x2 = np.array(x2, copy=False)
y2 = np.array(y2, copy=False)
z2 = np.array(z2, copy=False)
# check
if x1.shape != y1.shape or x1.shape != z1.shape:
raise ValueError('x1 and y1/z1 do not match!')
if x2.shape != y2.shape or x2.shape != z2.shape:
raise ValueError('x2 and y2/z2 do not match!')
# this is equivalent to, but faster than just doing np.array([x1, y1])
coords1 = np.empty((x1.size, 2))
coords1[:, 0] = x1
coords1[:, 1] = y1
# this is equivalent to, but faster than just doing np.array([x1, y1])
coords2 = np.empty((x2.size, 2))
coords2[:, 0] = x2
coords2[:, 1] = y2
# set kdt for coord2
kdt = KDT(coords2)
# ---
# Match using kdt
# ---
idxs2_within_balls = kdt.query_ball_point(
coords1, stol) # find the neighbors within a ball
n_within_ball = np.array(map(len, idxs2_within_balls),
dtype=np.int) # counts within each ball
zero_within_ball = np.where(
n_within_ball == 0)[0] # find which one does not have neighbors
nonzero_within_ball = np.where(
n_within_ball > 0)[0] # find which one has neighbors
# declare the distance / idxs2 for each element in nonzero_within_ball
# I use no-brain looping here, slow but seems to be acceptable
dz_within_ball = [] # the distance
idxs2 = []
for i in nonzero_within_ball:
#print i, len(idxs2_within_balls[i]), z1[i], z2[ idxs2_within_balls[i] ]
# Another sub-kdt within a ball, but this times we use kdt.query to find the nearest one
dz_temp, matched_id_temp = KDT(
np.transpose([z2[idxs2_within_balls[i]]
])).query(np.transpose([z1[i]]))
matched_id_temp = idxs2_within_balls[i][matched_id_temp]
# append
dz_within_ball.append(
dz_temp) # the distance of the nearest neighbor within the ball
idxs2.append(
matched_id_temp
) # the index in array2 of the nearest neighbor within the ball
# index for coord1 - only using the object with non-zero neighbor in the ball
idxs1 = np.arange(x1.size)[nonzero_within_ball]
idxs2 = np.array(idxs2, dtype=np.int)
dz_within_ball = np.array(dz_within_ball, dtype=np.float)
# clean
del dz_temp, matched_id_temp
# msk to clean the object with dz > ztol
ztol = np.array(ztol, ndmin=1)
if len(ztol) == 1:
msk = (dz_within_ball < ztol)
elif len(ztol) == len(x1):
msk = (dz_within_ball < ztol[nonzero_within_ball])
else:
raise ValueError(
"The length of ztol has to be 1 (float) or as the same as input x1/y1. len(ztol):",
len(ztol))
# only keep the matches which have dz < ztol
idxs1 = idxs1[msk]
idxs2 = idxs2[msk]
ds = np.hypot(x1[idxs1] - x2[idxs2], y1[idxs1] - y2[idxs2])
dz = dz_within_ball[msk]
return idxs1, idxs2, ds, dz
def carte2dmatch(x1, y1, x2, y2, tol=None, nnearest=1):
"""
Finds matches in one catalog to another.
Parameters
x1 : array-like
Cartesian coordinate x of the first catalog
y1 : array-like
Cartesian coordinate y of the first catalog (shape of array must match `x1`)
x2 : array-like
Cartesian coordinate x of the second catalog
y2 : array-like
Cartesian coordinate y of the second catalog (shape of array must match `x2`)
tol : float or None, optional
How close (in the unit of the cartesian coordinate) a match has to be to count as a match. If None,
all nearest neighbors for the first catalog will be returned.
nnearest : int, optional
The nth neighbor to find. E.g., 1 for the nearest nearby, 2 for the
second nearest neighbor, etc. Particularly useful if you want to get
the nearest *non-self* neighbor of a catalog. To do this, use:
``carte2dmatch(x, y, x, y, nnearest=2)``
Returns
-------
idx1 : int array
Indecies into the first catalog of the matches. Will never be
larger than `x1`/`y1`.
idx2 : int array
Indecies into the second catalog of the matches. Will never be
larger than `x1`/`y1`.
ds : float array
Distance (in the unit of the cartesian coordinate) between the matches
"""
# sanitize
x1 = np.array(x1, copy=False)
y1 = np.array(y1, copy=False)
x2 = np.array(x2, copy=False)
y2 = np.array(y2, copy=False)
# check
if x1.shape != y1.shape:
raise ValueError('x1 and y1 do not match!')
if x2.shape != y2.shape:
raise ValueError('x2 and y2 do not match!')
# this is equivalent to, but faster than just doing np.array([x1, y1, z1])
coords1 = np.empty((x1.size, 2))
coords1[:, 0] = x1
coords1[:, 1] = y1
# this is equivalent to, but faster than just doing np.array([x1, y1, z1])
coords2 = np.empty((x2.size, 2))
coords2[:, 0] = x2
coords2[:, 1] = y2
# set kdt for coord2
kdt = KDT(coords2)
if nnearest == 1:
idxs2 = kdt.query(coords1)[1]
elif nnearest > 1:
idxs2 = kdt.query(coords1, nnearest)[1][:, -1]
else:
raise ValueError('invalid nnearest ' + str(nnearest))
# calc the distance
ds = np.hypot(x1 - x2[idxs2], y1 - y2[idxs2])
# index for coord1
idxs1 = np.arange(x1.size)
if tol is not None:
msk = ds < tol
idxs1 = idxs1[msk]
idxs2 = idxs2[msk]
ds = ds[msk]
return idxs1, idxs2, ds
def run_xsc_phot(bcdphot_out_path, mosaic_path):
replaced = {}
for cat in find_files(bcdphot_out_path, "*_combined_hdr_catalog.txt"):
print("\n======================================================")
print("\nadjusting photometry in: {}".format(cat.split('/')[-1]))
print("------------------------------------------------------")
outpath = cat.replace('combined_hdr_catalog.txt', '2mass_xsc.tbl')
# retrieve 2mass data if file doesn't already exist (from previous run)
if not os.path.isfile(outpath):
# get url and retrieve data
url = query_2mass_xsc_polygon(*get_region_corners(cat))
print("\ndownloading 2MASS photometry from: {}".format(url))
text = urllib2.urlopen(url).read()
# write to disk
with open(outpath, 'w') as f:
f.write(text)
print("\ncreated file: {}".format(outpath))
# read back in as recarray
print("\nreading: {}".format(outpath))
names = open(outpath).read().split('\n')[76].split('|')[1:-1]
da = np.recfromtxt(outpath, skip_header=80, names=names)
# write input file for xsc_phot.pro
infile_outpath = '/'.join(cat.split('/')[:-1]) + '/xsc.txt'
with open(infile_outpath, 'w') as w:
for i in range(da.shape[0]):
w.write("{} {} {} {}\n".format(da.designation[i], da.ra[i],
da.dec[i], da.r_ext[i]))
print(
"\ncreated input file for xsc_phot.pro: {}".format(infile_outpath))
# locate the FITS mosaic file for xsc_phot.pro to do photometry on
reg, ch = cat.split('/')[-1].split('_')[:2]
mosaicfile = filter(lambda x: 'dirbe{}/ch{}/long/full/Combine'\
.format(reg,ch) in x, find_files(mosaic_path, '*mosaic.fits'))[0]
print("\nfound mosaic file: {}".format(mosaicfile))
# spawn IDL subprocess running xsc_phot.pro and catch stdout in file
outpath = infile_outpath.replace('xsc.txt', 'xsc_phot_out.txt')
if not os.path.isfile(outpath):
outfile = open(outpath, 'w')
print("\nspawning xsc_phot.pro IDL subprocess")
cmd = "xsc_phot,'" + mosaicfile + "','" + infile_outpath + "','long'"
rc = subprocess.call(
['/usr/local/itt/idl71/bin/idl', '-quiet', '-e', cmd],
stderr=subprocess.PIPE,
stdout=outfile)
outfile.close()
# read in output to recarray
print("\nreading: {}".format(outpath))
phot = np.recfromtxt(outpath,
names=['id', 'flux', 'unc', 'sky', 'skyunc'])
# make sure rows are aligned
assert (da.designation == phot.id).all()
# ignore xsc sources we got a NaN or negative flux for
bad = np.isnan(phot.flux) | (phot.flux < 0)
print("\naper.pro returned NaN or negative flux for {} sources".format(
bad.sum()))
if bad.sum() > 0:
for i in phot[bad].id:
print(i)
outpath = cat.replace('combined_hdr_catalog.txt',
'xsc_nan_phot.csv')
with open(outpath, 'w') as f:
w = csv.writer(f)
w.writerow(da.dtype.names)
w.writerows(da[bad].tolist())
print('\ncreated file: {}'.format(outpath))
phot = phot[~bad]
da = da[~bad]
# read in pipeline catalog
print("\nreading: {}".format(cat))
names = open(cat).readline().split()[1:]
c = np.recfromtxt(cat, names=names)
# loop through xsc sources and find matches in pipeline catalog
print(
"\nfinding records associated with XSC sources in pipeline catalog"
)
c_flux_total = []
n_in_aper = []
c_idx = []
coords = radec_to_coords(c.ra, c.dec)
kdt = KDT(coords)
for i in range(phot.size):
radius = da.r_ext[i] / 3600.
# idx1, idx2, ds = spherematch(da.ra[i], da.dec[i],
# c.ra, c.dec, tolerance=radius)
idx, ds = spherematch2(da.ra[i],
da.dec[i],
c.ra,
c.dec,
kdt,
tolerance=radius,
k=500)
# c_flux_total.append(c.flux[idx2].sum())
# n_in_aper.append(c.flux[idx2].size)
# c_idx.append(idx2.tolist())
c_flux_total.append(c.flux[idx].sum())
n_in_aper.append(ds.size)
c_idx.append(idx.tolist())
print("\nhistogram of source counts in r_ext aperture")
for i in [(i, n_in_aper.count(i)) for i in set(n_in_aper)]:
print i
# create new version of catalog file with xsc-associated entries replaced
c_idx = np.array(flatten(c_idx))
print("\nremoving {}, adding {}".format(c_idx.size, phot.size))
replaced[cat] = {'old': c_idx.size, 'new': phot.size}
replaced[cat]['hist'] = [(i, n_in_aper.count(i))
for i in set(n_in_aper)]
c = np.delete(c, c_idx)
newrows = np.rec.array([(-i, da.ra[i], da.dec[i],
phot.flux[i], phot.unc[i], 1) for i in \
range(phot.size)], dtype=c.dtype)
newcat = np.hstack((c, newrows))
# write new version of catalog to disk
fmt = ['%i'] + ['%0.8f'] * 2 + ['%.4e'] * 2 + ['%i']
outpath = cat.replace('catalog.txt', 'catalog_xsc_cor.txt')
np.savetxt(outpath, newcat, fmt=fmt, header=' '.join(names))
print('\ncreated file: {}'.format(outpath))
# make plot of total old vs. new flux
plt.scatter(c_flux_total, phot.flux)
ylim = plt.gca().get_ylim()
plt.xlim(*ylim)
max_y = ylim[1]
plt.plot(ylim, ylim, 'r-')
plt.xlabel('old flux [mJy]')
plt.ylabel('new flux [mJy]')
name = ' '.join(cat.split('/')[-1].split('_')[:2])
plt.title(name)
outpath = cat.replace('combined_hdr_catalog.txt',
'xsc_new_vs_old_phot.png')
plt.savefig(outpath, dpi=200)
plt.close()
print('\ncreated file: {}'.format(outpath))
outfile = 'xsc_replaced.json'
json.dump(replaced, open(outfile, 'w'))
print("\ncreated file: {}".format(outfile))
print("\nremoved / added")
for k, v in replaced.iteritems():
print k.split('/')[-1], v['old'], v['new']
m = np.mean([i['old'] / float(i['new']) for i in replaced.values()])
print("average ratio: {}".format(m))
print("\nK mag and r_ext of sources with NaN photometry:")
for i in find_files(bcdphot_out_path, "*xsc_nan_phot.csv"):
reg = i.split('/')[-1]
rec = np.recfromcsv(i)
bad_id = rec.designation.tolist()
bad_k = rec.k_m_k20fe.tolist()
bad_r_ext = rec.r_ext.tolist()
print reg
print("\tid\t\t\tKmag\tr_ext")
if type(bad_id) is list:
seq = sorted(zip(bad_id, bad_k, bad_r_ext), key=lambda x: x[0])
for j, k, l in seq:
print("\t{}\t{}\t{}".format(j, k, l))
else:
print("\t{}\t{}\t{}".format(bad_id, bad_k, bad_r_ext))
def spherematch(ra1, dec1, ra2, dec2, tol=None, nnearest=1, nthreads=1):
"""
Determines the matches between two catalogues of sources with
<ra, dec> coordinates.
Parameters
----------
ra1, dec1 : array_like
Right ascension and declination of the 1st catalogue.
Units are in `degrees`.
ra2, dec2 : array_like
Right ascension and declination of the 2nd catalogue.
Units are in `degrees`.
tol : float or None, optional
How close (in degrees) a match has to be to count as a match.
If None, all nearest neighbors for the 1st catalogue will be returned.
nnearest : int, optional
The nth neighbor to find. E.g. 1 for the nearest nearby, 2 for the
second nearest neighbor, etc. Partcularly useful if you want to get
the nearest *non-self* neighbor of a catalogue.
To do this use::
``spherematch(ra, dec, ra, dec, nnearest=2)``
if `nnearest == 0`, all matches are returned.
nthreads : int, optional
Number of threads to use for calculation. This variable is set to
1 by default. Must be larger than 1.
Returns
----------
idx1 : int `numpy.ndarray`
Indices of the 1st catalogue of the matches. Will never be larger
than `ra1`/`dec1`.
idx2 : int `numpy.ndarray`
Indices of the 2nd catalogue of the matches. Will never be larger
than `ra1`/`dec1`.
ds : float `numpy.ndarray`
Distance (in degrees) between the matches.
"""
file_msg = fd.Program_Msg(__file__)
## Checking input arguments
valid_types = (list, np.ndarray)
# `ra1`
if not (isinstance(ra1, valid_types)):
msg = '{0} `ra1` ({1}) is not a valid type!'.format(file_msg, type(ra1))
raise LSSUtils_Error(msg)
# `dec1`
if not (isinstance(dec1, valid_types)):
msg = '{0} `dec1` ({1}) is not a valid type!'.format(file_msg, type(dec1))
raise LSSUtils_Error(msg)
# `ra2`
if not (isinstance(ra2, valid_types)):
msg = '{0} `ra2` ({1}) is not a valid type!'.format(file_msg, type(ra2))
raise LSSUtils_Error(msg)
# `dec2`
if not (isinstance(dec2, valid_types)):
msg = '{0} `dec2` ({1}) is not a valid type!'.format(file_msg, type(dec2))
raise LSSUtils_Error(msg)
# `nnearest`
if nnearest < 0:
msg = '{0} `nnearest` ({1}) must be larger than `0`!'.format(file_msg,
nnearest)
raise LSSUtils_Error(msg)
# `threads`
if nthreads < 1:
msg = '{0} `nthreads` ({1}) must be larger than `1`!'.format(file_msg,
nthreads)
raise LSSUtils_Error(msg)
##
## Converting arguments into arrays for ease of use
ra1 = np.array(ra1 , copy=False)
dec1 = np.array(dec1, copy=False)
ra2 = np.array(ra2 , copy=False)
dec2 = np.array(dec2, copy=False)
## Checking shape
# 1st catalogue
if ra1.shape != dec1.shape:
msg = '{0} The shape of `ra1` ({1}) does not mathc that of `dec1` ({2}).'
msg = msg.format(file_msg, ra1.shape, dec1.shape)
raise LSSUtils_Error(msg)
# 2nd catalogue
if ra2.shape != dec2.shape:
msg = '{0} The shape of `ra2` ({1}) does not mathc that of `dec2` ({2}).'
msg = msg.format(file_msg, ra2.shape, dec2.shape)
raise LSSUtils_Error(msg)
##
## Converting spherical coordinates into cartesian coordinates
# 1st catalogue
x1, y1, z1 = _spherical_to_cartesian_fast( ra1.ravel(),
dec1.ravel(),
nthreads)
coords1 = np.empty((x1.size,3))
coords1[:, 0] = x1
coords1[:, 1] = y1
coords1[:, 2] = z1
# 2nd catalogue
x2, y2, z2 = _spherical_to_cartesian_fast( ra2.ravel(),
dec2.ravel(),
nthreads)
coords2 = np.empty((x2.size,3))
coords2[:, 0] = x2
coords2[:, 1] = y2
coords2[:, 2] = z2
##
## Finding nearest neighbors
kdt = KDT(coords2)
# Finding neighbors
if nnearest == 1:
idx_s2 = kdt.query(coords1)[1]
elif (nnearest == 0) and (tol is not None): # if you want ALL matches!
p1_x, p1_y, p1_z = _spherical_to_cartesian_fast(90., 0 , nthreads)
p2_x, p2_y, p2_z = _spherical_to_cartesian_fast(90., tol, nthreads)
# Converting to floats
p1_x = float(p1_x)
p1_y = float(p1_y)
p1_z = float(p1_z)
p2_x = float(p2_x)
p2_y = float(p2_y)
p2_z = float(p2_z)
r = np.sqrt((p2_x - p1_x)**2 + (p2_y - p1_y)**2 + (p2_z - p1_z)**2)
idx_s2 = kdt.query_ball_point(coords1, r)[0]
elif nnearest > 1:
idx_s2 = kdt.query(coords1, nnearest)[1][:, -1]
else:
msg = '{0} Invalid `nnearest` ({1})!'.format(file_msg, nnearest)
raise LSSUtils_Error(msg)
##
## Calculating distance between matches
ds = _great_circle_distance_fast( ra1 ,
dec1 ,
ra2[idx_s2] ,
dec2[idx_s2],
nthreads )
##
## If `tol` is None, then all objects will have a match.
idx_s1 = np.arange(ra1.size)
##
## Remove matches that are `beyond` the tolerance separation
if (tol is not None) and (nnearest != 0):
mask = ds < tol
idx_s1 = idx_s1[mask]
idx_s2 = idx_s2[mask]
ds = ds [mask]
return idx_s1, idx_s2, ds
def get_bcd_list(metadata):
"""
Metadata is a dict with keys:
name, radecfile, data_dir, out_dir, work_dir, aors, channel,
bcd_dict_path, max_cov
"""
radecfile = metadata['radecfile']
work_dir = metadata['work_dir']
aors = metadata['aors']
max_cov = metadata['max_cov']
# split the RA/Dec into two arrays
radec = np.genfromtxt(radecfile)
ra = radec[:, 0]
dec = radec[:, 1]
# read the region/ch/hdr specific bcd_dict in the work_dir for efficiency
bcd_dict = json.load(open(metadata['bcd_dict_path']))
filenames, filepaths = [np.array(i) for i in unzip(bcd_dict.items())]
# extract center pixel coordinates
files_ra = np.zeros(filepaths.size)
files_dec = np.zeros(filepaths.size)
for i, fp in enumerate(filepaths):
hdr = pyfits.getheader(fp)
files_ra[i] = hdr['CRVAL1']
files_dec[i] = hdr['CRVAL2']
# make array of coordinates and grow the tree
kdt = KDT(radec_to_coords(files_ra, files_dec))
# spawn processes using multiprocessing to check for images containing,
# the source, using the tree to find only the closest BCDs to check
ncpus = multiprocessing.cpu_count()
pool = multiprocessing.Pool(processes=ncpus)
# print "using %i CPUs" % ncpus
max_num_images = 0
sources = []
for i in range(len(ra)):
# create internal source ID and associate with each RA/Dec pair
d = {'id': i, 'ra': ra[i], 'dec': dec[i]}
message = 'finding files associated with source {} at ({}, {})'
print(message.format(i, ra[i], dec[i]))
# get the subset of BCDs to search
idx = get_k_closest_bcd_idx(ra[i], dec[i], kdt, k=max_cov)
n_files = filepaths[idx].size
filepaths_subset = filepaths[idx]
filenames_subset = filenames[idx]
argslist = zip([ra[i]] * n_files, [dec[i]] * n_files, filepaths_subset)
# send jobs to the pool
results = pool.map(source_in_image, argslist)
# unzip the results and extract the boolean array and pixel coordinates
results_unzipped = unzip(results)
bool_arr = np.array(results_unzipped[0])
# if none found, continue to next source
if np.sum(bool_arr) == 0:
continue
x = results_unzipped[1]
y = results_unzipped[2]
pix_coord = np.array(zip(x, y))[bool_arr].tolist()
# get the names of the files associated with the source
good_bcds = filenames_subset[bool_arr].tolist()
# compare the number of associated images to the previous maximum
num_images = len(good_bcds)
print('\t{} images'.format(num_images))
if num_images > max_num_images:
max_num_images = num_images
# store results in source dict and append to source list
d['files'] = good_bcds
d['pixels'] = pix_coord
sources.append(d)
outfile = 'bcd_list.json'
outfilepath = '/'.join([work_dir, outfile])
with open(outfilepath, 'w') as w:
json.dump(sources, w, indent=4 * ' ')
print('created file: {}'.format(outfilepath))
message = 'maximum number of images associated with a source: {}'
print(message.format(max_num_images))
coordsP[:, 0] = x_psf coordsP[:, 1] = y_psf coordsP[:, 2] = z_psf coordsF[:, 0] = x_flc coordsF[:, 1] = y_flc coordsF[:, 2] = z_flc ######################################################################## # kdt = KDT(coordsF) # idxsF = kdt.query(coordsP)[1] # ds = distArr(x_psf,y_psf,z_psf,x_flc[idxsF],y_flc[idxsF],z_flc[idxsF]) kdt = KDT(coordsP) idxsP = kdt.query(coordsF)[1] ds = distArr(x_flc, y_flc, z_flc, x_psf[idxsP], y_psf[idxsP], z_psf[idxsP]) # print(len(ds)) idxsF = np.arange(x_flc.size) msk = ds < matchtol idxsF = idxsF[msk] idxsP = idxsP[msk] ds = ds[msk] # print(len(idxs1))
filter = filters[ff]
x_drc_low = low_x[ff]
y_drc_low = low_y[ff]
xm_flc_low = flc_all['xdrc_low_'+filter]
ym_flc_low = flc_all['ydrc_low_'+filter]
coords1low = np.empty((xm_flc_low.size,2))
coords2low = np.empty((x_drc_low.size,2))
coords1low[:,0] = xm_flc_low
coords1low[:,1] = ym_flc_low
coords2low[:,0] = x_drc_low
coords2low[:,1] = y_drc_low
kdt = KDT(coords2low)
idxs2 = kdt.query(coords1low)[1]
ds = distArr(xm_flc_low,ym_flc_low,x_drc_low[idxs2],y_drc_low[idxs2])
idxs1 = np.arange(xm_flc_low.size)
msk = ds < matchtol
idxs1 = idxs1[msk]
idxs2 = idxs2[msk]
ds = ds[msk]
outfile = outDir+'hor-I-cut_drc_low_'+filter+'_tol1.txt'
np.savetxt(outfile, idxs2, fmt='%4i')
outfile = outDir+'hor-I-cut_flc_low_'+filter+'_tol1.txt'
def xymatch(x1, y1, x2, y2, tol=None, nnearest=1):
"""
Finds matches in one catalog to another.
Parameters
x1 : array-like
X-coordinates of first catalog
y1 : array-like
Y-coordinates of first catalog
x2 : array-like
X-coordinates of second catalog
y2 : array-like
Y-coordinates of second catalog
tol : float or None, optional
How close a match has to be to count as a match. If None,
all nearest neighbors for the first catalog will be returned.
nnearest : int, optional
The nth neighbor to find. E.g., 1 for the nearest nearby, 2 for the
second nearest neighbor, etc. Particularly useful if you want to get
the nearest *non-self* neighbor of a catalog. To do this, use:
``spherematch(x, y, x, y, nnearest=2)``
Returns
-------
idx1 : int array
Indecies into the first catalog of the matches. Will never be
larger than `x1`/`y1`.
idx2 : int array
Indecies into the second catalog of the matches. Will never be
larger than `x1`/`y1`.
ds : float array
Distance between the matches
"""
x1 = np.array(x1, copy=False)
y1 = np.array(y1, copy=False)
x2 = np.array(x2, copy=False)
y2 = np.array(y2, copy=False)
if x1.shape != y1.shape:
raise ValueError('x1 and y1 do not match!')
if x2.shape != y2.shape:
raise ValueError('x2 and y2 do not match!')
# this is equivalent to, but faster than just doing np.array([x1, y1])
coords1 = np.empty((x1.size, 2))
coords1[:, 0] = x1
coords1[:, 1] = y1
# this is equivalent to, but faster than just doing np.array([x2, y2])
coords2 = np.empty((x2.size, 2))
coords2[:, 0] = x2
coords2[:, 1] = y2
kdt = KDT(coords2)
if nnearest == 1:
ds,idxs2 = kdt.query(coords1)
elif nnearest > 1:
retval = kdt.query(coords1, nnearest)
ds = retval[0]
idxs2 = retval[1][:, -1]
else:
raise ValueError('invalid nnearest ' + str(nnearest))
idxs1 = np.arange(x1.size)
if tol is not None:
msk = ds < tol
idxs1 = idxs1[msk]
idxs2 = idxs2[msk]
ds = ds[msk]
return idxs1, idxs2, ds
def crossmatch_gaia(self, plate_solution=None, star_catalog=None):
"""
Crossmatch sources with Gaia objects, considering multiple solutions.
Parameters:
-----------
plate_solution : :class:`solve.PlateSolution`
Plate solution with one or more astrometric solutions
star_catalog : :class:`catalog.StarCatalog`
External star catalog with Gaia data
"""
from .solve import PlateSolution
from .catalog import StarCatalog
self.log.write('Crossmatching sources with Gaia objects',
level=3, event=44)
if plate_solution is None or plate_solution.num_solutions == 0:
self.log.write('Cannot crossmatch sources with Gaia objects '
'due to missing astrometric solutions!',
level=2, event=44)
return
if star_catalog is None:
self.log.write('Cannot crossmatch sources with Gaia objects '
'due to missing Gaia catalog data!',
level=2, event=44)
return
assert isinstance(plate_solution, PlateSolution)
assert isinstance(star_catalog, StarCatalog)
# Take parameters from plate_solution
num_solutions = plate_solution.num_solutions
solutions = plate_solution.solutions
mean_pixscale = plate_solution.mean_pixel_scale
# Number of Gaia stars
num_gaia = len(star_catalog)
self.log.write('Number of Gaia stars: {:d}'.format(num_gaia),
level=4, event=44, double_newline=False)
# Calculate RA and Dec for the plate epoch
ra_ref = (star_catalog['ra']
+ (self.plate_epoch - star_catalog['ref_epoch'])
* star_catalog['pmra']
/ np.cos(star_catalog['dec'] * np.pi / 180.) / 3600000.)
dec_ref = (star_catalog['dec']
+ (self.plate_epoch - star_catalog['ref_epoch'])
* star_catalog['pmdec'] / 3600000.)
#catalog = SkyCoord(ra_ref, dec_ref, frame='icrs')
xy_ref = np.empty((0, 2))
sol_ref = np.empty((0,), dtype=np.int8)
index_ref = np.empty((0,), dtype=np.int32)
# Build a list of Gaia stars in image coordinates
for i in np.arange(plate_solution.num_solutions):
solution = solutions[i]
# If there is a column named 'solution_num', then take only
# reference stars with the current solution number
if 'solution_num' in star_catalog.columns:
mask_sol = star_catalog['solution_num'] == i + 1
else:
mask_sol = np.full(num_gaia, True)
w = wcs.WCS(solution['header_wcs'])
try:
xr,yr = w.all_world2pix(ra_ref[mask_sol], dec_ref[mask_sol], 1)
except wcs.NoConvergence as e:
self.log.write('Failed to convert sky coordinates to '
'pixel coordinates for solution {:d}: {}'
.format(i + 1, e))
continue
mask_inside = ((xr > 0.5) & (xr < plate_solution.imwidth) &
(yr > 0.5) & (yr < plate_solution.imheight))
num_inside = mask_inside.sum()
xyr = np.vstack((xr[mask_inside], yr[mask_inside])).T
xy_ref = np.vstack((xy_ref, xyr))
sol_ref = np.hstack((sol_ref, np.full(num_inside, i + 1)))
index_ref = np.hstack((index_ref,
np.arange(num_gaia)[mask_sol][mask_inside]))
# Calculate mean astrometric error
sigma1 = u.Quantity([sol['scamp_sigma_1'] for sol in solutions
if sol['scamp_sigma_1'] is not None])
sigma2 = u.Quantity([sol['scamp_sigma_2'] for sol in solutions
if sol['scamp_sigma_2'] is not None])
if len(sigma1) > 0 and len(sigma2) > 0:
mean_scamp_sigma = np.sqrt(sigma1.mean()**2 + sigma2.mean()**2)
else:
mean_scamp_sigma = 2. * u.arcsec
# Crossmatch sources and Gaia stars
coords_plate = np.vstack((self['x_source'], self['y_source'])).T
tolerance = ((5. * mean_scamp_sigma / mean_pixscale)
.to(u.pixel).value)
#if (5. * mean_scamp_sigma) < 2 * u.arcsec:
# tolerance = ((2 * u.arcsec / mean_pixscale)
# .to(u.pixel).value)
tolerance_arcsec = (5. * mean_scamp_sigma).to(u.arcsec).value
self.log.write('Crossmatch tolerance: {:.2f} arcsec ({:.2f} pixels)'
.format(tolerance_arcsec, tolerance), level=4, event=44,
double_newline=False)
ind_plate, ind_ref, ds = crossmatch_cartesian(coords_plate, xy_ref,
tolerance=tolerance)
dist_arcsec = (ds * u.pixel * mean_pixscale).to(u.arcsec).value
ind_gaia = index_ref[ind_ref]
self['solution_num'][ind_plate] = sol_ref[ind_ref]
self['match_radius'][ind_plate] = tolerance_arcsec
self['gaiaedr3_id'][ind_plate] = star_catalog['source_id'][ind_gaia]
self['gaiaedr3_gmag'][ind_plate] = star_catalog['mag'][ind_gaia]
self['gaiaedr3_bpmag'][ind_plate] = star_catalog['mag1'][ind_gaia]
self['gaiaedr3_rpmag'][ind_plate] = star_catalog['mag2'][ind_gaia]
self['gaiaedr3_bp_rp'][ind_plate] = star_catalog['color_index'][ind_gaia]
self['gaiaedr3_dist'][ind_plate] = dist_arcsec
self.num_crossmatch_gaia = len(ind_plate)
# Mask nan values in listed columns
for col in ['gaiaedr3_gmag', 'gaiaedr3_bpmag', 'gaiaedr3_rpmag',
'gaiaedr3_bp_rp', 'gaiaedr3_dist']:
self[col] = MaskedColumn(self[col], mask=np.isnan(self[col]))
# Mask zeros in the ID column
col = 'gaiaedr3_id'
self[col] = MaskedColumn(self[col], mask=(self[col] == 0))
# Store number of crossmatched sources to each solution
grp = self.group_by('solution_num').groups
tab_grp = Table(grp.aggregate(len)['solution_num', 'source_num'])
tab_grp.rename_column('source_num', 'num_gaia_edr3')
for i in np.arange(plate_solution.num_solutions):
solution = solutions[i]
m = tab_grp['solution_num'] == i + 1
if m.sum() > 0:
num_gaia_edr3 = tab_grp['num_gaia_edr3'][m].data[0]
solution['num_gaia_edr3'] = num_gaia_edr3
else:
solution['num_gaia_edr3'] = 0
# Crossmatch: find all neighbours for sources
kdt_ref = KDT(xy_ref)
kdt_plate = KDT(coords_plate)
max_distance = ((20. * mean_scamp_sigma / mean_pixscale)
.to(u.pixel).value)
if (20. * mean_scamp_sigma) < 5 * u.arcsec:
max_distance = (5 * u.arcsec / mean_pixscale).to(u.pixel).value
max_dist_arcsec = (max_distance * u.pixel * mean_pixscale).to(u.arcsec).value
self.log.write('Finding all reference stars around sources within '
'the radius of {:.2f} arcsec ({:.2f} pixels)'
.format(max_dist_arcsec, max_distance),
level=4, event=44)
mtrx = kdt_plate.sparse_distance_matrix(kdt_ref, max_distance)
mtrx_keys = np.array([a for a in mtrx.keys()])
# Check if there are neighbors at all
if len(mtrx_keys) > 0:
k_plate = mtrx_keys[:,0]
k_ref = mtrx_keys[:,1]
dist = np.fromiter(mtrx.values(), dtype=float) * u.pixel
# Construct neighbors table
nbs = Table()
nbs['source_num'] = self['source_num'][k_plate]
nbs['gaiaedr3_id'] = star_catalog['source_id'][index_ref[k_ref]]
nbs['dist'] = dist
nbs['solution_num'] = sol_ref[k_ref]
nbs['x_gaia'] = xy_ref[k_ref,0]
nbs['y_gaia'] = xy_ref[k_ref,1]
# Create the flag_xmatch column by joining the neighbors table
# with the source table
tab = Table()
tab['source_num'] = self['source_num']
tab['gaiaedr3_id'] = MaskedColumn(self['gaiaedr3_id']).filled(0)
tab['flag_xmatch'] = np.int8(1)
jtab = join(nbs, tab, keys=('source_num', 'gaiaedr3_id'),
join_type='left')
jtab['flag_xmatch'] = MaskedColumn(jtab['flag_xmatch']).filled(0)
self.neighbors_gaia = jtab
# Calculate neighbor counts
source_num, cnt = np.unique(nbs['source_num'].data, return_counts=True)
mask = np.isin(self['source_num'], source_num)
ind_mask = np.where(mask)[0]
self['gaiaedr3_neighbors'][ind_mask] = cnt
else:
# Create empty neighbors table
nbs = Table(names=('source_num', 'gaiaedr3_id', 'dist',
'solution_num', 'x_gaia', 'y_gaia',
'flag_xmatch'),
dtype=('i4', 'i8', 'f4', 'i2', 'f8', 'f8', 'i1'))
self.neighbors_gaia = nbs
# Process coordinates again, because solution_num assignments may have changed
self.process_coordinates(plate_solution=plate_solution)
def runPixMatch(outpre, filter):
if filter == 'f606w':
let = 'v'
else:
let = 'i'
if outpre == 'lower':
x_drc_low = drc_low['x_' + let]
y_drc_low = drc_low['y_' + let]
xm_flc_low = flc_all['xdrc_low_' + filter]
ym_flc_low = flc_all['ydrc_low_' + filter]
coords1low = np.empty((xm_flc_low.size, 2))
coords2low = np.empty((x_drc_low.size, 2))
coords1low[:, 0] = xm_flc_low
coords1low[:, 1] = ym_flc_low
coords2low[:, 0] = x_drc_low
coords2low[:, 1] = y_drc_low
kdt = KDT(coords2low)
idxs2 = kdt.query(coords1low)[1]
ds = distArr(xm_flc_low, ym_flc_low, x_drc_low[idxs2],
y_drc_low[idxs2])
idxs1 = np.arange(xm_flc_low.size)
msk = ds < matchtol
idxs1 = idxs1[msk]
idxs2 = idxs2[msk]
ds = ds[msk]
else:
x_drc_up = drc_up['x_' + let]
y_drc_up = drc_up['y_' + let]
xm_flc_up = flc_all['xdrc_up_' + filter]
ym_flc_up = flc_all['ydrc_up_' + filter]
coords1up = np.empty((xm_flc_up.size, 2))
coords2up = np.empty((x_drc_up.size, 2))
coords1up[:, 0] = xm_flc_up
coords1up[:, 1] = ym_flc_up
coords2up[:, 0] = x_drc_up
coords2up[:, 1] = y_drc_up
kdt = KDT(coords2up)
idxs2 = kdt.query(coords1up)[1]
ds = distArr(xm_flc_up, ym_flc_up, x_drc_up[idxs2], y_drc_up[idxs2])
idxs1 = np.arange(xm_flc_up.size)
msk = ds < matchtol
idxs1 = idxs1[msk]
idxs2 = idxs2[msk]
ds = ds[msk]
print(len(idxs1))
outfile = main_dir + 'hor-I-cut_drc_' + outpre + '_' + filter + '_tol{0}_magCuts.txt'.format(
matchtol)
np.savetxt(outfile, idxs2, fmt='%4i')
outfile = main_dir + 'hor-I-cut_flc_' + outpre + '_' + filter + '_tol{0}_magCuts.txt'.format(
matchtol)
np.savetxt(outfile, idxs1, fmt='%4i')
# outfile = main_dir+'hor-I-cut_ds_'+outpre+'_'+filter+'_tol{0}.txt'.format(matchtol)
# np.savetxt(outfile, ds, fmt='%1.4f')
return None
def spherematch(ra1, dec1, ra2, dec2, tol=None, nnearest=1, threads=1):
"""
Determines the matches between two catalogues of sources with ra,dec coordinates
Parameters
ra1 : array-like
Right Ascension in degrees of the first catalog
dec1 : array-like
Declination in degrees of the first catalog (shape of array must match `ra1`)
ra2 : array-like
Right Ascension in degrees of the second catalog
dec2 : array-like
Declination in degrees of the second catalog (shape of array must match `ra2`)
tol : float or None, optional
How close (in degrees) a match has to be to count as a match. If None,
all nearest neighbors for the first catalog will be returned.
nnearest : int, optional
The nth neighbor to find. E.g., 1 for the nearest nearby, 2 for the
second nearest neighbor, etc. Particularly useful if you want to get
the nearest *non-self* neighbor of a catalog. To do this, use:
``spherematch(ra, dec, ra, dec, nnearest=2)``
if nnearest==0, all matches are returned
Returns
idx1 : int array
Indecies into the first catalog of the matches. Will never be
larger than `ra1`/`dec1`.
idx2 : int array
Indecies into the second catalog of the matches. Will never be
larger than `ra1`/`dec1`.
ds : float array
Distance (in degrees) between the matches
"""
#convert arguments into arrays for ease of use
ra1 = np.array(ra1, copy=False)
dec1 = np.array(dec1, copy=False)
ra2 = np.array(ra2, copy=False)
dec2 = np.array(dec2, copy=False)
#check to see if arguments are consistent
if ra1.shape != dec1.shape:
raise ValueError('ra1 and dec1 do not match!')
if ra2.shape != dec2.shape:
raise ValueError('ra2 and dec2 do not match!')
#convert spherical coordinates into cartesian coordinates
x1, y1, z1 = _spherical_to_cartesian_fast(ra1.ravel(), dec1.ravel(),
threads)
# this is equivalent to, but faster than just doing np.array([x1, y1, z1])
coords1 = np.empty((x1.size, 3))
coords1[:, 0] = x1
coords1[:, 1] = y1
coords1[:, 2] = z1
#convert spherical coordinates into cartesian coordinates
x2, y2, z2 = _spherical_to_cartesian_fast(ra2.ravel(), dec2.ravel(),
threads)
# this is equivalent to, but faster than just doing np.array([x1, y1, z1])
coords2 = np.empty((x2.size, 3))
coords2[:, 0] = x2
coords2[:, 1] = y2
coords2[:, 2] = z2
#create tree structure
kdt = KDT(coords2)
#find neighbors
if nnearest == 1:
idxs2 = kdt.query(coords1)[1]
elif nnearest == 0 and (tol is not None): #if you want all matches
p1x, p1y, p1z = _spherical_to_cartesian_fast(90, 0, threads)
p2x, p2y, p2z = _spherical_to_cartesian_fast(90, tol, threads)
p1x = float(p1x)
p2x = float(p2x)
p1y = float(p1y)
p2y = float(p2y)
p1z = float(p1z)
p2z = float(p2z)
r = np.sqrt(
(p2x - p1x)**2 + (p2y - p1y)**2 + (p2z - p1z)**2) #cartesian tol
idxs2 = kdt.query_ball_point(coords1, r)[0]
elif nnearest > 1:
idxs2 = kdt.query(coords1, nnearest)[1][:, -1]
else:
raise ValueError('invalid nnearest ' + str(nnearest))
#calculate distances between matches
ds = _great_circle_distance_fast(ra1, dec1, ra2[idxs2], dec2[idxs2],
threads)
#if tolerance is None, then all objects will have a match
idxs1 = np.arange(ra1.size)
#remove matches that are beyond the tolerance seperation
if (tol is not None) and nnearest != 0:
msk = ds < tol
idxs1 = idxs1[msk]
idxs2 = idxs2[msk]
ds = ds[msk]
return idxs1, idxs2, ds
def spherematch(ra1, dec1, ra2, dec2, tol=None, nnearest=1):
"""
Finds matches in one catalog to another.
Parameters
ra1 : array-like
Right Ascension in degrees of the first catalog
dec1 : array-like
Declination in degrees of the first catalog (shape of array must match `ra1`)
ra2 : array-like
Right Ascension in degrees of the second catalog
dec2 : array-like
Declination in degrees of the second catalog (shape of array must match `ra2`)
tol : float or None, optional
How close (in degrees) a match has to be to count as a match. If None,
all nearest neighbors for the first catalog will be returned.
nnearest : int, optional
The nth neighbor to find. E.g., 1 for the nearest nearby, 2 for the
second nearest neighbor, etc. Particularly useful if you want to get
the nearest *non-self* neighbor of a catalog. To do this, use:
``spherematch(ra, dec, ra, dec, nnearest=2)``
Returns
-------
idx1 : int array
Indecies into the first catalog of the matches. Will never be
larger than `ra1`/`dec1`.
idx2 : int array
Indecies into the second catalog of the matches. Will never be
larger than `ra1`/`dec1`.
ds : float array
Distance (in degrees) between the matches
"""
ra1 = np.array(ra1, copy=False)
dec1 = np.array(dec1, copy=False)
ra2 = np.array(ra2, copy=False)
dec2 = np.array(dec2, copy=False)
if ra1.shape != dec1.shape:
raise ValueError('ra1 and dec1 do not match!')
if ra2.shape != dec2.shape:
raise ValueError('ra2 and dec2 do not match!')
x1, y1, z1 = _spherical_to_cartesian(ra1.ravel(), dec1.ravel())
# this is equivalent to, but faster than just doing np.array([x1, y1, z1])
coords1 = np.empty((x1.size, 3))
coords1[:, 0] = x1
coords1[:, 1] = y1
coords1[:, 2] = z1
x2, y2, z2 = _spherical_to_cartesian(ra2.ravel(), dec2.ravel())
# this is equivalent to, but faster than just doing np.array([x1, y1, z1])
coords2 = np.empty((x2.size, 3))
coords2[:, 0] = x2
coords2[:, 1] = y2
coords2[:, 2] = z2
kdt = KDT(coords2)
if nnearest == 1:
idxs2 = kdt.query(coords1)[1]
elif nnearest > 1:
idxs2 = kdt.query(coords1, nnearest)[1][:, -1]
else:
raise ValueError('invalid nnearest ' + str(nnearest))
ds = _great_circle_distance(ra1, dec1, ra2[idxs2], dec2[idxs2])
idxs1 = np.arange(ra1.size)
if tol is not None:
msk = ds < tol
idxs1 = idxs1[msk]
idxs2 = idxs2[msk]
ds = ds[msk]
return idxs1, idxs2, ds
def spherematch(ra1, dec1, ra2, dec2, logger, tol=None, nnearest=1):
"""
Finds matches in one catalog to another.
Parameters
ra1 : array-like
Right Ascension in degrees of the first catalog
dec1 : array-like
Declination in degrees of the first catalog (shape of array must match `ra1`)
ra2 : array-like
Right Ascension in degrees of the second catalog
dec2 : array-like
Declination in degrees of the second catalog (shape of array must match `ra2`)
tol : float or None, optional
How close (in degrees) a match has to be to count as a match. If None,
all nearest neighbors for the first catalog will be returned.
nnearest : int, optional
The nth neighbor to find. E.g., 1 for the nearest nearby, 2 for the
second nearest neighbor, etc. Particularly useful if you want to get
the nearest *non-self* neighbor of a catalog. To do this, use:
``spherematch(ra, dec, ra, dec, nnearest=2)``
Returns
-------
idx1 : int array
Indecies into the first catalog of the matches. Will never be
larger than `ra1`/`dec1`.
idx2 : int array
Indecies into the second catalog of the matches. Will never be
larger than `ra1`/`dec1`.
ds : float array
Distance (in degrees) between the matches
"""
ra1 = np.array(ra1, copy=False)
dec1 = np.array(dec1, copy=False)
ra2 = np.array(ra2, copy=False)
dec2 = np.array(dec2, copy=False)
if ra1.shape != dec1.shape:
raise ValueError('ra1 and dec1 do not match!')
if ra2.shape != dec2.shape:
raise ValueError('ra2 and dec2 do not match!')
x1, y1, z1 = _spherical_to_cartesian(ra1.ravel(), dec1.ravel())
# this is equivalent to, but faster than just doing np.array([x1, y1, z1])
coords1 = np.empty((x1.size, 3))
coords1[:, 0] = x1
coords1[:, 1] = y1
coords1[:, 2] = z1
x2, y2, z2 = _spherical_to_cartesian(ra2.ravel(), dec2.ravel())
# this is equivalent to, but faster than just doing np.array([x1, y1, z1])
coords2 = np.empty((x2.size, 3))
coords2[:, 0] = x2
coords2[:, 1] = y2
coords2[:, 2] = z2
kdt = KDT(coords2)
if nnearest == 1:
idxs2 = kdt.query(coords1)[1]
elif nnearest > 1:
idxs2 = kdt.query(coords1, nnearest)[1][:, -1]
else:
raise ValueError('invalid nnearest ' + str(nnearest))
ds = _great_circle_distance(ra1, dec1, ra2[idxs2], dec2[idxs2])
idxs1 = np.arange(ra1.size)
if tol is not None:
msk = ds < tol
idxs1 = idxs1[msk]
idxs2 = idxs2[msk]
ds = ds[msk]
# The following removes duplicates, but with no distinction of merit
'''c = len(idxs1)
idxs2, idxs2_unique_indexes = np.unique(idxs2, return_index=True)
idxs1 = idxs1[idxs2_unique_indexes]
logger.info("(pipeline._XMatchSources.spherematch) Found " + str(c - len(idxs1)) + " duplicate(s) in spherematch")'''
# The following removes duplicates, only preserving the nearest match
c = len(idxs1)
for i in Counter(idxs2).most_common(
): # returns ([idx_that_is_the_duplicate, count), ..])
if i[1] < 2: # no more duplicates here
break
else:
duplicate_idxs = np.where(idxs2 == i[0])[
0] # what are the indexes of the duplicate entry in idxs2
duplicate_idxs_dist = ds[
duplicate_idxs] # find the corresponding distances
duplicate_idx_min = np.argmin(
duplicate_idxs_dist
) # find the index of the minimum of the above
keep = duplicate_idxs[
duplicate_idx_min] # this the index in duplicate_idxs to be retained
for d in duplicate_idxs: # delete all but "keep" index
if d == keep:
continue
else:
idxs1 = np.delete(idxs1, d)
idxs2 = np.delete(idxs2, d)
ds = np.delete(ds, d)
logger.info("(pipeline._XMatchSources.spherematch) Found " +
str(c - len(idxs1)) + " duplicate(s) in spherematch")
return idxs1, idxs2, ds