def _sanity_check_object_table_mapper(qresult, radius, explist):
all_objs = np.empty(0, dtype=np.uint64)
havedet_objs = np.empty(0, dtype=np.uint64)
for rows in qresult:
# Prep work for checking #3, at the end.
# Collect all objects
objs = np.unique(rows.obj_id) # Get unique IDs
objs = objs[~np.in1d(
objs, all_objs,
assume_unique=True)] # Keep only those never before seen
all_objs = np.append(all_objs, objs) # Append to a list of all objects
# Remove outer-joined rows
rows = rows[~rows.dnull]
# Prep work for checking #3, at the end.
# Collect objects with linked detections
objs = np.unique(rows.obj_id)
objs = objs[~np.in1d(
objs, havedet_objs,
assume_unique=True)] # Keep only those never before seen
havedet_objs = np.append(
havedet_objs, objs) # Append to a list of having a detection
# Remove all detections that did not originate from explist
if explist is not None:
rows = rows[np.in1d(rows.exp_id, explist)]
# Check radius (cond #4)
dist = gc_dist(rows.olon, rows.olat, rows.dlon, rows.dlat)
assert np.all(dist < radius)
# Regroup by det_id's cell and yield to reducer.
# We do this because some det_id's from other cells may be linked
# to an object from this cell
cells = qresult.pix.cell_for_id(rows.det_id)
for cell_id in np.unique(cells):
yield (cell_id, rows[("det_id", "obj_id")][cells == cell_id])
# Check that each object has exactly one detection (cond #3)
all_objs.sort()
havedet_objs.sort()
assert np.all(all_objs == havedet_objs)
def _sanity_check_object_table_mapper(qresult, radius, explist):
all_objs = np.empty(0, dtype=np.uint64)
havedet_objs = np.empty(0, dtype=np.uint64)
for rows in qresult:
# Prep work for checking #3, at the end.
# Collect all objects
objs = np.unique(rows.obj_id) # Get unique IDs
objs = objs[~np.in1d(objs, all_objs, assume_unique=True)] # Keep only those never before seen
all_objs = np.append(all_objs, objs) # Append to a list of all objects
# Remove outer-joined rows
rows = rows[~rows.dnull]
# Prep work for checking #3, at the end.
# Collect objects with linked detections
objs = np.unique(rows.obj_id)
objs = objs[~np.in1d(objs, havedet_objs, assume_unique=True)] # Keep only those never before seen
havedet_objs = np.append(havedet_objs, objs) # Append to a list of having a detection
# Remove all detections that did not originate from explist
if explist is not None:
rows = rows[np.in1d(rows.exp_id, explist)]
# Check radius (cond #4)
dist = gc_dist(rows.olon, rows.olat, rows.dlon, rows.dlat)
assert np.all(dist < radius)
# Regroup by det_id's cell and yield to reducer.
# We do this because some det_id's from other cells may be linked
# to an object from this cell
cells = qresult.pix.cell_for_id(rows.det_id)
for cell_id in np.unique(cells):
yield (cell_id, rows[("det_id", "obj_id")][cells == cell_id])
# Check that each object has exactly one detection (cond #3)
all_objs.sort()
havedet_objs.sort()
assert np.all(all_objs == havedet_objs)
# coding: utf-8
from sys import argv
from BeautifulSoup import BeautifulStoneSoup
from matplotlib import pyplot
from utils import gc_dist, elevation
gpx_file = argv[1]
data = open(gpx_file).read()
soup = BeautifulStoneSoup(data)
points = soup.findAll("trkpt")
route = [(float(p.attrs[0][1]), float(p.attrs[1][1])) for p in points]
dist_diffs = [0] + [gc_dist(route[i], route[i + 1]) for i in range(1, len(route) - 1)]
dist_from_src = [0]
for d in dist_diffs:
dist_from_src.append(dist_from_src[-1] + d)
elevs = elevation(route)
pyplot.plot(dist_from_src, elevs)
pyplot.show()
def _xmatch_mapper(qresult, tabname_to, radius, tabname_xm, n_neighbors):
"""
Mapper:
- given all objects in a cell, make an ANN tree
- load all objects in tabname_to (including neighbors), make an ANN tree, find matches
- store the output into an index table
"""
from scikits.ann import kdtree
db = qresult.db
pix = qresult.pix
table_xm = db.table(tabname_xm)
for rows in qresult:
cell_id = rows.info.cell_id
join = ColGroup(
dtype=[('_M1', 'u8'), ('_M2',
'u8'), ('_DIST',
'f4'), ('_NR',
'u1'), ('_LON',
'f8'), ('_LAT',
'f8')])
(id1, ra1, dec1) = rows.as_columns()
(id2, ra2,
dec2) = db.query('_ID, _LON, _LAT FROM %s' % tabname_to).fetch_cell(
cell_id, include_cached=True).as_columns()
if len(id2) != 0:
# Project to tangent plane around the center of the cell. We
# assume the cell is small enough for the distortions not to
# matter and Euclidian distances apply
bounds, _ = pix.cell_bounds(cell_id)
(clon, clat) = bhpix.deproj_bhealpix(*bounds.center())
xy1 = np.column_stack(gnomonic(ra1, dec1, clon, clat))
xy2 = np.column_stack(gnomonic(ra2, dec2, clon, clat))
# Construct kD-tree to find an object in table_to that is nearest
# to an object in table_from, for every object in table_from
tree = kdtree(xy2)
match_idxs, match_d2 = tree.knn(xy1, min(n_neighbors, len(xy2)))
del tree
# Create the index table array
join.resize(match_idxs.size)
for k in xrange(match_idxs.shape[1]):
match_idx = match_idxs[:, k]
join['_M1'][k::match_idxs.shape[1]] = id1
join['_M2'][k::match_idxs.shape[1]] = id2[match_idx]
join['_DIST'][k::match_idxs.shape[1]] = gc_dist(
ra1, dec1, ra2[match_idx], dec2[match_idx])
join['_LON'][k::match_idxs.shape[1]] = ra2[match_idx]
join['_LAT'][k::match_idxs.shape[1]] = dec2[match_idx]
join['_NR'][k::match_idxs.shape[1]] = k
# Remove matches beyond the xmatch radius
join = join[join['_DIST'] < radius]
if len(join):
# compute the cell_id part of the join table's
# IDs. While this is unimportant now (as we could
# just set all of them equal to cell_id part of
# cell_id), if we ever decide to change the
# pixelation of the table later on, this will
# allow us to correctly repixelize the join table as
# well.
#x, y, t, _ = pix._xyti_from_id(join['_M1']) # ... but at the spatial location given by the object table.
#join['_ID'] = pix._id_from_xyti(x, y, t, 0) # This will make the new IDs have zeros in the object part (so Table.append will autogen them)
# TODO: Allow the stuff above (in Table.append)
join['_ID'] = pix.cell_for_id(join['_M1'])
# TODO: Debugging, remove when happy
cid = np.unique(pix.cell_for_id(join['_ID']))
assert len(cid) == 1, len(cid)
assert cid[0] == cell_id, '%s %s' % (cid[0], cell_id)
####
table_xm.append(join)
yield len(id1), len(id2), len(join)
else:
yield len(rows), 0, 0
def _obj_det_match(cells, db, obj_tabname, det_tabname, o2d_tabname, radius, explist=None, _rematching=False):
"""
This kernel assumes:
a) det_table and obj_table have equal partitioning (equally
sized/enumerated spatial cells)
b) both det_table and obj_table have up-to-date neighbor caches
c) temporal det_table cells within this spatial cell are stored
local to this process (relevant for shared-nothing setups)
d) exposures don't stretch across temporal cells
Algorithm:
- fetch all existing static sky objects, including the cached ones (*)
- project them to tangent plane around the center of the cell
(we assume the cell is small enough for the distortions not to matter)
- construct a kD tree in (x, y) tangent space
- for each temporal cell, in sorted order (++):
1.) Fetch the detections, including the cached ones (+)
2.) Project to tangent plane
3.) for each exposure, in sorted order (++):
a.) Match agains the kD tree of objects
b.) Add those that didn't match to the list of objects
4.) For newly added objects: store to disk only those that
fall within this cell (the others will be matched and
stored in their parent cells)
5.) For matched detections: Drop detections matched to cached
objects (these will be matched and stored in the objects'
parent cell). Store the rest.
(+) It is allowed (and necessary to allow) for a cached detection
to be matched against an object within our cell. This
correctly matches cases when the object is right inside
the cell boundary, but the detection is just to the
outside.
(++) Having cells and detections sorted ensures that objects in overlapping
(cached) regions are seen by kernels in different cells in the same
order, thus resulting in the same matches. Note: this may fail in
extremely crowded region, but as of now it's not clear how big of
a problem (if any!) will this pose.
(*) Cached objects must be loaded and matched against to guard against
the case where an object is just outside the edge, while a detection
is just inside. If the cached object was not loaded, the detection
would not match and be proclamed to be a new object. However, in the
cached object's parent cell, the detection would match to the object
and be stored there as well.
The algorithm above ensures that such a detection will matched to
the cached object in this cell (and be dropped in step 5), preventing
it from being promoted into a new object.
TODO: The above algorithm ensures no detection is assigned to more than
one object. It also ensures that each detection links to an object.
Implement a consistency check to verify that.
"""
from scikits.ann import kdtree
# Input is a tuple of obj_cell, and det_cells falling under that obj_cell
obj_cell, det_cells = cells
det_cells.sort()
assert len(det_cells)
# Fetch the frequently used bits
obj_table = db.table(obj_tabname)
det_table = db.table(det_tabname)
o2d_table = db.table(o2d_tabname)
pix = obj_table.pix
# locate cell center (for gnomonic projection)
(bounds, tbounds) = pix.cell_bounds(obj_cell)
(clon, clat) = bhpix.deproj_bhealpix(*bounds.center())
# fetch existing static sky, convert to gnomonic
objs = db.query('_ID, _LON, _LAT FROM %s' % obj_tabname).fetch_cell(obj_cell, include_cached=True)
xyobj = np.column_stack(gnomonic(objs['_LON'], objs['_LAT'], clon, clat))
nobj = len(objs) # Total number of static sky objects
tree = None
nobj_old = 0
# for sanity checks/debugging (see below)
expseen = set()
## TODO: Debugging, remove when happy
assert (np.unique(sorted(det_cells)) == sorted(det_cells)).all()
##print "Det cells: ", det_cells
# Loop, xmatch, and store
if explist is not None:
explist = np.asarray(list(explist), dtype=np.uint64) # Ensure explist is a ndarray
det_query = db.query('_ID, _LON, _LAT, _EXP, _CACHED FROM %s' % det_tabname)
for det_cell in sorted(det_cells):
# fetch detections in this cell, convert to gnomonic coordinates
# keep only detections with _EXP in explist, unless explist is None
detections = det_query.fetch_cell(det_cell, include_cached=True)
# if there are no preexisting static sky objects, and all detections in this cell are cached,
# there's no way we'll get a match that will be kept in the end. Just continue to the
# next one if this is the case.
cachedonly = len(objs) == 0 and detections._CACHED.all()
if cachedonly:
# print "Skipping cached-only", len(cached)
yield (None, None, None, None, None, None) # Yield just to have the progress counter properly incremented
continue;
if explist is not None:
keep = np.in1d(detections._EXP, explist)
if not np.all(keep):
detections = detections[keep]
if len(detections) == 0:
yield (None, None, None, None, None, None) # Yield just to have the progress counter properly incremented
continue
_, ra2, dec2, exposures, cached = detections.as_columns()
detections.add_column('xy', np.column_stack(gnomonic(ra2, dec2, clon, clat)))
# prep join table
join = ColGroup(dtype=o2d_table.dtype_for(['_ID', '_M1', '_M2', '_DIST', '_LON', '_LAT']))
njoin = 0;
nobj0 = nobj;
##print "Cell", det_cell, " - Unique exposures: ", set(exposures)
# Process detections exposure-by-exposure, as detections from
# different exposures within a same temporal cell are allowed
# to belong to the same object
uexposures = set(exposures)
for exposure in sorted(uexposures):
# Sanity check: a consistent table cannot have two
# exposures stretching over more than one cell
assert exposure not in expseen
expseen.add(exposure);
# Extract objects belonging to this exposure only
detections2 = detections[exposures == exposure]
id2, ra2, dec2, _, _, xydet = detections2.as_columns()
ndet = len(xydet)
if len(xyobj) != 0:
# Construct kD-tree and find the object nearest to each
# detection from this cell
if tree is None or nobj_old != len(xyobj):
del tree
nobj_old = len(xyobj)
tree = kdtree(xyobj)
match_idx, match_d2 = tree.knn(xydet, 1)
match_idx = match_idx[:,0] # First neighbor only
####
#if np.uint64(13828114484734072082) in id2:
# np.savetxt('bla.%d.static=%d.txt' % (det_cell, pix.static_cell_for_cell(det_cell)), objs.as_ndarray(), fmt='%s')
# Compute accurate distances, and select detections not matched to existing objects
dist = gc_dist(objs['_LON'][match_idx], objs['_LAT'][match_idx], ra2, dec2)
unmatched = dist >= radius
else:
# All detections will become new objects (and therefore, dist=0)
dist = np.zeros(ndet, dtype='f4')
unmatched = np.ones(ndet, dtype=bool)
match_idx = np.empty(ndet, dtype='i4')
# x, y, t = pix._xyt_from_cell_id(det_cell)
# print "det_cell %s, MJD %s, Exposure %s == %d detections, %d objects, %d matched, %d unmatched" % (det_cell, t, exposure, len(detections2), nobj, len(unmatched)-unmatched.sum(), unmatched.sum())
# Promote unmatched detections to new objects
_, newra, newdec, _, _, newxy = detections2[unmatched].as_columns()
nunmatched = unmatched.sum()
reserve_space(objs, nobj+nunmatched)
objs['_LON'][nobj:nobj+nunmatched] = newra
objs['_LAT'][nobj:nobj+nunmatched] = newdec
dist[unmatched] = 0.
match_idx[unmatched] = np.arange(nobj, nobj+nunmatched, dtype='i4') # Set the indices of unmatched detections to newly created objects
# Join objects to their detections
reserve_space(join, njoin+ndet)
join['_M1'][njoin:njoin+ndet] = match_idx
join['_M2'][njoin:njoin+ndet] = id2
join['_DIST'][njoin:njoin+ndet] = dist
# TODO: For debugging; remove when happy
join['_LON'][njoin:njoin+ndet] = ra2
join['_LAT'][njoin:njoin+ndet] = dec2
njoin += ndet
# Prep for next loop
nobj += nunmatched
xyobj = np.append(xyobj, newxy, axis=0)
# TODO: Debugging: Final consistency check (remove when happy with the code)
dist = gc_dist( objs['_LON'][ join['_M1'][njoin-ndet:njoin] ],
objs['_LAT'][ join['_M1'][njoin-ndet:njoin] ], ra2, dec2)
assert (dist < radius).all()
# Truncate output tables to their actual number of elements
objs = objs[0:nobj]
join = join[0:njoin]
assert len(objs) >= nobj0
# Find the objects that fall outside of cell boundaries. These will
# be processed and stored by their parent cells. Also leave out the objects
# that are already stored in the database
(x, y) = bhpix.proj_bhealpix(objs['_LON'], objs['_LAT'])
in_ = bounds.isInsideV(x, y)
innew = in_.copy();
innew[:nobj0] = False # New objects in cell selector
ids = objs['_ID']
nobjadded = innew.sum()
if nobjadded:
# Append the new objects to the object table, obtaining their IDs.
assert not _rematching, 'cell_id=%s, nnew=%s\n%s' % (det_cell, nobjadded, objs[innew])
ids[innew] = obj_table.append(objs[('_LON', '_LAT')][innew])
# Set the indices of objects not in this cell to zero (== a value
# no valid object in the database can have). Therefore, all
# out-of-bounds links will have _M1 == 0 (#1), and will be removed
# by the np1d call (#2)
ids[~in_] = 0
# 1) Change the relative index to true obj_id in the join table
join['_M1'] = ids[join['_M1']]
# 2) Keep only the joins to objects inside the cell
join = join[ np.in1d(join['_M1'], ids[in_]) ]
# Append to the join table, in *dec_cell* of obj_table (!important!)
if len(join) != 0:
# compute the cell_id part of the join table's
# IDs.While this is unimportant now (as we could
# just set all of them equal to cell_id part of
# cell_id), if we ever decide to change the
# pixelation of the table later on, this will
# allow us to correctly split up the join table as
# well.
#_, _, t = pix._xyt_from_cell_id(det_cell) # This row points to a detection in the temporal cell ...
#x, y, _, _ = pix._xyti_from_id(join['_M1']) # ... but at the spatial location given by the object table.
#join['_ID'][:] = pix._id_from_xyti(x, y, t, 0) # This will make the new IDs have zeros in the object part (so Table.append will autogen them)
join['_ID'][:] = det_cell
o2d_table.append(join)
assert not cachedonly or (nobjadded == 0 and len(join) == 0)
# return: Number of exposures, number of objects before processing this cell, number of detections processed (incl. cached),
# number of newly added objects, number of detections xmatched, number of detection processed that weren't cached
# Note: some of the xmatches may be to newly added objects (e.g., if there are two
# overlapping exposures within a cell; first one will add new objects, second one will match agains them)
yield (len(uexposures), nobj0, len(detections), nobjadded, len(join), (cached == False).sum())
def _xmatch_mapper(qresult, tabname_to, radius, tabname_xm, n_neighbors):
"""
Mapper:
- given all objects in a cell, make an ANN tree
- load all objects in tabname_to (including neighbors), make an ANN tree, find matches
- store the output into an index table
"""
from scikits.ann import kdtree
db = qresult.db
pix = qresult.pix
table_xm = db.table(tabname_xm)
for rows in qresult:
cell_id = rows.info.cell_id
join = ColGroup(dtype=[('_M1', 'u8'), ('_M2', 'u8'), ('_DIST', 'f4'), ('_NR', 'u1'), ('_LON', 'f8'), ('_LAT', 'f8')])
(id1, ra1, dec1) = rows.as_columns()
(id2, ra2, dec2) = db.query('_ID, _LON, _LAT FROM %s' % tabname_to).fetch_cell(cell_id, include_cached=True).as_columns()
if len(id2) != 0:
# Project to tangent plane around the center of the cell. We
# assume the cell is small enough for the distortions not to
# matter and Euclidian distances apply
bounds, _ = pix.cell_bounds(cell_id)
(clon, clat) = bhpix.deproj_bhealpix(*bounds.center())
xy1 = np.column_stack(gnomonic(ra1, dec1, clon, clat))
xy2 = np.column_stack(gnomonic(ra2, dec2, clon, clat))
# Construct kD-tree to find an object in table_to that is nearest
# to an object in table_from, for every object in table_from
tree = kdtree(xy2)
match_idxs, match_d2 = tree.knn(xy1, min(n_neighbors, len(xy2)))
del tree
# Create the index table array
join.resize(match_idxs.size)
for k in xrange(match_idxs.shape[1]):
match_idx = match_idxs[:,k]
join['_M1'][k::match_idxs.shape[1]] = id1
join['_M2'][k::match_idxs.shape[1]] = id2[match_idx]
join['_DIST'][k::match_idxs.shape[1]] = gc_dist(ra1, dec1, ra2[match_idx], dec2[match_idx])
join['_LON'][k::match_idxs.shape[1]] = ra2[match_idx]
join['_LAT'][k::match_idxs.shape[1]] = dec2[match_idx]
join['_NR'][k::match_idxs.shape[1]] = k
# Remove matches beyond the xmatch radius
join = join[join['_DIST'] < radius]
if len(join):
# compute the cell_id part of the join table's
# IDs. While this is unimportant now (as we could
# just set all of them equal to cell_id part of
# cell_id), if we ever decide to change the
# pixelation of the table later on, this will
# allow us to correctly repixelize the join table as
# well.
#x, y, t, _ = pix._xyti_from_id(join['_M1']) # ... but at the spatial location given by the object table.
#join['_ID'] = pix._id_from_xyti(x, y, t, 0) # This will make the new IDs have zeros in the object part (so Table.append will autogen them)
# TODO: Allow the stuff above (in Table.append)
join['_ID'] = pix.cell_for_id(join['_M1'])
# TODO: Debugging, remove when happy
cid = np.unique(pix.cell_for_id(join['_ID']))
assert len(cid) == 1, len(cid)
assert cid[0] == cell_id, '%s %s' % (cid[0], cell_id)
####
table_xm.append(join)
yield len(id1), len(id2), len(join)
else:
yield len(rows), 0, 0
def _obj_det_match(cells,
db,
obj_tabname,
det_tabname,
o2d_tabname,
radius,
explist=None,
_rematching=False):
"""
This kernel assumes:
a) det_table and obj_table have equal partitioning (equally
sized/enumerated spatial cells)
b) both det_table and obj_table have up-to-date neighbor caches
c) temporal det_table cells within this spatial cell are stored
local to this process (relevant for shared-nothing setups)
d) exposures don't stretch across temporal cells
Algorithm:
- fetch all existing static sky objects, including the cached ones (*)
- project them to tangent plane around the center of the cell
(we assume the cell is small enough for the distortions not to matter)
- construct a kD tree in (x, y) tangent space
- for each temporal cell, in sorted order (++):
1.) Fetch the detections, including the cached ones (+)
2.) Project to tangent plane
3.) for each exposure, in sorted order (++):
a.) Match agains the kD tree of objects
b.) Add those that didn't match to the list of objects
4.) For newly added objects: store to disk only those that
fall within this cell (the others will be matched and
stored in their parent cells)
5.) For matched detections: Drop detections matched to cached
objects (these will be matched and stored in the objects'
parent cell). Store the rest.
(+) It is allowed (and necessary to allow) for a cached detection
to be matched against an object within our cell. This
correctly matches cases when the object is right inside
the cell boundary, but the detection is just to the
outside.
(++) Having cells and detections sorted ensures that objects in overlapping
(cached) regions are seen by kernels in different cells in the same
order, thus resulting in the same matches. Note: this may fail in
extremely crowded region, but as of now it's not clear how big of
a problem (if any!) will this pose.
(*) Cached objects must be loaded and matched against to guard against
the case where an object is just outside the edge, while a detection
is just inside. If the cached object was not loaded, the detection
would not match and be proclamed to be a new object. However, in the
cached object's parent cell, the detection would match to the object
and be stored there as well.
The algorithm above ensures that such a detection will matched to
the cached object in this cell (and be dropped in step 5), preventing
it from being promoted into a new object.
TODO: The above algorithm ensures no detection is assigned to more than
one object. It also ensures that each detection links to an object.
Implement a consistency check to verify that.
"""
from scikits.ann import kdtree
# Input is a tuple of obj_cell, and det_cells falling under that obj_cell
obj_cell, det_cells = cells
det_cells.sort()
assert len(det_cells)
# Fetch the frequently used bits
obj_table = db.table(obj_tabname)
det_table = db.table(det_tabname)
o2d_table = db.table(o2d_tabname)
pix = obj_table.pix
# locate cell center (for gnomonic projection)
(bounds, tbounds) = pix.cell_bounds(obj_cell)
(clon, clat) = bhpix.deproj_bhealpix(*bounds.center())
# fetch existing static sky, convert to gnomonic
objs = db.query('_ID, _LON, _LAT FROM %s' % obj_tabname).fetch_cell(
obj_cell, include_cached=True)
xyobj = np.column_stack(gnomonic(objs['_LON'], objs['_LAT'], clon, clat))
nobj = len(objs) # Total number of static sky objects
tree = None
nobj_old = 0
# for sanity checks/debugging (see below)
expseen = set()
## TODO: Debugging, remove when happy
assert (np.unique(sorted(det_cells)) == sorted(det_cells)).all()
##print "Det cells: ", det_cells
# Loop, xmatch, and store
if explist is not None:
explist = np.asarray(list(explist),
dtype=np.uint64) # Ensure explist is a ndarray
det_query = db.query('_ID, _LON, _LAT, _EXP, _CACHED FROM %s' %
det_tabname)
for det_cell in sorted(det_cells):
# fetch detections in this cell, convert to gnomonic coordinates
# keep only detections with _EXP in explist, unless explist is None
detections = det_query.fetch_cell(det_cell, include_cached=True)
# if there are no preexisting static sky objects, and all detections in this cell are cached,
# there's no way we'll get a match that will be kept in the end. Just continue to the
# next one if this is the case.
cachedonly = len(objs) == 0 and detections._CACHED.all()
if cachedonly:
# print "Skipping cached-only", len(cached)
yield (
None, None, None, None, None, None
) # Yield just to have the progress counter properly incremented
continue
if explist is not None:
keep = np.in1d(detections._EXP, explist)
if not np.all(keep):
detections = detections[keep]
if len(detections) == 0:
yield (
None, None, None, None, None, None
) # Yield just to have the progress counter properly incremented
continue
_, ra2, dec2, exposures, cached = detections.as_columns()
detections.add_column('xy',
np.column_stack(gnomonic(ra2, dec2, clon, clat)))
# prep join table
join = ColGroup(dtype=o2d_table.dtype_for(
['_ID', '_M1', '_M2', '_DIST', '_LON', '_LAT']))
njoin = 0
nobj0 = nobj
##print "Cell", det_cell, " - Unique exposures: ", set(exposures)
# Process detections exposure-by-exposure, as detections from
# different exposures within a same temporal cell are allowed
# to belong to the same object
uexposures = set(exposures)
for exposure in sorted(uexposures):
# Sanity check: a consistent table cannot have two
# exposures stretching over more than one cell
assert exposure not in expseen
expseen.add(exposure)
# Extract objects belonging to this exposure only
detections2 = detections[exposures == exposure]
id2, ra2, dec2, _, _, xydet = detections2.as_columns()
ndet = len(xydet)
if len(xyobj) != 0:
# Construct kD-tree and find the object nearest to each
# detection from this cell
if tree is None or nobj_old != len(xyobj):
del tree
nobj_old = len(xyobj)
tree = kdtree(xyobj)
match_idx, match_d2 = tree.knn(xydet, 1)
match_idx = match_idx[:, 0] # First neighbor only
####
#if np.uint64(13828114484734072082) in id2:
# np.savetxt('bla.%d.static=%d.txt' % (det_cell, pix.static_cell_for_cell(det_cell)), objs.as_ndarray(), fmt='%s')
# Compute accurate distances, and select detections not matched to existing objects
dist = gc_dist(objs['_LON'][match_idx],
objs['_LAT'][match_idx], ra2, dec2)
unmatched = dist >= radius
else:
# All detections will become new objects (and therefore, dist=0)
dist = np.zeros(ndet, dtype='f4')
unmatched = np.ones(ndet, dtype=bool)
match_idx = np.empty(ndet, dtype='i4')
# x, y, t = pix._xyt_from_cell_id(det_cell)
# print "det_cell %s, MJD %s, Exposure %s == %d detections, %d objects, %d matched, %d unmatched" % (det_cell, t, exposure, len(detections2), nobj, len(unmatched)-unmatched.sum(), unmatched.sum())
# Promote unmatched detections to new objects
_, newra, newdec, _, _, newxy = detections2[unmatched].as_columns()
nunmatched = unmatched.sum()
reserve_space(objs, nobj + nunmatched)
objs['_LON'][nobj:nobj + nunmatched] = newra
objs['_LAT'][nobj:nobj + nunmatched] = newdec
dist[unmatched] = 0.
match_idx[unmatched] = np.arange(
nobj, nobj + nunmatched, dtype='i4'
) # Set the indices of unmatched detections to newly created objects
# Join objects to their detections
reserve_space(join, njoin + ndet)
join['_M1'][njoin:njoin + ndet] = match_idx
join['_M2'][njoin:njoin + ndet] = id2
join['_DIST'][njoin:njoin + ndet] = dist
# TODO: For debugging; remove when happy
join['_LON'][njoin:njoin + ndet] = ra2
join['_LAT'][njoin:njoin + ndet] = dec2
njoin += ndet
# Prep for next loop
nobj += nunmatched
xyobj = np.append(xyobj, newxy, axis=0)
# TODO: Debugging: Final consistency check (remove when happy with the code)
dist = gc_dist(objs['_LON'][join['_M1'][njoin - ndet:njoin]],
objs['_LAT'][join['_M1'][njoin - ndet:njoin]], ra2,
dec2)
assert (dist < radius).all()
# Truncate output tables to their actual number of elements
objs = objs[0:nobj]
join = join[0:njoin]
assert len(objs) >= nobj0
# Find the objects that fall outside of cell boundaries. These will
# be processed and stored by their parent cells. Also leave out the objects
# that are already stored in the database
(x, y) = bhpix.proj_bhealpix(objs['_LON'], objs['_LAT'])
in_ = bounds.isInsideV(x, y)
innew = in_.copy()
innew[:nobj0] = False # New objects in cell selector
ids = objs['_ID']
nobjadded = innew.sum()
if nobjadded:
# Append the new objects to the object table, obtaining their IDs.
assert not _rematching, 'cell_id=%s, nnew=%s\n%s' % (
det_cell, nobjadded, objs[innew])
ids[innew] = obj_table.append(objs[('_LON', '_LAT')][innew])
# Set the indices of objects not in this cell to zero (== a value
# no valid object in the database can have). Therefore, all
# out-of-bounds links will have _M1 == 0 (#1), and will be removed
# by the np1d call (#2)
ids[~in_] = 0
# 1) Change the relative index to true obj_id in the join table
join['_M1'] = ids[join['_M1']]
# 2) Keep only the joins to objects inside the cell
join = join[np.in1d(join['_M1'], ids[in_])]
# Append to the join table, in *dec_cell* of obj_table (!important!)
if len(join) != 0:
# compute the cell_id part of the join table's
# IDs.While this is unimportant now (as we could
# just set all of them equal to cell_id part of
# cell_id), if we ever decide to change the
# pixelation of the table later on, this will
# allow us to correctly split up the join table as
# well.
#_, _, t = pix._xyt_from_cell_id(det_cell) # This row points to a detection in the temporal cell ...
#x, y, _, _ = pix._xyti_from_id(join['_M1']) # ... but at the spatial location given by the object table.
#join['_ID'][:] = pix._id_from_xyti(x, y, t, 0) # This will make the new IDs have zeros in the object part (so Table.append will autogen them)
join['_ID'][:] = det_cell
o2d_table.append(join)
assert not cachedonly or (nobjadded == 0 and len(join) == 0)
# return: Number of exposures, number of objects before processing this cell, number of detections processed (incl. cached),
# number of newly added objects, number of detections xmatched, number of detection processed that weren't cached
# Note: some of the xmatches may be to newly added objects (e.g., if there are two
# overlapping exposures within a cell; first one will add new objects, second one will match agains them)
yield (len(uexposures), nobj0, len(detections), nobjadded, len(join),
(cached == False).sum())