Beispiel #1
0
def _sanity_check_object_table_mapper(qresult, radius, explist):
    all_objs = np.empty(0, dtype=np.uint64)
    havedet_objs = np.empty(0, dtype=np.uint64)

    for rows in qresult:
        # Prep work for checking #3, at the end.
        # Collect all objects
        objs = np.unique(rows.obj_id)  # Get unique IDs
        objs = objs[~np.in1d(
            objs, all_objs,
            assume_unique=True)]  # Keep only those never before seen
        all_objs = np.append(all_objs, objs)  # Append to a list of all objects

        # Remove outer-joined rows
        rows = rows[~rows.dnull]

        # Prep work for checking #3, at the end.
        # Collect objects with linked detections
        objs = np.unique(rows.obj_id)
        objs = objs[~np.in1d(
            objs, havedet_objs,
            assume_unique=True)]  # Keep only those never before seen
        havedet_objs = np.append(
            havedet_objs, objs)  # Append to a list of having a detection

        # Remove all detections that did not originate from explist
        if explist is not None:
            rows = rows[np.in1d(rows.exp_id, explist)]

        # Check radius (cond #4)
        dist = gc_dist(rows.olon, rows.olat, rows.dlon, rows.dlat)
        assert np.all(dist < radius)

        # Regroup by det_id's cell and yield to reducer.
        # We do this because some det_id's from other cells may be linked
        # to an object from this cell
        cells = qresult.pix.cell_for_id(rows.det_id)
        for cell_id in np.unique(cells):
            yield (cell_id, rows[("det_id", "obj_id")][cells == cell_id])

    # Check that each object has exactly one detection (cond #3)
    all_objs.sort()
    havedet_objs.sort()
    assert np.all(all_objs == havedet_objs)
Beispiel #2
0
def _sanity_check_object_table_mapper(qresult, radius, explist):
	all_objs = np.empty(0, dtype=np.uint64)
	havedet_objs = np.empty(0, dtype=np.uint64)

	for rows in qresult:
		# Prep work for checking #3, at the end.
		# Collect all objects
		objs = np.unique(rows.obj_id)					# Get unique IDs
		objs = objs[~np.in1d(objs, all_objs, assume_unique=True)]	# Keep only those never before seen
		all_objs = np.append(all_objs, objs)					# Append to a list of all objects

		# Remove outer-joined rows
		rows = rows[~rows.dnull]

		# Prep work for checking #3, at the end.
		# Collect objects with linked detections
		objs = np.unique(rows.obj_id)
		objs = objs[~np.in1d(objs, havedet_objs, assume_unique=True)]	# Keep only those never before seen
		havedet_objs = np.append(havedet_objs, objs)			# Append to a list of having a detection

		# Remove all detections that did not originate from explist
		if explist is not None:
			rows = rows[np.in1d(rows.exp_id, explist)]

		# Check radius (cond #4)
		dist = gc_dist(rows.olon, rows.olat, rows.dlon, rows.dlat)
		assert np.all(dist < radius)

		# Regroup by det_id's cell and yield to reducer.
		# We do this because some det_id's from other cells may be linked
		# to an object from this cell
		cells = qresult.pix.cell_for_id(rows.det_id)
		for cell_id in np.unique(cells):
			yield (cell_id, rows[("det_id", "obj_id")][cells == cell_id])

	# Check that each object has exactly one detection (cond #3)
	all_objs.sort()
	havedet_objs.sort()
	assert np.all(all_objs == havedet_objs)
Beispiel #3
0
# coding: utf-8
from sys import argv
from BeautifulSoup import BeautifulStoneSoup
from matplotlib import pyplot
from utils import gc_dist, elevation

gpx_file = argv[1]
data = open(gpx_file).read()
soup = BeautifulStoneSoup(data)

points = soup.findAll("trkpt")
route = [(float(p.attrs[0][1]), float(p.attrs[1][1])) for p in points]

dist_diffs = [0] + [gc_dist(route[i], route[i + 1]) for i in range(1, len(route) - 1)]
dist_from_src = [0]

for d in dist_diffs:
    dist_from_src.append(dist_from_src[-1] + d)

elevs = elevation(route)

pyplot.plot(dist_from_src, elevs)
pyplot.show()
Beispiel #4
0
def _xmatch_mapper(qresult, tabname_to, radius, tabname_xm, n_neighbors):
    """
	    Mapper:
	    	- given all objects in a cell, make an ANN tree
	    	- load all objects in tabname_to (including neighbors), make an ANN tree, find matches
	    	- store the output into an index table
	"""
    from scikits.ann import kdtree

    db = qresult.db
    pix = qresult.pix
    table_xm = db.table(tabname_xm)

    for rows in qresult:
        cell_id = rows.info.cell_id

        join = ColGroup(
            dtype=[('_M1', 'u8'), ('_M2',
                                   'u8'), ('_DIST',
                                           'f4'), ('_NR',
                                                   'u1'), ('_LON',
                                                           'f8'), ('_LAT',
                                                                   'f8')])

        (id1, ra1, dec1) = rows.as_columns()
        (id2, ra2,
         dec2) = db.query('_ID, _LON, _LAT FROM %s' % tabname_to).fetch_cell(
             cell_id, include_cached=True).as_columns()

        if len(id2) != 0:
            # Project to tangent plane around the center of the cell. We
            # assume the cell is small enough for the distortions not to
            # matter and Euclidian distances apply
            bounds, _ = pix.cell_bounds(cell_id)
            (clon, clat) = bhpix.deproj_bhealpix(*bounds.center())
            xy1 = np.column_stack(gnomonic(ra1, dec1, clon, clat))
            xy2 = np.column_stack(gnomonic(ra2, dec2, clon, clat))

            # Construct kD-tree to find an object in table_to that is nearest
            # to an object in table_from, for every object in table_from
            tree = kdtree(xy2)
            match_idxs, match_d2 = tree.knn(xy1, min(n_neighbors, len(xy2)))
            del tree

            # Create the index table array
            join.resize(match_idxs.size)
            for k in xrange(match_idxs.shape[1]):
                match_idx = match_idxs[:, k]
                join['_M1'][k::match_idxs.shape[1]] = id1
                join['_M2'][k::match_idxs.shape[1]] = id2[match_idx]
                join['_DIST'][k::match_idxs.shape[1]] = gc_dist(
                    ra1, dec1, ra2[match_idx], dec2[match_idx])
                join['_LON'][k::match_idxs.shape[1]] = ra2[match_idx]
                join['_LAT'][k::match_idxs.shape[1]] = dec2[match_idx]
                join['_NR'][k::match_idxs.shape[1]] = k

            # Remove matches beyond the xmatch radius
            join = join[join['_DIST'] < radius]

        if len(join):
            # compute the cell_id part of the join table's
            # IDs. While this is unimportant now (as we could
            # just set all of them equal to cell_id part of
            # cell_id), if we ever decide to change the
            # pixelation of the table later on, this will
            # allow us to correctly repixelize the join table as
            # well.
            #x, y, t, _  = pix._xyti_from_id(join['_M1'])	# ... but at the spatial location given by the object table.
            #join['_ID'] = pix._id_from_xyti(x, y, t, 0)     # This will make the new IDs have zeros in the object part (so Table.append will autogen them)

            # TODO: Allow the stuff above (in Table.append)
            join['_ID'] = pix.cell_for_id(join['_M1'])

            # TODO: Debugging, remove when happy
            cid = np.unique(pix.cell_for_id(join['_ID']))
            assert len(cid) == 1, len(cid)
            assert cid[0] == cell_id, '%s %s' % (cid[0], cell_id)
            ####

            table_xm.append(join)

            yield len(id1), len(id2), len(join)
        else:
            yield len(rows), 0, 0
Beispiel #5
0
def _obj_det_match(cells, db, obj_tabname, det_tabname, o2d_tabname, radius, explist=None, _rematching=False):
	"""
	This kernel assumes:
	   a) det_table and obj_table have equal partitioning (equally
	      sized/enumerated spatial cells)
	   b) both det_table and obj_table have up-to-date neighbor caches
	   c) temporal det_table cells within this spatial cell are stored
	      local to this process (relevant for shared-nothing setups)
	   d) exposures don't stretch across temporal cells

	Algorithm:
	   - fetch all existing static sky objects, including the cached ones (*)
	   - project them to tangent plane around the center of the cell
	     (we assume the cell is small enough for the distortions not to matter)
	   - construct a kD tree in (x, y) tangent space
	   - for each temporal cell, in sorted order (++):
	   	1.) Fetch the detections, including the cached ones (+)
	   	2.) Project to tangent plane

	   	3.) for each exposure, in sorted order (++):
		    a.) Match agains the kD tree of objects
		    b.) Add those that didn't match to the list of objects 

		4.) For newly added objects: store to disk only those that
		    fall within this cell (the others will be matched and
		    stored in their parent cells)

		5.) For matched detections: Drop detections matched to cached
		    objects (these will be matched and stored in the objects'
		    parent cell). Store the rest.


	   (+) It is allowed (and necessary to allow) for a cached detection
		    to be matched against an object within our cell.  This
		    correctly matches cases when the object is right inside
		    the cell boundary, but the detection is just to the
		    outside.

	   (++) Having cells and detections sorted ensures that objects in overlapping
	        (cached) regions are seen by kernels in different cells in the same
	        order, thus resulting in the same matches. Note: this may fail in
	        extremely crowded region, but as of now it's not clear how big of
	        a problem (if any!) will this pose.

	   (*) Cached objects must be loaded and matched against to guard against
	       the case where an object is just outside the edge, while a detection
	       is just inside. If the cached object was not loaded, the detection
	       would not match and be proclamed to be a new object. However, in the
	       cached object's parent cell, the detection would match to the object
	       and be stored there as well.
	       
	       The algorithm above ensures that such a detection will matched to
	       the cached object in this cell (and be dropped in step 5), preventing
	       it from being promoted into a new object.

	   TODO: The above algorithm ensures no detection is assigned to more than
	   	one object. It also ensures that each detection links to an object.
	   	Implement a consistency check to verify that.
	"""

	from scikits.ann import kdtree

	# Input is a tuple of obj_cell, and det_cells falling under that obj_cell
	obj_cell, det_cells = cells
	det_cells.sort()
	assert len(det_cells)

	# Fetch the frequently used bits
	obj_table = db.table(obj_tabname)
	det_table = db.table(det_tabname)
	o2d_table = db.table(o2d_tabname)
	pix = obj_table.pix

	# locate cell center (for gnomonic projection)
	(bounds, tbounds)  = pix.cell_bounds(obj_cell)
	(clon, clat) = bhpix.deproj_bhealpix(*bounds.center())

	# fetch existing static sky, convert to gnomonic
	objs  = db.query('_ID, _LON, _LAT FROM %s' % obj_tabname).fetch_cell(obj_cell, include_cached=True)
	xyobj = np.column_stack(gnomonic(objs['_LON'], objs['_LAT'], clon, clat))
	nobj  = len(objs)	# Total number of static sky objects
	tree  = None
	nobj_old = 0

	# for sanity checks/debugging (see below)
	expseen = set()

	## TODO: Debugging, remove when happy
	assert (np.unique(sorted(det_cells)) == sorted(det_cells)).all()
	##print "Det cells: ", det_cells

	# Loop, xmatch, and store
	if explist is not None:
		explist = np.asarray(list(explist), dtype=np.uint64)	# Ensure explist is a ndarray
	det_query = db.query('_ID, _LON, _LAT, _EXP, _CACHED FROM %s' % det_tabname)
	for det_cell in sorted(det_cells):
		# fetch detections in this cell, convert to gnomonic coordinates
		# keep only detections with _EXP in explist, unless explist is None
		detections = det_query.fetch_cell(det_cell, include_cached=True)

		# if there are no preexisting static sky objects, and all detections in this cell are cached,
		# there's no way we'll get a match that will be kept in the end. Just continue to the
		# next one if this is the case.
		cachedonly = len(objs) == 0 and detections._CACHED.all()
		if cachedonly:
#			print "Skipping cached-only", len(cached)
			yield (None, None, None, None, None, None) # Yield just to have the progress counter properly incremented
			continue;

		if explist is not None:
			keep = np.in1d(detections._EXP, explist)
			if not np.all(keep):
				detections = detections[keep]
			if len(detections) == 0:
				yield (None, None, None, None, None, None) # Yield just to have the progress counter properly incremented
				continue
		_, ra2, dec2, exposures, cached = detections.as_columns()
		detections.add_column('xy', np.column_stack(gnomonic(ra2, dec2, clon, clat)))

		# prep join table
		join  = ColGroup(dtype=o2d_table.dtype_for(['_ID', '_M1', '_M2', '_DIST', '_LON', '_LAT']))
		njoin = 0;
		nobj0 = nobj;

		##print "Cell", det_cell, " - Unique exposures: ", set(exposures)

		# Process detections exposure-by-exposure, as detections from
		# different exposures within a same temporal cell are allowed
		# to belong to the same object
		uexposures = set(exposures)
		for exposure in sorted(uexposures):
			# Sanity check: a consistent table cannot have two
			# exposures stretching over more than one cell
			assert exposure not in expseen
			expseen.add(exposure);

			# Extract objects belonging to this exposure only
			detections2 = detections[exposures == exposure]
			id2, ra2, dec2, _, _, xydet = detections2.as_columns()
			ndet = len(xydet)

			if len(xyobj) != 0:
				# Construct kD-tree and find the object nearest to each
				# detection from this cell
				if tree is None or nobj_old != len(xyobj):
					del tree
					nobj_old = len(xyobj)
					tree = kdtree(xyobj)
				match_idx, match_d2 = tree.knn(xydet, 1)
				match_idx = match_idx[:,0]		# First neighbor only

				####
				#if np.uint64(13828114484734072082) in id2:
				#	np.savetxt('bla.%d.static=%d.txt' % (det_cell, pix.static_cell_for_cell(det_cell)), objs.as_ndarray(), fmt='%s')

				# Compute accurate distances, and select detections not matched to existing objects
				dist       = gc_dist(objs['_LON'][match_idx], objs['_LAT'][match_idx], ra2, dec2)
				unmatched  = dist >= radius
			else:
				# All detections will become new objects (and therefore, dist=0)
				dist       = np.zeros(ndet, dtype='f4')
				unmatched  = np.ones(ndet, dtype=bool)
				match_idx  = np.empty(ndet, dtype='i4')

#			x, y, t = pix._xyt_from_cell_id(det_cell)
#			print "det_cell %s, MJD %s, Exposure %s  ==  %d detections, %d objects, %d matched, %d unmatched" % (det_cell, t, exposure, len(detections2), nobj, len(unmatched)-unmatched.sum(), unmatched.sum())

			# Promote unmatched detections to new objects
			_, newra, newdec, _, _, newxy = detections2[unmatched].as_columns()
			nunmatched = unmatched.sum()
			reserve_space(objs, nobj+nunmatched)
			objs['_LON'][nobj:nobj+nunmatched] = newra
			objs['_LAT'][nobj:nobj+nunmatched] = newdec
			dist[unmatched]                    = 0.
			match_idx[unmatched] = np.arange(nobj, nobj+nunmatched, dtype='i4')	# Set the indices of unmatched detections to newly created objects

			# Join objects to their detections
			reserve_space(join, njoin+ndet)
			join['_M1'][njoin:njoin+ndet]   = match_idx
			join['_M2'][njoin:njoin+ndet]   =       id2
			join['_DIST'][njoin:njoin+ndet] =      dist
			# TODO: For debugging; remove when happy
			join['_LON'][njoin:njoin+ndet]  =       ra2
			join['_LAT'][njoin:njoin+ndet]  =      dec2
			njoin += ndet

			# Prep for next loop
			nobj  += nunmatched
			xyobj  = np.append(xyobj, newxy, axis=0)

			# TODO: Debugging: Final consistency check (remove when happy with the code)
			dist = gc_dist( objs['_LON'][  join['_M1'][njoin-ndet:njoin]  ],
					objs['_LAT'][  join['_M1'][njoin-ndet:njoin]  ], ra2, dec2)
			assert (dist < radius).all()

		# Truncate output tables to their actual number of elements
		objs = objs[0:nobj]
		join = join[0:njoin]
		assert len(objs) >= nobj0

		# Find the objects that fall outside of cell boundaries. These will
		# be processed and stored by their parent cells. Also leave out the objects
		# that are already stored in the database
		(x, y) = bhpix.proj_bhealpix(objs['_LON'], objs['_LAT'])
		in_    = bounds.isInsideV(x, y)
		innew  = in_.copy();
		innew[:nobj0] = False											# New objects in cell selector

		ids = objs['_ID']
		nobjadded = innew.sum()
		if nobjadded:
			# Append the new objects to the object table, obtaining their IDs.
			assert not _rematching, 'cell_id=%s, nnew=%s\n%s' % (det_cell, nobjadded, objs[innew])
			ids[innew] = obj_table.append(objs[('_LON', '_LAT')][innew])

		# Set the indices of objects not in this cell to zero (== a value
		# no valid object in the database can have). Therefore, all
		# out-of-bounds links will have _M1 == 0 (#1), and will be removed
		# by the np1d call (#2)
		ids[~in_] = 0

		# 1) Change the relative index to true obj_id in the join table
		join['_M1'] = ids[join['_M1']]

		# 2) Keep only the joins to objects inside the cell
		join = join[ np.in1d(join['_M1'], ids[in_]) ]

		# Append to the join table, in *dec_cell* of obj_table (!important!)
		if len(join) != 0:
			# compute the cell_id part of the join table's
			# IDs.While this is unimportant now (as we could
			# just set all of them equal to cell_id part of
			# cell_id), if we ever decide to change the
			# pixelation of the table later on, this will
			# allow us to correctly split up the join table as
			# well.
			#_, _, t    = pix._xyt_from_cell_id(det_cell)	# This row points to a detection in the temporal cell ...
			#x, y, _, _ = pix._xyti_from_id(join['_M1'])	# ... but at the spatial location given by the object table.
			#join['_ID'][:] = pix._id_from_xyti(x, y, t, 0) # This will make the new IDs have zeros in the object part (so Table.append will autogen them)
			join['_ID'][:] = det_cell

			o2d_table.append(join)

		assert not cachedonly or (nobjadded == 0 and len(join) == 0)

		# return: Number of exposures, number of objects before processing this cell, number of detections processed (incl. cached),
		#         number of newly added objects, number of detections xmatched, number of detection processed that weren't cached
		# Note: some of the xmatches may be to newly added objects (e.g., if there are two 
		#       overlapping exposures within a cell; first one will add new objects, second one will match agains them)
		yield (len(uexposures), nobj0, len(detections), nobjadded, len(join), (cached == False).sum())
Beispiel #6
0
def _xmatch_mapper(qresult, tabname_to, radius, tabname_xm, n_neighbors):
	"""
	    Mapper:
	    	- given all objects in a cell, make an ANN tree
	    	- load all objects in tabname_to (including neighbors), make an ANN tree, find matches
	    	- store the output into an index table
	"""
	from scikits.ann import kdtree

	db       = qresult.db
	pix      = qresult.pix
	table_xm = db.table(tabname_xm)

	for rows in qresult:
		cell_id  = rows.info.cell_id

		join = ColGroup(dtype=[('_M1', 'u8'), ('_M2', 'u8'), ('_DIST', 'f4'), ('_NR', 'u1'), ('_LON', 'f8'), ('_LAT', 'f8')])

		(id1, ra1, dec1) = rows.as_columns()
		(id2, ra2, dec2) = db.query('_ID, _LON, _LAT FROM %s' % tabname_to).fetch_cell(cell_id, include_cached=True).as_columns()

		if len(id2) != 0:
			# Project to tangent plane around the center of the cell. We
			# assume the cell is small enough for the distortions not to
			# matter and Euclidian distances apply
			bounds, _    = pix.cell_bounds(cell_id)
			(clon, clat) = bhpix.deproj_bhealpix(*bounds.center())
			xy1 = np.column_stack(gnomonic(ra1, dec1, clon, clat))
			xy2 = np.column_stack(gnomonic(ra2, dec2, clon, clat))

			# Construct kD-tree to find an object in table_to that is nearest
			# to an object in table_from, for every object in table_from
			tree = kdtree(xy2)
			match_idxs, match_d2 = tree.knn(xy1, min(n_neighbors, len(xy2)))
			del tree

			# Create the index table array
			join.resize(match_idxs.size)
			for k in xrange(match_idxs.shape[1]):
				match_idx = match_idxs[:,k]
				join['_M1'][k::match_idxs.shape[1]]   = id1
				join['_M2'][k::match_idxs.shape[1]]   = id2[match_idx]
				join['_DIST'][k::match_idxs.shape[1]] = gc_dist(ra1, dec1, ra2[match_idx], dec2[match_idx])
				join['_LON'][k::match_idxs.shape[1]]  = ra2[match_idx]
				join['_LAT'][k::match_idxs.shape[1]]  = dec2[match_idx]
				join['_NR'][k::match_idxs.shape[1]]   = k

			# Remove matches beyond the xmatch radius
			join = join[join['_DIST'] < radius]

		if len(join):
			# compute the cell_id part of the join table's
			# IDs. While this is unimportant now (as we could
			# just set all of them equal to cell_id part of
			# cell_id), if we ever decide to change the
			# pixelation of the table later on, this will
			# allow us to correctly repixelize the join table as
			# well.
			#x, y, t, _  = pix._xyti_from_id(join['_M1'])	# ... but at the spatial location given by the object table.
			#join['_ID'] = pix._id_from_xyti(x, y, t, 0)     # This will make the new IDs have zeros in the object part (so Table.append will autogen them)
			
			# TODO: Allow the stuff above (in Table.append)
			join['_ID'] = pix.cell_for_id(join['_M1'])

			# TODO: Debugging, remove when happy
			cid = np.unique(pix.cell_for_id(join['_ID']))
			assert len(cid) == 1, len(cid)
			assert cid[0] == cell_id, '%s %s' % (cid[0], cell_id)
			####

			table_xm.append(join)

			yield len(id1), len(id2), len(join)
		else:
			yield len(rows), 0, 0
Beispiel #7
0
def _obj_det_match(cells,
                   db,
                   obj_tabname,
                   det_tabname,
                   o2d_tabname,
                   radius,
                   explist=None,
                   _rematching=False):
    """
	This kernel assumes:
	   a) det_table and obj_table have equal partitioning (equally
	      sized/enumerated spatial cells)
	   b) both det_table and obj_table have up-to-date neighbor caches
	   c) temporal det_table cells within this spatial cell are stored
	      local to this process (relevant for shared-nothing setups)
	   d) exposures don't stretch across temporal cells

	Algorithm:
	   - fetch all existing static sky objects, including the cached ones (*)
	   - project them to tangent plane around the center of the cell
	     (we assume the cell is small enough for the distortions not to matter)
	   - construct a kD tree in (x, y) tangent space
	   - for each temporal cell, in sorted order (++):
	   	1.) Fetch the detections, including the cached ones (+)
	   	2.) Project to tangent plane

	   	3.) for each exposure, in sorted order (++):
		    a.) Match agains the kD tree of objects
		    b.) Add those that didn't match to the list of objects 

		4.) For newly added objects: store to disk only those that
		    fall within this cell (the others will be matched and
		    stored in their parent cells)

		5.) For matched detections: Drop detections matched to cached
		    objects (these will be matched and stored in the objects'
		    parent cell). Store the rest.


	   (+) It is allowed (and necessary to allow) for a cached detection
		    to be matched against an object within our cell.  This
		    correctly matches cases when the object is right inside
		    the cell boundary, but the detection is just to the
		    outside.

	   (++) Having cells and detections sorted ensures that objects in overlapping
	        (cached) regions are seen by kernels in different cells in the same
	        order, thus resulting in the same matches. Note: this may fail in
	        extremely crowded region, but as of now it's not clear how big of
	        a problem (if any!) will this pose.

	   (*) Cached objects must be loaded and matched against to guard against
	       the case where an object is just outside the edge, while a detection
	       is just inside. If the cached object was not loaded, the detection
	       would not match and be proclamed to be a new object. However, in the
	       cached object's parent cell, the detection would match to the object
	       and be stored there as well.
	       
	       The algorithm above ensures that such a detection will matched to
	       the cached object in this cell (and be dropped in step 5), preventing
	       it from being promoted into a new object.

	   TODO: The above algorithm ensures no detection is assigned to more than
	   	one object. It also ensures that each detection links to an object.
	   	Implement a consistency check to verify that.
	"""

    from scikits.ann import kdtree

    # Input is a tuple of obj_cell, and det_cells falling under that obj_cell
    obj_cell, det_cells = cells
    det_cells.sort()
    assert len(det_cells)

    # Fetch the frequently used bits
    obj_table = db.table(obj_tabname)
    det_table = db.table(det_tabname)
    o2d_table = db.table(o2d_tabname)
    pix = obj_table.pix

    # locate cell center (for gnomonic projection)
    (bounds, tbounds) = pix.cell_bounds(obj_cell)
    (clon, clat) = bhpix.deproj_bhealpix(*bounds.center())

    # fetch existing static sky, convert to gnomonic
    objs = db.query('_ID, _LON, _LAT FROM %s' % obj_tabname).fetch_cell(
        obj_cell, include_cached=True)
    xyobj = np.column_stack(gnomonic(objs['_LON'], objs['_LAT'], clon, clat))
    nobj = len(objs)  # Total number of static sky objects
    tree = None
    nobj_old = 0

    # for sanity checks/debugging (see below)
    expseen = set()

    ## TODO: Debugging, remove when happy
    assert (np.unique(sorted(det_cells)) == sorted(det_cells)).all()
    ##print "Det cells: ", det_cells

    # Loop, xmatch, and store
    if explist is not None:
        explist = np.asarray(list(explist),
                             dtype=np.uint64)  # Ensure explist is a ndarray
    det_query = db.query('_ID, _LON, _LAT, _EXP, _CACHED FROM %s' %
                         det_tabname)
    for det_cell in sorted(det_cells):
        # fetch detections in this cell, convert to gnomonic coordinates
        # keep only detections with _EXP in explist, unless explist is None
        detections = det_query.fetch_cell(det_cell, include_cached=True)

        # if there are no preexisting static sky objects, and all detections in this cell are cached,
        # there's no way we'll get a match that will be kept in the end. Just continue to the
        # next one if this is the case.
        cachedonly = len(objs) == 0 and detections._CACHED.all()
        if cachedonly:
            #			print "Skipping cached-only", len(cached)
            yield (
                None, None, None, None, None, None
            )  # Yield just to have the progress counter properly incremented
            continue

        if explist is not None:
            keep = np.in1d(detections._EXP, explist)
            if not np.all(keep):
                detections = detections[keep]
            if len(detections) == 0:
                yield (
                    None, None, None, None, None, None
                )  # Yield just to have the progress counter properly incremented
                continue
        _, ra2, dec2, exposures, cached = detections.as_columns()
        detections.add_column('xy',
                              np.column_stack(gnomonic(ra2, dec2, clon, clat)))

        # prep join table
        join = ColGroup(dtype=o2d_table.dtype_for(
            ['_ID', '_M1', '_M2', '_DIST', '_LON', '_LAT']))
        njoin = 0
        nobj0 = nobj

        ##print "Cell", det_cell, " - Unique exposures: ", set(exposures)

        # Process detections exposure-by-exposure, as detections from
        # different exposures within a same temporal cell are allowed
        # to belong to the same object
        uexposures = set(exposures)
        for exposure in sorted(uexposures):
            # Sanity check: a consistent table cannot have two
            # exposures stretching over more than one cell
            assert exposure not in expseen
            expseen.add(exposure)

            # Extract objects belonging to this exposure only
            detections2 = detections[exposures == exposure]
            id2, ra2, dec2, _, _, xydet = detections2.as_columns()
            ndet = len(xydet)

            if len(xyobj) != 0:
                # Construct kD-tree and find the object nearest to each
                # detection from this cell
                if tree is None or nobj_old != len(xyobj):
                    del tree
                    nobj_old = len(xyobj)
                    tree = kdtree(xyobj)
                match_idx, match_d2 = tree.knn(xydet, 1)
                match_idx = match_idx[:, 0]  # First neighbor only

                ####
                #if np.uint64(13828114484734072082) in id2:
                #	np.savetxt('bla.%d.static=%d.txt' % (det_cell, pix.static_cell_for_cell(det_cell)), objs.as_ndarray(), fmt='%s')

                # Compute accurate distances, and select detections not matched to existing objects
                dist = gc_dist(objs['_LON'][match_idx],
                               objs['_LAT'][match_idx], ra2, dec2)
                unmatched = dist >= radius
            else:
                # All detections will become new objects (and therefore, dist=0)
                dist = np.zeros(ndet, dtype='f4')
                unmatched = np.ones(ndet, dtype=bool)
                match_idx = np.empty(ndet, dtype='i4')

#			x, y, t = pix._xyt_from_cell_id(det_cell)
#			print "det_cell %s, MJD %s, Exposure %s  ==  %d detections, %d objects, %d matched, %d unmatched" % (det_cell, t, exposure, len(detections2), nobj, len(unmatched)-unmatched.sum(), unmatched.sum())

# Promote unmatched detections to new objects
            _, newra, newdec, _, _, newxy = detections2[unmatched].as_columns()
            nunmatched = unmatched.sum()
            reserve_space(objs, nobj + nunmatched)
            objs['_LON'][nobj:nobj + nunmatched] = newra
            objs['_LAT'][nobj:nobj + nunmatched] = newdec
            dist[unmatched] = 0.
            match_idx[unmatched] = np.arange(
                nobj, nobj + nunmatched, dtype='i4'
            )  # Set the indices of unmatched detections to newly created objects

            # Join objects to their detections
            reserve_space(join, njoin + ndet)
            join['_M1'][njoin:njoin + ndet] = match_idx
            join['_M2'][njoin:njoin + ndet] = id2
            join['_DIST'][njoin:njoin + ndet] = dist
            # TODO: For debugging; remove when happy
            join['_LON'][njoin:njoin + ndet] = ra2
            join['_LAT'][njoin:njoin + ndet] = dec2
            njoin += ndet

            # Prep for next loop
            nobj += nunmatched
            xyobj = np.append(xyobj, newxy, axis=0)

            # TODO: Debugging: Final consistency check (remove when happy with the code)
            dist = gc_dist(objs['_LON'][join['_M1'][njoin - ndet:njoin]],
                           objs['_LAT'][join['_M1'][njoin - ndet:njoin]], ra2,
                           dec2)
            assert (dist < radius).all()

        # Truncate output tables to their actual number of elements
        objs = objs[0:nobj]
        join = join[0:njoin]
        assert len(objs) >= nobj0

        # Find the objects that fall outside of cell boundaries. These will
        # be processed and stored by their parent cells. Also leave out the objects
        # that are already stored in the database
        (x, y) = bhpix.proj_bhealpix(objs['_LON'], objs['_LAT'])
        in_ = bounds.isInsideV(x, y)
        innew = in_.copy()
        innew[:nobj0] = False  # New objects in cell selector

        ids = objs['_ID']
        nobjadded = innew.sum()
        if nobjadded:
            # Append the new objects to the object table, obtaining their IDs.
            assert not _rematching, 'cell_id=%s, nnew=%s\n%s' % (
                det_cell, nobjadded, objs[innew])
            ids[innew] = obj_table.append(objs[('_LON', '_LAT')][innew])

        # Set the indices of objects not in this cell to zero (== a value
        # no valid object in the database can have). Therefore, all
        # out-of-bounds links will have _M1 == 0 (#1), and will be removed
        # by the np1d call (#2)
        ids[~in_] = 0

        # 1) Change the relative index to true obj_id in the join table
        join['_M1'] = ids[join['_M1']]

        # 2) Keep only the joins to objects inside the cell
        join = join[np.in1d(join['_M1'], ids[in_])]

        # Append to the join table, in *dec_cell* of obj_table (!important!)
        if len(join) != 0:
            # compute the cell_id part of the join table's
            # IDs.While this is unimportant now (as we could
            # just set all of them equal to cell_id part of
            # cell_id), if we ever decide to change the
            # pixelation of the table later on, this will
            # allow us to correctly split up the join table as
            # well.
            #_, _, t    = pix._xyt_from_cell_id(det_cell)	# This row points to a detection in the temporal cell ...
            #x, y, _, _ = pix._xyti_from_id(join['_M1'])	# ... but at the spatial location given by the object table.
            #join['_ID'][:] = pix._id_from_xyti(x, y, t, 0) # This will make the new IDs have zeros in the object part (so Table.append will autogen them)
            join['_ID'][:] = det_cell

            o2d_table.append(join)

        assert not cachedonly or (nobjadded == 0 and len(join) == 0)

        # return: Number of exposures, number of objects before processing this cell, number of detections processed (incl. cached),
        #         number of newly added objects, number of detections xmatched, number of detection processed that weren't cached
        # Note: some of the xmatches may be to newly added objects (e.g., if there are two
        #       overlapping exposures within a cell; first one will add new objects, second one will match agains them)
        yield (len(uexposures), nobj0, len(detections), nobjadded, len(join),
               (cached == False).sum())