def _get_cells_recursive(self, outcells, data_path, bounds_xy, bounds_t, pattern, x = 0., y = 0.): """ Helper for get_cells(). See documentation of get_cells() for usage """ lev = bhpix.get_pixel_level(x, y) dx = bhpix.pix_size(lev) # Check for nonzero overlap box = self._cell_bounds_xy(x, y, dx) bounds_xy = bounds_xy & box if not bounds_xy: return # Check if the cell directory exists (give up if it doesn't) path = data_path + '/' + self._path_to_cell_base_xy(x, y, lev) if not os.path.isdir(path): return # If re reached the bottom of the hierarchy if self._is_valid_cell_xy(x, y): # Get the cell_ids for leaf cells matching pattern xybounds = None if(bounds_xy.area() == box.area()) else bounds_xy sibling_cells = list(self._get_temporal_siblings(x, y, path, pattern)) for cell_id in sibling_cells: # Filter on time, add bounds x, y, t = self._xyt_from_cell_id(cell_id) if t != self.t0: # Cut on the time component tival = intervalset((t, t+self.dt)) tolap = bounds_t & tival if len(tolap): (l, r) = tolap[-1] # Get the right-most interval component if l == r == t+self.dt: # Is it a single point? tolap = intervalset(*tolap[:-1]) # Since objects in this cell have time in [t, t+dt), remove the t+dt point if len(tolap) == 0: # No overlap between the intervals -- skip this cell continue; # Return None if the cell is fully contained in the requested interval tbounds = None if tival == tolap else tolap else: # Static cell tbounds = bounds_t assert len(sibling_cells) == 1, "There can be only one static cell (cell_id=%s)" % cell_id # Add to output if tbounds is None: outcells[cell_id][xybounds] = None elif xybounds not in outcells[cell_id]: outcells[cell_id][xybounds] = tbounds elif outcells[cell_id][xybounds] is not None: outcells[cell_id][xybounds] |= tbounds else: # Recursively subdivide the four subpixels dx = dx / 2 for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5)]): (x2, y2) = (x, y) + dx*d self._get_cells_recursive(outcells, data_path, bounds_xy & box, bounds_t, pattern, x2, y2)
def __part_to_xy_t(part):
""" Internal: Used by canonicize_bounds """
bounds_xy = Polygon.Polygon()
bounds_t = intervalset()
for v in part:
if isinstance(v, Polygon.Polygon):
bounds_xy |= v
elif isinstance(v, intervalset):
bounds_t |= v
else:
raise Exception('Incorrect part specification')
if not bounds_xy: bounds_xy = ALLSKY
if len(bounds_t) == 0: bounds_t = intervalset((-np.inf, np.inf))
return (bounds_xy, bounds_t)
def get_cells(self, bounds=None, return_bounds=False, include_cached=True):
""" Return a list of (cell_id, bounds) tuples completely
covering the requested bounds.
bounds must be a list of (Polygon, intervalset) tuples.
Output is a list of (cell_id, xybounds, tbounds) tuples,
unless return_bounds=False when the output is just a
list of cell_ids.
"""
self._include_cached = include_cached
# Special case of bounds=None (all sky)
if bounds == None:
bounds = [(bn.ALLSKY, intervalset((-np.inf, np.inf)))]
# Find all existing cells satisfying the bounds
cells = defaultdict(dict)
if False:
# Single-threaded implementation
for bounds_xy, bounds_t in bounds:
self._get_cells_recursive(cells, bounds_xy, bounds_t, 0, 0, 1,
1.)
self._get_cells_recursive(cells, bounds_xy, bounds_t, 0, 1, 1,
1.)
self._get_cells_recursive(cells, bounds_xy, bounds_t, 1, 0, 1,
1.)
self._get_cells_recursive(cells, bounds_xy, bounds_t, 1, 1, 1,
1.)
else:
# Multi-process implementation (appears to be as good or better than single thread in
# nearly all cases of interest)
pool = pool2.Pool()
lev = min(4, self._pix.level)
ij = np.indices(
(2**lev, 2**lev)).reshape(2, -1).T # List of i,j coordinates
for cells_ in pool.imap_unordered(
ij,
_get_cells_kernel, (lev, self, bounds),
progress_callback=pool2.progress_pass):
for cell_id, b in cells_.iteritems():
for xyb, tb in b.iteritems():
_add_bounds(cells, cell_id, xyb, tb)
del pool
if len(cells):
# Transform (x, y, t) tuples to cell_ids
xyt = np.array(cells.keys()).transpose()
cell_ids = self._pix._cell_id_for_xyt(xyt[0], xyt[1], xyt[2])
# Reorder cells to be a dict of cell: [(poly, time), (poly, time)] entries
cells = dict(
((cell_id, v.items())
for (cell_id, (k, v)) in izip(cell_ids, cells.iteritems())))
if not return_bounds:
return cells.keys()
else:
return cells
def __part_to_xy_t(part):
""" Internal: Used by canonicize_bounds """
bounds_xy = Polygon.Polygon()
bounds_t = intervalset()
for v in part:
if isinstance(v, Polygon.Polygon):
bounds_xy |= v
elif isinstance(v, intervalset):
bounds_t |= v
else:
raise Exception("Incorrect part specification")
if not bounds_xy:
bounds_xy = ALLSKY
if len(bounds_t) == 0:
bounds_t = intervalset((-np.inf, np.inf))
return (bounds_xy, bounds_t)
def get_cells_with_dets_from_exps(db,
explist,
exp_tabname,
det_tabname,
fovradius=None):
""" Get all cells that may contain detections from within exposures
listed in explist. If given, use fovradius (in degrees) as the
radius of the field of view of each exposure, to limit the number
of cells that are considered. If fovradius is None, assume the field
of view of each exposure may extend to the entire sky.
"""
explist = frozenset(explist)
bb = {}
ti = intervalset()
for t, exp, lon, lat in db.query("_TIME, _EXP, _LON, _LAT from %s" %
exp_tabname).iterate():
if exp not in explist:
continue
if fovradius is not None:
tt = np.floor(t)
#fov = bounds.beam(lon, lat, fovradius*(2./np.sqrt(3.)), npts=8) # An octagon circumscribing the FoV
#assert fov.nPoints() == 8
fov = bounds.beam(lon, lat, fovradius)
try:
bb[tt] |= fov
except KeyError:
bb[tt] = fov
else:
ti.add((t - 0.2, t + 0.2))
if fovradius is None:
b = [(bounds.ALLSKY, ti)]
else:
b = []
for t0, fov in bb.iteritems():
b += [(fov, intervalset((t0, t0 + 1.)))]
#print t0, fov.area()*(180./np.pi)**2., fov.nPoints()
# Fetch all detection table cells that are within this interval
det_cells = db.table(det_tabname).get_cells(bounds=b)
return det_cells
def cell_bounds(self, cell_id): """ Return the bounding polygon and time intervalset for a given cell. """ x, y, t, _ = self._xyti_from_id(cell_id) assert _ == u0xFFFFFFFF bounds = self._cell_bounds_xy(x, y) return (bounds, intervalset((t, t+self.dt)))
def cell_bounds(self, cell_id):
"""
Return the bounding polygon and time intervalset
for a given cell.
"""
x, y, t, _ = self._xyti_from_id(cell_id)
assert _ == u0xFFFFFFFF
bounds = self._cell_bounds_xy(x, y)
return (bounds, intervalset((t, t + self.dt)))
def get_cells_with_dets_from_exps(db, explist, exp_tabname, det_tabname, fovradius=None):
""" Get all cells that may contain detections from within exposures
listed in explist. If given, use fovradius (in degrees) as the
radius of the field of view of each exposure, to limit the number
of cells that are considered. If fovradius is None, assume the field
of view of each exposure may extend to the entire sky.
"""
explist = frozenset(explist)
bb = {}
ti = intervalset()
for t, exp, lon, lat in db.query("_TIME, _EXP, _LON, _LAT from %s" % exp_tabname).iterate():
if exp not in explist:
continue
if fovradius is not None:
tt = np.floor(t)
#fov = bounds.beam(lon, lat, fovradius*(2./np.sqrt(3.)), npts=8) # An octagon circumscribing the FoV
#assert fov.nPoints() == 8
fov = bounds.beam(lon, lat, fovradius)
try:
bb[tt] |= fov
except KeyError:
bb[tt] = fov
else:
ti.add((t-0.2, t+0.2))
if fovradius is None:
b = [(bounds.ALLSKY, ti)]
else:
b = []
for t0, fov in bb.iteritems():
b += [ (fov, intervalset((t0, t0+1.))) ]
#print t0, fov.area()*(180./np.pi)**2., fov.nPoints()
# Fetch all detection table cells that are within this interval
det_cells = db.table(det_tabname).get_cells(bounds=b)
return det_cells
def get_cells(self, bounds=None, return_bounds=False, include_cached=True): """ Return a list of (cell_id, bounds) tuples completely covering the requested bounds. bounds must be a list of (Polygon, intervalset) tuples. Output is a list of (cell_id, xybounds, tbounds) tuples, unless return_bounds=False when the output is just a list of cell_ids. """ self._include_cached = include_cached # Special case of bounds=None (all sky) if bounds == None: bounds = [(bn.ALLSKY, intervalset((-np.inf, np.inf)))] # Find all existing cells satisfying the bounds cells = defaultdict(dict) if False: # Single-threaded implementation for bounds_xy, bounds_t in bounds: self._get_cells_recursive(cells, bounds_xy, bounds_t, 0, 0, 1, 1.) self._get_cells_recursive(cells, bounds_xy, bounds_t, 0, 1, 1, 1.) self._get_cells_recursive(cells, bounds_xy, bounds_t, 1, 0, 1, 1.) self._get_cells_recursive(cells, bounds_xy, bounds_t, 1, 1, 1, 1.) else: # Multi-process implementation (appears to be as good or better than single thread in # nearly all cases of interest) pool = pool2.Pool() lev = min(4, self._pix.level) ij = np.indices((2**lev,2**lev)).reshape(2, -1).T # List of i,j coordinates for cells_ in pool.imap_unordered(ij, _get_cells_kernel, (lev, self, bounds), progress_callback=pool2.progress_pass): for cell_id, b in cells_.iteritems(): for xyb, tb in b.iteritems(): _add_bounds(cells, cell_id, xyb, tb) del pool if len(cells): # Transform (x, y, t) tuples to cell_ids xyt = np.array(cells.keys()).transpose() cell_ids = self._pix._cell_id_for_xyt(xyt[0], xyt[1], xyt[2]) # Reorder cells to be a dict of cell: [(poly, time), (poly, time)] entries cells = dict(( (cell_id, v.items()) for (cell_id, (k, v)) in izip(cell_ids, cells.iteritems()) )) if not return_bounds: return cells.keys() else: return cells
def get_cells(self, data_path, pattern, bounds, return_bounds=False):
""" Return a list of (cell_id, bounds) tuples completely
covering the requested bounds.
bounds must be a list of (Polygon, intervalset) tuples.
Output is a list of (cell_id, xybounds, tbounds) tuples,
unless return_bounds=False when the output is just a
list of cell_ids.
"""
# Special case of bounds=None (all sky)
if bounds == None:
bounds = [(bn.ALLSKY, intervalset((-np.inf, np.inf)))]
# Find all existing cells satisfying the bounds
cells = defaultdict(dict)
for bounds_xy, bounds_t in bounds:
self._get_cells_recursive(cells, data_path, bounds_xy, bounds_t,
pattern)
# Reorder cells to be a dict of cell: [(poly, time), (poly, time)] entries
cells = dict(((k, v.items()) for k, v in cells.iteritems()))
if False:
# TODO: Debugging, remove when happy
for k, bounds in cells.iteritems():
print k, ':', str(self._xyti_from_id(k)),
for xy, t in bounds:
print(xy.area() if xy is not None else None, t),
print ''
print len(cells)
exit()
if not return_bounds:
return cells.keys()
else:
return cells
def get_cells(self, data_path, pattern, bounds, return_bounds=False): """ Return a list of (cell_id, bounds) tuples completely covering the requested bounds. bounds must be a list of (Polygon, intervalset) tuples. Output is a list of (cell_id, xybounds, tbounds) tuples, unless return_bounds=False when the output is just a list of cell_ids. """ # Special case of bounds=None (all sky) if bounds == None: bounds = [(bn.ALLSKY, intervalset((-np.inf, np.inf)))] # Find all existing cells satisfying the bounds cells = defaultdict(dict) for bounds_xy, bounds_t in bounds: self._get_cells_recursive(cells, data_path, bounds_xy, bounds_t, pattern) # Reorder cells to be a dict of cell: [(poly, time), (poly, time)] entries cells = dict(( (k, v.items()) for k, v in cells.iteritems() )) if False: # TODO: Debugging, remove when happy for k, bounds in cells.iteritems(): print k, ':', str(self._xyti_from_id(k)), for xy, t in bounds: print (xy.area() if xy is not None else None, t), print '' print len(cells) exit() if not return_bounds: return cells.keys() else: return cells
def _get_cells_recursive(self,
outcells,
bounds_xy,
bounds_t,
i=0,
j=0,
lev=0,
dx=2.):
""" Helper for get_cells(). See documentation of
get_cells() for usage
"""
w2 = 1 << (lev - 1)
x, y = (i - w2 + 0.5) * dx, (j - w2 + 0.5) * dx
# Check for nonzero overlap
box = self._pix._cell_bounds_xy(x, y, dx)
bounds_xy = bounds_xy & box
if not bounds_xy:
return
# Check if the cell directory exists (give up if it doesn't)
bmap = self._bmaps[lev]
offs = bmap[i, j]
if not offs:
return
# If re reached the bottom of the hierarchy
if offs > 1:
# Get the cell_ids for leaf cells matching pattern
xybounds = None if (bounds_xy.area() == box.area()) else bounds_xy
next = 0
while next != END_MARKER:
(t, _, _, next) = self._leaves[offs]
has_data = next > 0
next = abs(next)
if next != END_MARKER: # Not really necessary, but numpy warns of overflow otherwise.
offs += next
if not has_data and not self._include_cached:
continue
if t != self._pix.t0:
# Cut on the time component
tival = intervalset((t, t + self._pix.dt))
tolap = bounds_t & tival
if len(tolap):
(l, r
) = tolap[-1] # Get the right-most interval component
if l == r == t + self._pix.dt: # Is it a single point?
tolap = intervalset(
*tolap[:-1]
) # Since objects in this cell have time in [t, t+dt), remove the t+dt point
if len(
tolap
) == 0: # No overlap between the intervals -- skip this cell
continue
# Return None if the cell is fully contained in the requested interval
tbounds = None if tival == tolap else tolap
else:
# Static cell
tbounds = bounds_t
assert next == END_MARKER, "There can be only one static cell (x,y,t=%s,%s,%s)" % (
x, y, t)
# Compute cell ID
cell_id = (x, y, t)
# Add to output
_add_bounds(outcells, cell_id, xybounds, tbounds)
else:
# Recursively subdivide the four subpixels
for (di, dj) in [(0, 0), (0, 1), (1, 0), (1, 1)]:
i2 = i * 2 + di
j2 = j * 2 + dj
self._get_cells_recursive(outcells, bounds_xy & box, bounds_t,
i2, j2, lev + 1, dx / 2)
def _get_cells_recursive(self, outcells, bounds_xy, bounds_t, i = 0, j = 0, lev = 0, dx = 2.): """ Helper for get_cells(). See documentation of get_cells() for usage """ w2 = 1 << (lev-1) x, y = (i - w2 + 0.5)*dx, (j - w2 + 0.5)*dx # Check for nonzero overlap box = self._pix._cell_bounds_xy(x, y, dx) bounds_xy = bounds_xy & box if not bounds_xy: return # Check if the cell directory exists (give up if it doesn't) bmap = self._bmaps[lev] offs = bmap[i, j] if not offs: return # If re reached the bottom of the hierarchy if offs > 1: # Get the cell_ids for leaf cells matching pattern xybounds = None if(bounds_xy.area() == box.area()) else bounds_xy next = 0 while next != END_MARKER: (t, _, _, next) = self._leaves[offs] has_data = next > 0 next = abs(next) if next != END_MARKER: # Not really necessary, but numpy warns of overflow otherwise. offs += next if not has_data and not self._include_cached: continue if t != self._pix.t0: # Cut on the time component tival = intervalset((t, t+self._pix.dt)) tolap = bounds_t & tival if len(tolap): (l, r) = tolap[-1] # Get the right-most interval component if l == r == t+self._pix.dt: # Is it a single point? tolap = intervalset(*tolap[:-1]) # Since objects in this cell have time in [t, t+dt), remove the t+dt point if len(tolap) == 0: # No overlap between the intervals -- skip this cell continue; # Return None if the cell is fully contained in the requested interval tbounds = None if tival == tolap else tolap else: # Static cell tbounds = bounds_t assert next == END_MARKER, "There can be only one static cell (x,y,t=%s,%s,%s)" % (x, y, t) # Compute cell ID cell_id = (x, y, t) # Add to output _add_bounds(outcells, cell_id, xybounds, tbounds) else: # Recursively subdivide the four subpixels for (di, dj) in [(0, 0), (0, 1), (1, 0), (1, 1)]: i2 = i*2 + di j2 = j*2 + dj self._get_cells_recursive(outcells, bounds_xy & box, bounds_t, i2, j2, lev+1, dx/2)
def _get_cells_recursive(self,
outcells,
data_path,
bounds_xy,
bounds_t,
pattern,
x=0.,
y=0.):
""" Helper for get_cells(). See documentation of
get_cells() for usage
"""
lev = bhpix.get_pixel_level(x, y)
dx = bhpix.pix_size(lev)
# Check for nonzero overlap
box = self._cell_bounds_xy(x, y, dx)
bounds_xy = bounds_xy & box
if not bounds_xy:
return
# Check if the cell directory exists (give up if it doesn't)
path = data_path + '/' + self._path_to_cell_base_xy(x, y, lev)
if not os.path.isdir(path):
return
# If re reached the bottom of the hierarchy
if self._is_valid_cell_xy(x, y):
# Get the cell_ids for leaf cells matching pattern
xybounds = None if (bounds_xy.area() == box.area()) else bounds_xy
sibling_cells = list(
self._get_temporal_siblings(x, y, path, pattern))
for cell_id in sibling_cells:
# Filter on time, add bounds
x, y, t = self._xyt_from_cell_id(cell_id)
if t != self.t0:
# Cut on the time component
tival = intervalset((t, t + self.dt))
tolap = bounds_t & tival
if len(tolap):
(l, r
) = tolap[-1] # Get the right-most interval component
if l == r == t + self.dt: # Is it a single point?
tolap = intervalset(
*tolap[:-1]
) # Since objects in this cell have time in [t, t+dt), remove the t+dt point
if len(
tolap
) == 0: # No overlap between the intervals -- skip this cell
continue
# Return None if the cell is fully contained in the requested interval
tbounds = None if tival == tolap else tolap
else:
# Static cell
tbounds = bounds_t
assert len(
sibling_cells
) == 1, "There can be only one static cell (cell_id=%s)" % cell_id
# Add to output
if tbounds is None:
outcells[cell_id][xybounds] = None
elif xybounds not in outcells[cell_id]:
outcells[cell_id][xybounds] = tbounds
elif outcells[cell_id][xybounds] is not None:
outcells[cell_id][xybounds] |= tbounds
else:
# Recursively subdivide the four subpixels
dx = dx / 2
for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5),
(0.5, -0.5)]):
(x2, y2) = (x, y) + dx * d
self._get_cells_recursive(outcells, data_path, bounds_xy & box,
bounds_t, pattern, x2, y2)