def _get_cells_kernel(ij, lev, cc, bounds):
i, j = ij
cells = defaultdict(dict)
for bounds_xy, bounds_t in bounds:
cc._get_cells_recursive(cells, bounds_xy, bounds_t, i, j, lev,
bhpix.pix_size(lev))
yield cells
def _get_cells_recursive(self, outcells, data_path, bounds_xy, bounds_t, pattern, x = 0., y = 0.): """ Helper for get_cells(). See documentation of get_cells() for usage """ lev = bhpix.get_pixel_level(x, y) dx = bhpix.pix_size(lev) # Check for nonzero overlap box = self._cell_bounds_xy(x, y, dx) bounds_xy = bounds_xy & box if not bounds_xy: return # Check if the cell directory exists (give up if it doesn't) path = data_path + '/' + self._path_to_cell_base_xy(x, y, lev) if not os.path.isdir(path): return # If re reached the bottom of the hierarchy if self._is_valid_cell_xy(x, y): # Get the cell_ids for leaf cells matching pattern xybounds = None if(bounds_xy.area() == box.area()) else bounds_xy sibling_cells = list(self._get_temporal_siblings(x, y, path, pattern)) for cell_id in sibling_cells: # Filter on time, add bounds x, y, t = self._xyt_from_cell_id(cell_id) if t != self.t0: # Cut on the time component tival = intervalset((t, t+self.dt)) tolap = bounds_t & tival if len(tolap): (l, r) = tolap[-1] # Get the right-most interval component if l == r == t+self.dt: # Is it a single point? tolap = intervalset(*tolap[:-1]) # Since objects in this cell have time in [t, t+dt), remove the t+dt point if len(tolap) == 0: # No overlap between the intervals -- skip this cell continue; # Return None if the cell is fully contained in the requested interval tbounds = None if tival == tolap else tolap else: # Static cell tbounds = bounds_t assert len(sibling_cells) == 1, "There can be only one static cell (cell_id=%s)" % cell_id # Add to output if tbounds is None: outcells[cell_id][xybounds] = None elif xybounds not in outcells[cell_id]: outcells[cell_id][xybounds] = tbounds elif outcells[cell_id][xybounds] is not None: outcells[cell_id][xybounds] |= tbounds else: # Recursively subdivide the four subpixels dx = dx / 2 for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5)]): (x2, y2) = (x, y) + dx*d self._get_cells_recursive(outcells, data_path, bounds_xy & box, bounds_t, pattern, x2, y2)
def _scan_recursive(self, bmap, lev=0, x=0., y=0.):
# Check if the cell directory exists in any allowable snapshot
# Snapshots are ordered newest-to-oldest, so we can avoid touching the
# siblings of older snapshots if they exists in newer ones
dx = bhpix.pix_size(lev)
# .../snapshots/XXXXXX/tablets/-0.5+0.5/..../
paths = [(snapid, '%s/tablets/%s' %
(snapshot_path, self._pix._path_to_cell_base_xy(x, y, lev)))
for (snapid, snapshot_path) in self.__snapshots]
paths = [(snapid, path) for (snapid, path) in paths
if os.path.isdir(path)] # Keep only those that exist
if not len(paths): # Dead end
return
if lev != self._pix.level:
# Recursively subdivide the four subpixels
dx = dx / 2
for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5),
(0.5, -0.5)]):
(x2, y2) = (x, y) + dx * d
self._scan_recursive(bmap, lev + 1, x2, y2)
else:
# Leaf
w2 = bhpix.width(lev) // 2
i, j = (x // dx + w2, y // dx + w2)
# Collect the data we have, and add them to the set of cells that exist
siblings = dict()
for snapid, path in paths:
for tcell, fn in self._get_temporal_siblings(
path, self.__pattern):
if tcell not in siblings: # Add only if there's no newer version
# check if there are any non-cached data in here
try:
with tables.open_file(fn) as fp:
has_data = len(fp.root.main.table) > 0
except tables.exceptions.NoSuchNodeError:
has_data = False
siblings[tcell] = snapid, has_data
# Add any relevant pre-existing data
i = int(i)
j = int(j)
offs = self._bmaps[self._pix.level][i, j]
if offs != 0:
for mjd, snapid, _, next in iter_siblings(self._leaves, offs):
if mjd not in siblings:
siblings[mjd] = snapid, next > 0
# Add this list to bitmap
assert bmap[i, j] == 0
bmap[i,
j] = [(tcell, snapid, self._pix._cell_id_for_xyt(x, y, tcell),
has_data)
for (tcell, (snapid, has_data)) in siblings.iteritems()]
def _scan_recursive(self, bmap, lev=0, x=0., y=0.): # Check if the cell directory exists in any allowable snapshot # Snapshots are ordered newest-to-oldest, so we can avoid touching the # siblings of older snapshots if they exists in newer ones dx = bhpix.pix_size(lev) # .../snapshots/XXXXXX/tablets/-0.5+0.5/..../ paths = [ (snapid, '%s/tablets/%s' % (snapshot_path, self._pix._path_to_cell_base_xy(x, y, lev))) for (snapid, snapshot_path) in self.__snapshots ] paths = [ (snapid, path) for (snapid, path) in paths if os.path.isdir(path) ] # Keep only those that exist if not len(paths): # Dead end return if lev != self._pix.level: # Recursively subdivide the four subpixels dx = dx / 2 for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5)]): (x2, y2) = (x, y) + dx*d self._scan_recursive(bmap, lev+1, x2, y2) else: # Leaf w2 = bhpix.width(lev) // 2 i, j = (x // dx + w2, y // dx + w2) # Collect the data we have, and add them to the set of cells that exist siblings = dict() for snapid, path in paths: for tcell, fn in self._get_temporal_siblings(path, self.__pattern): if tcell not in siblings: # Add only if there's no newer version # check if there are any non-cached data in here try: with tables.openFile(fn) as fp: has_data = len(fp.root.main.table) > 0 except tables.exceptions.NoSuchNodeError: has_data = False siblings[tcell] = snapid, has_data # Add any relevant pre-existing data offs = self._bmaps[self._pix.level][i, j] if offs != 0: for mjd, snapid, _, next in iter_siblings(self._leaves, offs): if mjd not in siblings: siblings[mjd] = snapid, next > 0 # Add this list to bitmap assert bmap[i, j] == 0 bmap[i, j] = [ (tcell, snapid, self._pix._cell_id_for_xyt(x, y, tcell), has_data) for (tcell, (snapid, has_data)) in siblings.iteritems() ]
def _cell_bounds_xy(self, x, y, dx = None): """ Return a bounding polygon for a given cell center x,y """ if dx is None: lev = bhpix.get_pixel_level(x, y) dx = bhpix.pix_size(lev) ##dx = bhpix.pix_size(self.level) bounds = Polygon.Shapes.Rectangle(dx) bounds.shift(x - 0.5*dx, y - 0.5*dx); if fabs(fabs(x) - fabs(y)) == 0.5: # If it's a "halfpixel", return a triangle # by clipping agains the sky bounds &= bn.ALLSKY return bounds
def _cell_bounds_xy(self, x, y, dx=None):
""" Return a bounding polygon for a given
cell center x,y
"""
if dx is None:
lev = bhpix.get_pixel_level(x, y)
dx = bhpix.pix_size(lev)
##dx = bhpix.pix_size(self.level)
bounds = Polygon.Shapes.Rectangle(dx)
bounds.shift(x - 0.5 * dx, y - 0.5 * dx)
if fabs(fabs(x) - fabs(y)) == 0.5:
# If it's a "halfpixel", return a triangle
# by clipping agains the sky
bounds &= bn.ALLSKY
return bounds
def _update(self, table_path, snapid):
# Find what we already have loaded
prevsnap = np.max(
self._leaves['snapid']) if len(self._leaves) > 2 else None
assert prevsnap <= snapid, "Cannot update a catalog to an older snapshot"
## Enumerate all existing snapshots older or equal to snapid, and newer than prevsnap, and sort them, newest first
self.__snapshots = sorted(isnapshots(table_path,
first=prevsnap,
last=snapid,
no_first=True,
return_path=True),
reverse=True)
# Add the snapid snapshot by hand, if not in the list of committed snapshots
# (because it is permissible to update to a yet-uncommitted snapshot)
if len(self.__snapshots) == 0 or self.__snapshots[0][0] != snapid:
self.__snapshots.insert(
0, (snapid, get_snapshot_path(table_path, snapid)))
# Recursively scan, in parallel
w = bhpix.width(self._pix.level)
bmap = np.zeros((w, w), dtype=object)
lev = min(4, self._pix.level)
dx = bhpix.pix_size(lev)
i, j = np.indices(
(2**lev, 2**lev)).reshape(2, -1) # List of i,j coordinates
w2 = 1 << (lev - 1)
x, y = (i - w2 + 0.5) * dx, (j - w2 + 0.5) * dx
pool = pool2.Pool()
for bmap2 in pool.imap_unordered(zip(x, y), _scan_recursive_kernel,
(lev, self)):
assert not np.any((bmap != 0) & (bmap2 != 0))
mask = bmap2 != 0
bmap[mask] = bmap2[mask]
del pool
# Add data about cells that were not touched by this update
bmap_cur = self._bmaps[self._pix.level]
mask_cur = (bmap_cur != 0) & (bmap == 0)
lists_cur = [[(mjd, snapid, cell_id, next > 0)
for (mjd, snapid, cell_id,
next) in iter_siblings(self._leaves, offs)]
for offs in bmap_cur[mask_cur]]
try:
bmap[mask_cur] = lists_cur
except ValueError:
# Workaround for a numpy 1.6.0 bug (http://projects.scipy.org/numpy/ticket/1870)
coords = np.mgrid[
0:w, 0:
w][:,
mask_cur].T # List of coordinates corresponding to True entries in mask_cur
for (ii, jj), vv in izip(coords, lists_cur):
bmap[ii, jj] = vv
# Repack the temporal siblings to a single numpy array, emulating a linked list
lists = bmap[bmap != 0]
llens = np.fromiter((len(l) for l in lists), dtype=np.int32)
leaves = np.empty(np.sum(llens) + 2,
dtype=[('mjd', 'f4'), ('snapid', object),
('cell_id', 'u8'), ('next', 'i4')])
leaves[:2] = [
(np.inf, 0, 0, END_MARKER)
] * 2 # We start with two dummy entries, so that offs=0 and 1 are invalid and can take other meanings.
seen = dict()
at = 2
for l in lists:
last_i = len(l) - 1
for (i, (mjd, snapid, cell_id, has_data)) in enumerate(l):
# Make equal strings refer to the same string object
try:
snapid = seen[snapid]
except KeyError:
seen[snapid] = snapid
next = 1 if has_data else -1
if i == last_i:
next *= END_MARKER
leaves[at] = (mjd, snapid, cell_id, next)
at += 1
# Construct bmap that has offsets to head of the linked list of siblings
offs = np.zeros(len(llens), dtype=np.int64)
offs[1:] = np.cumsum(llens)[:-1]
offs += 2 # Pointers to beginnings of individual lists
assert np.all(abs(leaves['next'][offs - 1]) == END_MARKER)
obmap = np.zeros(bmap.shape, dtype=np.int32)
obmap[bmap != 0] = offs
# Recompute mipmaps
bmaps = self._compute_mipmaps(obmap)
return bmaps, leaves
def _get_cells_kernel(ij, lev, cc, bounds):
i, j = ij
cells = defaultdict(dict)
for bounds_xy, bounds_t in bounds:
cc._get_cells_recursive(cells, bounds_xy, bounds_t, i, j, lev, bhpix.pix_size(lev))
yield cells
def _update(self, table_path, snapid):
# Find what we already have loaded
prevsnap = np.max(self._leaves['snapid']) if len(self._leaves) > 2 else None
assert prevsnap <= snapid, "Cannot update a catalog to an older snapshot"
## Enumerate all existing snapshots older or equal to snapid, and newer than prevsnap, and sort them, newest first
self.__snapshots = sorted(isnapshots(table_path, first=prevsnap, last=snapid, no_first=True, return_path=True), reverse=True)
# Add the snapid snapshot by hand, if not in the list of committed snapshots
# (because it is permissible to update to a yet-uncommitted snapshot)
if len(self.__snapshots) == 0 or self.__snapshots[0][0] != snapid:
self.__snapshots.insert(0, (snapid, get_snapshot_path(table_path, snapid)))
# Recursively scan, in parallel
w = bhpix.width(self._pix.level)
bmap = np.zeros((w, w), dtype=object)
lev = min(4, self._pix.level)
dx = bhpix.pix_size(lev)
i, j = np.indices((2**lev,2**lev)).reshape(2, -1) # List of i,j coordinates
w2 = 1 << (lev-1)
x, y = (i - w2 + 0.5)*dx, (j - w2 + 0.5)*dx
pool = pool2.Pool()
for bmap2 in pool.imap_unordered(zip(x, y), _scan_recursive_kernel, (lev, self)):
assert not np.any((bmap != 0) & (bmap2 != 0))
mask = bmap2 != 0
bmap[mask] = bmap2[mask]
del pool
# Add data about cells that were not touched by this update
bmap_cur = self._bmaps[self._pix.level]
mask_cur = (bmap_cur != 0) & (bmap == 0)
lists_cur = [ [ (mjd, snapid, cell_id, next > 0) for (mjd, snapid, cell_id, next) in iter_siblings(self._leaves, offs) ] for offs in bmap_cur[mask_cur] ]
try:
bmap[mask_cur] = lists_cur
except ValueError:
# Workaround for a numpy 1.6.0 bug (http://projects.scipy.org/numpy/ticket/1870)
coords = np.mgrid[0:w,0:w][:,mask_cur].T # List of coordinates corresponding to True entries in mask_cur
for (ii, jj), vv in izip(coords, lists_cur):
bmap[ii, jj] = vv
# Repack the temporal siblings to a single numpy array, emulating a linked list
lists = bmap[bmap != 0]
llens = np.fromiter( (len(l) for l in lists), dtype=np.int32 )
leaves = np.empty(np.sum(llens)+2, dtype=[('mjd', 'f4'), ('snapid', object), ('cell_id', 'u8'), ('next', 'i4')])
leaves[:2] = [(np.inf, 0, 0, END_MARKER)]*2 # We start with two dummy entries, so that offs=0 and 1 are invalid and can take other meanings.
seen = dict()
at = 2
for l in lists:
last_i = len(l) - 1
for (i, (mjd, snapid, cell_id, has_data)) in enumerate(l):
# Make equal strings refer to the same string object
try:
snapid = seen[snapid]
except KeyError:
seen[snapid] = snapid
next = 1 if has_data else -1
if i == last_i:
next *= END_MARKER
leaves[at] = (mjd, snapid, cell_id, next)
at += 1
# Construct bmap that has offsets to head of the linked list of siblings
offs = np.zeros(len(llens), dtype=np.int64)
offs[1:] = np.cumsum(llens)[:-1]
offs += 2 # Pointers to beginnings of individual lists
assert np.all(abs(leaves['next'][offs-1]) == END_MARKER)
obmap = np.zeros(bmap.shape, dtype=np.int32)
obmap[bmap != 0] = offs
# Recompute mipmaps
bmaps = self._compute_mipmaps(obmap)
return bmaps, leaves
def _get_cells_recursive(self,
outcells,
data_path,
bounds_xy,
bounds_t,
pattern,
x=0.,
y=0.):
""" Helper for get_cells(). See documentation of
get_cells() for usage
"""
lev = bhpix.get_pixel_level(x, y)
dx = bhpix.pix_size(lev)
# Check for nonzero overlap
box = self._cell_bounds_xy(x, y, dx)
bounds_xy = bounds_xy & box
if not bounds_xy:
return
# Check if the cell directory exists (give up if it doesn't)
path = data_path + '/' + self._path_to_cell_base_xy(x, y, lev)
if not os.path.isdir(path):
return
# If re reached the bottom of the hierarchy
if self._is_valid_cell_xy(x, y):
# Get the cell_ids for leaf cells matching pattern
xybounds = None if (bounds_xy.area() == box.area()) else bounds_xy
sibling_cells = list(
self._get_temporal_siblings(x, y, path, pattern))
for cell_id in sibling_cells:
# Filter on time, add bounds
x, y, t = self._xyt_from_cell_id(cell_id)
if t != self.t0:
# Cut on the time component
tival = intervalset((t, t + self.dt))
tolap = bounds_t & tival
if len(tolap):
(l, r
) = tolap[-1] # Get the right-most interval component
if l == r == t + self.dt: # Is it a single point?
tolap = intervalset(
*tolap[:-1]
) # Since objects in this cell have time in [t, t+dt), remove the t+dt point
if len(
tolap
) == 0: # No overlap between the intervals -- skip this cell
continue
# Return None if the cell is fully contained in the requested interval
tbounds = None if tival == tolap else tolap
else:
# Static cell
tbounds = bounds_t
assert len(
sibling_cells
) == 1, "There can be only one static cell (cell_id=%s)" % cell_id
# Add to output
if tbounds is None:
outcells[cell_id][xybounds] = None
elif xybounds not in outcells[cell_id]:
outcells[cell_id][xybounds] = tbounds
elif outcells[cell_id][xybounds] is not None:
outcells[cell_id][xybounds] |= tbounds
else:
# Recursively subdivide the four subpixels
dx = dx / 2
for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5),
(0.5, -0.5)]):
(x2, y2) = (x, y) + dx * d
self._get_cells_recursive(outcells, data_path, bounds_xy & box,
bounds_t, pattern, x2, y2)