def _scan_recursive_kernel(xy, lev, cc):
x, y = xy
w = bhpix.width(cc._pix.level)
bmap = np.zeros((w, w), dtype=object)
cc._scan_recursive(bmap, lev, x, y)
yield bmap
def _scan_recursive_kernel(xy, lev, cc): x, y = xy w = bhpix.width(cc._pix.level) bmap = np.zeros((w, w), dtype=object) cc._scan_recursive(bmap, lev, x, y) yield bmap
def clear(self):
# Initialize an empty table
w = bhpix.width(self._pix.level)
self._bmaps = self._compute_mipmaps(np.zeros((w, w), dtype=object))
self._leaves = np.empty(2, dtype=[('mjd', 'f4'), ('snapid', object), ('cell_id', 'u8'), ('next', 'i4')])
self._leaves[:2] = [(np.inf, 0, 0, END_MARKER)]*2
self._rebuild_internal_state()
def _scan_recursive(self, bmap, lev=0, x=0., y=0.):
# Check if the cell directory exists in any allowable snapshot
# Snapshots are ordered newest-to-oldest, so we can avoid touching the
# siblings of older snapshots if they exists in newer ones
dx = bhpix.pix_size(lev)
# .../snapshots/XXXXXX/tablets/-0.5+0.5/..../
paths = [(snapid, '%s/tablets/%s' %
(snapshot_path, self._pix._path_to_cell_base_xy(x, y, lev)))
for (snapid, snapshot_path) in self.__snapshots]
paths = [(snapid, path) for (snapid, path) in paths
if os.path.isdir(path)] # Keep only those that exist
if not len(paths): # Dead end
return
if lev != self._pix.level:
# Recursively subdivide the four subpixels
dx = dx / 2
for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5),
(0.5, -0.5)]):
(x2, y2) = (x, y) + dx * d
self._scan_recursive(bmap, lev + 1, x2, y2)
else:
# Leaf
w2 = bhpix.width(lev) // 2
i, j = (x // dx + w2, y // dx + w2)
# Collect the data we have, and add them to the set of cells that exist
siblings = dict()
for snapid, path in paths:
for tcell, fn in self._get_temporal_siblings(
path, self.__pattern):
if tcell not in siblings: # Add only if there's no newer version
# check if there are any non-cached data in here
try:
with tables.open_file(fn) as fp:
has_data = len(fp.root.main.table) > 0
except tables.exceptions.NoSuchNodeError:
has_data = False
siblings[tcell] = snapid, has_data
# Add any relevant pre-existing data
i = int(i)
j = int(j)
offs = self._bmaps[self._pix.level][i, j]
if offs != 0:
for mjd, snapid, _, next in iter_siblings(self._leaves, offs):
if mjd not in siblings:
siblings[mjd] = snapid, next > 0
# Add this list to bitmap
assert bmap[i, j] == 0
bmap[i,
j] = [(tcell, snapid, self._pix._cell_id_for_xyt(x, y, tcell),
has_data)
for (tcell, (snapid, has_data)) in siblings.iteritems()]
def clear(self):
# Initialize an empty table
w = bhpix.width(self._pix.level)
self._bmaps = self._compute_mipmaps(np.zeros((w, w), dtype=object))
self._leaves = np.empty(2,
dtype=[('mjd', 'f4'), ('snapid', object),
('cell_id', 'u8'), ('next', 'i4')])
self._leaves[:2] = [(np.inf, 0, 0, END_MARKER)] * 2
self._rebuild_internal_state()
def _compute_mipmaps(self, bmap):
# Create mip-maps
bmaps = {self._pix.level: bmap} # Mip-maps of bmap with True if the cell has data in its leaves, and False otherwise
for lev in xrange(self._pix.level-1, -1, -1):
w = bhpix.width(lev)
m0 = np.zeros((w, w), dtype=np.int32)
m1 = bmaps[lev+1]
for (i, j) in [(0, 0), (0, 1), (1, 0), (1, 1)]:
m0[:,:] |= m1[i::2, j::2]
m0 = m0 != 0
bmaps[lev] = m0
return bmaps
def _scan_recursive(self, bmap, lev=0, x=0., y=0.): # Check if the cell directory exists in any allowable snapshot # Snapshots are ordered newest-to-oldest, so we can avoid touching the # siblings of older snapshots if they exists in newer ones dx = bhpix.pix_size(lev) # .../snapshots/XXXXXX/tablets/-0.5+0.5/..../ paths = [ (snapid, '%s/tablets/%s' % (snapshot_path, self._pix._path_to_cell_base_xy(x, y, lev))) for (snapid, snapshot_path) in self.__snapshots ] paths = [ (snapid, path) for (snapid, path) in paths if os.path.isdir(path) ] # Keep only those that exist if not len(paths): # Dead end return if lev != self._pix.level: # Recursively subdivide the four subpixels dx = dx / 2 for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5)]): (x2, y2) = (x, y) + dx*d self._scan_recursive(bmap, lev+1, x2, y2) else: # Leaf w2 = bhpix.width(lev) // 2 i, j = (x // dx + w2, y // dx + w2) # Collect the data we have, and add them to the set of cells that exist siblings = dict() for snapid, path in paths: for tcell, fn in self._get_temporal_siblings(path, self.__pattern): if tcell not in siblings: # Add only if there's no newer version # check if there are any non-cached data in here try: with tables.openFile(fn) as fp: has_data = len(fp.root.main.table) > 0 except tables.exceptions.NoSuchNodeError: has_data = False siblings[tcell] = snapid, has_data # Add any relevant pre-existing data offs = self._bmaps[self._pix.level][i, j] if offs != 0: for mjd, snapid, _, next in iter_siblings(self._leaves, offs): if mjd not in siblings: siblings[mjd] = snapid, next > 0 # Add this list to bitmap assert bmap[i, j] == 0 bmap[i, j] = [ (tcell, snapid, self._pix._cell_id_for_xyt(x, y, tcell), has_data) for (tcell, (snapid, has_data)) in siblings.iteritems() ]
def _compute_mipmaps(self, bmap):
# Create mip-maps
bmaps = {
self._pix.level: bmap
} # Mip-maps of bmap with True if the cell has data in its leaves, and False otherwise
for lev in xrange(self._pix.level - 1, -1, -1):
w = bhpix.width(lev)
m0 = np.zeros((w, w), dtype=np.int32)
m1 = bmaps[lev + 1]
for (i, j) in [(0, 0), (0, 1), (1, 0), (1, 1)]:
m0[:, :] = m0[:, :] | m1[i::2, j::2]
m0 = m0 != 0
bmaps[lev] = m0
return bmaps
def _update(self, table_path, snapid):
# Find what we already have loaded
prevsnap = np.max(
self._leaves['snapid']) if len(self._leaves) > 2 else None
assert prevsnap <= snapid, "Cannot update a catalog to an older snapshot"
## Enumerate all existing snapshots older or equal to snapid, and newer than prevsnap, and sort them, newest first
self.__snapshots = sorted(isnapshots(table_path,
first=prevsnap,
last=snapid,
no_first=True,
return_path=True),
reverse=True)
# Add the snapid snapshot by hand, if not in the list of committed snapshots
# (because it is permissible to update to a yet-uncommitted snapshot)
if len(self.__snapshots) == 0 or self.__snapshots[0][0] != snapid:
self.__snapshots.insert(
0, (snapid, get_snapshot_path(table_path, snapid)))
# Recursively scan, in parallel
w = bhpix.width(self._pix.level)
bmap = np.zeros((w, w), dtype=object)
lev = min(4, self._pix.level)
dx = bhpix.pix_size(lev)
i, j = np.indices(
(2**lev, 2**lev)).reshape(2, -1) # List of i,j coordinates
w2 = 1 << (lev - 1)
x, y = (i - w2 + 0.5) * dx, (j - w2 + 0.5) * dx
pool = pool2.Pool()
for bmap2 in pool.imap_unordered(zip(x, y), _scan_recursive_kernel,
(lev, self)):
assert not np.any((bmap != 0) & (bmap2 != 0))
mask = bmap2 != 0
bmap[mask] = bmap2[mask]
del pool
# Add data about cells that were not touched by this update
bmap_cur = self._bmaps[self._pix.level]
mask_cur = (bmap_cur != 0) & (bmap == 0)
lists_cur = [[(mjd, snapid, cell_id, next > 0)
for (mjd, snapid, cell_id,
next) in iter_siblings(self._leaves, offs)]
for offs in bmap_cur[mask_cur]]
try:
bmap[mask_cur] = lists_cur
except ValueError:
# Workaround for a numpy 1.6.0 bug (http://projects.scipy.org/numpy/ticket/1870)
coords = np.mgrid[
0:w, 0:
w][:,
mask_cur].T # List of coordinates corresponding to True entries in mask_cur
for (ii, jj), vv in izip(coords, lists_cur):
bmap[ii, jj] = vv
# Repack the temporal siblings to a single numpy array, emulating a linked list
lists = bmap[bmap != 0]
llens = np.fromiter((len(l) for l in lists), dtype=np.int32)
leaves = np.empty(np.sum(llens) + 2,
dtype=[('mjd', 'f4'), ('snapid', object),
('cell_id', 'u8'), ('next', 'i4')])
leaves[:2] = [
(np.inf, 0, 0, END_MARKER)
] * 2 # We start with two dummy entries, so that offs=0 and 1 are invalid and can take other meanings.
seen = dict()
at = 2
for l in lists:
last_i = len(l) - 1
for (i, (mjd, snapid, cell_id, has_data)) in enumerate(l):
# Make equal strings refer to the same string object
try:
snapid = seen[snapid]
except KeyError:
seen[snapid] = snapid
next = 1 if has_data else -1
if i == last_i:
next *= END_MARKER
leaves[at] = (mjd, snapid, cell_id, next)
at += 1
# Construct bmap that has offsets to head of the linked list of siblings
offs = np.zeros(len(llens), dtype=np.int64)
offs[1:] = np.cumsum(llens)[:-1]
offs += 2 # Pointers to beginnings of individual lists
assert np.all(abs(leaves['next'][offs - 1]) == END_MARKER)
obmap = np.zeros(bmap.shape, dtype=np.int32)
obmap[bmap != 0] = offs
# Recompute mipmaps
bmaps = self._compute_mipmaps(obmap)
return bmaps, leaves
def _update(self, table_path, snapid):
# Find what we already have loaded
prevsnap = np.max(self._leaves['snapid']) if len(self._leaves) > 2 else None
assert prevsnap <= snapid, "Cannot update a catalog to an older snapshot"
## Enumerate all existing snapshots older or equal to snapid, and newer than prevsnap, and sort them, newest first
self.__snapshots = sorted(isnapshots(table_path, first=prevsnap, last=snapid, no_first=True, return_path=True), reverse=True)
# Add the snapid snapshot by hand, if not in the list of committed snapshots
# (because it is permissible to update to a yet-uncommitted snapshot)
if len(self.__snapshots) == 0 or self.__snapshots[0][0] != snapid:
self.__snapshots.insert(0, (snapid, get_snapshot_path(table_path, snapid)))
# Recursively scan, in parallel
w = bhpix.width(self._pix.level)
bmap = np.zeros((w, w), dtype=object)
lev = min(4, self._pix.level)
dx = bhpix.pix_size(lev)
i, j = np.indices((2**lev,2**lev)).reshape(2, -1) # List of i,j coordinates
w2 = 1 << (lev-1)
x, y = (i - w2 + 0.5)*dx, (j - w2 + 0.5)*dx
pool = pool2.Pool()
for bmap2 in pool.imap_unordered(zip(x, y), _scan_recursive_kernel, (lev, self)):
assert not np.any((bmap != 0) & (bmap2 != 0))
mask = bmap2 != 0
bmap[mask] = bmap2[mask]
del pool
# Add data about cells that were not touched by this update
bmap_cur = self._bmaps[self._pix.level]
mask_cur = (bmap_cur != 0) & (bmap == 0)
lists_cur = [ [ (mjd, snapid, cell_id, next > 0) for (mjd, snapid, cell_id, next) in iter_siblings(self._leaves, offs) ] for offs in bmap_cur[mask_cur] ]
try:
bmap[mask_cur] = lists_cur
except ValueError:
# Workaround for a numpy 1.6.0 bug (http://projects.scipy.org/numpy/ticket/1870)
coords = np.mgrid[0:w,0:w][:,mask_cur].T # List of coordinates corresponding to True entries in mask_cur
for (ii, jj), vv in izip(coords, lists_cur):
bmap[ii, jj] = vv
# Repack the temporal siblings to a single numpy array, emulating a linked list
lists = bmap[bmap != 0]
llens = np.fromiter( (len(l) for l in lists), dtype=np.int32 )
leaves = np.empty(np.sum(llens)+2, dtype=[('mjd', 'f4'), ('snapid', object), ('cell_id', 'u8'), ('next', 'i4')])
leaves[:2] = [(np.inf, 0, 0, END_MARKER)]*2 # We start with two dummy entries, so that offs=0 and 1 are invalid and can take other meanings.
seen = dict()
at = 2
for l in lists:
last_i = len(l) - 1
for (i, (mjd, snapid, cell_id, has_data)) in enumerate(l):
# Make equal strings refer to the same string object
try:
snapid = seen[snapid]
except KeyError:
seen[snapid] = snapid
next = 1 if has_data else -1
if i == last_i:
next *= END_MARKER
leaves[at] = (mjd, snapid, cell_id, next)
at += 1
# Construct bmap that has offsets to head of the linked list of siblings
offs = np.zeros(len(llens), dtype=np.int64)
offs[1:] = np.cumsum(llens)[:-1]
offs += 2 # Pointers to beginnings of individual lists
assert np.all(abs(leaves['next'][offs-1]) == END_MARKER)
obmap = np.zeros(bmap.shape, dtype=np.int32)
obmap[bmap != 0] = offs
# Recompute mipmaps
bmaps = self._compute_mipmaps(obmap)
return bmaps, leaves