def _get_cells_kernel(ij, lev, cc, bounds): i, j = ij cells = defaultdict(dict) for bounds_xy, bounds_t in bounds: cc._get_cells_recursive(cells, bounds_xy, bounds_t, i, j, lev, bhpix.pix_size(lev)) yield cells
def _get_cells_recursive(self, outcells, data_path, bounds_xy, bounds_t, pattern, x = 0., y = 0.): """ Helper for get_cells(). See documentation of get_cells() for usage """ lev = bhpix.get_pixel_level(x, y) dx = bhpix.pix_size(lev) # Check for nonzero overlap box = self._cell_bounds_xy(x, y, dx) bounds_xy = bounds_xy & box if not bounds_xy: return # Check if the cell directory exists (give up if it doesn't) path = data_path + '/' + self._path_to_cell_base_xy(x, y, lev) if not os.path.isdir(path): return # If re reached the bottom of the hierarchy if self._is_valid_cell_xy(x, y): # Get the cell_ids for leaf cells matching pattern xybounds = None if(bounds_xy.area() == box.area()) else bounds_xy sibling_cells = list(self._get_temporal_siblings(x, y, path, pattern)) for cell_id in sibling_cells: # Filter on time, add bounds x, y, t = self._xyt_from_cell_id(cell_id) if t != self.t0: # Cut on the time component tival = intervalset((t, t+self.dt)) tolap = bounds_t & tival if len(tolap): (l, r) = tolap[-1] # Get the right-most interval component if l == r == t+self.dt: # Is it a single point? tolap = intervalset(*tolap[:-1]) # Since objects in this cell have time in [t, t+dt), remove the t+dt point if len(tolap) == 0: # No overlap between the intervals -- skip this cell continue; # Return None if the cell is fully contained in the requested interval tbounds = None if tival == tolap else tolap else: # Static cell tbounds = bounds_t assert len(sibling_cells) == 1, "There can be only one static cell (cell_id=%s)" % cell_id # Add to output if tbounds is None: outcells[cell_id][xybounds] = None elif xybounds not in outcells[cell_id]: outcells[cell_id][xybounds] = tbounds elif outcells[cell_id][xybounds] is not None: outcells[cell_id][xybounds] |= tbounds else: # Recursively subdivide the four subpixels dx = dx / 2 for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5)]): (x2, y2) = (x, y) + dx*d self._get_cells_recursive(outcells, data_path, bounds_xy & box, bounds_t, pattern, x2, y2)
def _scan_recursive(self, bmap, lev=0, x=0., y=0.): # Check if the cell directory exists in any allowable snapshot # Snapshots are ordered newest-to-oldest, so we can avoid touching the # siblings of older snapshots if they exists in newer ones dx = bhpix.pix_size(lev) # .../snapshots/XXXXXX/tablets/-0.5+0.5/..../ paths = [(snapid, '%s/tablets/%s' % (snapshot_path, self._pix._path_to_cell_base_xy(x, y, lev))) for (snapid, snapshot_path) in self.__snapshots] paths = [(snapid, path) for (snapid, path) in paths if os.path.isdir(path)] # Keep only those that exist if not len(paths): # Dead end return if lev != self._pix.level: # Recursively subdivide the four subpixels dx = dx / 2 for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5)]): (x2, y2) = (x, y) + dx * d self._scan_recursive(bmap, lev + 1, x2, y2) else: # Leaf w2 = bhpix.width(lev) // 2 i, j = (x // dx + w2, y // dx + w2) # Collect the data we have, and add them to the set of cells that exist siblings = dict() for snapid, path in paths: for tcell, fn in self._get_temporal_siblings( path, self.__pattern): if tcell not in siblings: # Add only if there's no newer version # check if there are any non-cached data in here try: with tables.open_file(fn) as fp: has_data = len(fp.root.main.table) > 0 except tables.exceptions.NoSuchNodeError: has_data = False siblings[tcell] = snapid, has_data # Add any relevant pre-existing data i = int(i) j = int(j) offs = self._bmaps[self._pix.level][i, j] if offs != 0: for mjd, snapid, _, next in iter_siblings(self._leaves, offs): if mjd not in siblings: siblings[mjd] = snapid, next > 0 # Add this list to bitmap assert bmap[i, j] == 0 bmap[i, j] = [(tcell, snapid, self._pix._cell_id_for_xyt(x, y, tcell), has_data) for (tcell, (snapid, has_data)) in siblings.iteritems()]
def _scan_recursive(self, bmap, lev=0, x=0., y=0.): # Check if the cell directory exists in any allowable snapshot # Snapshots are ordered newest-to-oldest, so we can avoid touching the # siblings of older snapshots if they exists in newer ones dx = bhpix.pix_size(lev) # .../snapshots/XXXXXX/tablets/-0.5+0.5/..../ paths = [ (snapid, '%s/tablets/%s' % (snapshot_path, self._pix._path_to_cell_base_xy(x, y, lev))) for (snapid, snapshot_path) in self.__snapshots ] paths = [ (snapid, path) for (snapid, path) in paths if os.path.isdir(path) ] # Keep only those that exist if not len(paths): # Dead end return if lev != self._pix.level: # Recursively subdivide the four subpixels dx = dx / 2 for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5)]): (x2, y2) = (x, y) + dx*d self._scan_recursive(bmap, lev+1, x2, y2) else: # Leaf w2 = bhpix.width(lev) // 2 i, j = (x // dx + w2, y // dx + w2) # Collect the data we have, and add them to the set of cells that exist siblings = dict() for snapid, path in paths: for tcell, fn in self._get_temporal_siblings(path, self.__pattern): if tcell not in siblings: # Add only if there's no newer version # check if there are any non-cached data in here try: with tables.openFile(fn) as fp: has_data = len(fp.root.main.table) > 0 except tables.exceptions.NoSuchNodeError: has_data = False siblings[tcell] = snapid, has_data # Add any relevant pre-existing data offs = self._bmaps[self._pix.level][i, j] if offs != 0: for mjd, snapid, _, next in iter_siblings(self._leaves, offs): if mjd not in siblings: siblings[mjd] = snapid, next > 0 # Add this list to bitmap assert bmap[i, j] == 0 bmap[i, j] = [ (tcell, snapid, self._pix._cell_id_for_xyt(x, y, tcell), has_data) for (tcell, (snapid, has_data)) in siblings.iteritems() ]
def _cell_bounds_xy(self, x, y, dx = None): """ Return a bounding polygon for a given cell center x,y """ if dx is None: lev = bhpix.get_pixel_level(x, y) dx = bhpix.pix_size(lev) ##dx = bhpix.pix_size(self.level) bounds = Polygon.Shapes.Rectangle(dx) bounds.shift(x - 0.5*dx, y - 0.5*dx); if fabs(fabs(x) - fabs(y)) == 0.5: # If it's a "halfpixel", return a triangle # by clipping agains the sky bounds &= bn.ALLSKY return bounds
def _cell_bounds_xy(self, x, y, dx=None): """ Return a bounding polygon for a given cell center x,y """ if dx is None: lev = bhpix.get_pixel_level(x, y) dx = bhpix.pix_size(lev) ##dx = bhpix.pix_size(self.level) bounds = Polygon.Shapes.Rectangle(dx) bounds.shift(x - 0.5 * dx, y - 0.5 * dx) if fabs(fabs(x) - fabs(y)) == 0.5: # If it's a "halfpixel", return a triangle # by clipping agains the sky bounds &= bn.ALLSKY return bounds
def _update(self, table_path, snapid): # Find what we already have loaded prevsnap = np.max( self._leaves['snapid']) if len(self._leaves) > 2 else None assert prevsnap <= snapid, "Cannot update a catalog to an older snapshot" ## Enumerate all existing snapshots older or equal to snapid, and newer than prevsnap, and sort them, newest first self.__snapshots = sorted(isnapshots(table_path, first=prevsnap, last=snapid, no_first=True, return_path=True), reverse=True) # Add the snapid snapshot by hand, if not in the list of committed snapshots # (because it is permissible to update to a yet-uncommitted snapshot) if len(self.__snapshots) == 0 or self.__snapshots[0][0] != snapid: self.__snapshots.insert( 0, (snapid, get_snapshot_path(table_path, snapid))) # Recursively scan, in parallel w = bhpix.width(self._pix.level) bmap = np.zeros((w, w), dtype=object) lev = min(4, self._pix.level) dx = bhpix.pix_size(lev) i, j = np.indices( (2**lev, 2**lev)).reshape(2, -1) # List of i,j coordinates w2 = 1 << (lev - 1) x, y = (i - w2 + 0.5) * dx, (j - w2 + 0.5) * dx pool = pool2.Pool() for bmap2 in pool.imap_unordered(zip(x, y), _scan_recursive_kernel, (lev, self)): assert not np.any((bmap != 0) & (bmap2 != 0)) mask = bmap2 != 0 bmap[mask] = bmap2[mask] del pool # Add data about cells that were not touched by this update bmap_cur = self._bmaps[self._pix.level] mask_cur = (bmap_cur != 0) & (bmap == 0) lists_cur = [[(mjd, snapid, cell_id, next > 0) for (mjd, snapid, cell_id, next) in iter_siblings(self._leaves, offs)] for offs in bmap_cur[mask_cur]] try: bmap[mask_cur] = lists_cur except ValueError: # Workaround for a numpy 1.6.0 bug (http://projects.scipy.org/numpy/ticket/1870) coords = np.mgrid[ 0:w, 0: w][:, mask_cur].T # List of coordinates corresponding to True entries in mask_cur for (ii, jj), vv in izip(coords, lists_cur): bmap[ii, jj] = vv # Repack the temporal siblings to a single numpy array, emulating a linked list lists = bmap[bmap != 0] llens = np.fromiter((len(l) for l in lists), dtype=np.int32) leaves = np.empty(np.sum(llens) + 2, dtype=[('mjd', 'f4'), ('snapid', object), ('cell_id', 'u8'), ('next', 'i4')]) leaves[:2] = [ (np.inf, 0, 0, END_MARKER) ] * 2 # We start with two dummy entries, so that offs=0 and 1 are invalid and can take other meanings. seen = dict() at = 2 for l in lists: last_i = len(l) - 1 for (i, (mjd, snapid, cell_id, has_data)) in enumerate(l): # Make equal strings refer to the same string object try: snapid = seen[snapid] except KeyError: seen[snapid] = snapid next = 1 if has_data else -1 if i == last_i: next *= END_MARKER leaves[at] = (mjd, snapid, cell_id, next) at += 1 # Construct bmap that has offsets to head of the linked list of siblings offs = np.zeros(len(llens), dtype=np.int64) offs[1:] = np.cumsum(llens)[:-1] offs += 2 # Pointers to beginnings of individual lists assert np.all(abs(leaves['next'][offs - 1]) == END_MARKER) obmap = np.zeros(bmap.shape, dtype=np.int32) obmap[bmap != 0] = offs # Recompute mipmaps bmaps = self._compute_mipmaps(obmap) return bmaps, leaves
def _update(self, table_path, snapid): # Find what we already have loaded prevsnap = np.max(self._leaves['snapid']) if len(self._leaves) > 2 else None assert prevsnap <= snapid, "Cannot update a catalog to an older snapshot" ## Enumerate all existing snapshots older or equal to snapid, and newer than prevsnap, and sort them, newest first self.__snapshots = sorted(isnapshots(table_path, first=prevsnap, last=snapid, no_first=True, return_path=True), reverse=True) # Add the snapid snapshot by hand, if not in the list of committed snapshots # (because it is permissible to update to a yet-uncommitted snapshot) if len(self.__snapshots) == 0 or self.__snapshots[0][0] != snapid: self.__snapshots.insert(0, (snapid, get_snapshot_path(table_path, snapid))) # Recursively scan, in parallel w = bhpix.width(self._pix.level) bmap = np.zeros((w, w), dtype=object) lev = min(4, self._pix.level) dx = bhpix.pix_size(lev) i, j = np.indices((2**lev,2**lev)).reshape(2, -1) # List of i,j coordinates w2 = 1 << (lev-1) x, y = (i - w2 + 0.5)*dx, (j - w2 + 0.5)*dx pool = pool2.Pool() for bmap2 in pool.imap_unordered(zip(x, y), _scan_recursive_kernel, (lev, self)): assert not np.any((bmap != 0) & (bmap2 != 0)) mask = bmap2 != 0 bmap[mask] = bmap2[mask] del pool # Add data about cells that were not touched by this update bmap_cur = self._bmaps[self._pix.level] mask_cur = (bmap_cur != 0) & (bmap == 0) lists_cur = [ [ (mjd, snapid, cell_id, next > 0) for (mjd, snapid, cell_id, next) in iter_siblings(self._leaves, offs) ] for offs in bmap_cur[mask_cur] ] try: bmap[mask_cur] = lists_cur except ValueError: # Workaround for a numpy 1.6.0 bug (http://projects.scipy.org/numpy/ticket/1870) coords = np.mgrid[0:w,0:w][:,mask_cur].T # List of coordinates corresponding to True entries in mask_cur for (ii, jj), vv in izip(coords, lists_cur): bmap[ii, jj] = vv # Repack the temporal siblings to a single numpy array, emulating a linked list lists = bmap[bmap != 0] llens = np.fromiter( (len(l) for l in lists), dtype=np.int32 ) leaves = np.empty(np.sum(llens)+2, dtype=[('mjd', 'f4'), ('snapid', object), ('cell_id', 'u8'), ('next', 'i4')]) leaves[:2] = [(np.inf, 0, 0, END_MARKER)]*2 # We start with two dummy entries, so that offs=0 and 1 are invalid and can take other meanings. seen = dict() at = 2 for l in lists: last_i = len(l) - 1 for (i, (mjd, snapid, cell_id, has_data)) in enumerate(l): # Make equal strings refer to the same string object try: snapid = seen[snapid] except KeyError: seen[snapid] = snapid next = 1 if has_data else -1 if i == last_i: next *= END_MARKER leaves[at] = (mjd, snapid, cell_id, next) at += 1 # Construct bmap that has offsets to head of the linked list of siblings offs = np.zeros(len(llens), dtype=np.int64) offs[1:] = np.cumsum(llens)[:-1] offs += 2 # Pointers to beginnings of individual lists assert np.all(abs(leaves['next'][offs-1]) == END_MARKER) obmap = np.zeros(bmap.shape, dtype=np.int32) obmap[bmap != 0] = offs # Recompute mipmaps bmaps = self._compute_mipmaps(obmap) return bmaps, leaves
def _get_cells_recursive(self, outcells, data_path, bounds_xy, bounds_t, pattern, x=0., y=0.): """ Helper for get_cells(). See documentation of get_cells() for usage """ lev = bhpix.get_pixel_level(x, y) dx = bhpix.pix_size(lev) # Check for nonzero overlap box = self._cell_bounds_xy(x, y, dx) bounds_xy = bounds_xy & box if not bounds_xy: return # Check if the cell directory exists (give up if it doesn't) path = data_path + '/' + self._path_to_cell_base_xy(x, y, lev) if not os.path.isdir(path): return # If re reached the bottom of the hierarchy if self._is_valid_cell_xy(x, y): # Get the cell_ids for leaf cells matching pattern xybounds = None if (bounds_xy.area() == box.area()) else bounds_xy sibling_cells = list( self._get_temporal_siblings(x, y, path, pattern)) for cell_id in sibling_cells: # Filter on time, add bounds x, y, t = self._xyt_from_cell_id(cell_id) if t != self.t0: # Cut on the time component tival = intervalset((t, t + self.dt)) tolap = bounds_t & tival if len(tolap): (l, r ) = tolap[-1] # Get the right-most interval component if l == r == t + self.dt: # Is it a single point? tolap = intervalset( *tolap[:-1] ) # Since objects in this cell have time in [t, t+dt), remove the t+dt point if len( tolap ) == 0: # No overlap between the intervals -- skip this cell continue # Return None if the cell is fully contained in the requested interval tbounds = None if tival == tolap else tolap else: # Static cell tbounds = bounds_t assert len( sibling_cells ) == 1, "There can be only one static cell (cell_id=%s)" % cell_id # Add to output if tbounds is None: outcells[cell_id][xybounds] = None elif xybounds not in outcells[cell_id]: outcells[cell_id][xybounds] = tbounds elif outcells[cell_id][xybounds] is not None: outcells[cell_id][xybounds] |= tbounds else: # Recursively subdivide the four subpixels dx = dx / 2 for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5)]): (x2, y2) = (x, y) + dx * d self._get_cells_recursive(outcells, data_path, bounds_xy & box, bounds_t, pattern, x2, y2)