def _get_cells_recursive(self, outcells, data_path, bounds_xy, bounds_t, pattern, x = 0., y = 0.): """ Helper for get_cells(). See documentation of get_cells() for usage """ lev = bhpix.get_pixel_level(x, y) dx = bhpix.pix_size(lev) # Check for nonzero overlap box = self._cell_bounds_xy(x, y, dx) bounds_xy = bounds_xy & box if not bounds_xy: return # Check if the cell directory exists (give up if it doesn't) path = data_path + '/' + self._path_to_cell_base_xy(x, y, lev) if not os.path.isdir(path): return # If re reached the bottom of the hierarchy if self._is_valid_cell_xy(x, y): # Get the cell_ids for leaf cells matching pattern xybounds = None if(bounds_xy.area() == box.area()) else bounds_xy sibling_cells = list(self._get_temporal_siblings(x, y, path, pattern)) for cell_id in sibling_cells: # Filter on time, add bounds x, y, t = self._xyt_from_cell_id(cell_id) if t != self.t0: # Cut on the time component tival = intervalset((t, t+self.dt)) tolap = bounds_t & tival if len(tolap): (l, r) = tolap[-1] # Get the right-most interval component if l == r == t+self.dt: # Is it a single point? tolap = intervalset(*tolap[:-1]) # Since objects in this cell have time in [t, t+dt), remove the t+dt point if len(tolap) == 0: # No overlap between the intervals -- skip this cell continue; # Return None if the cell is fully contained in the requested interval tbounds = None if tival == tolap else tolap else: # Static cell tbounds = bounds_t assert len(sibling_cells) == 1, "There can be only one static cell (cell_id=%s)" % cell_id # Add to output if tbounds is None: outcells[cell_id][xybounds] = None elif xybounds not in outcells[cell_id]: outcells[cell_id][xybounds] = tbounds elif outcells[cell_id][xybounds] is not None: outcells[cell_id][xybounds] |= tbounds else: # Recursively subdivide the four subpixels dx = dx / 2 for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5)]): (x2, y2) = (x, y) + dx*d self._get_cells_recursive(outcells, data_path, bounds_xy & box, bounds_t, pattern, x2, y2)
def __part_to_xy_t(part): """ Internal: Used by canonicize_bounds """ bounds_xy = Polygon.Polygon() bounds_t = intervalset() for v in part: if isinstance(v, Polygon.Polygon): bounds_xy |= v elif isinstance(v, intervalset): bounds_t |= v else: raise Exception('Incorrect part specification') if not bounds_xy: bounds_xy = ALLSKY if len(bounds_t) == 0: bounds_t = intervalset((-np.inf, np.inf)) return (bounds_xy, bounds_t)
def get_cells(self, bounds=None, return_bounds=False, include_cached=True): """ Return a list of (cell_id, bounds) tuples completely covering the requested bounds. bounds must be a list of (Polygon, intervalset) tuples. Output is a list of (cell_id, xybounds, tbounds) tuples, unless return_bounds=False when the output is just a list of cell_ids. """ self._include_cached = include_cached # Special case of bounds=None (all sky) if bounds == None: bounds = [(bn.ALLSKY, intervalset((-np.inf, np.inf)))] # Find all existing cells satisfying the bounds cells = defaultdict(dict) if False: # Single-threaded implementation for bounds_xy, bounds_t in bounds: self._get_cells_recursive(cells, bounds_xy, bounds_t, 0, 0, 1, 1.) self._get_cells_recursive(cells, bounds_xy, bounds_t, 0, 1, 1, 1.) self._get_cells_recursive(cells, bounds_xy, bounds_t, 1, 0, 1, 1.) self._get_cells_recursive(cells, bounds_xy, bounds_t, 1, 1, 1, 1.) else: # Multi-process implementation (appears to be as good or better than single thread in # nearly all cases of interest) pool = pool2.Pool() lev = min(4, self._pix.level) ij = np.indices( (2**lev, 2**lev)).reshape(2, -1).T # List of i,j coordinates for cells_ in pool.imap_unordered( ij, _get_cells_kernel, (lev, self, bounds), progress_callback=pool2.progress_pass): for cell_id, b in cells_.iteritems(): for xyb, tb in b.iteritems(): _add_bounds(cells, cell_id, xyb, tb) del pool if len(cells): # Transform (x, y, t) tuples to cell_ids xyt = np.array(cells.keys()).transpose() cell_ids = self._pix._cell_id_for_xyt(xyt[0], xyt[1], xyt[2]) # Reorder cells to be a dict of cell: [(poly, time), (poly, time)] entries cells = dict( ((cell_id, v.items()) for (cell_id, (k, v)) in izip(cell_ids, cells.iteritems()))) if not return_bounds: return cells.keys() else: return cells
def __part_to_xy_t(part): """ Internal: Used by canonicize_bounds """ bounds_xy = Polygon.Polygon() bounds_t = intervalset() for v in part: if isinstance(v, Polygon.Polygon): bounds_xy |= v elif isinstance(v, intervalset): bounds_t |= v else: raise Exception("Incorrect part specification") if not bounds_xy: bounds_xy = ALLSKY if len(bounds_t) == 0: bounds_t = intervalset((-np.inf, np.inf)) return (bounds_xy, bounds_t)
def get_cells_with_dets_from_exps(db, explist, exp_tabname, det_tabname, fovradius=None): """ Get all cells that may contain detections from within exposures listed in explist. If given, use fovradius (in degrees) as the radius of the field of view of each exposure, to limit the number of cells that are considered. If fovradius is None, assume the field of view of each exposure may extend to the entire sky. """ explist = frozenset(explist) bb = {} ti = intervalset() for t, exp, lon, lat in db.query("_TIME, _EXP, _LON, _LAT from %s" % exp_tabname).iterate(): if exp not in explist: continue if fovradius is not None: tt = np.floor(t) #fov = bounds.beam(lon, lat, fovradius*(2./np.sqrt(3.)), npts=8) # An octagon circumscribing the FoV #assert fov.nPoints() == 8 fov = bounds.beam(lon, lat, fovradius) try: bb[tt] |= fov except KeyError: bb[tt] = fov else: ti.add((t - 0.2, t + 0.2)) if fovradius is None: b = [(bounds.ALLSKY, ti)] else: b = [] for t0, fov in bb.iteritems(): b += [(fov, intervalset((t0, t0 + 1.)))] #print t0, fov.area()*(180./np.pi)**2., fov.nPoints() # Fetch all detection table cells that are within this interval det_cells = db.table(det_tabname).get_cells(bounds=b) return det_cells
def cell_bounds(self, cell_id): """ Return the bounding polygon and time intervalset for a given cell. """ x, y, t, _ = self._xyti_from_id(cell_id) assert _ == u0xFFFFFFFF bounds = self._cell_bounds_xy(x, y) return (bounds, intervalset((t, t+self.dt)))
def cell_bounds(self, cell_id): """ Return the bounding polygon and time intervalset for a given cell. """ x, y, t, _ = self._xyti_from_id(cell_id) assert _ == u0xFFFFFFFF bounds = self._cell_bounds_xy(x, y) return (bounds, intervalset((t, t + self.dt)))
def get_cells_with_dets_from_exps(db, explist, exp_tabname, det_tabname, fovradius=None): """ Get all cells that may contain detections from within exposures listed in explist. If given, use fovradius (in degrees) as the radius of the field of view of each exposure, to limit the number of cells that are considered. If fovradius is None, assume the field of view of each exposure may extend to the entire sky. """ explist = frozenset(explist) bb = {} ti = intervalset() for t, exp, lon, lat in db.query("_TIME, _EXP, _LON, _LAT from %s" % exp_tabname).iterate(): if exp not in explist: continue if fovradius is not None: tt = np.floor(t) #fov = bounds.beam(lon, lat, fovradius*(2./np.sqrt(3.)), npts=8) # An octagon circumscribing the FoV #assert fov.nPoints() == 8 fov = bounds.beam(lon, lat, fovradius) try: bb[tt] |= fov except KeyError: bb[tt] = fov else: ti.add((t-0.2, t+0.2)) if fovradius is None: b = [(bounds.ALLSKY, ti)] else: b = [] for t0, fov in bb.iteritems(): b += [ (fov, intervalset((t0, t0+1.))) ] #print t0, fov.area()*(180./np.pi)**2., fov.nPoints() # Fetch all detection table cells that are within this interval det_cells = db.table(det_tabname).get_cells(bounds=b) return det_cells
def get_cells(self, bounds=None, return_bounds=False, include_cached=True): """ Return a list of (cell_id, bounds) tuples completely covering the requested bounds. bounds must be a list of (Polygon, intervalset) tuples. Output is a list of (cell_id, xybounds, tbounds) tuples, unless return_bounds=False when the output is just a list of cell_ids. """ self._include_cached = include_cached # Special case of bounds=None (all sky) if bounds == None: bounds = [(bn.ALLSKY, intervalset((-np.inf, np.inf)))] # Find all existing cells satisfying the bounds cells = defaultdict(dict) if False: # Single-threaded implementation for bounds_xy, bounds_t in bounds: self._get_cells_recursive(cells, bounds_xy, bounds_t, 0, 0, 1, 1.) self._get_cells_recursive(cells, bounds_xy, bounds_t, 0, 1, 1, 1.) self._get_cells_recursive(cells, bounds_xy, bounds_t, 1, 0, 1, 1.) self._get_cells_recursive(cells, bounds_xy, bounds_t, 1, 1, 1, 1.) else: # Multi-process implementation (appears to be as good or better than single thread in # nearly all cases of interest) pool = pool2.Pool() lev = min(4, self._pix.level) ij = np.indices((2**lev,2**lev)).reshape(2, -1).T # List of i,j coordinates for cells_ in pool.imap_unordered(ij, _get_cells_kernel, (lev, self, bounds), progress_callback=pool2.progress_pass): for cell_id, b in cells_.iteritems(): for xyb, tb in b.iteritems(): _add_bounds(cells, cell_id, xyb, tb) del pool if len(cells): # Transform (x, y, t) tuples to cell_ids xyt = np.array(cells.keys()).transpose() cell_ids = self._pix._cell_id_for_xyt(xyt[0], xyt[1], xyt[2]) # Reorder cells to be a dict of cell: [(poly, time), (poly, time)] entries cells = dict(( (cell_id, v.items()) for (cell_id, (k, v)) in izip(cell_ids, cells.iteritems()) )) if not return_bounds: return cells.keys() else: return cells
def get_cells(self, data_path, pattern, bounds, return_bounds=False): """ Return a list of (cell_id, bounds) tuples completely covering the requested bounds. bounds must be a list of (Polygon, intervalset) tuples. Output is a list of (cell_id, xybounds, tbounds) tuples, unless return_bounds=False when the output is just a list of cell_ids. """ # Special case of bounds=None (all sky) if bounds == None: bounds = [(bn.ALLSKY, intervalset((-np.inf, np.inf)))] # Find all existing cells satisfying the bounds cells = defaultdict(dict) for bounds_xy, bounds_t in bounds: self._get_cells_recursive(cells, data_path, bounds_xy, bounds_t, pattern) # Reorder cells to be a dict of cell: [(poly, time), (poly, time)] entries cells = dict(((k, v.items()) for k, v in cells.iteritems())) if False: # TODO: Debugging, remove when happy for k, bounds in cells.iteritems(): print k, ':', str(self._xyti_from_id(k)), for xy, t in bounds: print(xy.area() if xy is not None else None, t), print '' print len(cells) exit() if not return_bounds: return cells.keys() else: return cells
def get_cells(self, data_path, pattern, bounds, return_bounds=False): """ Return a list of (cell_id, bounds) tuples completely covering the requested bounds. bounds must be a list of (Polygon, intervalset) tuples. Output is a list of (cell_id, xybounds, tbounds) tuples, unless return_bounds=False when the output is just a list of cell_ids. """ # Special case of bounds=None (all sky) if bounds == None: bounds = [(bn.ALLSKY, intervalset((-np.inf, np.inf)))] # Find all existing cells satisfying the bounds cells = defaultdict(dict) for bounds_xy, bounds_t in bounds: self._get_cells_recursive(cells, data_path, bounds_xy, bounds_t, pattern) # Reorder cells to be a dict of cell: [(poly, time), (poly, time)] entries cells = dict(( (k, v.items()) for k, v in cells.iteritems() )) if False: # TODO: Debugging, remove when happy for k, bounds in cells.iteritems(): print k, ':', str(self._xyti_from_id(k)), for xy, t in bounds: print (xy.area() if xy is not None else None, t), print '' print len(cells) exit() if not return_bounds: return cells.keys() else: return cells
def _get_cells_recursive(self, outcells, bounds_xy, bounds_t, i=0, j=0, lev=0, dx=2.): """ Helper for get_cells(). See documentation of get_cells() for usage """ w2 = 1 << (lev - 1) x, y = (i - w2 + 0.5) * dx, (j - w2 + 0.5) * dx # Check for nonzero overlap box = self._pix._cell_bounds_xy(x, y, dx) bounds_xy = bounds_xy & box if not bounds_xy: return # Check if the cell directory exists (give up if it doesn't) bmap = self._bmaps[lev] offs = bmap[i, j] if not offs: return # If re reached the bottom of the hierarchy if offs > 1: # Get the cell_ids for leaf cells matching pattern xybounds = None if (bounds_xy.area() == box.area()) else bounds_xy next = 0 while next != END_MARKER: (t, _, _, next) = self._leaves[offs] has_data = next > 0 next = abs(next) if next != END_MARKER: # Not really necessary, but numpy warns of overflow otherwise. offs += next if not has_data and not self._include_cached: continue if t != self._pix.t0: # Cut on the time component tival = intervalset((t, t + self._pix.dt)) tolap = bounds_t & tival if len(tolap): (l, r ) = tolap[-1] # Get the right-most interval component if l == r == t + self._pix.dt: # Is it a single point? tolap = intervalset( *tolap[:-1] ) # Since objects in this cell have time in [t, t+dt), remove the t+dt point if len( tolap ) == 0: # No overlap between the intervals -- skip this cell continue # Return None if the cell is fully contained in the requested interval tbounds = None if tival == tolap else tolap else: # Static cell tbounds = bounds_t assert next == END_MARKER, "There can be only one static cell (x,y,t=%s,%s,%s)" % ( x, y, t) # Compute cell ID cell_id = (x, y, t) # Add to output _add_bounds(outcells, cell_id, xybounds, tbounds) else: # Recursively subdivide the four subpixels for (di, dj) in [(0, 0), (0, 1), (1, 0), (1, 1)]: i2 = i * 2 + di j2 = j * 2 + dj self._get_cells_recursive(outcells, bounds_xy & box, bounds_t, i2, j2, lev + 1, dx / 2)
def _get_cells_recursive(self, outcells, bounds_xy, bounds_t, i = 0, j = 0, lev = 0, dx = 2.): """ Helper for get_cells(). See documentation of get_cells() for usage """ w2 = 1 << (lev-1) x, y = (i - w2 + 0.5)*dx, (j - w2 + 0.5)*dx # Check for nonzero overlap box = self._pix._cell_bounds_xy(x, y, dx) bounds_xy = bounds_xy & box if not bounds_xy: return # Check if the cell directory exists (give up if it doesn't) bmap = self._bmaps[lev] offs = bmap[i, j] if not offs: return # If re reached the bottom of the hierarchy if offs > 1: # Get the cell_ids for leaf cells matching pattern xybounds = None if(bounds_xy.area() == box.area()) else bounds_xy next = 0 while next != END_MARKER: (t, _, _, next) = self._leaves[offs] has_data = next > 0 next = abs(next) if next != END_MARKER: # Not really necessary, but numpy warns of overflow otherwise. offs += next if not has_data and not self._include_cached: continue if t != self._pix.t0: # Cut on the time component tival = intervalset((t, t+self._pix.dt)) tolap = bounds_t & tival if len(tolap): (l, r) = tolap[-1] # Get the right-most interval component if l == r == t+self._pix.dt: # Is it a single point? tolap = intervalset(*tolap[:-1]) # Since objects in this cell have time in [t, t+dt), remove the t+dt point if len(tolap) == 0: # No overlap between the intervals -- skip this cell continue; # Return None if the cell is fully contained in the requested interval tbounds = None if tival == tolap else tolap else: # Static cell tbounds = bounds_t assert next == END_MARKER, "There can be only one static cell (x,y,t=%s,%s,%s)" % (x, y, t) # Compute cell ID cell_id = (x, y, t) # Add to output _add_bounds(outcells, cell_id, xybounds, tbounds) else: # Recursively subdivide the four subpixels for (di, dj) in [(0, 0), (0, 1), (1, 0), (1, 1)]: i2 = i*2 + di j2 = j*2 + dj self._get_cells_recursive(outcells, bounds_xy & box, bounds_t, i2, j2, lev+1, dx/2)
def _get_cells_recursive(self, outcells, data_path, bounds_xy, bounds_t, pattern, x=0., y=0.): """ Helper for get_cells(). See documentation of get_cells() for usage """ lev = bhpix.get_pixel_level(x, y) dx = bhpix.pix_size(lev) # Check for nonzero overlap box = self._cell_bounds_xy(x, y, dx) bounds_xy = bounds_xy & box if not bounds_xy: return # Check if the cell directory exists (give up if it doesn't) path = data_path + '/' + self._path_to_cell_base_xy(x, y, lev) if not os.path.isdir(path): return # If re reached the bottom of the hierarchy if self._is_valid_cell_xy(x, y): # Get the cell_ids for leaf cells matching pattern xybounds = None if (bounds_xy.area() == box.area()) else bounds_xy sibling_cells = list( self._get_temporal_siblings(x, y, path, pattern)) for cell_id in sibling_cells: # Filter on time, add bounds x, y, t = self._xyt_from_cell_id(cell_id) if t != self.t0: # Cut on the time component tival = intervalset((t, t + self.dt)) tolap = bounds_t & tival if len(tolap): (l, r ) = tolap[-1] # Get the right-most interval component if l == r == t + self.dt: # Is it a single point? tolap = intervalset( *tolap[:-1] ) # Since objects in this cell have time in [t, t+dt), remove the t+dt point if len( tolap ) == 0: # No overlap between the intervals -- skip this cell continue # Return None if the cell is fully contained in the requested interval tbounds = None if tival == tolap else tolap else: # Static cell tbounds = bounds_t assert len( sibling_cells ) == 1, "There can be only one static cell (cell_id=%s)" % cell_id # Add to output if tbounds is None: outcells[cell_id][xybounds] = None elif xybounds not in outcells[cell_id]: outcells[cell_id][xybounds] = tbounds elif outcells[cell_id][xybounds] is not None: outcells[cell_id][xybounds] |= tbounds else: # Recursively subdivide the four subpixels dx = dx / 2 for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5)]): (x2, y2) = (x, y) + dx * d self._get_cells_recursive(outcells, data_path, bounds_xy & box, bounds_t, pattern, x2, y2)