示例#1
0
def _scan_recursive_kernel(xy, lev, cc):
    x, y = xy

    w = bhpix.width(cc._pix.level)
    bmap = np.zeros((w, w), dtype=object)
    cc._scan_recursive(bmap, lev, x, y)

    yield bmap
示例#2
0
def _scan_recursive_kernel(xy, lev, cc):
	x, y = xy

	w = bhpix.width(cc._pix.level)
	bmap = np.zeros((w, w), dtype=object)
	cc._scan_recursive(bmap, lev, x, y)

	yield bmap
示例#3
0
	def clear(self):
		# Initialize an empty table
		w = bhpix.width(self._pix.level)
		self._bmaps = self._compute_mipmaps(np.zeros((w, w), dtype=object))
		self._leaves = np.empty(2, dtype=[('mjd', 'f4'), ('snapid', object), ('cell_id', 'u8'), ('next', 'i4')])
		self._leaves[:2] = [(np.inf, 0, 0, END_MARKER)]*2
		
		self._rebuild_internal_state()
示例#4
0
    def _scan_recursive(self, bmap, lev=0, x=0., y=0.):
        # Check if the cell directory exists in any allowable snapshot
        # Snapshots are ordered newest-to-oldest, so we can avoid touching the
        # siblings of older snapshots if they exists in newer ones
        dx = bhpix.pix_size(lev)

        # .../snapshots/XXXXXX/tablets/-0.5+0.5/..../
        paths = [(snapid, '%s/tablets/%s' %
                  (snapshot_path, self._pix._path_to_cell_base_xy(x, y, lev)))
                 for (snapid, snapshot_path) in self.__snapshots]
        paths = [(snapid, path) for (snapid, path) in paths
                 if os.path.isdir(path)]  # Keep only those that exist

        if not len(paths):  # Dead end
            return

        if lev != self._pix.level:
            # Recursively subdivide the four subpixels
            dx = dx / 2
            for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5),
                               (0.5, -0.5)]):
                (x2, y2) = (x, y) + dx * d
                self._scan_recursive(bmap, lev + 1, x2, y2)
        else:
            # Leaf
            w2 = bhpix.width(lev) // 2
            i, j = (x // dx + w2, y // dx + w2)

            # Collect the data we have, and add them to the set of cells that exist
            siblings = dict()
            for snapid, path in paths:
                for tcell, fn in self._get_temporal_siblings(
                        path, self.__pattern):
                    if tcell not in siblings:  # Add only if there's no newer version
                        # check if there are any non-cached data in here
                        try:
                            with tables.open_file(fn) as fp:
                                has_data = len(fp.root.main.table) > 0
                        except tables.exceptions.NoSuchNodeError:
                            has_data = False

                        siblings[tcell] = snapid, has_data

            # Add any relevant pre-existing data
            i = int(i)
            j = int(j)
            offs = self._bmaps[self._pix.level][i, j]
            if offs != 0:
                for mjd, snapid, _, next in iter_siblings(self._leaves, offs):
                    if mjd not in siblings:
                        siblings[mjd] = snapid, next > 0

            # Add this list to bitmap
            assert bmap[i, j] == 0
            bmap[i,
                 j] = [(tcell, snapid, self._pix._cell_id_for_xyt(x, y, tcell),
                        has_data)
                       for (tcell, (snapid, has_data)) in siblings.iteritems()]
示例#5
0
    def clear(self):
        # Initialize an empty table
        w = bhpix.width(self._pix.level)
        self._bmaps = self._compute_mipmaps(np.zeros((w, w), dtype=object))
        self._leaves = np.empty(2,
                                dtype=[('mjd', 'f4'), ('snapid', object),
                                       ('cell_id', 'u8'), ('next', 'i4')])
        self._leaves[:2] = [(np.inf, 0, 0, END_MARKER)] * 2

        self._rebuild_internal_state()
示例#6
0
	def _compute_mipmaps(self, bmap):
		# Create mip-maps
		bmaps = {self._pix.level: bmap}	# Mip-maps of bmap with True if the cell has data in its leaves, and False otherwise
		for lev in xrange(self._pix.level-1, -1, -1):
			w = bhpix.width(lev)
			m0 = np.zeros((w, w), dtype=np.int32)
			m1 = bmaps[lev+1]
			for (i, j) in [(0, 0), (0, 1), (1, 0), (1, 1)]:
				m0[:,:] |= m1[i::2, j::2]
			m0 = m0 != 0

			bmaps[lev] = m0

		return bmaps
示例#7
0
	def _scan_recursive(self, bmap, lev=0, x=0., y=0.):
		# Check if the cell directory exists in any allowable snapshot
		# Snapshots are ordered newest-to-oldest, so we can avoid touching the
		# siblings of older snapshots if they exists in newer ones
		dx = bhpix.pix_size(lev)

		# .../snapshots/XXXXXX/tablets/-0.5+0.5/..../
		paths = [ (snapid, '%s/tablets/%s' % (snapshot_path,  self._pix._path_to_cell_base_xy(x, y, lev))) for (snapid, snapshot_path) in self.__snapshots ]
		paths = [ (snapid, path) for (snapid, path) in paths if os.path.isdir(path) ]	# Keep only those that exist

		if not len(paths): # Dead end
			return

		if lev != self._pix.level:
			# Recursively subdivide the four subpixels
			dx = dx / 2
			for d in np.array([(0.5, 0.5), (-0.5, 0.5), (-0.5, -0.5), (0.5, -0.5)]):
				(x2, y2) = (x, y) + dx*d
				self._scan_recursive(bmap, lev+1, x2, y2)
		else:
			# Leaf
			w2 = bhpix.width(lev) // 2
			i, j = (x // dx + w2, y // dx + w2)

			# Collect the data we have, and add them to the set of cells that exist
			siblings = dict()
			for snapid, path in paths:
				for tcell, fn in self._get_temporal_siblings(path, self.__pattern):
					if tcell not in siblings: # Add only if there's no newer version
						# check if there are any non-cached data in here
						try:
							with tables.openFile(fn) as fp:
								has_data = len(fp.root.main.table) > 0
						except tables.exceptions.NoSuchNodeError:
							has_data = False
						
						siblings[tcell] = snapid, has_data

			# Add any relevant pre-existing data
			offs = self._bmaps[self._pix.level][i, j]
			if offs != 0:
				for mjd, snapid, _, next in iter_siblings(self._leaves, offs):
					if mjd not in siblings:
						siblings[mjd] = snapid, next > 0

			# Add this list to bitmap
			assert bmap[i, j] == 0
			bmap[i, j] = [ (tcell, snapid, self._pix._cell_id_for_xyt(x, y, tcell), has_data) for (tcell, (snapid, has_data)) in siblings.iteritems() ]
示例#8
0
    def _compute_mipmaps(self, bmap):
        # Create mip-maps
        bmaps = {
            self._pix.level: bmap
        }  # Mip-maps of bmap with True if the cell has data in its leaves, and False otherwise
        for lev in xrange(self._pix.level - 1, -1, -1):
            w = bhpix.width(lev)
            m0 = np.zeros((w, w), dtype=np.int32)
            m1 = bmaps[lev + 1]
            for (i, j) in [(0, 0), (0, 1), (1, 0), (1, 1)]:
                m0[:, :] = m0[:, :] | m1[i::2, j::2]
            m0 = m0 != 0

            bmaps[lev] = m0

        return bmaps
示例#9
0
    def _update(self, table_path, snapid):
        # Find what we already have loaded
        prevsnap = np.max(
            self._leaves['snapid']) if len(self._leaves) > 2 else None
        assert prevsnap <= snapid, "Cannot update a catalog to an older snapshot"

        ## Enumerate all existing snapshots older or equal to snapid, and newer than prevsnap, and sort them, newest first
        self.__snapshots = sorted(isnapshots(table_path,
                                             first=prevsnap,
                                             last=snapid,
                                             no_first=True,
                                             return_path=True),
                                  reverse=True)

        # Add the snapid snapshot by hand, if not in the list of committed snapshots
        # (because it is permissible to update to a yet-uncommitted snapshot)
        if len(self.__snapshots) == 0 or self.__snapshots[0][0] != snapid:
            self.__snapshots.insert(
                0, (snapid, get_snapshot_path(table_path, snapid)))

        # Recursively scan, in parallel
        w = bhpix.width(self._pix.level)
        bmap = np.zeros((w, w), dtype=object)

        lev = min(4, self._pix.level)
        dx = bhpix.pix_size(lev)
        i, j = np.indices(
            (2**lev, 2**lev)).reshape(2, -1)  # List of i,j coordinates
        w2 = 1 << (lev - 1)
        x, y = (i - w2 + 0.5) * dx, (j - w2 + 0.5) * dx

        pool = pool2.Pool()
        for bmap2 in pool.imap_unordered(zip(x, y), _scan_recursive_kernel,
                                         (lev, self)):
            assert not np.any((bmap != 0) & (bmap2 != 0))
            mask = bmap2 != 0
            bmap[mask] = bmap2[mask]
        del pool

        # Add data about cells that were not touched by this update
        bmap_cur = self._bmaps[self._pix.level]
        mask_cur = (bmap_cur != 0) & (bmap == 0)
        lists_cur = [[(mjd, snapid, cell_id, next > 0)
                      for (mjd, snapid, cell_id,
                           next) in iter_siblings(self._leaves, offs)]
                     for offs in bmap_cur[mask_cur]]
        try:
            bmap[mask_cur] = lists_cur
        except ValueError:
            # Workaround for a numpy 1.6.0 bug (http://projects.scipy.org/numpy/ticket/1870)
            coords = np.mgrid[
                0:w, 0:
                w][:,
                   mask_cur].T  # List of coordinates corresponding to True entries in mask_cur
            for (ii, jj), vv in izip(coords, lists_cur):
                bmap[ii, jj] = vv

        # Repack the temporal siblings to a single numpy array, emulating a linked list
        lists = bmap[bmap != 0]
        llens = np.fromiter((len(l) for l in lists), dtype=np.int32)
        leaves = np.empty(np.sum(llens) + 2,
                          dtype=[('mjd', 'f4'), ('snapid', object),
                                 ('cell_id', 'u8'), ('next', 'i4')])
        leaves[:2] = [
            (np.inf, 0, 0, END_MARKER)
        ] * 2  # We start with two dummy entries, so that offs=0 and 1 are invalid and can take other meanings.
        seen = dict()
        at = 2
        for l in lists:
            last_i = len(l) - 1
            for (i, (mjd, snapid, cell_id, has_data)) in enumerate(l):
                # Make equal strings refer to the same string object
                try:
                    snapid = seen[snapid]
                except KeyError:
                    seen[snapid] = snapid

                next = 1 if has_data else -1
                if i == last_i:
                    next *= END_MARKER

                leaves[at] = (mjd, snapid, cell_id, next)
                at += 1

        # Construct bmap that has offsets to head of the linked list of siblings
        offs = np.zeros(len(llens), dtype=np.int64)
        offs[1:] = np.cumsum(llens)[:-1]
        offs += 2  # Pointers to beginnings of individual lists
        assert np.all(abs(leaves['next'][offs - 1]) == END_MARKER)
        obmap = np.zeros(bmap.shape, dtype=np.int32)
        obmap[bmap != 0] = offs

        # Recompute mipmaps
        bmaps = self._compute_mipmaps(obmap)

        return bmaps, leaves
示例#10
0
	def _update(self, table_path, snapid):
		# Find what we already have loaded
		prevsnap = np.max(self._leaves['snapid']) if len(self._leaves) > 2 else None
		assert prevsnap <= snapid, "Cannot update a catalog to an older snapshot"

		## Enumerate all existing snapshots older or equal to snapid, and newer than prevsnap, and sort them, newest first
		self.__snapshots = sorted(isnapshots(table_path, first=prevsnap, last=snapid, no_first=True, return_path=True), reverse=True)

		# Add the snapid snapshot by hand, if not in the list of committed snapshots
		# (because it is permissible to update to a yet-uncommitted snapshot)
		if len(self.__snapshots) == 0 or self.__snapshots[0][0] != snapid:
			self.__snapshots.insert(0, (snapid, get_snapshot_path(table_path, snapid)))

		# Recursively scan, in parallel
		w = bhpix.width(self._pix.level)
		bmap = np.zeros((w, w), dtype=object)

		lev = min(4, self._pix.level)
		dx = bhpix.pix_size(lev)
		i, j = np.indices((2**lev,2**lev)).reshape(2, -1) # List of i,j coordinates
		w2 = 1 << (lev-1)
		x, y =  (i - w2 + 0.5)*dx, (j - w2 + 0.5)*dx

		pool = pool2.Pool()
		for bmap2 in pool.imap_unordered(zip(x, y), _scan_recursive_kernel, (lev, self)):
			assert not np.any((bmap != 0) & (bmap2 != 0))
			mask = bmap2 != 0
			bmap[mask] = bmap2[mask]
		del pool

		# Add data about cells that were not touched by this update
		bmap_cur = self._bmaps[self._pix.level]
		mask_cur = (bmap_cur != 0) & (bmap == 0)
		lists_cur = [ [ (mjd, snapid, cell_id, next > 0) for (mjd, snapid, cell_id, next) in iter_siblings(self._leaves, offs) ] for offs in bmap_cur[mask_cur] ]
		try:
			bmap[mask_cur] = lists_cur
		except ValueError:
			# Workaround for a numpy 1.6.0 bug (http://projects.scipy.org/numpy/ticket/1870)
			coords = np.mgrid[0:w,0:w][:,mask_cur].T	# List of coordinates corresponding to True entries in mask_cur
			for (ii, jj), vv in izip(coords, lists_cur):
				bmap[ii, jj] = vv

		# Repack the temporal siblings to a single numpy array, emulating a linked list
		lists = bmap[bmap != 0]
		llens = np.fromiter( (len(l) for l in lists), dtype=np.int32 )
		leaves = np.empty(np.sum(llens)+2, dtype=[('mjd', 'f4'), ('snapid', object), ('cell_id', 'u8'), ('next', 'i4')])
		leaves[:2] = [(np.inf, 0, 0, END_MARKER)]*2	# We start with two dummy entries, so that offs=0 and 1 are invalid and can take other meanings.
		seen = dict()
		at = 2
		for l in lists:
			last_i = len(l) - 1
			for (i, (mjd, snapid, cell_id, has_data)) in enumerate(l):
				# Make equal strings refer to the same string object
				try:
					snapid = seen[snapid]
				except KeyError:
					seen[snapid] = snapid

				next = 1 if has_data else -1
				if i == last_i:
					next *= END_MARKER

				leaves[at] = (mjd, snapid, cell_id, next)
				at += 1

		# Construct bmap that has offsets to head of the linked list of siblings
		offs     = np.zeros(len(llens), dtype=np.int64)
		offs[1:] = np.cumsum(llens)[:-1]
		offs     += 2				# Pointers to beginnings of individual lists
		assert np.all(abs(leaves['next'][offs-1]) == END_MARKER)
		obmap = np.zeros(bmap.shape, dtype=np.int32)
		obmap[bmap != 0] = offs

		# Recompute mipmaps
		bmaps = self._compute_mipmaps(obmap)

		return bmaps, leaves