Exemplo n.º 1
0
    def propagateDirty(self, slot, subindex, roi):
        shape = self.Input.meta.shape
        key = roi.toSlice()

        if slot == self.inputs["Input"]:
            start, stop = sliceToRoi(key, shape)

            with self._lock:
                if self._blockState is not None:
                    blockStart = numpy.floor(1.0 * start / self._blockShape)
                    blockStop = numpy.ceil(1.0 * stop / self._blockShape)
                    blockKey = roiToSlice(blockStart, blockStop)
                    if self._fixed:
                        # Remember that this block became dirty while we were fixed
                        #  so we can notify downstream operators when we become unfixed.
                        self._blockState[blockKey] = OpArrayCache.FIXED_DIRTY
                        self._has_fixed_dirty_blocks = True
                    else:
                        self._blockState[blockKey] = OpArrayCache.DIRTY

            if not self._fixed:
                self.outputs["Output"].setDirty(key)
        if slot == self.inputs["fixAtCurrent"]:
            if self.inputs["fixAtCurrent"].ready():
                self._fixed = self.inputs["fixAtCurrent"].value
                if not self._fixed and self.Output.meta.shape is not None and self._has_fixed_dirty_blocks:
                    # We've become unfixed, so we need to notify downstream
                    #  operators of every block that became dirty while we were fixed.
                    # Convert all FIXED_DIRTY states into DIRTY states
                    with self._lock:
                        cond = (
                            self._blockState[...] == OpArrayCache.FIXED_DIRTY)
                        self._blockState[...] = fastWhere(
                            cond, OpArrayCache.DIRTY, self._blockState,
                            numpy.uint8)
                        self._has_fixed_dirty_blocks = False
                    newDirtyBlocks = numpy.transpose(numpy.nonzero(cond))

                    # To avoid lots of setDirty notifications, we simply merge all the dirtyblocks into one single superblock.
                    # This should be the best option in most cases, but could be bad in some cases.
                    # TODO: Optimize this by merging the dirty blocks via connected components or something.
                    cacheShape = numpy.array(self.Output.meta.shape)
                    dirtyStart = cacheShape
                    dirtyStop = [0] * len(cacheShape)
                    for index in newDirtyBlocks:
                        blockStart = index * self._blockShape
                        blockStop = numpy.minimum(
                            blockStart + self._blockShape, cacheShape)

                        dirtyStart = numpy.minimum(dirtyStart, blockStart)
                        dirtyStop = numpy.maximum(dirtyStop, blockStop)

                    if len(newDirtyBlocks > 0):
                        self.Output.setDirty(dirtyStart, dirtyStop)
Exemplo n.º 2
0
    def propagateDirty(self, slot, subindex, roi):
        shape = self.Output.meta.shape

        key = roi.toSlice()
        if slot == self.inputs["Input"]:
            start, stop = sliceToRoi(key, shape)

            with self._lock:
                if self._blockState is not None:
                    blockStart = numpy.floor(1.0 * start / self._blockShape)
                    blockStop = numpy.ceil(1.0 * stop / self._blockShape)
                    blockKey = roiToSlice(blockStart, blockStop)
                    if self._fixed:
                        # Remember that this block became dirty while we were fixed
                        #  so we can notify downstream operators when we become unfixed.
                        self._blockState[blockKey] = OpArrayCache.FIXED_DIRTY
                        self._has_fixed_dirty_blocks = True
                    else:
                        self._blockState[blockKey] = OpArrayCache.DIRTY

            if not self._fixed:
                self.outputs["Output"].setDirty(key)
        if slot == self.inputs["fixAtCurrent"]:
            if self.inputs["fixAtCurrent"].ready():
                self._fixed = self.inputs["fixAtCurrent"].value
                if not self._fixed and self.Output.meta.shape is not None and self._has_fixed_dirty_blocks:
                    # We've become unfixed, so we need to notify downstream
                    #  operators of every block that became dirty while we were fixed.
                    # Convert all FIXED_DIRTY states into DIRTY states
                    with self._lock:
                        cond = (
                            self._blockState[...] == OpArrayCache.FIXED_DIRTY)
                        self._blockState[...] = fastWhere(
                            cond, OpArrayCache.DIRTY, self._blockState,
                            numpy.uint8)
                        self._has_fixed_dirty_blocks = False
                    newDirtyBlocks = numpy.transpose(numpy.nonzero(cond))

                    # To avoid lots of setDirty notifications, we simply merge all the dirtyblocks into one single superblock.
                    # This should be the best option in most cases, but could be bad in some cases.
                    # TODO: Optimize this by merging the dirty blocks via connected components or something.
                    cacheShape = numpy.array(self.Output.meta.shape)
                    dirtyStart = cacheShape
                    dirtyStop = [0] * len(cacheShape)
                    for index in newDirtyBlocks:
                        blockStart = index * self._blockShape
                        blockStop = numpy.minimum(
                            blockStart + self._blockShape, cacheShape)

                        dirtyStart = numpy.minimum(dirtyStart, blockStart)
                        dirtyStop = numpy.maximum(dirtyStop, blockStop)

                    if len(newDirtyBlocks > 0):
                        self.Output.setDirty(dirtyStart, dirtyStop)
Exemplo n.º 3
0
    def _executeOutput(self, slot, subindex, roi, result):
        t = time.time()
        key = roi.toSlice()

        shape = self.Output.meta.shape
        start, stop = sliceToRoi(key, shape)

        with self._lock:
            ch = self._cacheHits
            ch += 1
            self._cacheHits = ch

            self._running += 1

            if (self._cache is None
                    or self._cache.shape != self.Output.meta.shape):
                self._allocateCache()

            cacheView = self._cache[:]  #prevent freeing of cache during running this function

            blockStart = (1.0 * start / self._blockShape).floor()
            blockStop = (1.0 * stop / self._blockShape).ceil()
            blockKey = roiToSlice(blockStart, blockStop)

            blockSet = self._blockState[blockKey]

            # this is a little optimization to shortcut
            # many lines of python code when all data is
            # is already in the cache:
            if numpy.logical_or(blockSet == OpArrayCache.CLEAN,
                                blockSet == OpArrayCache.FIXED_DIRTY).all():
                cache_result = self._cache[roiToSlice(start, stop)]
                self.Output.stype.copy_data(result, cache_result)

                self._running -= 1
                self._updatePriority()
                cacheView = None
                return

            extracted = numpy.extract(blockSet == OpArrayCache.IN_PROCESS,
                                      self._blockQuery[blockKey])
            inProcessQueries = numpy.unique(extracted)

            cond = (blockSet == OpArrayCache.DIRTY)
            tileWeights = fastWhere(cond, 1, 128**3, numpy.uint32)
            trueDirtyIndices = numpy.nonzero(cond)

            if has_drtile:
                tileArray = drtile.test_DRTILE(tileWeights, 128**3).swapaxes(
                    0, 1)
            else:
                tileStartArray = numpy.array(trueDirtyIndices)
                tileStopArray = 1 + tileStartArray
                tileArray = numpy.concatenate((tileStartArray, tileStopArray),
                                              axis=0)

            dirtyRois = []
            half = tileArray.shape[0] // 2
            dirtyPool = RequestPool()

            for i in range(tileArray.shape[1]):

                drStart3 = tileArray[:half, i]
                drStop3 = tileArray[half:, i]
                drStart2 = drStart3 + blockStart
                drStop2 = drStop3 + blockStart
                drStart = drStart2 * self._blockShape
                drStop = drStop2 * self._blockShape

                shape = self.Output.meta.shape
                drStop = numpy.minimum(drStop, shape)
                drStart = numpy.minimum(drStart, shape)

                key2 = roiToSlice(drStart2, drStop2)

                key = roiToSlice(drStart, drStop)

                if not self._fixed:
                    dirtyRois.append([drStart, drStop])

                    req = self.inputs["Input"][key].writeInto(self._cache[key])
                    req.uncancellable = True  #FIXME

                    dirtyPool.add(req)

                    self._blockQuery[key2] = weakref.ref(req)

                    #sanity check:
                    if (self._blockState[key2] != OpArrayCache.DIRTY).any():
                        logger.warning("original condition" + str(cond))
                        logger.warning("original tilearray {} {}".format(
                            tileArray, tileArray.shape))
                        logger.warning("original tileWeights {} {}".format(
                            tileWeights, tileWeights.shape))
                        logger.warning("sub condition {}".format(
                            self._blockState[key2] == OpArrayCache.DIRTY))
                        logger.warning("START={}, STOP={}".format(
                            drStart2, drStop2))
                        import h5py
                        with h5py.File("test.h5", "w") as f:
                            f.create_dataset("data", data=tileWeights)
                            logger.warning(
                                "%r \n %r \n %r\n %r\n %r \n%r" %
                                (key2, blockKey, self._blockState[key2],
                                 self._blockState[blockKey][trueDirtyIndices],
                                 self._blockState[blockKey], tileWeights))
                        assert False
                    self._blockState[key2] = OpArrayCache.IN_PROCESS

            # indicate the inprocessing state, by setting array to 0 (i.e. IN_PROCESS)
            if not self._fixed:
                blockSet[:] = fastWhere(cond, OpArrayCache.IN_PROCESS,
                                        blockSet, numpy.uint8)
            else:
                # Someone asked for some dirty blocks while we were fixed.
                # Mark these blocks to be signaled as dirty when we become unfixed
                blockSet[:] = fastWhere(cond, OpArrayCache.FIXED_DIRTY,
                                        blockSet, numpy.uint8)
                self._has_fixed_dirty_blocks = True

        temp = itertools.count(0)

        #wait for all requests to finish
        something_updated = len(dirtyPool) > 0
        dirtyPool.wait()
        if something_updated:
            # Signal that something was updated.
            # Note that we don't need to do this for the 'in process' queries (below)
            #  because they are already in the dirtyPool in some other thread
            self.Output._sig_value_changed()

        # indicate the finished inprocess state (i.e. CLEAN)
        if not self._fixed and temp.next() == 0:
            with self._lock:
                blockSet[:] = fastWhere(cond, OpArrayCache.CLEAN, blockSet,
                                        numpy.uint8)
                self._blockQuery[blockKey] = fastWhere(
                    cond, None, self._blockQuery[blockKey], object)

        # Wait for all in-process queries.
        # Can't use RequestPool here because these requests have already started.
        for req in inProcessQueries:
            req = req()  # get original req object from weakref
            if req is not None:
                req.wait()

        # finally, store results in result area
        with self._lock:
            if self._cache is not None:
                cache_result = self._cache[roiToSlice(start, stop)]
                self.Output.stype.copy_data(result, cache_result)
            else:
                self.inputs["Input"][roiToSlice(
                    start, stop)].writeInto(result).wait()
            self._running -= 1
            self._updatePriority()
            cacheView = None
        self.logger.debug(
            "read %s took %f sec." % (roi.pprint(), time.time() - t))
Exemplo n.º 4
0
    def _executeOutput(self, slot, subindex, roi, result):
        t = time.time()
        key = roi.toSlice()

        shape = self.Output.meta.shape
        start, stop = sliceToRoi(key, shape)

        self._lock.acquire()

        ch = self._cacheHits
        ch += 1
        self._cacheHits = ch

        self._running += 1

        if self._cache is None:
            self._allocateCache()

        cacheView = self._cache[:]  #prevent freeing of cache during running this function

        blockStart = (1.0 * start / self._blockShape).floor()
        blockStop = (1.0 * stop / self._blockShape).ceil()
        blockKey = roiToSlice(blockStart, blockStop)

        blockSet = self._blockState[blockKey]

        # this is a little optimization to shortcut
        # many lines of python code when all data is
        # is already in the cache:
        if numpy.logical_or(blockSet == OpArrayCache.CLEAN,
                            blockSet == OpArrayCache.FIXED_DIRTY).all():
            result[:] = self._cache[roiToSlice(start, stop)]
            self._running -= 1
            self._updatePriority()
            cacheView = None
            self._lock.release()
            return

        inProcessQueries = numpy.unique(
            numpy.extract(blockSet == OpArrayCache.IN_PROCESS,
                          self._blockQuery[blockKey]))

        cond = (blockSet == OpArrayCache.DIRTY)
        tileWeights = fastWhere(cond, 1, 128**3, numpy.uint32)
        trueDirtyIndices = numpy.nonzero(cond)

        tileArray = drtile.test_DRTILE(tileWeights, 128**3).swapaxes(0, 1)

        dirtyRois = []
        half = tileArray.shape[0] / 2
        dirtyPool = RequestPool()

        for i in range(tileArray.shape[1]):

            drStart3 = tileArray[:half, i]
            drStop3 = tileArray[half:, i]
            drStart2 = drStart3 + blockStart
            drStop2 = drStop3 + blockStart
            drStart = drStart2 * self._blockShape
            drStop = drStop2 * self._blockShape

            shape = self.Output.meta.shape
            drStop = numpy.minimum(drStop, shape)
            drStart = numpy.minimum(drStart, shape)

            key3 = roiToSlice(drStart3, drStop3)
            key2 = roiToSlice(drStart2, drStop2)

            key = roiToSlice(drStart, drStop)

            if not self._fixed:
                dirtyRois.append([drStart, drStop])

                req = self.inputs["Input"][key].writeInto(self._cache[key])

                req.uncancellable = True  #FIXME

                dirtyPool.add(req)

                self._blockQuery[key2] = weakref.ref(req)

                #sanity check:
                if (self._blockState[key2] != OpArrayCache.DIRTY).any():
                    logger.warning("original condition" + str(cond))
                    logger.warning("original tilearray {} {}".format(
                        tileArray, tileArray.shape))
                    logger.warning("original tileWeights {} {}".format(
                        tileWeights, tileWeights.shape))
                    logger.warning("sub condition {}".format(
                        self._blockState[key2] == OpArrayCache.DIRTY))
                    logger.warning("START={}, STOP={}".format(
                        drStart2, drStop2))
                    import h5py
                    with h5py.File("test.h5", "w") as f:
                        f.create_dataset("data", data=tileWeights)
                        logger.warning(
                            "%r \n %r \n %r\n %r\n %r \n%r" %
                            (key2, blockKey, self._blockState[key2],
                             self._blockState[blockKey][trueDirtyIndices],
                             self._blockState[blockKey], tileWeights))
                    assert False
                self._blockState[key2] = OpArrayCache.IN_PROCESS

        # indicate the inprocessing state, by setting array to 0 (i.e. IN_PROCESS)
        if not self._fixed:
            blockSet[:] = fastWhere(cond, OpArrayCache.IN_PROCESS, blockSet,
                                    numpy.uint8)
        else:
            # Someone asked for some dirty blocks while we were fixed.
            # Mark these blocks to be signaled as dirty when we become unfixed
            blockSet[:] = fastWhere(cond, OpArrayCache.FIXED_DIRTY, blockSet,
                                    numpy.uint8)
            self._has_fixed_dirty_blocks = True
        self._lock.release()

        temp = itertools.count(0)

        #wait for all requests to finish
        dirtyPool.wait()
        if len(dirtyPool) > 0:
            # Signal that something was updated.
            # Note that we don't need to do this for the 'in process' queries (below)
            #  because they are already in the dirtyPool in some other thread
            self.Output._sig_value_changed()
        dirtyPool.clean()

        # indicate the finished inprocess state (i.e. CLEAN)
        if not self._fixed and temp.next() == 0:
            with self._lock:
                blockSet[:] = fastWhere(cond, OpArrayCache.CLEAN, blockSet,
                                        numpy.uint8)
                self._blockQuery[blockKey] = fastWhere(
                    cond, None, self._blockQuery[blockKey], object)

        inProcessPool = RequestPool()
        #wait for all in process queries
        for req in inProcessQueries:
            req = req()  # get original req object from weakref
            if req is not None:
                inProcessPool.add(req)

        inProcessPool.wait()
        inProcessPool.clean()

        # finally, store results in result area
        self._lock.acquire()
        if self._cache is not None:
            result[:] = self._cache[roiToSlice(start, stop)]
        else:
            self.inputs["Input"][roiToSlice(start,
                                            stop)].writeInto(result).wait()
        self._running -= 1
        self._updatePriority()
        cacheView = None

        self._lock.release()
        self.logger.debug("read %s took %f sec." %
                          (roi.pprint(), time.time() - t))
Exemplo n.º 5
0
    def _executeOutput(self, slot, subindex, roi, result):
        key = roi.toSlice()

        shape = self.Output.meta.shape
        start, stop = sliceToRoi(key, shape)

        self.traceLogger.debug("Acquiring ArrayCache lock...")
        self._lock.acquire()
        self.traceLogger.debug("ArrayCache lock acquired.")

        ch = self._cacheHits
        ch += 1
        self._cacheHits = ch

        self._running += 1

        if self._cache is None:
            self._allocateCache()

        cacheView = self._cache[:] #prevent freeing of cache during running this function


        blockStart = (1.0 * start / self._blockShape).floor()
        blockStop = (1.0 * stop / self._blockShape).ceil()
        blockKey = roiToSlice(blockStart,blockStop)

        blockSet = self._blockState[blockKey]

        # this is a little optimization to shortcut
        # many lines of python code when all data is
        # is already in the cache:
        if numpy.logical_or(blockSet == OpArrayCache.CLEAN, blockSet == OpArrayCache.FIXED_DIRTY).all():
            result[:] = self._cache[roiToSlice(start, stop)]
            self._running -= 1
            self._updatePriority()
            cacheView = None
            self._lock.release()
            return

        inProcessQueries = numpy.unique(numpy.extract( blockSet == OpArrayCache.IN_PROCESS, self._blockQuery[blockKey]))

        cond = (blockSet == OpArrayCache.DIRTY)
        tileWeights = fastWhere(cond, 1, 128**3, numpy.uint32)
        trueDirtyIndices = numpy.nonzero(cond)

        tileArray = drtile.test_DRTILE(tileWeights, 128**3).swapaxes(0,1)

        dirtyRois = []
        half = tileArray.shape[0]/2
        dirtyPool = RequestPool()

        def onCancel(req):
            return False # indicate that this request cannot be canceled

        self.traceLogger.debug("Creating cache input requests")
        for i in range(tileArray.shape[1]):

            drStart3 = tileArray[:half,i]
            drStop3 = tileArray[half:,i]
            drStart2 = drStart3 + blockStart
            drStop2 = drStop3 + blockStart
            drStart = drStart2*self._blockShape
            drStop = drStop2*self._blockShape

            shape = self.Output.meta.shape
            drStop = numpy.minimum(drStop, shape)
            drStart = numpy.minimum(drStart, shape)

            key3 = roiToSlice(drStart3,drStop3)
            key2 = roiToSlice(drStart2,drStop2)

            key = roiToSlice(drStart,drStop)

            if not self._fixed:
                dirtyRois.append([drStart,drStop])

                req = self.inputs["Input"][key].writeInto(self._cache[key])

                req.uncancellable = True #FIXME
                
                dirtyPool.add(req)

                self._blockQuery[key2] = weakref.ref(req)

                #sanity check:
                if (self._blockState[key2] != OpArrayCache.DIRTY).any():
                    print "original condition", cond
                    print "original tilearray", tileArray, tileArray.shape
                    print "original tileWeights", tileWeights, tileWeights.shape
                    print "sub condition", self._blockState[key2] == OpArrayCache.DIRTY
                    print "START, STOP", drStart2, drStop2
                    import h5py
                    with h5py.File("test.h5", "w") as f:
                        f.create_dataset("data",data = tileWeights)
                        print "%r \n %r \n %r\n %r\n %r \n%r" % (key2, blockKey,self._blockState[key2], self._blockState[blockKey][trueDirtyIndices],self._blockState[blockKey],tileWeights)
                    assert False
                self._blockState[key2] = OpArrayCache.IN_PROCESS

        # indicate the inprocessing state, by setting array to 0 (i.e. IN_PROCESS)
        if not self._fixed:
            blockSet[:]  = fastWhere(cond, OpArrayCache.IN_PROCESS, blockSet, numpy.uint8)
        else:
            # Someone asked for some dirty blocks while we were fixed.
            # Mark these blocks to be signaled as dirty when we become unfixed
            blockSet[:]  = fastWhere(cond, OpArrayCache.FIXED_DIRTY, blockSet, numpy.uint8)
            self._has_fixed_dirty_blocks = True
        self._lock.release()

        temp = itertools.count(0)

        #wait for all requests to finish
        self.traceLogger.debug( "Firing all {} cache input requests...".format(len(dirtyPool)) )
        dirtyPool.wait()
        if len( dirtyPool ) > 0:
            # Signal that something was updated.
            # Note that we don't need to do this for the 'in process' queries (below)  
            #  because they are already in the dirtyPool in some other thread
            self.Output._sig_value_changed()
        dirtyPool.clean()
        self.traceLogger.debug( "All cache input requests received." )

        # indicate the finished inprocess state (i.e. CLEAN)
        if not self._fixed and temp.next() == 0:
            with self._lock:
                blockSet[:] = fastWhere(cond, OpArrayCache.CLEAN, blockSet, numpy.uint8)
                self._blockQuery[blockKey] = fastWhere(cond, None, self._blockQuery[blockKey], object)

        inProcessPool = RequestPool()
        #wait for all in process queries
        for req in inProcessQueries:
            req = req() # get original req object from weakref
            if req is not None:
                inProcessPool.add(req) 

        inProcessPool.wait()
        inProcessPool.clean()

        # finally, store results in result area
        self._lock.acquire()
        if self._cache is not None:
            result[:] = self._cache[roiToSlice(start, stop)]
        else:
            self.traceLogger.debug( "WAITING FOR INPUT WITH THE CACHE LOCK LOCKED!" )
            self.inputs["Input"][roiToSlice(start, stop)].writeInto(result).wait()
            self.traceLogger.debug( "INPUT RECEIVED WITH THE CACHE LOCK LOCKED." )
        self._running -= 1
        self._updatePriority()
        cacheView = None

        self._lock.release()