Ejemplo n.º 1
0
    def execute(self, slot, subindex, roi, result):
        featMatrix=[]
        labelsMatrix=[]
        for i,labels in enumerate(self.inputs["Labels"]):
            if labels.meta.shape is not None:
                labels=labels[:].allocate().wait()

                indexes=numpy.nonzero(labels[...,0].view(numpy.ndarray))
                #Maybe later request only part of the region?

                image=self.inputs["Images"][i][:].allocate().wait()

                features=image[indexes]
                labels=labels[indexes]

                featMatrix.append(features)
                labelsMatrix.append(labels)


        featMatrix=numpy.concatenate(featMatrix,axis=0)
        labelsMatrix=numpy.concatenate(labelsMatrix,axis=0)

        # train and store self._forest_count forests in parallel
        pool = Pool()
        for i in range(self._forest_count):
            def train_and_store(number):
                result[number] = vigra.learning.RandomForest(self._tree_count)
                result[number].learnRF(featMatrix.astype(numpy.float32),labelsMatrix.astype(numpy.uint32))
            req = pool.request(partial(train_and_store, i))

        pool.wait()

        return result
Ejemplo n.º 2
0
    def execute(self, slot, subindex, roi, result):
        t1 = time.time()
        key = roi.toSlice()
        nlabels=self.inputs["LabelsCount"].value

        traceLogger.debug("OpPredictRandomForest: Requesting classifier. roi={}".format(roi))
        forests=self.inputs["Classifier"][:].wait()

        if forests is None:
            # Training operator may return 'None' if there was no data to train with
            return numpy.zeros(numpy.subtract(roi.stop, roi.start), dtype=numpy.float32)[...]

        traceLogger.debug("OpPredictRandomForest: Got classifier")
        #assert RF.labelCount() == nlabels, "ERROR: OpPredictRandomForest, labelCount differs from true labelCount! %r vs. %r" % (RF.labelCount(), nlabels)

        newKey = key[:-1]
        newKey += (slice(0,self.inputs["Image"].meta.shape[-1],None),)

        res = self.inputs["Image"][newKey].wait()

        shape=res.shape
        prod = numpy.prod(shape[:-1])
        res.shape = (prod, shape[-1])
        features=res

        predictions = [0]*len(forests)

        def predict_forest(number):
            predictions[number] = forests[number].predictProbabilities(numpy.asarray(features, dtype=numpy.float32))

        t2 = time.time()

        # predict the data with all the forests in parallel
        pool = Pool()

        for i,f in enumerate(forests):
            req = pool.request(partial(predict_forest, i))

        pool.wait()
        pool.clean()

        prediction=numpy.dstack(predictions)
        prediction = numpy.average(prediction, axis=2)
        prediction.shape =  shape[:-1] + (forests[0].labelCount(),)
        #prediction = prediction.reshape(*(shape[:-1] + (forests[0].labelCount(),)))

        # If our LabelsCount is higher than the number of labels in the training set,
        # then our results aren't really valid.  FIXME !!!
        # Duplicate the last label's predictions
        for c in range(result.shape[-1]):
            result[...,c] = prediction[...,min(c+key[-1].start, prediction.shape[-1]-1)]

        t3 = time.time()

        # logger.info("Predict took %fseconds, actual RF time was %fs, feature time was %fs" % (t3-t1, t3-t2, t2-t1))
        return result
Ejemplo n.º 3
0
    def execute(self, slot, subindex, roi, result):
        progress = 0
        self.progressSignal(progress)
        numImages = len(self.Images)

        key = roi.toSlice()
        featMatrix=[]
        labelsMatrix=[]
        for i,labels in enumerate(self.inputs["Labels"]):
            if labels.meta.shape is not None:
                #labels=labels[:].allocate().wait()
                blocks = self.inputs["nonzeroLabelBlocks"][i][0].allocate().wait()

                progress += 10/numImages
                self.progressSignal(progress)

                reqlistlabels = []
                reqlistfeat = []
                traceLogger.debug("Sending requests for {} non-zero blocks (labels and data)".format( len(blocks[0])) )
                for b in blocks[0]:

                    request = labels[b].allocate()
                    featurekey = list(b)
                    featurekey[-1] = slice(None, None, None)
                    request2 = self.inputs["Images"][i][featurekey].allocate()

                    reqlistlabels.append(request)
                    reqlistfeat.append(request2)

                traceLogger.debug("Requests prepared")

                numLabelBlocks = len(reqlistlabels)
                progress_outer = [progress] # Store in list for closure access
                if numLabelBlocks > 0:
                    progressInc = (80-10)/numLabelBlocks/numImages

                def progressNotify(req):
                    # Note: If we wanted perfect progress reporting, we could use lock here
                    #       to protect the progress from being incremented simultaneously.
                    #       But that would slow things down and imperfect reporting is okay for our purposes.
                    progress_outer[0] += progressInc/2
                    self.progressSignal(progress_outer[0])

                for ir, req in enumerate(reqlistfeat):
                    image = req.notify(progressNotify)

                for ir, req in enumerate(reqlistlabels):
                    labblock = req.notify(progressNotify)

                traceLogger.debug("Requests fired")

                for ir, req in enumerate(reqlistlabels):
                    traceLogger.debug("Waiting for a label block...")
                    labblock = req.wait()

                    traceLogger.debug("Waiting for an image block...")
                    image = reqlistfeat[ir].wait()

                    indexes=numpy.nonzero(labblock[...,0].view(numpy.ndarray))
                    features=image[indexes]
                    labbla=labblock[indexes]

                    featMatrix.append(features)
                    labelsMatrix.append(labbla)

                progress = progress_outer[0]

                traceLogger.debug("Requests processed")

        self.progressSignal(80/numImages)

        if len(featMatrix) == 0 or len(labelsMatrix) == 0:
            # If there was no actual data for the random forest to train with, we return None
            result[:] = None
        else:
            featMatrix=numpy.concatenate(featMatrix,axis=0)
            labelsMatrix=numpy.concatenate(labelsMatrix,axis=0)
            maxLabel = self.inputs["MaxLabel"].value
            labelList = range(1, maxLabel+1) if maxLabel > 0 else list()

            try:
                logger.debug("Learning with Vigra...")
                # train and store self._forest_count forests in parallel
                pool = Pool()

                for i in range(self._forest_count):
                    def train_and_store(number):
                        result[number] = vigra.learning.RandomForest(self._tree_count, labels=labelList)
                        result[number].learnRF( numpy.asarray(featMatrix, dtype=numpy.float32),
                                                numpy.asarray(labelsMatrix, dtype=numpy.uint32))
                    req = pool.request(partial(train_and_store, i))

                pool.wait()
                pool.clean()

                logger.debug("Vigra finished")
            except:
                logger.error( "ERROR: could not learn classifier" )
                logger.error( "featMatrix shape={}, max={}, dtype={}".format(featMatrix.shape, featMatrix.max(), featMatrix.dtype) )
                logger.error( "labelsMatrix shape={}, max={}, dtype={}".format(labelsMatrix.shape, labelsMatrix.max(), labelsMatrix.dtype ) )
                raise
            finally:
                self.progressSignal(100)

        return result
Ejemplo n.º 4
0
    def _extract(self, image, labels):
        assert len(image.shape) == len(labels.shape) == 3, "Images must be 3D.  Shapes were: {} and {}".format( image.shape, labels.shape )
        xAxis = image.axistags.index('x')
        yAxis = image.axistags.index('y')
        zAxis = image.axistags.index('z')
        image = numpy.asarray(image, dtype=numpy.float32)
        labels = numpy.asarray(labels, dtype=numpy.uint32)
        
        feature_names_first = [feat for feat in self._vigraFeatureNames if feat in self.OffsetSensitiveFeatures]
        feature_names_second = [feat for feat in self._vigraFeatureNames if not feat in self.OffsetSensitiveFeatures]
        
        if not "Coord<Minimum>" in feature_names_first:
            feature_names_first.append("Coord<Minimum>")
        if not "Coord<Maximum>" in feature_names_first:
            feature_names_first.append("Coord<Maximum>")
        if not "Count" in feature_names_first:
            feature_names_first.append("Count")
            
        features_first = vigra.analysis.extractRegionFeatures(image, labels, feature_names_first, ignoreLabel=0)
        
        feature_dict = {}
        for key in features_first.keys():
            feature_dict[key] = features_first[key]
       
        mins = features_first["Coord<Minimum>"]
        maxs = features_first["Coord<Maximum>"]
        counts = features_first["Count"]
        
        nobj = mins.shape[0]
        features_obj = [None] #don't compute for the 0-th object (the background)
        features_incl = [None]
        features_excl = [None]
        first_good = 1
        pool = Pool()
        otherFeatures_dict = {}
        if len(self._otherFeatureNames)>0:
            #there are non-vigra features. let's make room for them
            #we can't do that for vigra features, because vigra computes more than 
            #we specify in featureNames and we want to keep that
            otherFeatures_dict = {}
            for key in self._otherFeatureNames:
                otherFeatures_dict[key]=[None]
            
        for i in range(1,nobj):
            print "processing object ", i
            #find the bounding box
            minx = max(mins[i][xAxis]-self.MARGIN, 0)
            miny = max(mins[i][yAxis]-self.MARGIN, 0)
            minz = max(mins[i][zAxis], 0)
            # Coord<Minimum> and Coord<Maximum> give us the [min,max] 
            # coords of the object, but we want the bounding box: [min,max), so add 1
            maxx = min(maxs[i][xAxis]+1+self.MARGIN, image.shape[xAxis])
            maxy = min(maxs[i][yAxis]+1+self.MARGIN, image.shape[yAxis])
            maxz = min(maxs[i][zAxis]+1, image.shape[zAxis])
            
            #FIXME: there must be a better way
            key = 3*[None]
            key[xAxis] = slice(minx, maxx, None)
            key[yAxis] = slice(miny, maxy, None)
            key[zAxis] = slice(minz, maxz, None)
            key = tuple(key)
            
            ccbbox = labels[key]
            rawbbox = image[key]
            ccbboxobject = numpy.where(ccbbox==i, 1, 0)
            
            #find the context area around the object
            bboxshape = 3*[None]
            bboxshape[xAxis] = maxx-minx
            bboxshape[yAxis] = maxy-miny
            bboxshape[zAxis] = maxz-minz
            bboxshape = tuple(bboxshape)
            passed = numpy.zeros(bboxshape, dtype=bool)
            
            for iz in range(maxz-minz):
                #FIXME: shoot me, axistags
                bboxkey = 3*[None]
                bboxkey[xAxis] = slice(None, None, None)
                bboxkey[yAxis] = slice(None, None, None)
                bboxkey[zAxis] = iz
                bboxkey = tuple(bboxkey)
                #TODO: Ulli once mentioned that distance transform can be made anisotropic in 3D
                dt = vigra.filters.distanceTransform2D( numpy.asarray(ccbbox[bboxkey], dtype=numpy.float32) )
                passed[bboxkey] = dt<self.MARGIN
                
            ccbboxexcl = passed-ccbboxobject
            if "bad_slices" in self._otherFeatureNames:
                #compute the quality score of an object - 
                #count the number of fully black slices inside its bbox
                #FIXME: the interpolation part is not tested at all...
                nbadslices = 0
                badslices = []
                area = rawbbox.shape[xAxis]*rawbbox.shape[yAxis]
                bboxkey = 3*[None]
                bboxkey[xAxis] = slice(None, None, None)
                bboxkey[yAxis] = slice(None, None, None)
                for iz in range(maxz-minz):
                    bboxkey[zAxis] = iz
                    nblack = numpy.sum(rawbbox[tuple(bboxkey)]==0)
                    if nblack>0.5*area:
                        nbadslices = nbadslices+1
                        badslices.append(iz)
                
                otherFeatures_dict["bad_slices"].append(numpy.array([nbadslices]))
                    
            labeled_bboxes = [passed, ccbboxexcl, ccbboxobject]
            feats = [None, None, None]
            for ibox, bbox in enumerate(labeled_bboxes):
                def extractObjectFeatures(ibox):
                    feats[ibox] = vigra.analysis.extractRegionFeatures(numpy.asarray(rawbbox, dtype=numpy.float32), \
                                                                    numpy.asarray(labeled_bboxes[ibox], dtype=numpy.uint32), \
                                                                    feature_names_second, \
                                                                    histogramRange=[0, 255], \
                                                                    binCount = 10,\
                                                                    ignoreLabel=0)
                req = pool.request(partial(extractObjectFeatures, ibox))
            pool.wait()

            features_incl.append(feats[0])
            features_excl.append(feats[1])
            features_obj.append(feats[2])


            if "lbp" in self._otherFeatureNames:
                #FIXME: there is a mess about which of the lbp features are computed (obj, excl or incl)
              
                #compute lbp features
                import skimage.feature as ft
                P=8
                R=1
                lbp_total = numpy.zeros(passed.shape)
                for iz in range(maxz-minz): 
                    #an lbp image
                    bboxkey = 3*[None]
                    bboxkey[xAxis] = slice(None, None, None)
                    bboxkey[yAxis] = slice(None, None, None)
                    bboxkey[zAxis] = iz
                    bboxkey = tuple(bboxkey)
                    lbp_total[bboxkey] = ft.local_binary_pattern(rawbbox[bboxkey], P, R, "uniform")
                #extract relevant parts
                #print "computed lbp for volume:", lbp_total.shape,
                #print "extracting pieces:", passed.shape, ccbboxexcl.shape, ccbboxobject.shape
                lbp_incl = lbp_total[passed]
                lbp_excl = lbp_total[ccbboxexcl.astype(bool)]
                lbp_obj = lbp_total[ccbboxobject.astype(bool)]
                #print "extracted pieces", lbp_incl.shape, lbp_excl.shape, lbp_obj.shape
                lbp_hist_incl, _ = numpy.histogram(lbp_incl, normed=True, bins=P+2, range=(0, P+2))
                lbp_hist_excl, _ = numpy.histogram(lbp_excl, normed=True, bins=P+2, range=(0, P+2))
                lbp_hist_obj, _ = numpy.histogram(lbp_obj, normed=True, bins=P+2, range=(0, P+2))
                #print "computed histogram"
                otherFeatures_dict["lbp_incl"].append(lbp_hist_incl)
                otherFeatures_dict["lbp_excl"].append(lbp_hist_excl)
                otherFeatures_dict["lbp"].append(lbp_hist_obj)

            if "lapl" in self._otherFeatureNames:
                #compute mean and variance of laplacian in the object and its neighborhood
                lapl = None
                try:
                    lapl = vigra.filters.laplacianOfGaussian(rawbbox)
                except RuntimeError:
                    #kernel longer than line. who cares?
                    otherFeatures_dict["lapl_incl"].append(None)
                    otherFeatures_dict["lapl_excl"].append(None)
                    otherFeatures_dict["lapl"].append(None)
                else:
                    lapl_incl = lapl[passed]
                    lapl_excl = lapl[ccbboxexcl.astype(bool)]
                    lapl_obj = lapl[ccbboxobject.astype(bool)]
                    lapl_mean_incl = numpy.mean(lapl_incl)
                    lapl_var_incl = numpy.var(lapl_incl)
                    lapl_mean_excl = numpy.mean(lapl_excl)
                    lapl_var_excl = numpy.var(lapl_excl)
                    lapl_mean_obj = numpy.mean(lapl_obj)
                    lapl_var_obj = numpy.var(lapl_obj)
                    otherFeatures_dict["lapl_incl"].append(numpy.array([lapl_mean_incl, lapl_var_incl]))
                    otherFeatures_dict["lapl_excl"].append(numpy.array([lapl_mean_excl, lapl_var_excl]))
                    otherFeatures_dict["lapl"].append(numpy.array([lapl_mean_obj, lapl_var_obj]))
           
        feature_keys = features_incl[first_good].keys()
        
        #copy over non-vigra features and turn them into numpy arrays
        for key in otherFeatures_dict.keys():
            #print otherFeatures_dict[key]
            #find the number of channels
            feature = otherFeatures_dict[key]
            nchannels = feature[first_good].shape[0]
            for irow, row in enumerate(feature):
                if row is None:
                    #print "NaNs in row", irow
                    feature[irow]=numpy.zeros((nchannels,))
            
            feature_dict[key]=numpy.vstack(otherFeatures_dict[key])
            assert feature_dict[key].shape[0]==nobj, "didn't compute features for all objects {}".format(key)
            #print key, feature_dict[key].shape
            
        for key in feature_keys:
            if key in feature_names_first:
                continue
            
            nchannels = 0
            #we always have two objects, background is first
            #unless, of course, it's a global measurement, and then it's just one element, grrrh
            
            #sometimes, vigra returns one-dimensional features as (nobj, 1) and sometimes as (nobj,)
            #the following try-except is for this case
            
            try:
                nchannels = len(features_incl[first_good][key][0])
            except TypeError:
                nchannels = 1
            #print "assembling key:", key, "nchannels:", nchannels
            #print "feature arrays:", len(features_incl), len(features_excl), len(features_obj)
            #FIXME: find the maximum number of channels and pre-allocate
            feature_obj = numpy.zeros((nobj, nchannels))
            feature_incl = numpy.zeros((nobj, nchannels))
            feature_excl = numpy.zeros((nobj, nchannels))
            
            for i in range(nobj):
                if features_obj[i] is not None:
                    try:
                        feature_obj[i] = features_obj[i][key][1]
                        feature_incl[i] = features_incl[i][key][1]
                        feature_excl[i] = features_excl[i][key][1]
                    except:
                        #global number, not a list, haha
                        feature_obj[i] = features_obj[i][key]
                        feature_incl[i] = features_incl[i][key]
                        feature_excl[i] = features_excl[i][key]
            
            feature_dict[key]=feature_obj
            feature_dict[key+"_incl"]=feature_incl
            feature_dict[key+"_excl"]=feature_excl
            #print key, feature_obj.shape, feature_incl.shape, feature_excl.shape
        end1 = time.clock()
        #print "computed the following features:", feature_dict.keys()
        return feature_dict
Ejemplo n.º 5
0
    def execute(self, slot, subindex, rroi, result):
        assert slot == self.Features or slot == self.Output
        if slot == self.Features:
            key = roiToSlice(rroi.start, rroi.stop)
            index = subindex[0]
            subslot = self.Features[index]
            key = list(key)
            channelIndex = self.Input.meta.axistags.index('c')
            
            # Translate channel slice to the correct location for the output slot.
            key[channelIndex] = slice(self.featureOutputChannels[index][0] + key[channelIndex].start,
                                      self.featureOutputChannels[index][0] + key[channelIndex].stop)
            rroi = SubRegion(subslot, pslice=key)
    
            # Get output slot region for this channel
            return self.execute(self.Output, (), rroi, result)
        elif slot == self.outputs["Output"]:
            key = rroi.toSlice()
            cnt = 0
            written = 0
            assert (rroi.stop<=self.outputs["Output"].meta.shape).all()
            flag = 'c'
            channelAxis=self.inputs["Input"].meta.axistags.index('c')
            axisindex = channelAxis
            oldkey = list(key)
            oldkey.pop(axisindex)


            inShape  = self.inputs["Input"].meta.shape

            shape = self.outputs["Output"].meta.shape

            axistags = self.inputs["Input"].meta.axistags

            result = result.view(vigra.VigraArray)
            result.axistags = copy.copy(axistags)


            hasTimeAxis = self.inputs["Input"].meta.axistags.axisTypeCount(vigra.AxisType.Time)
            timeAxis=self.inputs["Input"].meta.axistags.index('t')

            subkey = popFlagsFromTheKey(key,axistags,'c')
            subshape=popFlagsFromTheKey(shape,axistags,'c')
            at2 = copy.copy(axistags)
            at2.dropChannelAxis()
            subshape=popFlagsFromTheKey(subshape,at2,'t')
            subkey = popFlagsFromTheKey(subkey,at2,'t')

            oldstart, oldstop = roi.sliceToRoi(key, shape)

            start, stop = roi.sliceToRoi(subkey,subkey)
            maxSigma = max(0.7,self.maxSigma)

            # The region of the smoothed image we need to give to the feature filter (in terms of INPUT coordinates)
            vigOpSourceStart, vigOpSourceStop = roi.extendSlice(start, stop, subshape, 0.7, window = 2)
            
            # The region of the input that we need to give to the smoothing operator (in terms of INPUT coordinates)
            newStart, newStop = roi.extendSlice(vigOpSourceStart, vigOpSourceStop, subshape, maxSigma, window = 3.5)

            # Translate coordinates (now in terms of smoothed image coordinates)
            vigOpSourceStart = roi.TinyVector(vigOpSourceStart - newStart)
            vigOpSourceStop = roi.TinyVector(vigOpSourceStop - newStart)

            readKey = roi.roiToSlice(newStart, newStop)


            writeNewStart = start - newStart
            writeNewStop = writeNewStart +  stop - start


            treadKey=list(readKey)

            if hasTimeAxis:
                if timeAxis < channelAxis:
                    treadKey.insert(timeAxis, key[timeAxis])
                else:
                    treadKey.insert(timeAxis-1, key[timeAxis])
            if  self.inputs["Input"].meta.axistags.axisTypeCount(vigra.AxisType.Channels) == 0:
                treadKey =  popFlagsFromTheKey(treadKey,axistags,'c')
            else:
                treadKey.insert(channelAxis, slice(None,None,None))

            treadKey=tuple(treadKey)

            req = self.inputs["Input"][treadKey].allocate()

            sourceArray = req.wait()
            req.result = None
            req.destination = None
            if sourceArray.dtype != numpy.float32:
                sourceArrayF = sourceArray.astype(numpy.float32)
                sourceArray.resize((1,), refcheck = False)
                del sourceArray
                sourceArray = sourceArrayF
            sourceArrayV = sourceArray.view(vigra.VigraArray)
            sourceArrayV.axistags =  copy.copy(axistags)





            dimCol = len(self.scales)
            dimRow = self.matrix.shape[0]


            sourceArraysForSigmas = [None]*dimCol

            #connect individual operators
            for j in range(dimCol):
                hasScale = False
                for i in range(dimRow):
                    if self.matrix[i,j]:
                        hasScale = True
                if not hasScale:
                    continue
                destSigma = 1.0
                if self.scales[j] > destSigma:
                    tempSigma = math.sqrt(self.scales[j]**2 - destSigma**2)
                else:
                    destSigma = 0.0
                    tempSigma = self.scales[j]
                vigOpSourceShape = list(vigOpSourceStop - vigOpSourceStart)
                if hasTimeAxis:

                    if timeAxis < channelAxis:
                        vigOpSourceShape.insert(timeAxis, ( oldstop - oldstart)[timeAxis])
                    else:
                        vigOpSourceShape.insert(timeAxis-1, ( oldstop - oldstart)[timeAxis])
                    vigOpSourceShape.insert(channelAxis, inShape[channelAxis])

                    sourceArraysForSigmas[j] = numpy.ndarray(tuple(vigOpSourceShape),numpy.float32)
                    for i,vsa in enumerate(sourceArrayV.timeIter()):
                        droi = (tuple(vigOpSourceStart._asint()), tuple(vigOpSourceStop._asint()))
                        tmp_key = getAllExceptAxis(len(sourceArraysForSigmas[j].shape),timeAxis, i)
                        sourceArraysForSigmas[j][tmp_key] = vigra.filters.gaussianSmoothing(vsa,tempSigma, roi = droi, window_size = 3.5 )
                else:
                    droi = (tuple(vigOpSourceStart._asint()), tuple(vigOpSourceStop._asint()))
                    #print droi, sourceArray.shape, tempSigma,self.scales[j]
                    sourceArraysForSigmas[j] = vigra.filters.gaussianSmoothing(sourceArrayV, sigma = tempSigma, roi = droi, window_size = 3.5)
                    #sourceArrayForSigma = sourceArrayForSigma.view(numpy.ndarray)

            del sourceArrayV
            try:
                sourceArray.resize((1,), refcheck = False)
            except ValueError:
                # Sometimes this fails, but that's okay.
                logger.debug("Failed to free array memory.")                
            del sourceArray

            
            closures = []

            #connect individual operators
            for i in range(dimRow):
                for j in range(dimCol):
                    val=self.matrix[i,j]
                    if val:
                        vop= self.featureOps[i][j]
                        oslot = vop.outputs["Output"]
                        req = None
                        inTagKeys = [ax.key for ax in oslot.meta.axistags]
                        if flag in inTagKeys:
                            slices = oslot.meta.shape[axisindex]
                            if cnt + slices >= rroi.start[axisindex] and rroi.start[axisindex]-cnt<slices and rroi.start[axisindex]+written<rroi.stop[axisindex]:
                                begin = 0
                                if cnt < rroi.start[axisindex]:
                                    begin = rroi.start[axisindex] - cnt
                                end = slices
                                if cnt + end > rroi.stop[axisindex]:
                                    end -= cnt + end - rroi.stop[axisindex]
                                key_ = copy.copy(oldkey)
                                key_.insert(axisindex, slice(begin, end, None))
                                reskey = [slice(None, None, None) for x in range(len(result.shape))]
                                reskey[axisindex] = slice(written, written+end-begin, None)

                                destArea = result[tuple(reskey)]
                                roi_ = SubRegion(self.Input, pslice=key_)                                
                                closure = partial(oslot.operator.execute, oslot, (), roi_, destArea, sourceArray = sourceArraysForSigmas[j])
                                closures.append(closure)

                                written += end - begin
                            cnt += slices
                        else:
                            if cnt>=rroi.start[axisindex] and rroi.start[axisindex] + written < rroi.stop[axisindex]:
                                reskey = copy.copy(oldkey)
                                reskey.insert(axisindex, written)
                                #print "key: ", key, "reskey: ", reskey, "oldkey: ", oldkey
                                #print "result: ", result.shape, "inslot:", inSlot.shape

                                destArea = result[tuple(reskey)]
                                logger.debug(oldkey, destArea.shape, sourceArraysForSigmas[j].shape)
                                oldroi = SubRegion(self.Input, pslice=oldkey)
                                closure = partial(oslot.operator.execute, oslot, (), oldroi, destArea, sourceArray = sourceArraysForSigmas[j])
                                closures.append(closure)

                                written += 1
                            cnt += 1
            pool = Pool()
            for c in closures:
                r = pool.request(c)
            pool.wait()
            pool.clean()

            for i in range(len(sourceArraysForSigmas)):
                if sourceArraysForSigmas[i] is not None:
                    try:
                        sourceArraysForSigmas[i].resize((1,))
                    except:
                        sourceArraysForSigmas[i] = None
Ejemplo n.º 6
0
        req = Request(functools.partial(empty_func, b = 11))
        req.submit()

    for r in requests:
        r.wait()

# Make sure this test occurs entirely within greenlets.
req = Request( functools.partial( lots_of_work ) )
req.submit()
req.wait()

t2 = time.time()
print "\n\n"
print "LAZYFLOW REQUEST WAIT:   %f seconds for %d iterations" % (t2-t1,mcount)
print "                                %0.3fms latency" % ((t2-t1)*1e3/mcount,)


t1 = time.time()

pool = Pool()

for i in range(50000):
    pool.request(functools.partial(empty_func, b = 11))

pool.wait()

t2 = time.time()
print "\n\n"
print "LAZYFLOW POOL WAIT:   %f seconds for %d iterations" % (t2-t1,mcount)
print "                                %0.3fms latency" % ((t2-t1)*1e3/mcount,)
Ejemplo n.º 7
0
t1 = time.time()

requests = []
for i in range(50000):
    req = Request(empty_func, b = 11)
    req.submit()

for r in requests:
    r.wait()

t2 = time.time()
print "\n\n"
print "LAZYFLOW REQUEST WAIT:   %f seconds for %d iterations" % (t2-t1,mcount)
print "                                %0.3fms latency" % ((t2-t1)*1e3/mcount,)


t1 = time.time()

pool = Pool()

for i in range(50000):
    pool.request(empty_func, b = 11)

pool.wait()

t2 = time.time()
print "\n\n"
print "LAZYFLOW POOL WAIT:   %f seconds for %d iterations" % (t2-t1,mcount)
print "                                %0.3fms latency" % ((t2-t1)*1e3/mcount,)