def test_interior_pixel_generator(self): b = 10 # b := border size Z = np.zeros((2, 100, 100), dtype=np.int32) for idx, pct in emlib.interior_pixel_generator(Z, b, 30): Z[idx[:, 0], idx[:, 1], idx[:, 2]] += 1 self.assertTrue(np.all(Z[:, b:-b, b:-b] == 1)) Z[:, b:-b, b:-b] = 0 self.assertTrue(np.all(Z == 0)) # Make sure it works with 4d tensors as well Z = np.zeros((2, 3, 100, 100), dtype=np.int32) for idx, pct in emlib.interior_pixel_generator(Z, b, 30): Z[idx[:, 0], :, idx[:, 1], idx[:, 2]] += 1 self.assertTrue(np.all(Z[:, :, b:-b, b:-b] == 1)) Z[:, :, b:-b, b:-b] = 0 self.assertTrue(np.all(Z == 0))
def test_interior_pixel_generator(self): b = 10 # b := border size Z = np.zeros((2,100,100), dtype=np.int32) for idx, pct in emlib.interior_pixel_generator(Z,b,30): Z[idx[:,0],idx[:,1],idx[:,2]] += 1 self.assertTrue(np.all(Z[:,b:-b,b:-b]==1)) Z[:,b:-b,b:-b] = 0 self.assertTrue(np.all(Z==0))
def _eval_cube(net, X, M, batchDim, bandSpec): assert(batchDim[2] == batchDim[3]) # tiles must be square # some variables and storage needed in the processing loop below tileRadius = int(batchDim[2]/2) Xi = np.zeros(batchDim, dtype=np.float32) yDummy = np.zeros((batchDim[0],), dtype=np.float32) cnnTime = 0.0 # time spent doing core CNN operations Yhat = None # process the cube tic = time.time() lastChatter = None for Idx, pct in emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=M): # populate the mini-batch buffer for jj in range(Idx.shape[0]): a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ] _tmp = time.time() net.set_input_arrays(Xi, yDummy) # XXX: could call preprocess() here? out = net.forward() yiHat = out['prob'] cnnTime += time.time() - _tmp nClasses = yiHat.shape[1] if Yhat is None: # on first iteration, create Yhat with the appropriate shape Yhat = -1*np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2])) # store the per-class probability estimates. # Note that on the final iteration, the size of yiHat may not match # the remaining space in Yhat (unless we get lucky and the data cube # size is a multiple of the mini-batch size). This is why we slice # yijHat before assigning to Yhat. for jj in range(nClasses): yijHat = np.squeeze(yiHat[:,jj,:,:]) # get slice containing probabilities for class j assert(len(yijHat.shape)==1) # should be a single vector now Yhat[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = yijHat[:Idx.shape[0]] # provide feedback on progress so far elapsed = (time.time() - tic) / 60. if (lastChatter is None) or ((elapsed - lastChatter) > 2): lastChatter = elapsed print "[deploy]: processed pixel at index %s (%0.2f min elapsed; %0.2f CNN min)" % (str(Idx[-1,:]), elapsed, cnnTime/60.) sys.stdout.flush() print('[deploy]: Finished processing cube. Net time was: %0.2f min (%0.2f CNN min)' % (elapsed, cnnTime/60.)) return Yhat
def _evaluate(model, X, log=None, batchSize=100, evalPct=1.0): """Evaluate model on held-out data. Returns: Prob : a tensor of per-pixel probability estimates with dimensions: (#layers, #classes, width, height) """ #---------------------------------------- # Pre-allocate some variables & storage. #---------------------------------------- nChannels, tileRows, tileCols = model.input_shape[1:4] ste = emlib.SimpleTileExtractor(tileRows, X) lastChatter = -2 startTime = time.time() # identify subset of volume to evaluate Mask = _downsample_mask(X, evalPct) if log: log.info('after masking, will evaluate %0.2f%% of data' % (100.0 * np.sum(Mask) / Mask.size)) # Create storage for class probabilities. # Note that we store all class probabilities, even if this # is a binary classification problem (in which case p(1) = 1 - p(0)). # We do this to support multiclass classification seamlessly. [numZ, numChan, numRows, numCols] = X.shape numClasses = model.output_shape[-1] Prob = -1 * np.ones([numZ, numClasses, numRows, numCols], dtype=np.float32) #---------------------------------------- # Loop over mini-batches #---------------------------------------- # note: set tileRadius to 0 so we evaluate whole volume it = emlib.interior_pixel_generator(X, 0, batchSize, mask=Mask) for mbIdx, (Idx, epochPct) in enumerate(it): n = Idx.shape[0] # may be < batchSize on final iteration Xi = ste.extract(Idx) prob = model.predict_on_batch(Xi) # updated based on Keras API changes #Prob[Idx[:,0], :, Idx[:,1], Idx[:,2]] = prob[0][:n,:] Prob[Idx[:, 0], :, Idx[:, 1], Idx[:, 2]] = prob[:n, :] # notify user re. progress elapsed = (time.time() - startTime) / 60.0 if (lastChatter + 2) < elapsed: lastChatter = elapsed if log: log.info(" last pixel %s (%0.2f%% complete)" % (str(Idx[-1, :]), 100. * epochPct)) return Prob
def _evaluate(model, X, log=None, batchSize=100, evalPct=1.0): """Evaluate model on held-out data. Returns: Prob : a tensor of per-pixel probability estimates with dimensions: (#layers, #classes, width, height) """ #---------------------------------------- # Pre-allocate some variables & storage. #---------------------------------------- nChannels, tileRows, tileCols = model.input_shape[1:4] ste = emlib.SimpleTileExtractor(tileRows, X) lastChatter = -2 startTime = time.time() # identify subset of volume to evaluate Mask = _downsample_mask(X, evalPct) if log: log.info('after masking, will evaluate %0.2f%% of data' % (100.0*np.sum(Mask)/Mask.size)) # Create storage for class probabilities. # Note that we store all class probabilities, even if this # is a binary classification problem (in which case p(1) = 1 - p(0)). # We do this to support multiclass classification seamlessly. [numZ, numChan, numRows, numCols] = X.shape numClasses = model.output_shape[-1] Prob = -1*np.ones([numZ, numClasses, numRows, numCols], dtype=np.float32) #---------------------------------------- # Loop over mini-batches #---------------------------------------- # note: set tileRadius to 0 so we evaluate whole volume it = emlib.interior_pixel_generator(X, 0, batchSize, mask=Mask) for mbIdx, (Idx, epochPct) in enumerate(it): n = Idx.shape[0] # may be < batchSize on final iteration Xi = ste.extract(Idx) prob = model.predict_on_batch(Xi) # updated based on Keras API changes #Prob[Idx[:,0], :, Idx[:,1], Idx[:,2]] = prob[0][:n,:] Prob[Idx[:,0], :, Idx[:,1], Idx[:,2]] = prob[:n,:] # notify user re. progress elapsed = (time.time() - startTime) / 60.0 if (lastChatter+2) < elapsed: lastChatter = elapsed if log: log.info(" last pixel %s (%0.2f%% complete)" % (str(Idx[-1,:]), 100.*epochPct)) return Prob
def _evaluate(model, X, Y, omitLabels=[], batchSize=100, log=None): """Evaluate model on held-out data. Here, used to periodically report performance on validation data. """ #---------------------------------------- # Pre-allocate some variables & storage. #---------------------------------------- nChannels, tileRows, tileCols = model.input_shape[1:4] tileRadius = int(tileRows/2) ste = emlib.SimpleTileExtractor(tileRows, X) numClasses = model.output_shape[-1] [numZ, numChan, numRows, numCols] = X.shape Prob = np.nan * np.ones([numZ, numClasses, numRows, numCols], dtype=np.float32) #---------------------------------------- # Loop over mini-batches #---------------------------------------- it = emlib.interior_pixel_generator(X, tileRadius, batchSize) for mbIdx, (Idx, epochPct) in enumerate(it): n = Idx.shape[0] # may be < batchSize on final iteration Xi = ste.extract(Idx) prob = model.predict_on_batch(Xi) # Note: it seems the Keras API has changed... #Prob[Idx[:,0], :, Idx[:,1], Idx[:,2]] = prob[0][:n,:] Prob[Idx[:,0], :, Idx[:,1], Idx[:,2]] = prob[:n,:] # Evaluate accuracy only on the subset of pixels that: # o were actually provided to the CNN (not downsampled) # o have a label that should be evaluated # # The mask tensor M will indicate which pixels to consider. M = np.all(np.isfinite(Prob), axis=1) for om in omitLabels: M[Y==om] = False Yhat = np.argmax(Prob, axis=1) # probabilities -> class labels acc = 100.0 * np.sum(Yhat[M] == Y[M]) / np.sum(M) return Prob, acc
def _training_loop(solver, X, Y, M, solverParam, batchDim, outDir, omitLabels=[], Xvalid=None, Yvalid=None, syn_func=None): """Main CNN training loop. """ assert (batchDim[2] == batchDim[3]) # tiles must be square # Some variables and storage that we'll use in the loop below # tileRadius = int(batchDim[2] / 2) Xi = np.zeros(batchDim, dtype=np.float32) yi = np.zeros((batchDim[0], ), dtype=np.float32) yMax = np.max(Y).astype(np.int32) losses = np.zeros((solverParam.max_iter, )) acc = np.zeros((solverParam.max_iter, )) currIter = 0 currEpoch = 0 # SGD parameters. SGD with momentum is of the form: # # V_{t+1} = \mu V_t - \alpha \nablaL(W_t) # W_{t+1} = W_t + V_{t+1} # # where W are the weights and V the previous update. # Ref: http://caffe.berkeleyvision.org/tutorial/solver.html # alpha = solverParam.base_lr # alpha := learning rate mu = solverParam.momentum # mu := momentum gamma = solverParam.gamma # gamma := step factor isModeStep = (solverParam.lr_policy == u'step') isTypeSGD = ( solverParam.solver_type == solverParam.SolverType.Value('SGD')) Vall = {} # stores previous SGD steps (for all layers) if not (isModeStep and isTypeSGD): raise RuntimeError( 'Sorry - only support SGD "step" mode at the present') # TODO: weight decay # TODO: layer-specific weights cnnTime = 0.0 # time spent doing core CNN operations tic = time.time() while currIter < solverParam.max_iter: #-------------------------------------------------- # Each generator provides a single epoch's worth of data. # However, Caffe doesn't really recognize the notion of an epoch; instead, # they specify a number of training "iterations" (mini-batch evaluations, I assume). # So the inner loop below is for a single epoch, which we may terminate # early if the max # of iterations is reached. #-------------------------------------------------- currEpoch += 1 it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], mask=M, omitLabels=omitLabels) for Idx, epochPct in it: # Map the indices Idx -> tiles Xi and labels yi # # Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) a few examples # from the previous minibatch will be "recycled" here. This is intentional # (to keep batch sizes consistent even if data set size is not a multiple # of the minibatch size). # for jj in range(Idx.shape[0]): yi[jj] = Y[Idx[jj, 0], Idx[jj, 1], Idx[jj, 2]] a = Idx[jj, 1] - tileRadius b = Idx[jj, 1] + tileRadius + 1 c = Idx[jj, 2] - tileRadius d = Idx[jj, 2] + tileRadius + 1 Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d] # label-preserving data transformation (synthetic data generation) if syn_func is not None: #Xi = _xform_minibatch(Xi) Xi = syn_func(Xi) #---------------------------------------- # one forward/backward pass and update weights # (SGD with momentum term) #---------------------------------------- _tmp = time.time() solver.net.set_input_arrays(Xi, yi) # XXX: could call preprocess() here? rv = solver.net.forward() solver.net.backward() for lIdx, layer in enumerate(solver.net.layers): for bIdx, blob in enumerate(layer.blobs): key = (lIdx, bIdx) V = Vall.get(key, 0.0) Vnext = mu * V - alpha * blob.diff blob.data[...] += Vnext Vall[key] = Vnext cnnTime += time.time() - _tmp # update running list of losses with the loss from this mini batch losses[currIter] = np.squeeze(rv['loss']) acc[currIter] = np.squeeze(rv['accuracy']) currIter += 1 #---------------------------------------- # Some events occur on mini-batch intervals. # Deal with those now. #---------------------------------------- if (currIter % solverParam.snapshot) == 0: fn = os.path.join(outDir, 'iter_%06d.caffemodel' % (currIter)) solver.net.save(str(fn)) if isModeStep and ((currIter % solverParam.stepsize) == 0): alpha *= gamma if (currIter % solverParam.display) == 1: elapsed = (time.time() - tic) / 60. print "[train]: completed iteration %d (of %d; %0.2f min elapsed; %0.2f CNN min)" % ( currIter, solverParam.max_iter, elapsed, cnnTime / 60.) print "[train]: epoch: %d (%0.2f), loss: %0.3f, acc: %0.3f, learn rate: %0.3e" % ( currEpoch, 100 * epochPct, np.mean(losses[max(0, currIter - 10):currIter]), np.mean(acc[max(0, currIter - 10):currIter]), alpha) sys.stdout.flush() if currIter >= solverParam.max_iter: break # in case we hit max_iter on a non-epoch boundary #-------------------------------------------------- # After each training epoch is complete, if we have validation # data, evaluate it. # Note: this only requires forward passes through the network #-------------------------------------------------- if (Xvalid is not None) and (Xvalid.size != 0) and ( Yvalid is not None) and (Yvalid.size != 0): # Mask out pixels whose label we don't care about. Mvalid = np.ones(Yvalid.shape, dtype=bool) for yIgnore in omitLabels: Mvalid[Yvalid == yIgnore] = False print "[train]: Evaluating on validation data (%d pixels)..." % np.sum( Mvalid) Confusion = np.zeros((yMax + 1, yMax + 1)) # confusion matrix it = emlib.interior_pixel_generator(Yvalid, tileRadius, batchDim[0], mask=Mvalid) for Idx, epochPct in it: # Extract subtiles from validation data set for jj in range(Idx.shape[0]): yi[jj] = Yvalid[Idx[jj, 0], Idx[jj, 1], Idx[jj, 2]] a = Idx[jj, 1] - tileRadius b = Idx[jj, 1] + tileRadius + 1 c = Idx[jj, 2] - tileRadius d = Idx[jj, 2] + tileRadius + 1 Xi[jj, 0, :, :] = Xvalid[Idx[jj, 0], a:b, c:d] #---------------------------------------- # one forward pass; no backward pass #---------------------------------------- solver.net.set_input_arrays(Xi, yi) # XXX: could call preprocess() here? rv = solver.net.forward() # extract statistics Prob = np.squeeze( rv['prob'] ) # matrix of estimated probabilities for each object yHat = np.argmax( Prob, 1) # estimated class is highest probability in vector for yTmp in range( yMax + 1 ): # note: assumes class labels are in {0, 1,..,n_classes-1} bits = (yi.astype(np.int32) == yTmp) for jj in range(yMax + 1): Confusion[yTmp, jj] += np.sum(yHat[bits] == jj) print '[train]: Validation results:' print ' %s' % str(Confusion) if yMax == 1: # Assume a binary classification problem where 0 is non-target and 1 is target. # # precision := TP / (TP + FP) # recall := TP / (TP + FN) # # Confusion Matrix: # yHat=0 yHat=1 # y=0 TN FP # y=1 FN TP # precision = (1.0 * Confusion[1, 1]) / np.sum(Confusion[:, 1]) recall = (1.0 * Confusion[1, 1]) / np.sum(Confusion[1, :]) f1Score = (2.0 * precision * recall) / (precision + recall) print ' precision=%0.3f, recall=%0.3f' % (precision, recall) print ' F1=%0.3f' % f1Score else: print '[train]: Not using a validation data set' sys.stdout.flush() # ----- end one epoch ----- # complete print "[train]: all done!" return losses, acc
def main(args): tileRadius = np.floor(args.tileSize/2) nMiniBatch = 1000 # here, a "mini-batch" specifies LMDB transaction size # make sure we don't clobber an existing output if os.path.exists(args.outDir): raise RuntimeError('Output path "%s" already exists; please move out of the way and try again' % args.outDir) # load the data volumes (EM image and labels, if any) print('[make_lmdb]: loading EM data file: %s' % args.emFileName) X = emlib.load_cube(args.emFileName, np.float32) if args.labelsFileName: print('[make_lmdb]: loading labels file: %s' % args.labelsFileName) Y = emlib.load_cube(args.labelsFileName, np.float32) Y = emlib.fix_class_labels(Y, eval(args.omitLabels)) assert(Y.shape == X.shape) else: print('[make_lmdb]: no labels file; assuming this is a test volume') Y = np.zeros(X.shape) # usually we expect fewer slices in Z than pixels in X or Y. # Make sure the dimensions look ok before proceeding. assert(X.shape[0] < X.shape[1]) assert(X.shape[0] < X.shape[2]) # Identify the subset of the data to use for training. # (default is to use it all) if len(args.slicesExpr): sliceIdx = eval(args.slicesExpr) X = X[sliceIdx, :, :] # python puts the z dimension first... Y = Y[sliceIdx, :, :] X = X.astype(np.uint8) # critical!! otherwise, Caffe just flails... print('[make_lmdb]: EM volume shape: %s' % str(X.shape)) print('[make_lmdb]: yAll is %s' % np.unique(Y)) print('[make_lmdb]: %0.2f%% pixels will be omitted' % (100.0*np.sum(Y==-1)/numel(Y))) print('[make_lmdb]: writing results to: %s' % args.outDir) print('') sys.stdout.flush() # Create the output database. # Multiply the actual size by a fudge factor to get a safe upper bound dbSize = (X.nbytes * args.tileSize * args.tileSize + Y.nbytes) * 10 env = lmdb.open(args.outDir, map_size=dbSize) # Extract all possible tiles. # This corresponds to extracting one "epoch" worth of tiles. tileId = 0 lastChatter = -1 tic = time.time() yCnt = np.zeros(sum(np.unique(Y) >= 0)) if np.any(Y > 0): # generates a balanced training data set (subsamples and shuffles) it = emlib.stratified_interior_pixel_generator(Y, tileRadius, nMiniBatch, omitLabels=[-1]) else: # enumerates all possible tiles in order (no shuffling) it = emlib.interior_pixel_generator(X, tileRadius, nMiniBatch) for Idx, epochPct in it: # respect upper bound on number of examples if tileId > args.maxNumExamples: print('[make_lmdb]: stopping at %d (max number of examples reached\n)' % (tileId-1)) break # Each mini-batch will be added to the database as a single transaction. with env.begin(write=True) as txn: # Translate indices Idx -> tiles Xi and labels yi. for jj in range(Idx.shape[0]): yi = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ] yi = int(yi) a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi = X[ Idx[jj,0], a:b, c:d ] assert(Xi.shape == (args.tileSize, args.tileSize)) datum = caffe.proto.caffe_pb2.Datum() datum.channels = 1 datum.height = Xi.shape[0] datum.width = Xi.shape[1] datum.data = Xi.tostring() # use tobytes() for newer numpy datum.label = yi strId = '{:08}'.format(tileId) txn.put(strId.encode('ascii'), datum.SerializeToString()) tileId += 1 yCnt[yi] += 1 # check early termination conditions if tileId > args.maxNumExamples: break #if np.floor(epochPct) > lastChatter: print('[make_lmdb] %% %0.2f done (%0.2f min; yCnt=%s)' % ((100*epochPct), (time.time() - tic)/60, str(yCnt))) lastChatter = epochPct
def _eval_cube(net, X, M, batchDim): """Uses Caffe to make predictions for the EM cube X.""" assert (batchDim[2] == batchDim[3]) # currently we assume tiles are square #-------------------------------------------------- # initialize variables and storage needed in the processing loop below #-------------------------------------------------- tileRadius = int(batchDim[2] / 2) Xi = np.zeros(batchDim, dtype=np.float32) yDummy = np.zeros((batchDim[0], ), dtype=np.float32) cnnTime = 0.0 # time spent doing core CNN operations nClasses = net.blobs['prob'].data.shape[ 1] # *** Assumes a layer called "prob" # allocate memory for return values # if we don't evaluate all pixels, the # ones not evaluated will have label -1 Yhat = -1 * np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2])) print "[deploy]: Yhat shape: %s" % str(Yhat.shape) sys.stdout.flush() #-------------------------------------------------- # process the cube #-------------------------------------------------- tic = time.time() lastChatter = None for Idx, pct in emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=M): # populate the mini-batch buffer for jj in range(Idx.shape[0]): a = Idx[jj, 1] - tileRadius b = Idx[jj, 1] + tileRadius + 1 c = Idx[jj, 2] - tileRadius d = Idx[jj, 2] + tileRadius + 1 Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d] # CNN forward pass _tmp = time.time() net.set_input_arrays(Xi, yDummy) out = net.forward() yiHat = out['prob'] cnnTime += time.time() - _tmp # On some version of Caffe, yiHat is (batchSize, nClasses, 1, 1) # On newer versions, it is natively (batchSize, nClasses) # The squeeze here is to accommodate older versions yiHat = np.squeeze(yiHat) # store the per-class probability estimates. # # * Note that on the final iteration, the size of yiHat may not match # the remaining space in Yhat (unless we get lucky and the data cube # size is a multiple of the mini-batch size). This is why we slice # yijHat before assigning to Yhat. for jj in range(nClasses): yijHat = yiHat[:, jj] # get slice containing probabilities for class j assert (len(yijHat.shape) == 1) # should be a vector (vs tensor) Yhat[jj, Idx[:, 0], Idx[:, 1], Idx[:, 2]] = yijHat[:Idx.shape[0]] # (*) # provide feedback on progress so far elapsed = (time.time() - tic) / 60. if (lastChatter is None) or ((elapsed - lastChatter) > 2): print( '[deploy]: %0.2f min elapsed (%0.2f CNN min, %0.2f%% complete)' % (elapsed, cnnTime / 60., 100. * pct)) sys.stdout.flush() lastChatter = elapsed # all done! print('[deploy]: Finished processing cube.') print('[deploy]: Net time was: %0.2f min (%0.2f CNN min)' % (elapsed, cnnTime / 60.)) return Yhat
def _eval_cube(net, X, M, batchDim, extractFeat=''): """ PARAMETERS: net - a Caffe network object X - A 3d tensor of data to evaluate M - A boolean 3d tensor mask; 1 := evaluate the corresponding pixel batchDim - Length 2 vector of tile [width,height]. RETURN VALUES: Yhat - a tensor with dimensions (#classes, ...) where "..." denotes data cube dimensions Xprime - a tensor with dimensions (#features, ...) where "..." denotes data cube dimensions """ assert (batchDim[2] == batchDim[3]) # currently we assume tiles are square #-------------------------------------------------- # initialize variables and storage needed in the processing loop below #-------------------------------------------------- tileRadius = int(batchDim[2] / 2) Xi = np.zeros(batchDim, dtype=np.float32) yDummy = np.zeros((batchDim[0], ), dtype=np.float32) cnnTime = 0.0 # time spent doing core CNN operations nClasses = net.blobs['prob'].data.shape[ 1] # *** Assumes a layer called "prob" # allocate memory for return values Yhat = -1 * np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2])) if len(extractFeat): nFeats = net.blobs[extractFeat].data.shape[1] Xprime = np.zeros((nFeats, X.shape[0], X.shape[1], X.shape[2])) else: Xprime = None print "[deploy]: Yhat shape: %s" % str(Yhat.shape) if Xprime is not None: print "[deploy]: Extracting features from layer: %s" % extractFeat print "[deploy]: Xprime shape: %s" % str( Xprime.shape) sys.stdout.flush() #-------------------------------------------------- # process the cube #-------------------------------------------------- tic = time.time() lastChatter = None for Idx, pct in emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=M): # populate the mini-batch buffer for jj in range(Idx.shape[0]): a = Idx[jj, 1] - tileRadius b = Idx[jj, 1] + tileRadius + 1 c = Idx[jj, 2] - tileRadius d = Idx[jj, 2] + tileRadius + 1 Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d] # CNN forward pass _tmp = time.time() net.set_input_arrays(Xi, yDummy) out = net.forward() yiHat = out['prob'] cnnTime += time.time() - _tmp # On some version of Caffe, yiHat is (batchSize, nClasses, 1, 1) # On newer versions, it is natively (batchSize, nClasses) # The squeeze here is to accommodate older versions yiHat = np.squeeze(yiHat) # store the per-class probability estimates. # # * Note that on the final iteration, the size of yiHat may not match # the remaining space in Yhat (unless we get lucky and the data cube # size is a multiple of the mini-batch size). This is why we slice # yijHat before assigning to Yhat. for jj in range(nClasses): yijHat = yiHat[:, jj] # get slice containing probabilities for class j assert (len(yijHat.shape) == 1) # should be a vector (vs tensor) Yhat[jj, Idx[:, 0], Idx[:, 1], Idx[:, 2]] = yijHat[:Idx.shape[0]] # (*) # for features: net.blobs['ip1'].data should be (100,200,1,1) for batch size 100, ip output size 200 if Xprime is not None: for jj in range(nFeats): Xprimejj = np.squeeze(net.blobs[extractFeat].data[:, jj, :, :] ) # feature jj, all objects assert (len(Xprimejj.shape) == 1 ) # should be a vector (vs tensor) Xprime[jj, Idx[:, 0], Idx[:, 1], Idx[:, 2]] = Xprimejj[:Idx.shape[0]] # provide feedback on progress so far elapsed = (time.time() - tic) / 60. if (lastChatter is None) or ((elapsed - lastChatter) > 2): lastChatter = elapsed print "[deploy]: processed pixel at index %s (%0.2f min elapsed; %0.2f CNN min)" % ( str(Idx[-1, :]), elapsed, cnnTime / 60.) sys.stdout.flush() print( '[deploy]: Finished processing cube. Net time was: %0.2f min (%0.2f CNN min)' % (elapsed, cnnTime / 60.)) return Yhat, Xprime
def predict(net, X, Mask, batchDim): """Generates predictions for a data volume. The data volume is assumed to be a tensor with shape: (#slices, width, height) """ # *** This code assumes a layer called "prob" if 'prob' not in net.blobs: raise RuntimeError("Can't find a layer with output called 'prob'") # Pre-allocate some variables & storage. # tileRadius = int(batchDim[2] / 2) Xi = np.zeros(batchDim, dtype=np.float32) yi = np.zeros((batchDim[0], ), dtype=np.float32) nClasses = net.blobs['prob'].data.shape[1] # if we don't evaluate all pixels, the # ones not evaluated will have label -1 Prob = -1 * np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2])) print "[emCNN]: Evaluating %0.2f%% of cube" % (100.0 * np.sum(Mask) / numel(Mask)) # do it tic = time.time() cnnTime = 0 lastChatter = -2 it = emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=Mask) for Idx, epochPct in it: # Extract subtiles from validation data set for jj in range(Idx.shape[0]): a = Idx[jj, 1] - tileRadius b = Idx[jj, 1] + tileRadius + 1 c = Idx[jj, 2] - tileRadius d = Idx[jj, 2] + tileRadius + 1 Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d] yi[jj] = 0 # this is just a dummy value #---------------------------------------- # one forward pass; no backward pass #---------------------------------------- _tmp = time.time() net.set_input_arrays(Xi, yi) out = net.forward() cnnTime += time.time() - _tmp # On some version of Caffe, Prob is (batchSize, nClasses, 1, 1) # On newer versions, it is natively (batchSize, nClasses) # The squeeze here is to accommodate older versions ProbBatch = np.squeeze(out['prob']) # store the per-class probability estimates. # # * Note that on the final iteration, the size of Prob may not match # the remaining space in Yhat (unless we get lucky and the data cube # size is a multiple of the mini-batch size). This is why we slice # yijHat before assigning to Yhat. for jj in range(nClasses): pj = ProbBatch[:, jj] # get probabilities for class j assert (len(pj.shape) == 1) # should be a vector (vs tensor) Prob[jj, Idx[:, 0], Idx[:, 1], Idx[:, 2]] = pj[:Idx.shape[0]] # (*) elapsed = time.time() - tic if (lastChatter + 2) < (elapsed / 60.): # notify progress every 2 min lastChatter = elapsed / 60. print('[emCNN]: elapsed=%0.2f min; %0.2f%% complete' % (elapsed / 60., 100. * epochPct)) # done elapsed = time.time() - tic print('[emCNN]: Total time to evaluate cube: %0.2f min (%0.2f CNN min)' % (elapsed / 60., cnnTime / 60.)) return Prob
def predict(net, X, Mask, batchDim, nMC=0): """Generates predictions for a data volume. PARAMETERS: X : a data volume/tensor with dimensions (#slices, height, width) Mask : a boolean tensor with the same size as X. Only positive elements will be classified. The prediction for all negative elements will be -1. Use this to run predictions on a subset of the volume. batchDim : a tuple of the form (#classes, minibatchSize, height, width) """ # *** This code assumes a layer called "prob" if 'prob' not in net.blobs: raise RuntimeError("Can't find a layer called 'prob'") print "[emCNN]: Evaluating %0.2f%% of cube" % (100.0*np.sum(Mask)/numel(Mask)) # Pre-allocate some variables & storage. # tileRadius = int(batchDim[2]/2) Xi = np.zeros(batchDim, dtype=np.float32) yi = np.zeros((batchDim[0],), dtype=np.float32) nClasses = net.blobs['prob'].data.shape[1] # if we don't evaluate all pixels, the # ones not evaluated will have label -1 if nMC <= 0: Prob = -1*np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2])) else: Prob = -1*np.ones((nMC, X.shape[0], X.shape[1], X.shape[2])) print "[emCNN]: Generating %d MC samples for class 0" % nMC if nClasses > 2: print "[emCNN]: !!!WARNING!!! nClasses > 2 but we are only extracting MC samples for class 0 at this time..." # do it startTime = time.time() cnnTime = 0 lastChatter = -2 it = emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=Mask) for Idx, epochPct in it: # Extract subtiles from validation data set for jj in range(Idx.shape[0]): a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ] yi[jj] = 0 # this is just a dummy value #---------------------------------------- # forward pass only (i.e. no backward pass) #---------------------------------------- if nMC <= 0: # this is the typical case - just one forward pass _tmp = time.time() net.set_input_arrays(Xi, yi) out = net.forward() cnnTime += time.time() - _tmp # On some version of Caffe, Prob is (batchSize, nClasses, 1, 1) # On newer versions, it is natively (batchSize, nClasses) # The squeeze here is to accommodate older versions ProbBatch = np.squeeze(out['prob']) # store the per-class probability estimates. # # * On the final iteration, the size of Prob may not match # the remaining space in Yhat (unless we get lucky and the # data cube size is a multiple of the mini-batch size). # This is why we slice yijHat before assigning to Yhat. for jj in range(nClasses): pj = ProbBatch[:,jj] # get probabilities for class j assert(len(pj.shape)==1) # should be a vector (vs tensor) Prob[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = pj[:Idx.shape[0]] # (*) else: # Generate MC-based uncertainty estimates # (instead of just a single point estimate) _tmp = time.time() net.set_input_arrays(Xi, yi) # do nMC forward passes and save the probability estimate # for class 0. for ii in range(nMC): out = net.forward() ProbBatch = np.squeeze(out['prob']) p0 = ProbBatch[:,0] # get probabilities for class 0 assert(len(p0.shape)==1) # should be a vector (vs tensor) Prob[ii, Idx[:,0], Idx[:,1], Idx[:,2]] = p0[:Idx.shape[0]] # (*) cnnTime += time.time() - _tmp elapsed = (time.time() - startTime) / 60.0 if (lastChatter+2) < elapsed: # notify progress every 2 min lastChatter = elapsed print('[emCNN]: elapsed=%0.2f min; %0.2f%% complete' % (elapsed, 100.*epochPct)) sys.stdout.flush() # done print('[emCNN]: Total time to evaluate cube: %0.2f min (%0.2f CNN min)' % (elapsed, cnnTime/60.)) return Prob
def _eval_cube(net, X, M, batchDim, extractFeat=''): """ PARAMETERS: net - a Caffe network object X - A 3d tensor of data to evaluate M - A boolean 3d tensor mask; 1 := evaluate the corresponding pixel batchDim - Length 2 vector of tile [width,height]. RETURN VALUES: Yhat - a tensor with dimensions (#classes, ...) where "..." denotes data cube dimensions Xprime - a tensor with dimensions (#features, ...) where "..." denotes data cube dimensions """ assert(batchDim[2] == batchDim[3]) # currently we assume tiles are square #-------------------------------------------------- # initialize variables and storage needed in the processing loop below #-------------------------------------------------- tileRadius = int(batchDim[2]/2) Xi = np.zeros(batchDim, dtype=np.float32) yDummy = np.zeros((batchDim[0],), dtype=np.float32) cnnTime = 0.0 # time spent doing core CNN operations nClasses = net.blobs['prob'].data.shape[1] # *** Assumes a layer called "prob" # allocate memory for return values Yhat = -1*np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2])) if len(extractFeat): nFeats = net.blobs[extractFeat].data.shape[1] Xprime = np.zeros((nFeats, X.shape[0], X.shape[1], X.shape[2])) else: Xprime = None print "[deploy]: Yhat shape: %s" % str(Yhat.shape) if Xprime is not None: print "[deploy]: Extracting features from layer: %s" % extractFeat print "[deploy]: Xprime shape: %s" % str(Xprime.shape) sys.stdout.flush() #-------------------------------------------------- # process the cube #-------------------------------------------------- tic = time.time() lastChatter = None for Idx, pct in emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=M): # populate the mini-batch buffer for jj in range(Idx.shape[0]): a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ] # CNN forward pass _tmp = time.time() net.set_input_arrays(Xi, yDummy) out = net.forward() yiHat = out['prob'] cnnTime += time.time() - _tmp # On some version of Caffe, yiHat is (batchSize, nClasses, 1, 1) # On newer versions, it is natively (batchSize, nClasses) # The squeeze here is to accommodate older versions yiHat = np.squeeze(yiHat) # store the per-class probability estimates. # # * Note that on the final iteration, the size of yiHat may not match # the remaining space in Yhat (unless we get lucky and the data cube # size is a multiple of the mini-batch size). This is why we slice # yijHat before assigning to Yhat. for jj in range(nClasses): yijHat = yiHat[:,jj] # get slice containing probabilities for class j assert(len(yijHat.shape)==1) # should be a vector (vs tensor) Yhat[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = yijHat[:Idx.shape[0]] # (*) # for features: net.blobs['ip1'].data should be (100,200,1,1) for batch size 100, ip output size 200 if Xprime is not None: for jj in range(nFeats): Xprimejj = np.squeeze(net.blobs[extractFeat].data[:,jj,:,:]) # feature jj, all objects assert(len(Xprimejj.shape)==1) # should be a vector (vs tensor) Xprime[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = Xprimejj[:Idx.shape[0]] # provide feedback on progress so far elapsed = (time.time() - tic) / 60. if (lastChatter is None) or ((elapsed - lastChatter) > 2): lastChatter = elapsed print "[deploy]: processed pixel at index %s (%0.2f min elapsed; %0.2f CNN min)" % (str(Idx[-1,:]), elapsed, cnnTime/60.) sys.stdout.flush() print('[deploy]: Finished processing cube. Net time was: %0.2f min (%0.2f CNN min)' % (elapsed, cnnTime/60.)) return Yhat, Xprime
def _training_loop(solver, X, Y, M, solverParam, batchDim, outDir, omitLabels=[], Xvalid=None, Yvalid=None, syn_func=None): """Main CNN training loop. """ assert(batchDim[2] == batchDim[3]) # tiles must be square # Some variables and storage that we'll use in the loop below # tileRadius = int(batchDim[2]/2) Xi = np.zeros(batchDim, dtype=np.float32) yi = np.zeros((batchDim[0],), dtype=np.float32) yMax = np.max(Y).astype(np.int32) losses = np.zeros((solverParam.max_iter,)) acc = np.zeros((solverParam.max_iter,)) currIter = 0 currEpoch = 0 # SGD parameters. SGD with momentum is of the form: # # V_{t+1} = \mu V_t - \alpha \nablaL(W_t) # W_{t+1} = W_t + V_{t+1} # # where W are the weights and V the previous update. # Ref: http://caffe.berkeleyvision.org/tutorial/solver.html # alpha = solverParam.base_lr # alpha := learning rate mu = solverParam.momentum # mu := momentum gamma = solverParam.gamma # gamma := step factor isModeStep = (solverParam.lr_policy == u'step') isTypeSGD = (solverParam.solver_type == solverParam.SolverType.Value('SGD')) Vall = {} # stores previous SGD steps (for all layers) if not (isModeStep and isTypeSGD): raise RuntimeError('Sorry - only support SGD "step" mode at the present') # TODO: weight decay # TODO: layer-specific weights cnnTime = 0.0 # time spent doing core CNN operations tic = time.time() while currIter < solverParam.max_iter: #-------------------------------------------------- # Each generator provides a single epoch's worth of data. # However, Caffe doesn't really recognize the notion of an epoch; instead, # they specify a number of training "iterations" (mini-batch evaluations, I assume). # So the inner loop below is for a single epoch, which we may terminate # early if the max # of iterations is reached. #-------------------------------------------------- currEpoch += 1 it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], mask=M, omitLabels=omitLabels) for Idx, epochPct in it: # Map the indices Idx -> tiles Xi and labels yi # # Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) a few examples # from the previous minibatch will be "recycled" here. This is intentional # (to keep batch sizes consistent even if data set size is not a multiple # of the minibatch size). # for jj in range(Idx.shape[0]): yi[jj] = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ] a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ] # label-preserving data transformation (synthetic data generation) if syn_func is not None: #Xi = _xform_minibatch(Xi) Xi = syn_func(Xi) #---------------------------------------- # one forward/backward pass and update weights # (SGD with momentum term) #---------------------------------------- _tmp = time.time() solver.net.set_input_arrays(Xi, yi) # XXX: could call preprocess() here? rv = solver.net.forward() solver.net.backward() for lIdx, layer in enumerate(solver.net.layers): for bIdx, blob in enumerate(layer.blobs): key = (lIdx, bIdx) V = Vall.get(key, 0.0) Vnext = mu*V - alpha * blob.diff blob.data[...] += Vnext Vall[key] = Vnext cnnTime += time.time() - _tmp # update running list of losses with the loss from this mini batch losses[currIter] = np.squeeze(rv['loss']) acc[currIter] = np.squeeze(rv['accuracy']) currIter += 1 #---------------------------------------- # Some events occur on mini-batch intervals. # Deal with those now. #---------------------------------------- if (currIter % solverParam.snapshot) == 0: fn = os.path.join(outDir, 'iter_%06d.caffemodel' % (currIter)) solver.net.save(str(fn)) if isModeStep and ((currIter % solverParam.stepsize) == 0): alpha *= gamma if (currIter % solverParam.display) == 1: elapsed = (time.time() - tic)/60. print "[train]: completed iteration %d (of %d; %0.2f min elapsed; %0.2f CNN min)" % (currIter, solverParam.max_iter, elapsed, cnnTime/60.) print "[train]: epoch: %d (%0.2f), loss: %0.3f, acc: %0.3f, learn rate: %0.3e" % (currEpoch, 100*epochPct, np.mean(losses[max(0,currIter-10):currIter]), np.mean(acc[max(0,currIter-10):currIter]), alpha) sys.stdout.flush() if currIter >= solverParam.max_iter: break # in case we hit max_iter on a non-epoch boundary #-------------------------------------------------- # After each training epoch is complete, if we have validation # data, evaluate it. # Note: this only requires forward passes through the network #-------------------------------------------------- if (Xvalid is not None) and (Xvalid.size != 0) and (Yvalid is not None) and (Yvalid.size != 0): # Mask out pixels whose label we don't care about. Mvalid = np.ones(Yvalid.shape, dtype=bool) for yIgnore in omitLabels: Mvalid[Yvalid==yIgnore] = False print "[train]: Evaluating on validation data (%d pixels)..." % np.sum(Mvalid) Confusion = np.zeros((yMax+1, yMax+1)) # confusion matrix it = emlib.interior_pixel_generator(Yvalid, tileRadius, batchDim[0], mask=Mvalid) for Idx, epochPct in it: # Extract subtiles from validation data set for jj in range(Idx.shape[0]): yi[jj] = Yvalid[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ] a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = Xvalid[ Idx[jj,0], a:b, c:d ] #---------------------------------------- # one forward pass; no backward pass #---------------------------------------- solver.net.set_input_arrays(Xi, yi) # XXX: could call preprocess() here? rv = solver.net.forward() # extract statistics Prob = np.squeeze(rv['prob']) # matrix of estimated probabilities for each object yHat = np.argmax(Prob,1) # estimated class is highest probability in vector for yTmp in range(yMax+1): # note: assumes class labels are in {0, 1,..,n_classes-1} bits = (yi.astype(np.int32) == yTmp) for jj in range(yMax+1): Confusion[yTmp,jj] += np.sum(yHat[bits]==jj) print '[train]: Validation results:' print ' %s' % str(Confusion) if yMax == 1: # Assume a binary classification problem where 0 is non-target and 1 is target. # # precision := TP / (TP + FP) # recall := TP / (TP + FN) # # Confusion Matrix: # yHat=0 yHat=1 # y=0 TN FP # y=1 FN TP # precision = (1.0*Confusion[1,1]) / np.sum(Confusion[:,1]) recall = (1.0*Confusion[1,1]) / np.sum(Confusion[1,:]) f1Score = (2.0 * precision * recall) / (precision + recall); print ' precision=%0.3f, recall=%0.3f' % (precision, recall) print ' F1=%0.3f' % f1Score else: print '[train]: Not using a validation data set' sys.stdout.flush() # ----- end one epoch ----- # complete print "[train]: all done!" return losses, acc
def predict(net, X, Mask, batchDim): """Generates predictions for a data volume. The data volume is assumed to be a tensor with shape: (#slices, width, height) """ # *** This code assumes a layer called "prob" if 'prob' not in net.blobs: raise RuntimeError("Can't find a layer with output called 'prob'") # Pre-allocate some variables & storage. # tileRadius = int(batchDim[2]/2) Xi = np.zeros(batchDim, dtype=np.float32) yi = np.zeros((batchDim[0],), dtype=np.float32) nClasses = net.blobs['prob'].data.shape[1] # if we don't evaluate all pixels, the # ones not evaluated will have label -1 Prob = -1*np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2])) print "[emCNN]: Evaluating %0.2f%% of cube" % (100.0*np.sum(Mask)/numel(Mask)) # do it tic = time.time() cnnTime = 0 lastChatter = -2 it = emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=Mask) for Idx, epochPct in it: # Extract subtiles from validation data set for jj in range(Idx.shape[0]): a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ] yi[jj] = 0 # this is just a dummy value #---------------------------------------- # one forward pass; no backward pass #---------------------------------------- _tmp = time.time() net.set_input_arrays(Xi, yi) out = net.forward() cnnTime += time.time() - _tmp # On some version of Caffe, Prob is (batchSize, nClasses, 1, 1) # On newer versions, it is natively (batchSize, nClasses) # The squeeze here is to accommodate older versions ProbBatch = np.squeeze(out['prob']) # store the per-class probability estimates. # # * Note that on the final iteration, the size of Prob may not match # the remaining space in Yhat (unless we get lucky and the data cube # size is a multiple of the mini-batch size). This is why we slice # yijHat before assigning to Yhat. for jj in range(nClasses): pj = ProbBatch[:,jj] # get probabilities for class j assert(len(pj.shape)==1) # should be a vector (vs tensor) Prob[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = pj[:Idx.shape[0]] # (*) elapsed = time.time() - tic if (lastChatter+2) < (elapsed/60.): # notify progress every 2 min lastChatter = elapsed/60. print('[emCNN]: elapsed=%0.2f min; %0.2f%% complete' % (elapsed/60., 100.*epochPct)) # done elapsed = time.time() - tic print('[emCNN]: Total time to evaluate cube: %0.2f min (%0.2f CNN min)' % (elapsed/60., cnnTime/60.)) return Prob
def _eval_cube(net, X, M, batchDim): """Uses Caffe to make predictions for the EM cube X.""" assert(batchDim[2] == batchDim[3]) # currently we assume tiles are square #-------------------------------------------------- # initialize variables and storage needed in the processing loop below #-------------------------------------------------- tileRadius = int(batchDim[2]/2) Xi = np.zeros(batchDim, dtype=np.float32) yDummy = np.zeros((batchDim[0],), dtype=np.float32) cnnTime = 0.0 # time spent doing core CNN operations nClasses = net.blobs['prob'].data.shape[1] # *** Assumes a layer called "prob" # allocate memory for return values # if we don't evaluate all pixels, the # ones not evaluated will have label -1 Yhat = -1*np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2])) print "[deploy]: Yhat shape: %s" % str(Yhat.shape) sys.stdout.flush() #-------------------------------------------------- # process the cube #-------------------------------------------------- tic = time.time() lastChatter = None for Idx, pct in emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=M): # populate the mini-batch buffer for jj in range(Idx.shape[0]): a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ] # CNN forward pass _tmp = time.time() net.set_input_arrays(Xi, yDummy) out = net.forward() yiHat = out['prob'] cnnTime += time.time() - _tmp # On some version of Caffe, yiHat is (batchSize, nClasses, 1, 1) # On newer versions, it is natively (batchSize, nClasses) # The squeeze here is to accommodate older versions yiHat = np.squeeze(yiHat) # store the per-class probability estimates. # # * Note that on the final iteration, the size of yiHat may not match # the remaining space in Yhat (unless we get lucky and the data cube # size is a multiple of the mini-batch size). This is why we slice # yijHat before assigning to Yhat. for jj in range(nClasses): yijHat = yiHat[:,jj] # get slice containing probabilities for class j assert(len(yijHat.shape)==1) # should be a vector (vs tensor) Yhat[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = yijHat[:Idx.shape[0]] # (*) # provide feedback on progress so far elapsed = (time.time() - tic) / 60. if (lastChatter is None) or ((elapsed - lastChatter) > 2): print('[deploy]: %0.2f min elapsed (%0.2f CNN min, %0.2f%% complete)' % (elapsed, cnnTime/60., 100.*pct)) sys.stdout.flush() lastChatter = elapsed # all done! print('[deploy]: Finished processing cube.') print('[deploy]: Net time was: %0.2f min (%0.2f CNN min)' % (elapsed, cnnTime/60.)) return Yhat