def test_interior_pixel_generator(self):
b = 10 # b := border size
Z = np.zeros((2, 100, 100), dtype=np.int32)
for idx, pct in emlib.interior_pixel_generator(Z, b, 30):
Z[idx[:, 0], idx[:, 1], idx[:, 2]] += 1
self.assertTrue(np.all(Z[:, b:-b, b:-b] == 1))
Z[:, b:-b, b:-b] = 0
self.assertTrue(np.all(Z == 0))
# Make sure it works with 4d tensors as well
Z = np.zeros((2, 3, 100, 100), dtype=np.int32)
for idx, pct in emlib.interior_pixel_generator(Z, b, 30):
Z[idx[:, 0], :, idx[:, 1], idx[:, 2]] += 1
self.assertTrue(np.all(Z[:, :, b:-b, b:-b] == 1))
Z[:, :, b:-b, b:-b] = 0
self.assertTrue(np.all(Z == 0))
def test_interior_pixel_generator(self):
b = 10 # b := border size
Z = np.zeros((2,100,100), dtype=np.int32)
for idx, pct in emlib.interior_pixel_generator(Z,b,30):
Z[idx[:,0],idx[:,1],idx[:,2]] += 1
self.assertTrue(np.all(Z[:,b:-b,b:-b]==1))
Z[:,b:-b,b:-b] = 0
self.assertTrue(np.all(Z==0))
def _eval_cube(net, X, M, batchDim, bandSpec):
assert(batchDim[2] == batchDim[3]) # tiles must be square
# some variables and storage needed in the processing loop below
tileRadius = int(batchDim[2]/2)
Xi = np.zeros(batchDim, dtype=np.float32)
yDummy = np.zeros((batchDim[0],), dtype=np.float32)
cnnTime = 0.0 # time spent doing core CNN operations
Yhat = None
# process the cube
tic = time.time()
lastChatter = None
for Idx, pct in emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=M):
# populate the mini-batch buffer
for jj in range(Idx.shape[0]):
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ]
_tmp = time.time()
net.set_input_arrays(Xi, yDummy)
# XXX: could call preprocess() here?
out = net.forward()
yiHat = out['prob']
cnnTime += time.time() - _tmp
nClasses = yiHat.shape[1]
if Yhat is None: # on first iteration, create Yhat with the appropriate shape
Yhat = -1*np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2]))
# store the per-class probability estimates.
# Note that on the final iteration, the size of yiHat may not match
# the remaining space in Yhat (unless we get lucky and the data cube
# size is a multiple of the mini-batch size). This is why we slice
# yijHat before assigning to Yhat.
for jj in range(nClasses):
yijHat = np.squeeze(yiHat[:,jj,:,:]) # get slice containing probabilities for class j
assert(len(yijHat.shape)==1) # should be a single vector now
Yhat[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = yijHat[:Idx.shape[0]]
# provide feedback on progress so far
elapsed = (time.time() - tic) / 60.
if (lastChatter is None) or ((elapsed - lastChatter) > 2):
lastChatter = elapsed
print "[deploy]: processed pixel at index %s (%0.2f min elapsed; %0.2f CNN min)" % (str(Idx[-1,:]), elapsed, cnnTime/60.)
sys.stdout.flush()
print('[deploy]: Finished processing cube. Net time was: %0.2f min (%0.2f CNN min)' % (elapsed, cnnTime/60.))
return Yhat
def _evaluate(model, X, log=None, batchSize=100, evalPct=1.0):
"""Evaluate model on held-out data.
Returns:
Prob : a tensor of per-pixel probability estimates with dimensions:
(#layers, #classes, width, height)
"""
#----------------------------------------
# Pre-allocate some variables & storage.
#----------------------------------------
nChannels, tileRows, tileCols = model.input_shape[1:4]
ste = emlib.SimpleTileExtractor(tileRows, X)
lastChatter = -2
startTime = time.time()
# identify subset of volume to evaluate
Mask = _downsample_mask(X, evalPct)
if log:
log.info('after masking, will evaluate %0.2f%% of data' %
(100.0 * np.sum(Mask) / Mask.size))
# Create storage for class probabilities.
# Note that we store all class probabilities, even if this
# is a binary classification problem (in which case p(1) = 1 - p(0)).
# We do this to support multiclass classification seamlessly.
[numZ, numChan, numRows, numCols] = X.shape
numClasses = model.output_shape[-1]
Prob = -1 * np.ones([numZ, numClasses, numRows, numCols], dtype=np.float32)
#----------------------------------------
# Loop over mini-batches
#----------------------------------------
# note: set tileRadius to 0 so we evaluate whole volume
it = emlib.interior_pixel_generator(X, 0, batchSize, mask=Mask)
for mbIdx, (Idx, epochPct) in enumerate(it):
n = Idx.shape[0] # may be < batchSize on final iteration
Xi = ste.extract(Idx)
prob = model.predict_on_batch(Xi)
# updated based on Keras API changes
#Prob[Idx[:,0], :, Idx[:,1], Idx[:,2]] = prob[0][:n,:]
Prob[Idx[:, 0], :, Idx[:, 1], Idx[:, 2]] = prob[:n, :]
# notify user re. progress
elapsed = (time.time() - startTime) / 60.0
if (lastChatter + 2) < elapsed:
lastChatter = elapsed
if log:
log.info(" last pixel %s (%0.2f%% complete)" %
(str(Idx[-1, :]), 100. * epochPct))
return Prob
def _evaluate(model, X, log=None, batchSize=100, evalPct=1.0):
"""Evaluate model on held-out data.
Returns:
Prob : a tensor of per-pixel probability estimates with dimensions:
(#layers, #classes, width, height)
"""
#----------------------------------------
# Pre-allocate some variables & storage.
#----------------------------------------
nChannels, tileRows, tileCols = model.input_shape[1:4]
ste = emlib.SimpleTileExtractor(tileRows, X)
lastChatter = -2
startTime = time.time()
# identify subset of volume to evaluate
Mask = _downsample_mask(X, evalPct)
if log: log.info('after masking, will evaluate %0.2f%% of data' % (100.0*np.sum(Mask)/Mask.size))
# Create storage for class probabilities.
# Note that we store all class probabilities, even if this
# is a binary classification problem (in which case p(1) = 1 - p(0)).
# We do this to support multiclass classification seamlessly.
[numZ, numChan, numRows, numCols] = X.shape
numClasses = model.output_shape[-1]
Prob = -1*np.ones([numZ, numClasses, numRows, numCols], dtype=np.float32)
#----------------------------------------
# Loop over mini-batches
#----------------------------------------
# note: set tileRadius to 0 so we evaluate whole volume
it = emlib.interior_pixel_generator(X, 0, batchSize, mask=Mask)
for mbIdx, (Idx, epochPct) in enumerate(it):
n = Idx.shape[0] # may be < batchSize on final iteration
Xi = ste.extract(Idx)
prob = model.predict_on_batch(Xi)
# updated based on Keras API changes
#Prob[Idx[:,0], :, Idx[:,1], Idx[:,2]] = prob[0][:n,:]
Prob[Idx[:,0], :, Idx[:,1], Idx[:,2]] = prob[:n,:]
# notify user re. progress
elapsed = (time.time() - startTime) / 60.0
if (lastChatter+2) < elapsed:
lastChatter = elapsed
if log: log.info(" last pixel %s (%0.2f%% complete)" % (str(Idx[-1,:]), 100.*epochPct))
return Prob
def _evaluate(model, X, Y, omitLabels=[], batchSize=100, log=None):
"""Evaluate model on held-out data. Here, used to periodically
report performance on validation data.
"""
#----------------------------------------
# Pre-allocate some variables & storage.
#----------------------------------------
nChannels, tileRows, tileCols = model.input_shape[1:4]
tileRadius = int(tileRows/2)
ste = emlib.SimpleTileExtractor(tileRows, X)
numClasses = model.output_shape[-1]
[numZ, numChan, numRows, numCols] = X.shape
Prob = np.nan * np.ones([numZ, numClasses, numRows, numCols],
dtype=np.float32)
#----------------------------------------
# Loop over mini-batches
#----------------------------------------
it = emlib.interior_pixel_generator(X, tileRadius, batchSize)
for mbIdx, (Idx, epochPct) in enumerate(it):
n = Idx.shape[0] # may be < batchSize on final iteration
Xi = ste.extract(Idx)
prob = model.predict_on_batch(Xi)
# Note: it seems the Keras API has changed...
#Prob[Idx[:,0], :, Idx[:,1], Idx[:,2]] = prob[0][:n,:]
Prob[Idx[:,0], :, Idx[:,1], Idx[:,2]] = prob[:n,:]
# Evaluate accuracy only on the subset of pixels that:
# o were actually provided to the CNN (not downsampled)
# o have a label that should be evaluated
#
# The mask tensor M will indicate which pixels to consider.
M = np.all(np.isfinite(Prob), axis=1)
for om in omitLabels:
M[Y==om] = False
Yhat = np.argmax(Prob, axis=1) # probabilities -> class labels
acc = 100.0 * np.sum(Yhat[M] == Y[M]) / np.sum(M)
return Prob, acc
def _training_loop(solver,
X,
Y,
M,
solverParam,
batchDim,
outDir,
omitLabels=[],
Xvalid=None,
Yvalid=None,
syn_func=None):
"""Main CNN training loop.
"""
assert (batchDim[2] == batchDim[3]) # tiles must be square
# Some variables and storage that we'll use in the loop below
#
tileRadius = int(batchDim[2] / 2)
Xi = np.zeros(batchDim, dtype=np.float32)
yi = np.zeros((batchDim[0], ), dtype=np.float32)
yMax = np.max(Y).astype(np.int32)
losses = np.zeros((solverParam.max_iter, ))
acc = np.zeros((solverParam.max_iter, ))
currIter = 0
currEpoch = 0
# SGD parameters. SGD with momentum is of the form:
#
# V_{t+1} = \mu V_t - \alpha \nablaL(W_t)
# W_{t+1} = W_t + V_{t+1}
#
# where W are the weights and V the previous update.
# Ref: http://caffe.berkeleyvision.org/tutorial/solver.html
#
alpha = solverParam.base_lr # alpha := learning rate
mu = solverParam.momentum # mu := momentum
gamma = solverParam.gamma # gamma := step factor
isModeStep = (solverParam.lr_policy == u'step')
isTypeSGD = (
solverParam.solver_type == solverParam.SolverType.Value('SGD'))
Vall = {} # stores previous SGD steps (for all layers)
if not (isModeStep and isTypeSGD):
raise RuntimeError(
'Sorry - only support SGD "step" mode at the present')
# TODO: weight decay
# TODO: layer-specific weights
cnnTime = 0.0 # time spent doing core CNN operations
tic = time.time()
while currIter < solverParam.max_iter:
#--------------------------------------------------
# Each generator provides a single epoch's worth of data.
# However, Caffe doesn't really recognize the notion of an epoch; instead,
# they specify a number of training "iterations" (mini-batch evaluations, I assume).
# So the inner loop below is for a single epoch, which we may terminate
# early if the max # of iterations is reached.
#--------------------------------------------------
currEpoch += 1
it = emlib.stratified_interior_pixel_generator(Y,
tileRadius,
batchDim[0],
mask=M,
omitLabels=omitLabels)
for Idx, epochPct in it:
# Map the indices Idx -> tiles Xi and labels yi
#
# Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) a few examples
# from the previous minibatch will be "recycled" here. This is intentional
# (to keep batch sizes consistent even if data set size is not a multiple
# of the minibatch size).
#
for jj in range(Idx.shape[0]):
yi[jj] = Y[Idx[jj, 0], Idx[jj, 1], Idx[jj, 2]]
a = Idx[jj, 1] - tileRadius
b = Idx[jj, 1] + tileRadius + 1
c = Idx[jj, 2] - tileRadius
d = Idx[jj, 2] + tileRadius + 1
Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d]
# label-preserving data transformation (synthetic data generation)
if syn_func is not None:
#Xi = _xform_minibatch(Xi)
Xi = syn_func(Xi)
#----------------------------------------
# one forward/backward pass and update weights
# (SGD with momentum term)
#----------------------------------------
_tmp = time.time()
solver.net.set_input_arrays(Xi, yi)
# XXX: could call preprocess() here?
rv = solver.net.forward()
solver.net.backward()
for lIdx, layer in enumerate(solver.net.layers):
for bIdx, blob in enumerate(layer.blobs):
key = (lIdx, bIdx)
V = Vall.get(key, 0.0)
Vnext = mu * V - alpha * blob.diff
blob.data[...] += Vnext
Vall[key] = Vnext
cnnTime += time.time() - _tmp
# update running list of losses with the loss from this mini batch
losses[currIter] = np.squeeze(rv['loss'])
acc[currIter] = np.squeeze(rv['accuracy'])
currIter += 1
#----------------------------------------
# Some events occur on mini-batch intervals.
# Deal with those now.
#----------------------------------------
if (currIter % solverParam.snapshot) == 0:
fn = os.path.join(outDir, 'iter_%06d.caffemodel' % (currIter))
solver.net.save(str(fn))
if isModeStep and ((currIter % solverParam.stepsize) == 0):
alpha *= gamma
if (currIter % solverParam.display) == 1:
elapsed = (time.time() - tic) / 60.
print "[train]: completed iteration %d (of %d; %0.2f min elapsed; %0.2f CNN min)" % (
currIter, solverParam.max_iter, elapsed, cnnTime / 60.)
print "[train]: epoch: %d (%0.2f), loss: %0.3f, acc: %0.3f, learn rate: %0.3e" % (
currEpoch, 100 * epochPct,
np.mean(losses[max(0, currIter - 10):currIter]),
np.mean(acc[max(0, currIter - 10):currIter]), alpha)
sys.stdout.flush()
if currIter >= solverParam.max_iter:
break # in case we hit max_iter on a non-epoch boundary
#--------------------------------------------------
# After each training epoch is complete, if we have validation
# data, evaluate it.
# Note: this only requires forward passes through the network
#--------------------------------------------------
if (Xvalid is not None) and (Xvalid.size != 0) and (
Yvalid is not None) and (Yvalid.size != 0):
# Mask out pixels whose label we don't care about.
Mvalid = np.ones(Yvalid.shape, dtype=bool)
for yIgnore in omitLabels:
Mvalid[Yvalid == yIgnore] = False
print "[train]: Evaluating on validation data (%d pixels)..." % np.sum(
Mvalid)
Confusion = np.zeros((yMax + 1, yMax + 1)) # confusion matrix
it = emlib.interior_pixel_generator(Yvalid,
tileRadius,
batchDim[0],
mask=Mvalid)
for Idx, epochPct in it:
# Extract subtiles from validation data set
for jj in range(Idx.shape[0]):
yi[jj] = Yvalid[Idx[jj, 0], Idx[jj, 1], Idx[jj, 2]]
a = Idx[jj, 1] - tileRadius
b = Idx[jj, 1] + tileRadius + 1
c = Idx[jj, 2] - tileRadius
d = Idx[jj, 2] + tileRadius + 1
Xi[jj, 0, :, :] = Xvalid[Idx[jj, 0], a:b, c:d]
#----------------------------------------
# one forward pass; no backward pass
#----------------------------------------
solver.net.set_input_arrays(Xi, yi)
# XXX: could call preprocess() here?
rv = solver.net.forward()
# extract statistics
Prob = np.squeeze(
rv['prob']
) # matrix of estimated probabilities for each object
yHat = np.argmax(
Prob,
1) # estimated class is highest probability in vector
for yTmp in range(
yMax + 1
): # note: assumes class labels are in {0, 1,..,n_classes-1}
bits = (yi.astype(np.int32) == yTmp)
for jj in range(yMax + 1):
Confusion[yTmp, jj] += np.sum(yHat[bits] == jj)
print '[train]: Validation results:'
print ' %s' % str(Confusion)
if yMax == 1:
# Assume a binary classification problem where 0 is non-target and 1 is target.
#
# precision := TP / (TP + FP)
# recall := TP / (TP + FN)
#
# Confusion Matrix:
# yHat=0 yHat=1
# y=0 TN FP
# y=1 FN TP
#
precision = (1.0 * Confusion[1, 1]) / np.sum(Confusion[:, 1])
recall = (1.0 * Confusion[1, 1]) / np.sum(Confusion[1, :])
f1Score = (2.0 * precision * recall) / (precision + recall)
print ' precision=%0.3f, recall=%0.3f' % (precision, recall)
print ' F1=%0.3f' % f1Score
else:
print '[train]: Not using a validation data set'
sys.stdout.flush()
# ----- end one epoch -----
# complete
print "[train]: all done!"
return losses, acc
def main(args):
tileRadius = np.floor(args.tileSize/2)
nMiniBatch = 1000 # here, a "mini-batch" specifies LMDB transaction size
# make sure we don't clobber an existing output
if os.path.exists(args.outDir):
raise RuntimeError('Output path "%s" already exists; please move out of the way and try again' % args.outDir)
# load the data volumes (EM image and labels, if any)
print('[make_lmdb]: loading EM data file: %s' % args.emFileName)
X = emlib.load_cube(args.emFileName, np.float32)
if args.labelsFileName:
print('[make_lmdb]: loading labels file: %s' % args.labelsFileName)
Y = emlib.load_cube(args.labelsFileName, np.float32)
Y = emlib.fix_class_labels(Y, eval(args.omitLabels))
assert(Y.shape == X.shape)
else:
print('[make_lmdb]: no labels file; assuming this is a test volume')
Y = np.zeros(X.shape)
# usually we expect fewer slices in Z than pixels in X or Y.
# Make sure the dimensions look ok before proceeding.
assert(X.shape[0] < X.shape[1])
assert(X.shape[0] < X.shape[2])
# Identify the subset of the data to use for training.
# (default is to use it all)
if len(args.slicesExpr):
sliceIdx = eval(args.slicesExpr)
X = X[sliceIdx, :, :] # python puts the z dimension first...
Y = Y[sliceIdx, :, :]
X = X.astype(np.uint8) # critical!! otherwise, Caffe just flails...
print('[make_lmdb]: EM volume shape: %s' % str(X.shape))
print('[make_lmdb]: yAll is %s' % np.unique(Y))
print('[make_lmdb]: %0.2f%% pixels will be omitted' % (100.0*np.sum(Y==-1)/numel(Y)))
print('[make_lmdb]: writing results to: %s' % args.outDir)
print('')
sys.stdout.flush()
# Create the output database.
# Multiply the actual size by a fudge factor to get a safe upper bound
dbSize = (X.nbytes * args.tileSize * args.tileSize + Y.nbytes) * 10
env = lmdb.open(args.outDir, map_size=dbSize)
# Extract all possible tiles.
# This corresponds to extracting one "epoch" worth of tiles.
tileId = 0
lastChatter = -1
tic = time.time()
yCnt = np.zeros(sum(np.unique(Y) >= 0))
if np.any(Y > 0):
# generates a balanced training data set (subsamples and shuffles)
it = emlib.stratified_interior_pixel_generator(Y, tileRadius, nMiniBatch, omitLabels=[-1])
else:
# enumerates all possible tiles in order (no shuffling)
it = emlib.interior_pixel_generator(X, tileRadius, nMiniBatch)
for Idx, epochPct in it:
# respect upper bound on number of examples
if tileId > args.maxNumExamples:
print('[make_lmdb]: stopping at %d (max number of examples reached\n)' % (tileId-1))
break
# Each mini-batch will be added to the database as a single transaction.
with env.begin(write=True) as txn:
# Translate indices Idx -> tiles Xi and labels yi.
for jj in range(Idx.shape[0]):
yi = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ]
yi = int(yi)
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi = X[ Idx[jj,0], a:b, c:d ]
assert(Xi.shape == (args.tileSize, args.tileSize))
datum = caffe.proto.caffe_pb2.Datum()
datum.channels = 1
datum.height = Xi.shape[0]
datum.width = Xi.shape[1]
datum.data = Xi.tostring() # use tobytes() for newer numpy
datum.label = yi
strId = '{:08}'.format(tileId)
txn.put(strId.encode('ascii'), datum.SerializeToString())
tileId += 1
yCnt[yi] += 1
# check early termination conditions
if tileId > args.maxNumExamples:
break
#if np.floor(epochPct) > lastChatter:
print('[make_lmdb] %% %0.2f done (%0.2f min; yCnt=%s)' % ((100*epochPct), (time.time() - tic)/60, str(yCnt)))
lastChatter = epochPct
def _eval_cube(net, X, M, batchDim):
"""Uses Caffe to make predictions for the EM cube X."""
assert (batchDim[2] == batchDim[3]) # currently we assume tiles are square
#--------------------------------------------------
# initialize variables and storage needed in the processing loop below
#--------------------------------------------------
tileRadius = int(batchDim[2] / 2)
Xi = np.zeros(batchDim, dtype=np.float32)
yDummy = np.zeros((batchDim[0], ), dtype=np.float32)
cnnTime = 0.0 # time spent doing core CNN operations
nClasses = net.blobs['prob'].data.shape[
1] # *** Assumes a layer called "prob"
# allocate memory for return values
# if we don't evaluate all pixels, the
# ones not evaluated will have label -1
Yhat = -1 * np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2]))
print "[deploy]: Yhat shape: %s" % str(Yhat.shape)
sys.stdout.flush()
#--------------------------------------------------
# process the cube
#--------------------------------------------------
tic = time.time()
lastChatter = None
for Idx, pct in emlib.interior_pixel_generator(X,
tileRadius,
batchDim[0],
mask=M):
# populate the mini-batch buffer
for jj in range(Idx.shape[0]):
a = Idx[jj, 1] - tileRadius
b = Idx[jj, 1] + tileRadius + 1
c = Idx[jj, 2] - tileRadius
d = Idx[jj, 2] + tileRadius + 1
Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d]
# CNN forward pass
_tmp = time.time()
net.set_input_arrays(Xi, yDummy)
out = net.forward()
yiHat = out['prob']
cnnTime += time.time() - _tmp
# On some version of Caffe, yiHat is (batchSize, nClasses, 1, 1)
# On newer versions, it is natively (batchSize, nClasses)
# The squeeze here is to accommodate older versions
yiHat = np.squeeze(yiHat)
# store the per-class probability estimates.
#
# * Note that on the final iteration, the size of yiHat may not match
# the remaining space in Yhat (unless we get lucky and the data cube
# size is a multiple of the mini-batch size). This is why we slice
# yijHat before assigning to Yhat.
for jj in range(nClasses):
yijHat = yiHat[:,
jj] # get slice containing probabilities for class j
assert (len(yijHat.shape) == 1) # should be a vector (vs tensor)
Yhat[jj, Idx[:, 0], Idx[:, 1],
Idx[:, 2]] = yijHat[:Idx.shape[0]] # (*)
# provide feedback on progress so far
elapsed = (time.time() - tic) / 60.
if (lastChatter is None) or ((elapsed - lastChatter) > 2):
print(
'[deploy]: %0.2f min elapsed (%0.2f CNN min, %0.2f%% complete)'
% (elapsed, cnnTime / 60., 100. * pct))
sys.stdout.flush()
lastChatter = elapsed
# all done!
print('[deploy]: Finished processing cube.')
print('[deploy]: Net time was: %0.2f min (%0.2f CNN min)' %
(elapsed, cnnTime / 60.))
return Yhat
def _eval_cube(net, X, M, batchDim, extractFeat=''):
"""
PARAMETERS:
net - a Caffe network object
X - A 3d tensor of data to evaluate
M - A boolean 3d tensor mask; 1 := evaluate the corresponding pixel
batchDim - Length 2 vector of tile [width,height].
RETURN VALUES:
Yhat - a tensor with dimensions (#classes, ...) where "..." denotes data cube dimensions
Xprime - a tensor with dimensions (#features, ...) where "..." denotes data cube dimensions
"""
assert (batchDim[2] == batchDim[3]) # currently we assume tiles are square
#--------------------------------------------------
# initialize variables and storage needed in the processing loop below
#--------------------------------------------------
tileRadius = int(batchDim[2] / 2)
Xi = np.zeros(batchDim, dtype=np.float32)
yDummy = np.zeros((batchDim[0], ), dtype=np.float32)
cnnTime = 0.0 # time spent doing core CNN operations
nClasses = net.blobs['prob'].data.shape[
1] # *** Assumes a layer called "prob"
# allocate memory for return values
Yhat = -1 * np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2]))
if len(extractFeat):
nFeats = net.blobs[extractFeat].data.shape[1]
Xprime = np.zeros((nFeats, X.shape[0], X.shape[1], X.shape[2]))
else:
Xprime = None
print "[deploy]: Yhat shape: %s" % str(Yhat.shape)
if Xprime is not None:
print "[deploy]: Extracting features from layer: %s" % extractFeat
print "[deploy]: Xprime shape: %s" % str(
Xprime.shape)
sys.stdout.flush()
#--------------------------------------------------
# process the cube
#--------------------------------------------------
tic = time.time()
lastChatter = None
for Idx, pct in emlib.interior_pixel_generator(X,
tileRadius,
batchDim[0],
mask=M):
# populate the mini-batch buffer
for jj in range(Idx.shape[0]):
a = Idx[jj, 1] - tileRadius
b = Idx[jj, 1] + tileRadius + 1
c = Idx[jj, 2] - tileRadius
d = Idx[jj, 2] + tileRadius + 1
Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d]
# CNN forward pass
_tmp = time.time()
net.set_input_arrays(Xi, yDummy)
out = net.forward()
yiHat = out['prob']
cnnTime += time.time() - _tmp
# On some version of Caffe, yiHat is (batchSize, nClasses, 1, 1)
# On newer versions, it is natively (batchSize, nClasses)
# The squeeze here is to accommodate older versions
yiHat = np.squeeze(yiHat)
# store the per-class probability estimates.
#
# * Note that on the final iteration, the size of yiHat may not match
# the remaining space in Yhat (unless we get lucky and the data cube
# size is a multiple of the mini-batch size). This is why we slice
# yijHat before assigning to Yhat.
for jj in range(nClasses):
yijHat = yiHat[:,
jj] # get slice containing probabilities for class j
assert (len(yijHat.shape) == 1) # should be a vector (vs tensor)
Yhat[jj, Idx[:, 0], Idx[:, 1],
Idx[:, 2]] = yijHat[:Idx.shape[0]] # (*)
# for features: net.blobs['ip1'].data should be (100,200,1,1) for batch size 100, ip output size 200
if Xprime is not None:
for jj in range(nFeats):
Xprimejj = np.squeeze(net.blobs[extractFeat].data[:, jj, :, :]
) # feature jj, all objects
assert (len(Xprimejj.shape) == 1
) # should be a vector (vs tensor)
Xprime[jj, Idx[:, 0], Idx[:, 1],
Idx[:, 2]] = Xprimejj[:Idx.shape[0]]
# provide feedback on progress so far
elapsed = (time.time() - tic) / 60.
if (lastChatter is None) or ((elapsed - lastChatter) > 2):
lastChatter = elapsed
print "[deploy]: processed pixel at index %s (%0.2f min elapsed; %0.2f CNN min)" % (
str(Idx[-1, :]), elapsed, cnnTime / 60.)
sys.stdout.flush()
print(
'[deploy]: Finished processing cube. Net time was: %0.2f min (%0.2f CNN min)'
% (elapsed, cnnTime / 60.))
return Yhat, Xprime
def predict(net, X, Mask, batchDim):
"""Generates predictions for a data volume.
The data volume is assumed to be a tensor with shape:
(#slices, width, height)
"""
# *** This code assumes a layer called "prob"
if 'prob' not in net.blobs:
raise RuntimeError("Can't find a layer with output called 'prob'")
# Pre-allocate some variables & storage.
#
tileRadius = int(batchDim[2] / 2)
Xi = np.zeros(batchDim, dtype=np.float32)
yi = np.zeros((batchDim[0], ), dtype=np.float32)
nClasses = net.blobs['prob'].data.shape[1]
# if we don't evaluate all pixels, the
# ones not evaluated will have label -1
Prob = -1 * np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2]))
print "[emCNN]: Evaluating %0.2f%% of cube" % (100.0 * np.sum(Mask) /
numel(Mask))
# do it
tic = time.time()
cnnTime = 0
lastChatter = -2
it = emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=Mask)
for Idx, epochPct in it:
# Extract subtiles from validation data set
for jj in range(Idx.shape[0]):
a = Idx[jj, 1] - tileRadius
b = Idx[jj, 1] + tileRadius + 1
c = Idx[jj, 2] - tileRadius
d = Idx[jj, 2] + tileRadius + 1
Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d]
yi[jj] = 0 # this is just a dummy value
#----------------------------------------
# one forward pass; no backward pass
#----------------------------------------
_tmp = time.time()
net.set_input_arrays(Xi, yi)
out = net.forward()
cnnTime += time.time() - _tmp
# On some version of Caffe, Prob is (batchSize, nClasses, 1, 1)
# On newer versions, it is natively (batchSize, nClasses)
# The squeeze here is to accommodate older versions
ProbBatch = np.squeeze(out['prob'])
# store the per-class probability estimates.
#
# * Note that on the final iteration, the size of Prob may not match
# the remaining space in Yhat (unless we get lucky and the data cube
# size is a multiple of the mini-batch size). This is why we slice
# yijHat before assigning to Yhat.
for jj in range(nClasses):
pj = ProbBatch[:, jj] # get probabilities for class j
assert (len(pj.shape) == 1) # should be a vector (vs tensor)
Prob[jj, Idx[:, 0], Idx[:, 1], Idx[:,
2]] = pj[:Idx.shape[0]] # (*)
elapsed = time.time() - tic
if (lastChatter + 2) < (elapsed / 60.): # notify progress every 2 min
lastChatter = elapsed / 60.
print('[emCNN]: elapsed=%0.2f min; %0.2f%% complete' %
(elapsed / 60., 100. * epochPct))
# done
elapsed = time.time() - tic
print('[emCNN]: Total time to evaluate cube: %0.2f min (%0.2f CNN min)' %
(elapsed / 60., cnnTime / 60.))
return Prob
def predict(net, X, Mask, batchDim, nMC=0):
"""Generates predictions for a data volume.
PARAMETERS:
X : a data volume/tensor with dimensions (#slices, height, width)
Mask : a boolean tensor with the same size as X. Only positive
elements will be classified. The prediction for all
negative elements will be -1. Use this to run predictions
on a subset of the volume.
batchDim : a tuple of the form (#classes, minibatchSize, height, width)
"""
# *** This code assumes a layer called "prob"
if 'prob' not in net.blobs:
raise RuntimeError("Can't find a layer called 'prob'")
print "[emCNN]: Evaluating %0.2f%% of cube" % (100.0*np.sum(Mask)/numel(Mask))
# Pre-allocate some variables & storage.
#
tileRadius = int(batchDim[2]/2)
Xi = np.zeros(batchDim, dtype=np.float32)
yi = np.zeros((batchDim[0],), dtype=np.float32)
nClasses = net.blobs['prob'].data.shape[1]
# if we don't evaluate all pixels, the
# ones not evaluated will have label -1
if nMC <= 0:
Prob = -1*np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2]))
else:
Prob = -1*np.ones((nMC, X.shape[0], X.shape[1], X.shape[2]))
print "[emCNN]: Generating %d MC samples for class 0" % nMC
if nClasses > 2:
print "[emCNN]: !!!WARNING!!! nClasses > 2 but we are only extracting MC samples for class 0 at this time..."
# do it
startTime = time.time()
cnnTime = 0
lastChatter = -2
it = emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=Mask)
for Idx, epochPct in it:
# Extract subtiles from validation data set
for jj in range(Idx.shape[0]):
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ]
yi[jj] = 0 # this is just a dummy value
#----------------------------------------
# forward pass only (i.e. no backward pass)
#----------------------------------------
if nMC <= 0:
# this is the typical case - just one forward pass
_tmp = time.time()
net.set_input_arrays(Xi, yi)
out = net.forward()
cnnTime += time.time() - _tmp
# On some version of Caffe, Prob is (batchSize, nClasses, 1, 1)
# On newer versions, it is natively (batchSize, nClasses)
# The squeeze here is to accommodate older versions
ProbBatch = np.squeeze(out['prob'])
# store the per-class probability estimates.
#
# * On the final iteration, the size of Prob may not match
# the remaining space in Yhat (unless we get lucky and the
# data cube size is a multiple of the mini-batch size).
# This is why we slice yijHat before assigning to Yhat.
for jj in range(nClasses):
pj = ProbBatch[:,jj] # get probabilities for class j
assert(len(pj.shape)==1) # should be a vector (vs tensor)
Prob[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = pj[:Idx.shape[0]] # (*)
else:
# Generate MC-based uncertainty estimates
# (instead of just a single point estimate)
_tmp = time.time()
net.set_input_arrays(Xi, yi)
# do nMC forward passes and save the probability estimate
# for class 0.
for ii in range(nMC):
out = net.forward()
ProbBatch = np.squeeze(out['prob'])
p0 = ProbBatch[:,0] # get probabilities for class 0
assert(len(p0.shape)==1) # should be a vector (vs tensor)
Prob[ii, Idx[:,0], Idx[:,1], Idx[:,2]] = p0[:Idx.shape[0]] # (*)
cnnTime += time.time() - _tmp
elapsed = (time.time() - startTime) / 60.0
if (lastChatter+2) < elapsed: # notify progress every 2 min
lastChatter = elapsed
print('[emCNN]: elapsed=%0.2f min; %0.2f%% complete' % (elapsed, 100.*epochPct))
sys.stdout.flush()
# done
print('[emCNN]: Total time to evaluate cube: %0.2f min (%0.2f CNN min)' % (elapsed, cnnTime/60.))
return Prob
def _eval_cube(net, X, M, batchDim, extractFeat=''):
"""
PARAMETERS:
net - a Caffe network object
X - A 3d tensor of data to evaluate
M - A boolean 3d tensor mask; 1 := evaluate the corresponding pixel
batchDim - Length 2 vector of tile [width,height].
RETURN VALUES:
Yhat - a tensor with dimensions (#classes, ...) where "..." denotes data cube dimensions
Xprime - a tensor with dimensions (#features, ...) where "..." denotes data cube dimensions
"""
assert(batchDim[2] == batchDim[3]) # currently we assume tiles are square
#--------------------------------------------------
# initialize variables and storage needed in the processing loop below
#--------------------------------------------------
tileRadius = int(batchDim[2]/2)
Xi = np.zeros(batchDim, dtype=np.float32)
yDummy = np.zeros((batchDim[0],), dtype=np.float32)
cnnTime = 0.0 # time spent doing core CNN operations
nClasses = net.blobs['prob'].data.shape[1] # *** Assumes a layer called "prob"
# allocate memory for return values
Yhat = -1*np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2]))
if len(extractFeat):
nFeats = net.blobs[extractFeat].data.shape[1]
Xprime = np.zeros((nFeats, X.shape[0], X.shape[1], X.shape[2]))
else:
Xprime = None
print "[deploy]: Yhat shape: %s" % str(Yhat.shape)
if Xprime is not None:
print "[deploy]: Extracting features from layer: %s" % extractFeat
print "[deploy]: Xprime shape: %s" % str(Xprime.shape)
sys.stdout.flush()
#--------------------------------------------------
# process the cube
#--------------------------------------------------
tic = time.time()
lastChatter = None
for Idx, pct in emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=M):
# populate the mini-batch buffer
for jj in range(Idx.shape[0]):
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ]
# CNN forward pass
_tmp = time.time()
net.set_input_arrays(Xi, yDummy)
out = net.forward()
yiHat = out['prob']
cnnTime += time.time() - _tmp
# On some version of Caffe, yiHat is (batchSize, nClasses, 1, 1)
# On newer versions, it is natively (batchSize, nClasses)
# The squeeze here is to accommodate older versions
yiHat = np.squeeze(yiHat)
# store the per-class probability estimates.
#
# * Note that on the final iteration, the size of yiHat may not match
# the remaining space in Yhat (unless we get lucky and the data cube
# size is a multiple of the mini-batch size). This is why we slice
# yijHat before assigning to Yhat.
for jj in range(nClasses):
yijHat = yiHat[:,jj] # get slice containing probabilities for class j
assert(len(yijHat.shape)==1) # should be a vector (vs tensor)
Yhat[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = yijHat[:Idx.shape[0]] # (*)
# for features: net.blobs['ip1'].data should be (100,200,1,1) for batch size 100, ip output size 200
if Xprime is not None:
for jj in range(nFeats):
Xprimejj = np.squeeze(net.blobs[extractFeat].data[:,jj,:,:]) # feature jj, all objects
assert(len(Xprimejj.shape)==1) # should be a vector (vs tensor)
Xprime[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = Xprimejj[:Idx.shape[0]]
# provide feedback on progress so far
elapsed = (time.time() - tic) / 60.
if (lastChatter is None) or ((elapsed - lastChatter) > 2):
lastChatter = elapsed
print "[deploy]: processed pixel at index %s (%0.2f min elapsed; %0.2f CNN min)" % (str(Idx[-1,:]), elapsed, cnnTime/60.)
sys.stdout.flush()
print('[deploy]: Finished processing cube. Net time was: %0.2f min (%0.2f CNN min)' % (elapsed, cnnTime/60.))
return Yhat, Xprime
def _training_loop(solver, X, Y, M, solverParam, batchDim, outDir,
omitLabels=[], Xvalid=None, Yvalid=None, syn_func=None):
"""Main CNN training loop.
"""
assert(batchDim[2] == batchDim[3]) # tiles must be square
# Some variables and storage that we'll use in the loop below
#
tileRadius = int(batchDim[2]/2)
Xi = np.zeros(batchDim, dtype=np.float32)
yi = np.zeros((batchDim[0],), dtype=np.float32)
yMax = np.max(Y).astype(np.int32)
losses = np.zeros((solverParam.max_iter,))
acc = np.zeros((solverParam.max_iter,))
currIter = 0
currEpoch = 0
# SGD parameters. SGD with momentum is of the form:
#
# V_{t+1} = \mu V_t - \alpha \nablaL(W_t)
# W_{t+1} = W_t + V_{t+1}
#
# where W are the weights and V the previous update.
# Ref: http://caffe.berkeleyvision.org/tutorial/solver.html
#
alpha = solverParam.base_lr # alpha := learning rate
mu = solverParam.momentum # mu := momentum
gamma = solverParam.gamma # gamma := step factor
isModeStep = (solverParam.lr_policy == u'step')
isTypeSGD = (solverParam.solver_type == solverParam.SolverType.Value('SGD'))
Vall = {} # stores previous SGD steps (for all layers)
if not (isModeStep and isTypeSGD):
raise RuntimeError('Sorry - only support SGD "step" mode at the present')
# TODO: weight decay
# TODO: layer-specific weights
cnnTime = 0.0 # time spent doing core CNN operations
tic = time.time()
while currIter < solverParam.max_iter:
#--------------------------------------------------
# Each generator provides a single epoch's worth of data.
# However, Caffe doesn't really recognize the notion of an epoch; instead,
# they specify a number of training "iterations" (mini-batch evaluations, I assume).
# So the inner loop below is for a single epoch, which we may terminate
# early if the max # of iterations is reached.
#--------------------------------------------------
currEpoch += 1
it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], mask=M, omitLabels=omitLabels)
for Idx, epochPct in it:
# Map the indices Idx -> tiles Xi and labels yi
#
# Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) a few examples
# from the previous minibatch will be "recycled" here. This is intentional
# (to keep batch sizes consistent even if data set size is not a multiple
# of the minibatch size).
#
for jj in range(Idx.shape[0]):
yi[jj] = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ]
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ]
# label-preserving data transformation (synthetic data generation)
if syn_func is not None:
#Xi = _xform_minibatch(Xi)
Xi = syn_func(Xi)
#----------------------------------------
# one forward/backward pass and update weights
# (SGD with momentum term)
#----------------------------------------
_tmp = time.time()
solver.net.set_input_arrays(Xi, yi)
# XXX: could call preprocess() here?
rv = solver.net.forward()
solver.net.backward()
for lIdx, layer in enumerate(solver.net.layers):
for bIdx, blob in enumerate(layer.blobs):
key = (lIdx, bIdx)
V = Vall.get(key, 0.0)
Vnext = mu*V - alpha * blob.diff
blob.data[...] += Vnext
Vall[key] = Vnext
cnnTime += time.time() - _tmp
# update running list of losses with the loss from this mini batch
losses[currIter] = np.squeeze(rv['loss'])
acc[currIter] = np.squeeze(rv['accuracy'])
currIter += 1
#----------------------------------------
# Some events occur on mini-batch intervals.
# Deal with those now.
#----------------------------------------
if (currIter % solverParam.snapshot) == 0:
fn = os.path.join(outDir, 'iter_%06d.caffemodel' % (currIter))
solver.net.save(str(fn))
if isModeStep and ((currIter % solverParam.stepsize) == 0):
alpha *= gamma
if (currIter % solverParam.display) == 1:
elapsed = (time.time() - tic)/60.
print "[train]: completed iteration %d (of %d; %0.2f min elapsed; %0.2f CNN min)" % (currIter, solverParam.max_iter, elapsed, cnnTime/60.)
print "[train]: epoch: %d (%0.2f), loss: %0.3f, acc: %0.3f, learn rate: %0.3e" % (currEpoch, 100*epochPct, np.mean(losses[max(0,currIter-10):currIter]), np.mean(acc[max(0,currIter-10):currIter]), alpha)
sys.stdout.flush()
if currIter >= solverParam.max_iter:
break # in case we hit max_iter on a non-epoch boundary
#--------------------------------------------------
# After each training epoch is complete, if we have validation
# data, evaluate it.
# Note: this only requires forward passes through the network
#--------------------------------------------------
if (Xvalid is not None) and (Xvalid.size != 0) and (Yvalid is not None) and (Yvalid.size != 0):
# Mask out pixels whose label we don't care about.
Mvalid = np.ones(Yvalid.shape, dtype=bool)
for yIgnore in omitLabels:
Mvalid[Yvalid==yIgnore] = False
print "[train]: Evaluating on validation data (%d pixels)..." % np.sum(Mvalid)
Confusion = np.zeros((yMax+1, yMax+1)) # confusion matrix
it = emlib.interior_pixel_generator(Yvalid, tileRadius, batchDim[0], mask=Mvalid)
for Idx, epochPct in it:
# Extract subtiles from validation data set
for jj in range(Idx.shape[0]):
yi[jj] = Yvalid[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ]
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = Xvalid[ Idx[jj,0], a:b, c:d ]
#----------------------------------------
# one forward pass; no backward pass
#----------------------------------------
solver.net.set_input_arrays(Xi, yi)
# XXX: could call preprocess() here?
rv = solver.net.forward()
# extract statistics
Prob = np.squeeze(rv['prob']) # matrix of estimated probabilities for each object
yHat = np.argmax(Prob,1) # estimated class is highest probability in vector
for yTmp in range(yMax+1): # note: assumes class labels are in {0, 1,..,n_classes-1}
bits = (yi.astype(np.int32) == yTmp)
for jj in range(yMax+1):
Confusion[yTmp,jj] += np.sum(yHat[bits]==jj)
print '[train]: Validation results:'
print ' %s' % str(Confusion)
if yMax == 1:
# Assume a binary classification problem where 0 is non-target and 1 is target.
#
# precision := TP / (TP + FP)
# recall := TP / (TP + FN)
#
# Confusion Matrix:
# yHat=0 yHat=1
# y=0 TN FP
# y=1 FN TP
#
precision = (1.0*Confusion[1,1]) / np.sum(Confusion[:,1])
recall = (1.0*Confusion[1,1]) / np.sum(Confusion[1,:])
f1Score = (2.0 * precision * recall) / (precision + recall);
print ' precision=%0.3f, recall=%0.3f' % (precision, recall)
print ' F1=%0.3f' % f1Score
else:
print '[train]: Not using a validation data set'
sys.stdout.flush()
# ----- end one epoch -----
# complete
print "[train]: all done!"
return losses, acc
def predict(net, X, Mask, batchDim):
"""Generates predictions for a data volume.
The data volume is assumed to be a tensor with shape:
(#slices, width, height)
"""
# *** This code assumes a layer called "prob"
if 'prob' not in net.blobs:
raise RuntimeError("Can't find a layer with output called 'prob'")
# Pre-allocate some variables & storage.
#
tileRadius = int(batchDim[2]/2)
Xi = np.zeros(batchDim, dtype=np.float32)
yi = np.zeros((batchDim[0],), dtype=np.float32)
nClasses = net.blobs['prob'].data.shape[1]
# if we don't evaluate all pixels, the
# ones not evaluated will have label -1
Prob = -1*np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2]))
print "[emCNN]: Evaluating %0.2f%% of cube" % (100.0*np.sum(Mask)/numel(Mask))
# do it
tic = time.time()
cnnTime = 0
lastChatter = -2
it = emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=Mask)
for Idx, epochPct in it:
# Extract subtiles from validation data set
for jj in range(Idx.shape[0]):
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ]
yi[jj] = 0 # this is just a dummy value
#----------------------------------------
# one forward pass; no backward pass
#----------------------------------------
_tmp = time.time()
net.set_input_arrays(Xi, yi)
out = net.forward()
cnnTime += time.time() - _tmp
# On some version of Caffe, Prob is (batchSize, nClasses, 1, 1)
# On newer versions, it is natively (batchSize, nClasses)
# The squeeze here is to accommodate older versions
ProbBatch = np.squeeze(out['prob'])
# store the per-class probability estimates.
#
# * Note that on the final iteration, the size of Prob may not match
# the remaining space in Yhat (unless we get lucky and the data cube
# size is a multiple of the mini-batch size). This is why we slice
# yijHat before assigning to Yhat.
for jj in range(nClasses):
pj = ProbBatch[:,jj] # get probabilities for class j
assert(len(pj.shape)==1) # should be a vector (vs tensor)
Prob[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = pj[:Idx.shape[0]] # (*)
elapsed = time.time() - tic
if (lastChatter+2) < (elapsed/60.): # notify progress every 2 min
lastChatter = elapsed/60.
print('[emCNN]: elapsed=%0.2f min; %0.2f%% complete' % (elapsed/60., 100.*epochPct))
# done
elapsed = time.time() - tic
print('[emCNN]: Total time to evaluate cube: %0.2f min (%0.2f CNN min)' % (elapsed/60., cnnTime/60.))
return Prob
def _eval_cube(net, X, M, batchDim):
"""Uses Caffe to make predictions for the EM cube X."""
assert(batchDim[2] == batchDim[3]) # currently we assume tiles are square
#--------------------------------------------------
# initialize variables and storage needed in the processing loop below
#--------------------------------------------------
tileRadius = int(batchDim[2]/2)
Xi = np.zeros(batchDim, dtype=np.float32)
yDummy = np.zeros((batchDim[0],), dtype=np.float32)
cnnTime = 0.0 # time spent doing core CNN operations
nClasses = net.blobs['prob'].data.shape[1] # *** Assumes a layer called "prob"
# allocate memory for return values
# if we don't evaluate all pixels, the
# ones not evaluated will have label -1
Yhat = -1*np.ones((nClasses, X.shape[0], X.shape[1], X.shape[2]))
print "[deploy]: Yhat shape: %s" % str(Yhat.shape)
sys.stdout.flush()
#--------------------------------------------------
# process the cube
#--------------------------------------------------
tic = time.time()
lastChatter = None
for Idx, pct in emlib.interior_pixel_generator(X, tileRadius, batchDim[0], mask=M):
# populate the mini-batch buffer
for jj in range(Idx.shape[0]):
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ]
# CNN forward pass
_tmp = time.time()
net.set_input_arrays(Xi, yDummy)
out = net.forward()
yiHat = out['prob']
cnnTime += time.time() - _tmp
# On some version of Caffe, yiHat is (batchSize, nClasses, 1, 1)
# On newer versions, it is natively (batchSize, nClasses)
# The squeeze here is to accommodate older versions
yiHat = np.squeeze(yiHat)
# store the per-class probability estimates.
#
# * Note that on the final iteration, the size of yiHat may not match
# the remaining space in Yhat (unless we get lucky and the data cube
# size is a multiple of the mini-batch size). This is why we slice
# yijHat before assigning to Yhat.
for jj in range(nClasses):
yijHat = yiHat[:,jj] # get slice containing probabilities for class j
assert(len(yijHat.shape)==1) # should be a vector (vs tensor)
Yhat[jj, Idx[:,0], Idx[:,1], Idx[:,2]] = yijHat[:Idx.shape[0]] # (*)
# provide feedback on progress so far
elapsed = (time.time() - tic) / 60.
if (lastChatter is None) or ((elapsed - lastChatter) > 2):
print('[deploy]: %0.2f min elapsed (%0.2f CNN min, %0.2f%% complete)' % (elapsed, cnnTime/60., 100.*pct))
sys.stdout.flush()
lastChatter = elapsed
# all done!
print('[deploy]: Finished processing cube.')
print('[deploy]: Net time was: %0.2f min (%0.2f CNN min)' % (elapsed, cnnTime/60.))
return Yhat