def test_stratified_interior_pixel_generator(self):
b = 10 # b := border size
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# For a 50/50 split of pixels in the interior, the generator
# should reproduce the entire interior.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Y = np.zeros((2, 100, 100))
Y[:, 0:50, :] = 1
Z = np.zeros(Y.shape)
for idx, pct in emlib.stratified_interior_pixel_generator(Y, b, 30):
Z[idx[:, 0], idx[:, 1], idx[:, 2]] += 1
self.assertTrue(np.all(Z[:, b:-b, b:-b] == 1))
Z[:, b:-b, b:-b] = 0
self.assertTrue(np.all(Z == 0))
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# For a random input, should see a 50/50 split of class
# labels, but not necessarily hit the entire interior.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Y = np.random.rand(2, 100, 100) > 0.5
nOne = 0
nZero = 0
for idx, pct in emlib.stratified_interior_pixel_generator(Y, b, 30):
slices = idx[:, 0]
rows = idx[:, 1]
cols = idx[:, 2]
nOne += np.sum(Y[slices, rows, cols] == 1)
nZero += np.sum(Y[slices, rows, cols] == 0)
self.assertTrue(nOne == nZero)
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# For an input tensor with "no-ops", the sampler should only
# return pixels with a positive or negative label.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Y = np.zeros((2, 100, 100))
Y[:, 0:20, 0:20] = 1
Y[:, 50:70, 50:70] = -1
Z = np.zeros(Y.shape)
nPos = 0
nNeg = 0
nTotal = 0
for idx, pct in emlib.stratified_interior_pixel_generator(
Y, 0, 10, omitLabels=[0]):
slices = idx[:, 0]
rows = idx[:, 1]
cols = idx[:, 2]
Z[slices, rows, cols] = Y[slices, rows, cols]
nPos += np.sum(Y[slices, rows, cols] == 1)
nNeg += np.sum(Y[slices, rows, cols] == -1)
nTotal += len(slices)
self.assertTrue(nPos == 20 * 20 * 2)
self.assertTrue(nNeg == 20 * 20 * 2)
self.assertTrue(nTotal == 20 * 20 * 2 * 2)
self.assertTrue(np.all(Y == Z))
def test_stratified_interior_pixel_generator(self):
b = 10 # b := border size
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# For a 50/50 split of pixels in the interior, the generator
# should reproduce the entire interior.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Y = np.zeros((2,100,100))
Y[:,0:50,:] = 1
Z = np.zeros(Y.shape)
for idx,pct in emlib.stratified_interior_pixel_generator(Y,b,30):
Z[idx[:,0],idx[:,1],idx[:,2]] += 1
self.assertTrue(np.all(Z[:,b:-b,b:-b]==1))
Z[:,b:-b,b:-b] = 0
self.assertTrue(np.all(Z==0))
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# For a random input, should see a 50/50 split of class
# labels, but not necessarily hit the entire interior.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Y = np.random.rand(2,100,100) > 0.5
nOne=0; nZero=0;
for idx,pct in emlib.stratified_interior_pixel_generator(Y,b,30):
slices = idx[:,0]; rows = idx[:,1]; cols = idx[:,2]
nOne += np.sum(Y[slices,rows,cols] == 1)
nZero += np.sum(Y[slices,rows,cols] == 0)
self.assertTrue(nOne == nZero)
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# For an input tensor with "no-ops", the sampler should only
# return pixels with a positive or negative label.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Y = np.zeros((2,100,100))
Y[:,0:20,0:20] = 1
Y[:,50:70,50:70] = -1
Z = np.zeros(Y.shape)
nPos=0; nNeg=0; nTotal=0;
for idx,pct in emlib.stratified_interior_pixel_generator(Y,0,10,omitLabels=[0]):
slices = idx[:,0]; rows = idx[:,1]; cols = idx[:,2]
Z[slices,rows,cols] = Y[slices,rows,cols]
nPos += np.sum(Y[slices,rows,cols] == 1)
nNeg += np.sum(Y[slices,rows,cols] == -1)
nTotal += len(slices)
self.assertTrue(nPos == 20*20*2);
self.assertTrue(nNeg == 20*20*2);
self.assertTrue(nTotal == 20*20*2*2);
self.assertTrue(np.all(Y == Z))
def _training_loop(solver,
X,
Y,
M,
solverParam,
batchDim,
outDir,
omitLabels=[],
Xvalid=None,
Yvalid=None,
syn_func=None):
"""Main CNN training loop.
"""
assert (batchDim[2] == batchDim[3]) # tiles must be square
# Some variables and storage that we'll use in the loop below
#
tileRadius = int(batchDim[2] / 2)
Xi = np.zeros(batchDim, dtype=np.float32)
yi = np.zeros((batchDim[0], ), dtype=np.float32)
yMax = np.max(Y).astype(np.int32)
losses = np.zeros((solverParam.max_iter, ))
acc = np.zeros((solverParam.max_iter, ))
currIter = 0
currEpoch = 0
# SGD parameters. SGD with momentum is of the form:
#
# V_{t+1} = \mu V_t - \alpha \nablaL(W_t)
# W_{t+1} = W_t + V_{t+1}
#
# where W are the weights and V the previous update.
# Ref: http://caffe.berkeleyvision.org/tutorial/solver.html
#
alpha = solverParam.base_lr # alpha := learning rate
mu = solverParam.momentum # mu := momentum
gamma = solverParam.gamma # gamma := step factor
isModeStep = (solverParam.lr_policy == u'step')
isTypeSGD = (
solverParam.solver_type == solverParam.SolverType.Value('SGD'))
Vall = {} # stores previous SGD steps (for all layers)
if not (isModeStep and isTypeSGD):
raise RuntimeError(
'Sorry - only support SGD "step" mode at the present')
# TODO: weight decay
# TODO: layer-specific weights
cnnTime = 0.0 # time spent doing core CNN operations
tic = time.time()
while currIter < solverParam.max_iter:
#--------------------------------------------------
# Each generator provides a single epoch's worth of data.
# However, Caffe doesn't really recognize the notion of an epoch; instead,
# they specify a number of training "iterations" (mini-batch evaluations, I assume).
# So the inner loop below is for a single epoch, which we may terminate
# early if the max # of iterations is reached.
#--------------------------------------------------
currEpoch += 1
it = emlib.stratified_interior_pixel_generator(Y,
tileRadius,
batchDim[0],
mask=M,
omitLabels=omitLabels)
for Idx, epochPct in it:
# Map the indices Idx -> tiles Xi and labels yi
#
# Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) a few examples
# from the previous minibatch will be "recycled" here. This is intentional
# (to keep batch sizes consistent even if data set size is not a multiple
# of the minibatch size).
#
for jj in range(Idx.shape[0]):
yi[jj] = Y[Idx[jj, 0], Idx[jj, 1], Idx[jj, 2]]
a = Idx[jj, 1] - tileRadius
b = Idx[jj, 1] + tileRadius + 1
c = Idx[jj, 2] - tileRadius
d = Idx[jj, 2] + tileRadius + 1
Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d]
# label-preserving data transformation (synthetic data generation)
if syn_func is not None:
#Xi = _xform_minibatch(Xi)
Xi = syn_func(Xi)
#----------------------------------------
# one forward/backward pass and update weights
# (SGD with momentum term)
#----------------------------------------
_tmp = time.time()
solver.net.set_input_arrays(Xi, yi)
# XXX: could call preprocess() here?
rv = solver.net.forward()
solver.net.backward()
for lIdx, layer in enumerate(solver.net.layers):
for bIdx, blob in enumerate(layer.blobs):
key = (lIdx, bIdx)
V = Vall.get(key, 0.0)
Vnext = mu * V - alpha * blob.diff
blob.data[...] += Vnext
Vall[key] = Vnext
cnnTime += time.time() - _tmp
# update running list of losses with the loss from this mini batch
losses[currIter] = np.squeeze(rv['loss'])
acc[currIter] = np.squeeze(rv['accuracy'])
currIter += 1
#----------------------------------------
# Some events occur on mini-batch intervals.
# Deal with those now.
#----------------------------------------
if (currIter % solverParam.snapshot) == 0:
fn = os.path.join(outDir, 'iter_%06d.caffemodel' % (currIter))
solver.net.save(str(fn))
if isModeStep and ((currIter % solverParam.stepsize) == 0):
alpha *= gamma
if (currIter % solverParam.display) == 1:
elapsed = (time.time() - tic) / 60.
print "[train]: completed iteration %d (of %d; %0.2f min elapsed; %0.2f CNN min)" % (
currIter, solverParam.max_iter, elapsed, cnnTime / 60.)
print "[train]: epoch: %d (%0.2f), loss: %0.3f, acc: %0.3f, learn rate: %0.3e" % (
currEpoch, 100 * epochPct,
np.mean(losses[max(0, currIter - 10):currIter]),
np.mean(acc[max(0, currIter - 10):currIter]), alpha)
sys.stdout.flush()
if currIter >= solverParam.max_iter:
break # in case we hit max_iter on a non-epoch boundary
#--------------------------------------------------
# After each training epoch is complete, if we have validation
# data, evaluate it.
# Note: this only requires forward passes through the network
#--------------------------------------------------
if (Xvalid is not None) and (Xvalid.size != 0) and (
Yvalid is not None) and (Yvalid.size != 0):
# Mask out pixels whose label we don't care about.
Mvalid = np.ones(Yvalid.shape, dtype=bool)
for yIgnore in omitLabels:
Mvalid[Yvalid == yIgnore] = False
print "[train]: Evaluating on validation data (%d pixels)..." % np.sum(
Mvalid)
Confusion = np.zeros((yMax + 1, yMax + 1)) # confusion matrix
it = emlib.interior_pixel_generator(Yvalid,
tileRadius,
batchDim[0],
mask=Mvalid)
for Idx, epochPct in it:
# Extract subtiles from validation data set
for jj in range(Idx.shape[0]):
yi[jj] = Yvalid[Idx[jj, 0], Idx[jj, 1], Idx[jj, 2]]
a = Idx[jj, 1] - tileRadius
b = Idx[jj, 1] + tileRadius + 1
c = Idx[jj, 2] - tileRadius
d = Idx[jj, 2] + tileRadius + 1
Xi[jj, 0, :, :] = Xvalid[Idx[jj, 0], a:b, c:d]
#----------------------------------------
# one forward pass; no backward pass
#----------------------------------------
solver.net.set_input_arrays(Xi, yi)
# XXX: could call preprocess() here?
rv = solver.net.forward()
# extract statistics
Prob = np.squeeze(
rv['prob']
) # matrix of estimated probabilities for each object
yHat = np.argmax(
Prob,
1) # estimated class is highest probability in vector
for yTmp in range(
yMax + 1
): # note: assumes class labels are in {0, 1,..,n_classes-1}
bits = (yi.astype(np.int32) == yTmp)
for jj in range(yMax + 1):
Confusion[yTmp, jj] += np.sum(yHat[bits] == jj)
print '[train]: Validation results:'
print ' %s' % str(Confusion)
if yMax == 1:
# Assume a binary classification problem where 0 is non-target and 1 is target.
#
# precision := TP / (TP + FP)
# recall := TP / (TP + FN)
#
# Confusion Matrix:
# yHat=0 yHat=1
# y=0 TN FP
# y=1 FN TP
#
precision = (1.0 * Confusion[1, 1]) / np.sum(Confusion[:, 1])
recall = (1.0 * Confusion[1, 1]) / np.sum(Confusion[1, :])
f1Score = (2.0 * precision * recall) / (precision + recall)
print ' precision=%0.3f, recall=%0.3f' % (precision, recall)
print ' F1=%0.3f' % f1Score
else:
print '[train]: Not using a validation data set'
sys.stdout.flush()
# ----- end one epoch -----
# complete
print "[train]: all done!"
return losses, acc
def _train_one_epoch(model, X, Y,
omitLabels=[],
batchSize=100,
nBatches=sys.maxint,
log=None):
"""Trains the model for one epoch.
"""
#----------------------------------------
# Pre-allocate some variables & storage.
#----------------------------------------
nChannels, nRows, nCols = model.input_shape[1:4]
assert(nRows == nCols)
ste = emlib.SimpleTileExtractor(nRows, X, Y, omitLabels=omitLabels)
# some variables we'll use for reporting progress
lastChatter = -2
startTime = time.time()
gpuTime = 0
accBuffer = []
lossBuffer = []
#----------------------------------------
# Loop over mini-batches
#----------------------------------------
it = emlib.stratified_interior_pixel_generator(Y, 0, batchSize,
omitLabels=omitLabels,
stopAfter=nBatches*batchSize)
for mbIdx, (Idx, epochPct) in enumerate(it):
Xi, Yi = ste.extract(Idx)
# label-preserving data transformation (synthetic data generation)
Xi = _xform_minibatch(Xi)
assert(not np.any(np.isnan(Xi)))
assert(not np.any(np.isnan(Yi)))
# do training
tic = time.time()
loss, acc = model.train_on_batch(Xi, Yi)
gpuTime += time.time() - tic
accBuffer.append(acc); lossBuffer.append(loss)
#----------------------------------------
# Some events occur on regular intervals.
# Address these here.
#----------------------------------------
elapsed = (time.time() - startTime) / 60.0
if (lastChatter+2) < elapsed:
# notify progress every 2 min
lastChatter = elapsed
if len(accBuffer) < 10:
recentAcc = np.mean(accBuffer)
recentLoss = np.mean(lossBuffer)
else:
recentAcc = np.mean(accBuffer[-10:])
recentLoss = np.mean(lossBuffer[-10:])
if log:
log.info(" just completed mini-batch %d" % mbIdx)
log.info(" we are %0.2g%% complete with this epoch" % (100.*epochPct))
log.info(" recent accuracy, loss: %0.2f, %0.2f" % (recentAcc, recentLoss))
fracGPU = (gpuTime/60.)/elapsed
log.info(" pct. time spent on CNN ops.: %0.2f%%" % (100.*fracGPU))
log.info("")
# return statistics
return accBuffer, lossBuffer
def main(args):
tileRadius = np.floor(args.tileSize/2)
nMiniBatch = 1000 # here, a "mini-batch" specifies LMDB transaction size
# make sure we don't clobber an existing output
if os.path.exists(args.outDir):
raise RuntimeError('Output path "%s" already exists; please move out of the way and try again' % args.outDir)
# load the data volumes (EM image and labels, if any)
print('[make_lmdb]: loading EM data file: %s' % args.emFileName)
X = emlib.load_cube(args.emFileName, np.float32)
if args.labelsFileName:
print('[make_lmdb]: loading labels file: %s' % args.labelsFileName)
Y = emlib.load_cube(args.labelsFileName, np.float32)
Y = emlib.fix_class_labels(Y, eval(args.omitLabels))
assert(Y.shape == X.shape)
else:
print('[make_lmdb]: no labels file; assuming this is a test volume')
Y = np.zeros(X.shape)
# usually we expect fewer slices in Z than pixels in X or Y.
# Make sure the dimensions look ok before proceeding.
assert(X.shape[0] < X.shape[1])
assert(X.shape[0] < X.shape[2])
# Identify the subset of the data to use for training.
# (default is to use it all)
if len(args.slicesExpr):
sliceIdx = eval(args.slicesExpr)
X = X[sliceIdx, :, :] # python puts the z dimension first...
Y = Y[sliceIdx, :, :]
X = X.astype(np.uint8) # critical!! otherwise, Caffe just flails...
print('[make_lmdb]: EM volume shape: %s' % str(X.shape))
print('[make_lmdb]: yAll is %s' % np.unique(Y))
print('[make_lmdb]: %0.2f%% pixels will be omitted' % (100.0*np.sum(Y==-1)/numel(Y)))
print('[make_lmdb]: writing results to: %s' % args.outDir)
print('')
sys.stdout.flush()
# Create the output database.
# Multiply the actual size by a fudge factor to get a safe upper bound
dbSize = (X.nbytes * args.tileSize * args.tileSize + Y.nbytes) * 10
env = lmdb.open(args.outDir, map_size=dbSize)
# Extract all possible tiles.
# This corresponds to extracting one "epoch" worth of tiles.
tileId = 0
lastChatter = -1
tic = time.time()
yCnt = np.zeros(sum(np.unique(Y) >= 0))
if np.any(Y > 0):
# generates a balanced training data set (subsamples and shuffles)
it = emlib.stratified_interior_pixel_generator(Y, tileRadius, nMiniBatch, omitLabels=[-1])
else:
# enumerates all possible tiles in order (no shuffling)
it = emlib.interior_pixel_generator(X, tileRadius, nMiniBatch)
for Idx, epochPct in it:
# respect upper bound on number of examples
if tileId > args.maxNumExamples:
print('[make_lmdb]: stopping at %d (max number of examples reached\n)' % (tileId-1))
break
# Each mini-batch will be added to the database as a single transaction.
with env.begin(write=True) as txn:
# Translate indices Idx -> tiles Xi and labels yi.
for jj in range(Idx.shape[0]):
yi = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ]
yi = int(yi)
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi = X[ Idx[jj,0], a:b, c:d ]
assert(Xi.shape == (args.tileSize, args.tileSize))
datum = caffe.proto.caffe_pb2.Datum()
datum.channels = 1
datum.height = Xi.shape[0]
datum.width = Xi.shape[1]
datum.data = Xi.tostring() # use tobytes() for newer numpy
datum.label = yi
strId = '{:08}'.format(tileId)
txn.put(strId.encode('ascii'), datum.SerializeToString())
tileId += 1
yCnt[yi] += 1
# check early termination conditions
if tileId > args.maxNumExamples:
break
#if np.floor(epochPct) > lastChatter:
print('[make_lmdb] %% %0.2f done (%0.2f min; yCnt=%s)' % ((100*epochPct), (time.time() - tic)/60, str(yCnt)))
lastChatter = epochPct
def train_one_epoch(solver,
X,
Y,
trainInfo,
batchDim,
outDir='./',
omitLabels=[],
data_augment=None):
""" Trains a CNN for a single epoch.
batchDim := (nExamples, nFilters, width, height)
"""
# Pre-allocate some variables & storate.
#
tileRadius = int(batchDim[2] / 2)
Xi = np.zeros(batchDim, dtype=np.float32)
yi = np.zeros((batchDim[0], ), dtype=np.float32)
yMax = np.max(Y).astype(np.int32)
tic = time.time()
it = emlib.stratified_interior_pixel_generator(Y,
tileRadius,
batchDim[0],
omitLabels=omitLabels)
for Idx, epochPct in it:
# Map the indices Idx -> tiles Xi and labels yi
#
# Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch)
# a few examples from the previous minibatch will be "recycled" here.
# This is intentional (to keep batch sizes consistent even if data
# set size is not a multiple of the minibatch size).
#
for jj in range(Idx.shape[0]):
a = Idx[jj, 1] - tileRadius
b = Idx[jj, 1] + tileRadius + 1
c = Idx[jj, 2] - tileRadius
d = Idx[jj, 2] + tileRadius + 1
Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d]
yi[jj] = Y[Idx[jj, 0], Idx[jj, 1], Idx[jj, 2]]
# label-preserving data transformation (synthetic data generation)
if data_augment is not None:
Xi = data_augment(Xi)
#----------------------------------------
# one forward/backward pass and update weights
#----------------------------------------
_tmp = time.time()
solver.net.set_input_arrays(Xi, yi)
out = solver.net.forward()
solver.net.backward()
# SGD with momentum
for lIdx, layer in enumerate(solver.net.layers):
for bIdx, blob in enumerate(layer.blobs):
key = (lIdx, bIdx)
V = trainInfo.V.get(key, 0.0)
Vnext = (trainInfo.mu * V) - (trainInfo.alpha * blob.diff)
blob.data[...] += Vnext
trainInfo.V[key] = Vnext
# (try to) extract some useful info from the net
loss = out.get('loss', None)
acc = out.get('acc', None) # accuracy is not required...
# update run statistics
trainInfo.cnnTime += time.time() - _tmp
trainInfo.iter += 1
trainInfo.netTime += (time.time() - tic)
tic = time.time()
#----------------------------------------
# Some events occur on regular intervals.
# Address these here.
#----------------------------------------
if (trainInfo.iter % trainInfo.param.snapshot) == 0:
fn = os.path.join(outDir, 'iter_%06d.caffemodel' % trainInfo.iter)
solver.net.save(str(fn))
if trainInfo.isModeStep and ((trainInfo.iter %
trainInfo.param.stepsize) == 0):
trainInfo.alpha *= trainInfo.gamma
if (trainInfo.iter % trainInfo.param.display) == 1:
print "[emCNN]: completed iteration %d of %d (epoch=%0.2f);" % (
trainInfo.iter, trainInfo.param.max_iter,
trainInfo.epoch + epochPct)
print "[emCNN]: %0.2f min elapsed (%0.2f CNN min)" % (
trainInfo.netTime / 60., trainInfo.cnnTime / 60.)
if loss:
print "[emCNN]: loss=%0.2f" % loss
if acc:
print "[emCNN]: accuracy (train volume)=%0.2f" % acc
sys.stdout.flush()
if trainInfo.iter >= trainInfo.param.max_iter:
break # we hit max_iter on a non-epoch boundary...all done.
# all finished with this epoch
print "[emCNN]: epoch complete."
sys.stdout.flush()
return loss, acc
def train_one_epoch(solverMD, X, Y,
batchDim,
outDir='./',
omitLabels=[],
data_augment=None):
""" Trains a CNN for a single epoch.
PARAMETERS:
solverMD : an SGDSolverMemoryData object
X : a data volume/tensor with dimensions (#slices, height, width)
Y : a labels tensor with same size as X
batchDim : the tuple (#classes, minibatchSize, height, width)
outDir : output directory (e.g. for model snapshots)
omitLabels : class labels to skip during training (or [] for none)
data_agument : synthetic data augmentation function
"""
# Pre-allocate some variables & storage.
#
tileRadius = int(batchDim[2]/2)
Xi = np.zeros(batchDim, dtype=np.float32)
yi = np.zeros((batchDim[0],), dtype=np.float32)
yMax = np.max(Y).astype(np.int32)
lastChatter = -2
startTime = time.time()
it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], omitLabels=omitLabels)
for Idx, epochPct in it:
# Map the indices Idx -> tiles Xi and labels yi
#
# Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch)
# a few examples from the previous minibatch will be "recycled" here.
# This is intentional (to keep batch sizes consistent even if data
# set size is not a multiple of the minibatch size).
#
for jj in range(Idx.shape[0]):
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ]
yi[jj] = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ]
# label-preserving data transformation (synthetic data generation)
if data_augment is not None:
Xi = data_augment(Xi)
assert(not np.any(np.isnan(Xi)))
assert(not np.any(np.isnan(yi)))
#----------------------------------------
# one forward/backward pass and update weights
#----------------------------------------
out = solverMD.step(Xi, yi)
#----------------------------------------
# Some events occur on regular intervals.
# Address these here.
#----------------------------------------
if solverMD.is_time_for_snapshot():
fn = os.path.join(outDir, 'iter_%06d.caffemodel' % solverMD._iter)
solverMD._solver.net.save(str(fn))
print "[emCNN]: Saved snapshot."
if solverMD.is_training_complete():
break # we hit max_iter on a non-epoch boundary...all done.
elapsed = (time.time() - startTime) / 60.0
if (lastChatter+2) < elapsed: # notify progress every 2 min
lastChatter = elapsed
print('[emCNN]: we are %0.2f%% complete with this epoch' % (100.*epochPct))
sys.stdout.flush()
# all finished with this epoch
print "[emCNN]: epoch complete."
sys.stdout.flush()
def _training_loop(solver, X, Y, M, solverParam, batchDim, outDir,
omitLabels=[], Xvalid=None, Yvalid=None, syn_func=None):
"""Main CNN training loop.
"""
assert(batchDim[2] == batchDim[3]) # tiles must be square
# Some variables and storage that we'll use in the loop below
#
tileRadius = int(batchDim[2]/2)
Xi = np.zeros(batchDim, dtype=np.float32)
yi = np.zeros((batchDim[0],), dtype=np.float32)
yMax = np.max(Y).astype(np.int32)
losses = np.zeros((solverParam.max_iter,))
acc = np.zeros((solverParam.max_iter,))
currIter = 0
currEpoch = 0
# SGD parameters. SGD with momentum is of the form:
#
# V_{t+1} = \mu V_t - \alpha \nablaL(W_t)
# W_{t+1} = W_t + V_{t+1}
#
# where W are the weights and V the previous update.
# Ref: http://caffe.berkeleyvision.org/tutorial/solver.html
#
alpha = solverParam.base_lr # alpha := learning rate
mu = solverParam.momentum # mu := momentum
gamma = solverParam.gamma # gamma := step factor
isModeStep = (solverParam.lr_policy == u'step')
isTypeSGD = (solverParam.solver_type == solverParam.SolverType.Value('SGD'))
Vall = {} # stores previous SGD steps (for all layers)
if not (isModeStep and isTypeSGD):
raise RuntimeError('Sorry - only support SGD "step" mode at the present')
# TODO: weight decay
# TODO: layer-specific weights
cnnTime = 0.0 # time spent doing core CNN operations
tic = time.time()
while currIter < solverParam.max_iter:
#--------------------------------------------------
# Each generator provides a single epoch's worth of data.
# However, Caffe doesn't really recognize the notion of an epoch; instead,
# they specify a number of training "iterations" (mini-batch evaluations, I assume).
# So the inner loop below is for a single epoch, which we may terminate
# early if the max # of iterations is reached.
#--------------------------------------------------
currEpoch += 1
it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], mask=M, omitLabels=omitLabels)
for Idx, epochPct in it:
# Map the indices Idx -> tiles Xi and labels yi
#
# Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) a few examples
# from the previous minibatch will be "recycled" here. This is intentional
# (to keep batch sizes consistent even if data set size is not a multiple
# of the minibatch size).
#
for jj in range(Idx.shape[0]):
yi[jj] = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ]
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ]
# label-preserving data transformation (synthetic data generation)
if syn_func is not None:
#Xi = _xform_minibatch(Xi)
Xi = syn_func(Xi)
#----------------------------------------
# one forward/backward pass and update weights
# (SGD with momentum term)
#----------------------------------------
_tmp = time.time()
solver.net.set_input_arrays(Xi, yi)
# XXX: could call preprocess() here?
rv = solver.net.forward()
solver.net.backward()
for lIdx, layer in enumerate(solver.net.layers):
for bIdx, blob in enumerate(layer.blobs):
key = (lIdx, bIdx)
V = Vall.get(key, 0.0)
Vnext = mu*V - alpha * blob.diff
blob.data[...] += Vnext
Vall[key] = Vnext
cnnTime += time.time() - _tmp
# update running list of losses with the loss from this mini batch
losses[currIter] = np.squeeze(rv['loss'])
acc[currIter] = np.squeeze(rv['accuracy'])
currIter += 1
#----------------------------------------
# Some events occur on mini-batch intervals.
# Deal with those now.
#----------------------------------------
if (currIter % solverParam.snapshot) == 0:
fn = os.path.join(outDir, 'iter_%06d.caffemodel' % (currIter))
solver.net.save(str(fn))
if isModeStep and ((currIter % solverParam.stepsize) == 0):
alpha *= gamma
if (currIter % solverParam.display) == 1:
elapsed = (time.time() - tic)/60.
print "[train]: completed iteration %d (of %d; %0.2f min elapsed; %0.2f CNN min)" % (currIter, solverParam.max_iter, elapsed, cnnTime/60.)
print "[train]: epoch: %d (%0.2f), loss: %0.3f, acc: %0.3f, learn rate: %0.3e" % (currEpoch, 100*epochPct, np.mean(losses[max(0,currIter-10):currIter]), np.mean(acc[max(0,currIter-10):currIter]), alpha)
sys.stdout.flush()
if currIter >= solverParam.max_iter:
break # in case we hit max_iter on a non-epoch boundary
#--------------------------------------------------
# After each training epoch is complete, if we have validation
# data, evaluate it.
# Note: this only requires forward passes through the network
#--------------------------------------------------
if (Xvalid is not None) and (Xvalid.size != 0) and (Yvalid is not None) and (Yvalid.size != 0):
# Mask out pixels whose label we don't care about.
Mvalid = np.ones(Yvalid.shape, dtype=bool)
for yIgnore in omitLabels:
Mvalid[Yvalid==yIgnore] = False
print "[train]: Evaluating on validation data (%d pixels)..." % np.sum(Mvalid)
Confusion = np.zeros((yMax+1, yMax+1)) # confusion matrix
it = emlib.interior_pixel_generator(Yvalid, tileRadius, batchDim[0], mask=Mvalid)
for Idx, epochPct in it:
# Extract subtiles from validation data set
for jj in range(Idx.shape[0]):
yi[jj] = Yvalid[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ]
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = Xvalid[ Idx[jj,0], a:b, c:d ]
#----------------------------------------
# one forward pass; no backward pass
#----------------------------------------
solver.net.set_input_arrays(Xi, yi)
# XXX: could call preprocess() here?
rv = solver.net.forward()
# extract statistics
Prob = np.squeeze(rv['prob']) # matrix of estimated probabilities for each object
yHat = np.argmax(Prob,1) # estimated class is highest probability in vector
for yTmp in range(yMax+1): # note: assumes class labels are in {0, 1,..,n_classes-1}
bits = (yi.astype(np.int32) == yTmp)
for jj in range(yMax+1):
Confusion[yTmp,jj] += np.sum(yHat[bits]==jj)
print '[train]: Validation results:'
print ' %s' % str(Confusion)
if yMax == 1:
# Assume a binary classification problem where 0 is non-target and 1 is target.
#
# precision := TP / (TP + FP)
# recall := TP / (TP + FN)
#
# Confusion Matrix:
# yHat=0 yHat=1
# y=0 TN FP
# y=1 FN TP
#
precision = (1.0*Confusion[1,1]) / np.sum(Confusion[:,1])
recall = (1.0*Confusion[1,1]) / np.sum(Confusion[1,:])
f1Score = (2.0 * precision * recall) / (precision + recall);
print ' precision=%0.3f, recall=%0.3f' % (precision, recall)
print ' F1=%0.3f' % f1Score
else:
print '[train]: Not using a validation data set'
sys.stdout.flush()
# ----- end one epoch -----
# complete
print "[train]: all done!"
return losses, acc
def train_one_epoch(solver, X, Y,
trainInfo,
batchDim,
outDir='./',
omitLabels=[],
data_augment=None):
""" Trains a CNN for a single epoch.
batchDim := (nExamples, nFilters, width, height)
"""
# Pre-allocate some variables & storate.
#
tileRadius = int(batchDim[2]/2)
Xi = np.zeros(batchDim, dtype=np.float32)
yi = np.zeros((batchDim[0],), dtype=np.float32)
yMax = np.max(Y).astype(np.int32)
tic = time.time()
it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], omitLabels=omitLabels)
for Idx, epochPct in it:
# Map the indices Idx -> tiles Xi and labels yi
#
# Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch)
# a few examples from the previous minibatch will be "recycled" here.
# This is intentional (to keep batch sizes consistent even if data
# set size is not a multiple of the minibatch size).
#
for jj in range(Idx.shape[0]):
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ]
yi[jj] = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ]
# label-preserving data transformation (synthetic data generation)
if data_augment is not None:
Xi = data_augment(Xi)
#----------------------------------------
# one forward/backward pass and update weights
#----------------------------------------
_tmp = time.time()
solver.net.set_input_arrays(Xi, yi)
out = solver.net.forward()
solver.net.backward()
# SGD with momentum
for lIdx, layer in enumerate(solver.net.layers):
for bIdx, blob in enumerate(layer.blobs):
key = (lIdx, bIdx)
V = trainInfo.V.get(key, 0.0)
Vnext = (trainInfo.mu * V) - (trainInfo.alpha * blob.diff)
blob.data[...] += Vnext
trainInfo.V[key] = Vnext
# (try to) extract some useful info from the net
loss = out.get('loss', None)
acc = out.get('acc', None) # accuracy is not required...
# update run statistics
trainInfo.cnnTime += time.time() - _tmp
trainInfo.iter += 1
trainInfo.netTime += (time.time() - tic)
tic = time.time()
#----------------------------------------
# Some events occur on regular intervals.
# Address these here.
#----------------------------------------
if (trainInfo.iter % trainInfo.param.snapshot) == 0:
fn = os.path.join(outDir, 'iter_%06d.caffemodel' % trainInfo.iter)
solver.net.save(str(fn))
if trainInfo.isModeStep and ((trainInfo.iter % trainInfo.param.stepsize) ==0):
trainInfo.alpha *= trainInfo.gamma
if (trainInfo.iter % trainInfo.param.display) == 1:
print "[emCNN]: completed iteration %d of %d (epoch=%0.2f);" % (trainInfo.iter, trainInfo.param.max_iter, trainInfo.epoch+epochPct)
print "[emCNN]: %0.2f min elapsed (%0.2f CNN min)" % (trainInfo.netTime/60., trainInfo.cnnTime/60.)
if loss:
print "[emCNN]: loss=%0.2f" % loss
if acc:
print "[emCNN]: accuracy (train volume)=%0.2f" % acc
sys.stdout.flush()
if trainInfo.iter >= trainInfo.param.max_iter:
break # we hit max_iter on a non-epoch boundary...all done.
# all finished with this epoch
print "[emCNN]: epoch complete."
sys.stdout.flush()
return loss, acc
def _training_loop(solver, X, Y, M, solverParam, batchDim, outDir):
"""Performs CNN training.
"""
assert(batchDim[2] == batchDim[3]) # tiles must be square
# Some variables and storage that we'll use in the loop below
#
tileRadius = int(batchDim[2]/2)
Xi = np.zeros(batchDim, dtype=np.float32)
yi = np.zeros((batchDim[0],), dtype=np.float32)
losses = np.zeros((solverParam.max_iter,))
acc = np.zeros((solverParam.max_iter,))
currIter = 0
currEpoch = 0
# SGD parameters. SGD with momentum is of the form:
#
# V_{t+1} = \mu V_t - \alpha \nablaL(W_t)
# W_{t+1} = W_t + V_{t+1}
#
# where W are the weights and V the previous update.
# Ref: http://caffe.berkeleyvision.org/tutorial/solver.html
#
alpha = solverParam.base_lr # alpha := learning rate
mu = solverParam.momentum # mu := momentum
gamma = solverParam.gamma # gamma := step factor
isModeStep = (solverParam.lr_policy == u'step')
isTypeSGD = (solverParam.solver_type == solverParam.SolverType.Value('SGD'))
Vall = {} # stores previous SGD steps (for all layers)
if not (isModeStep and isTypeSGD):
raise RuntimeError('Sorry - only support SGD "step" mode at the present')
# TODO: weight decay
# TODO: layer-specific weights
# TODO: evaluate performance on valid slices instead of train slices?
# (complicated by the fact that pycaffe doesn't support test mode)
cnnTime = 0.0 # time spent doing core CNN operations
tic = time.time()
while currIter < solverParam.max_iter:
# Each generator provides a single epoch's worth of data.
# However, Caffe doesn't really recognize the notion of an epoch; instead,
# they specify a number of training "iterations" (mini-batch evaluations, I assume).
# So the inner loop below is for a single epoch, which we may terminate
# early if the max # of iterations is reached.
currEpoch += 1
it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], mask=M)
for Idx, epochPct in it:
# Map the indices Idx -> tiles Xi and labels yi
#
# Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) a few examples
# from the previous minibatch will be "recycled" here. This is intentional
# (to keep batch sizes consistent even if data set size is not a multiple
# of the minibatch size).
#
for jj in range(Idx.shape[0]):
yi[jj] = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ]
a = Idx[jj,1] - tileRadius
b = Idx[jj,1] + tileRadius + 1
c = Idx[jj,2] - tileRadius
d = Idx[jj,2] + tileRadius + 1
Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ]
# label-preserving data transformation (synthetic data generation)
Xi = _xform_minibatch(Xi)
#----------------------------------------
# one forward/backward pass and update weights
# (SGD with momentum term)
#----------------------------------------
_tmp = time.time()
solver.net.set_input_arrays(Xi, yi)
# XXX: could call preprocess() here?
rv = solver.net.forward()
solver.net.backward()
for lIdx, layer in enumerate(solver.net.layers):
for bIdx, blob in enumerate(layer.blobs):
key = (lIdx, bIdx)
V = Vall.get(key, 0.0)
Vnext = mu*V - alpha * blob.diff
blob.data[...] += Vnext
Vall[key] = Vnext
cnnTime += time.time() - _tmp
# update running list of losses with the loss from this mini batch
losses[currIter] = np.squeeze(rv['loss'])
acc[currIter] = np.squeeze(rv['accuracy'])
currIter += 1
#----------------------------------------
# Some events occur on mini-batch intervals.
# Deal with those now.
#----------------------------------------
if (currIter % solverParam.snapshot) == 0:
fn = os.path.join(outDir, 'iter_%05d.caffemodel' % (currIter))
solver.net.save(str(fn))
if isModeStep and ((currIter % solverParam.stepsize) == 0):
alpha *= gamma
if (currIter % solverParam.display) == 1:
elapsed = (time.time() - tic)/60.
print "[train]: completed iteration %d (of %d; %0.2f min elapsed; %0.2f CNN min)" % (currIter, solverParam.max_iter, elapsed, cnnTime/60.)
print "[train]: epoch: %d (%0.2f), loss: %0.3f, acc: %0.3f, learn rate: %0.3e" % (currEpoch, 100*epochPct, np.mean(losses[max(0,currIter-10):currIter]), np.mean(acc[max(0,currIter-10):currIter]), alpha)
sys.stdout.flush()
if currIter >= solverParam.max_iter:
break # in case we hit max_iter on a non-epoch boundary
return losses, acc
