def test_stratified_interior_pixel_generator(self): b = 10 # b := border size #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # For a 50/50 split of pixels in the interior, the generator # should reproduce the entire interior. #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Y = np.zeros((2, 100, 100)) Y[:, 0:50, :] = 1 Z = np.zeros(Y.shape) for idx, pct in emlib.stratified_interior_pixel_generator(Y, b, 30): Z[idx[:, 0], idx[:, 1], idx[:, 2]] += 1 self.assertTrue(np.all(Z[:, b:-b, b:-b] == 1)) Z[:, b:-b, b:-b] = 0 self.assertTrue(np.all(Z == 0)) #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # For a random input, should see a 50/50 split of class # labels, but not necessarily hit the entire interior. #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Y = np.random.rand(2, 100, 100) > 0.5 nOne = 0 nZero = 0 for idx, pct in emlib.stratified_interior_pixel_generator(Y, b, 30): slices = idx[:, 0] rows = idx[:, 1] cols = idx[:, 2] nOne += np.sum(Y[slices, rows, cols] == 1) nZero += np.sum(Y[slices, rows, cols] == 0) self.assertTrue(nOne == nZero) #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # For an input tensor with "no-ops", the sampler should only # return pixels with a positive or negative label. #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Y = np.zeros((2, 100, 100)) Y[:, 0:20, 0:20] = 1 Y[:, 50:70, 50:70] = -1 Z = np.zeros(Y.shape) nPos = 0 nNeg = 0 nTotal = 0 for idx, pct in emlib.stratified_interior_pixel_generator( Y, 0, 10, omitLabels=[0]): slices = idx[:, 0] rows = idx[:, 1] cols = idx[:, 2] Z[slices, rows, cols] = Y[slices, rows, cols] nPos += np.sum(Y[slices, rows, cols] == 1) nNeg += np.sum(Y[slices, rows, cols] == -1) nTotal += len(slices) self.assertTrue(nPos == 20 * 20 * 2) self.assertTrue(nNeg == 20 * 20 * 2) self.assertTrue(nTotal == 20 * 20 * 2 * 2) self.assertTrue(np.all(Y == Z))
def test_stratified_interior_pixel_generator(self): b = 10 # b := border size #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # For a 50/50 split of pixels in the interior, the generator # should reproduce the entire interior. #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Y = np.zeros((2,100,100)) Y[:,0:50,:] = 1 Z = np.zeros(Y.shape) for idx,pct in emlib.stratified_interior_pixel_generator(Y,b,30): Z[idx[:,0],idx[:,1],idx[:,2]] += 1 self.assertTrue(np.all(Z[:,b:-b,b:-b]==1)) Z[:,b:-b,b:-b] = 0 self.assertTrue(np.all(Z==0)) #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # For a random input, should see a 50/50 split of class # labels, but not necessarily hit the entire interior. #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Y = np.random.rand(2,100,100) > 0.5 nOne=0; nZero=0; for idx,pct in emlib.stratified_interior_pixel_generator(Y,b,30): slices = idx[:,0]; rows = idx[:,1]; cols = idx[:,2] nOne += np.sum(Y[slices,rows,cols] == 1) nZero += np.sum(Y[slices,rows,cols] == 0) self.assertTrue(nOne == nZero) #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # For an input tensor with "no-ops", the sampler should only # return pixels with a positive or negative label. #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Y = np.zeros((2,100,100)) Y[:,0:20,0:20] = 1 Y[:,50:70,50:70] = -1 Z = np.zeros(Y.shape) nPos=0; nNeg=0; nTotal=0; for idx,pct in emlib.stratified_interior_pixel_generator(Y,0,10,omitLabels=[0]): slices = idx[:,0]; rows = idx[:,1]; cols = idx[:,2] Z[slices,rows,cols] = Y[slices,rows,cols] nPos += np.sum(Y[slices,rows,cols] == 1) nNeg += np.sum(Y[slices,rows,cols] == -1) nTotal += len(slices) self.assertTrue(nPos == 20*20*2); self.assertTrue(nNeg == 20*20*2); self.assertTrue(nTotal == 20*20*2*2); self.assertTrue(np.all(Y == Z))
def _training_loop(solver, X, Y, M, solverParam, batchDim, outDir, omitLabels=[], Xvalid=None, Yvalid=None, syn_func=None): """Main CNN training loop. """ assert (batchDim[2] == batchDim[3]) # tiles must be square # Some variables and storage that we'll use in the loop below # tileRadius = int(batchDim[2] / 2) Xi = np.zeros(batchDim, dtype=np.float32) yi = np.zeros((batchDim[0], ), dtype=np.float32) yMax = np.max(Y).astype(np.int32) losses = np.zeros((solverParam.max_iter, )) acc = np.zeros((solverParam.max_iter, )) currIter = 0 currEpoch = 0 # SGD parameters. SGD with momentum is of the form: # # V_{t+1} = \mu V_t - \alpha \nablaL(W_t) # W_{t+1} = W_t + V_{t+1} # # where W are the weights and V the previous update. # Ref: http://caffe.berkeleyvision.org/tutorial/solver.html # alpha = solverParam.base_lr # alpha := learning rate mu = solverParam.momentum # mu := momentum gamma = solverParam.gamma # gamma := step factor isModeStep = (solverParam.lr_policy == u'step') isTypeSGD = ( solverParam.solver_type == solverParam.SolverType.Value('SGD')) Vall = {} # stores previous SGD steps (for all layers) if not (isModeStep and isTypeSGD): raise RuntimeError( 'Sorry - only support SGD "step" mode at the present') # TODO: weight decay # TODO: layer-specific weights cnnTime = 0.0 # time spent doing core CNN operations tic = time.time() while currIter < solverParam.max_iter: #-------------------------------------------------- # Each generator provides a single epoch's worth of data. # However, Caffe doesn't really recognize the notion of an epoch; instead, # they specify a number of training "iterations" (mini-batch evaluations, I assume). # So the inner loop below is for a single epoch, which we may terminate # early if the max # of iterations is reached. #-------------------------------------------------- currEpoch += 1 it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], mask=M, omitLabels=omitLabels) for Idx, epochPct in it: # Map the indices Idx -> tiles Xi and labels yi # # Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) a few examples # from the previous minibatch will be "recycled" here. This is intentional # (to keep batch sizes consistent even if data set size is not a multiple # of the minibatch size). # for jj in range(Idx.shape[0]): yi[jj] = Y[Idx[jj, 0], Idx[jj, 1], Idx[jj, 2]] a = Idx[jj, 1] - tileRadius b = Idx[jj, 1] + tileRadius + 1 c = Idx[jj, 2] - tileRadius d = Idx[jj, 2] + tileRadius + 1 Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d] # label-preserving data transformation (synthetic data generation) if syn_func is not None: #Xi = _xform_minibatch(Xi) Xi = syn_func(Xi) #---------------------------------------- # one forward/backward pass and update weights # (SGD with momentum term) #---------------------------------------- _tmp = time.time() solver.net.set_input_arrays(Xi, yi) # XXX: could call preprocess() here? rv = solver.net.forward() solver.net.backward() for lIdx, layer in enumerate(solver.net.layers): for bIdx, blob in enumerate(layer.blobs): key = (lIdx, bIdx) V = Vall.get(key, 0.0) Vnext = mu * V - alpha * blob.diff blob.data[...] += Vnext Vall[key] = Vnext cnnTime += time.time() - _tmp # update running list of losses with the loss from this mini batch losses[currIter] = np.squeeze(rv['loss']) acc[currIter] = np.squeeze(rv['accuracy']) currIter += 1 #---------------------------------------- # Some events occur on mini-batch intervals. # Deal with those now. #---------------------------------------- if (currIter % solverParam.snapshot) == 0: fn = os.path.join(outDir, 'iter_%06d.caffemodel' % (currIter)) solver.net.save(str(fn)) if isModeStep and ((currIter % solverParam.stepsize) == 0): alpha *= gamma if (currIter % solverParam.display) == 1: elapsed = (time.time() - tic) / 60. print "[train]: completed iteration %d (of %d; %0.2f min elapsed; %0.2f CNN min)" % ( currIter, solverParam.max_iter, elapsed, cnnTime / 60.) print "[train]: epoch: %d (%0.2f), loss: %0.3f, acc: %0.3f, learn rate: %0.3e" % ( currEpoch, 100 * epochPct, np.mean(losses[max(0, currIter - 10):currIter]), np.mean(acc[max(0, currIter - 10):currIter]), alpha) sys.stdout.flush() if currIter >= solverParam.max_iter: break # in case we hit max_iter on a non-epoch boundary #-------------------------------------------------- # After each training epoch is complete, if we have validation # data, evaluate it. # Note: this only requires forward passes through the network #-------------------------------------------------- if (Xvalid is not None) and (Xvalid.size != 0) and ( Yvalid is not None) and (Yvalid.size != 0): # Mask out pixels whose label we don't care about. Mvalid = np.ones(Yvalid.shape, dtype=bool) for yIgnore in omitLabels: Mvalid[Yvalid == yIgnore] = False print "[train]: Evaluating on validation data (%d pixels)..." % np.sum( Mvalid) Confusion = np.zeros((yMax + 1, yMax + 1)) # confusion matrix it = emlib.interior_pixel_generator(Yvalid, tileRadius, batchDim[0], mask=Mvalid) for Idx, epochPct in it: # Extract subtiles from validation data set for jj in range(Idx.shape[0]): yi[jj] = Yvalid[Idx[jj, 0], Idx[jj, 1], Idx[jj, 2]] a = Idx[jj, 1] - tileRadius b = Idx[jj, 1] + tileRadius + 1 c = Idx[jj, 2] - tileRadius d = Idx[jj, 2] + tileRadius + 1 Xi[jj, 0, :, :] = Xvalid[Idx[jj, 0], a:b, c:d] #---------------------------------------- # one forward pass; no backward pass #---------------------------------------- solver.net.set_input_arrays(Xi, yi) # XXX: could call preprocess() here? rv = solver.net.forward() # extract statistics Prob = np.squeeze( rv['prob'] ) # matrix of estimated probabilities for each object yHat = np.argmax( Prob, 1) # estimated class is highest probability in vector for yTmp in range( yMax + 1 ): # note: assumes class labels are in {0, 1,..,n_classes-1} bits = (yi.astype(np.int32) == yTmp) for jj in range(yMax + 1): Confusion[yTmp, jj] += np.sum(yHat[bits] == jj) print '[train]: Validation results:' print ' %s' % str(Confusion) if yMax == 1: # Assume a binary classification problem where 0 is non-target and 1 is target. # # precision := TP / (TP + FP) # recall := TP / (TP + FN) # # Confusion Matrix: # yHat=0 yHat=1 # y=0 TN FP # y=1 FN TP # precision = (1.0 * Confusion[1, 1]) / np.sum(Confusion[:, 1]) recall = (1.0 * Confusion[1, 1]) / np.sum(Confusion[1, :]) f1Score = (2.0 * precision * recall) / (precision + recall) print ' precision=%0.3f, recall=%0.3f' % (precision, recall) print ' F1=%0.3f' % f1Score else: print '[train]: Not using a validation data set' sys.stdout.flush() # ----- end one epoch ----- # complete print "[train]: all done!" return losses, acc
def _train_one_epoch(model, X, Y, omitLabels=[], batchSize=100, nBatches=sys.maxint, log=None): """Trains the model for one epoch. """ #---------------------------------------- # Pre-allocate some variables & storage. #---------------------------------------- nChannels, nRows, nCols = model.input_shape[1:4] assert(nRows == nCols) ste = emlib.SimpleTileExtractor(nRows, X, Y, omitLabels=omitLabels) # some variables we'll use for reporting progress lastChatter = -2 startTime = time.time() gpuTime = 0 accBuffer = [] lossBuffer = [] #---------------------------------------- # Loop over mini-batches #---------------------------------------- it = emlib.stratified_interior_pixel_generator(Y, 0, batchSize, omitLabels=omitLabels, stopAfter=nBatches*batchSize) for mbIdx, (Idx, epochPct) in enumerate(it): Xi, Yi = ste.extract(Idx) # label-preserving data transformation (synthetic data generation) Xi = _xform_minibatch(Xi) assert(not np.any(np.isnan(Xi))) assert(not np.any(np.isnan(Yi))) # do training tic = time.time() loss, acc = model.train_on_batch(Xi, Yi) gpuTime += time.time() - tic accBuffer.append(acc); lossBuffer.append(loss) #---------------------------------------- # Some events occur on regular intervals. # Address these here. #---------------------------------------- elapsed = (time.time() - startTime) / 60.0 if (lastChatter+2) < elapsed: # notify progress every 2 min lastChatter = elapsed if len(accBuffer) < 10: recentAcc = np.mean(accBuffer) recentLoss = np.mean(lossBuffer) else: recentAcc = np.mean(accBuffer[-10:]) recentLoss = np.mean(lossBuffer[-10:]) if log: log.info(" just completed mini-batch %d" % mbIdx) log.info(" we are %0.2g%% complete with this epoch" % (100.*epochPct)) log.info(" recent accuracy, loss: %0.2f, %0.2f" % (recentAcc, recentLoss)) fracGPU = (gpuTime/60.)/elapsed log.info(" pct. time spent on CNN ops.: %0.2f%%" % (100.*fracGPU)) log.info("") # return statistics return accBuffer, lossBuffer
def main(args): tileRadius = np.floor(args.tileSize/2) nMiniBatch = 1000 # here, a "mini-batch" specifies LMDB transaction size # make sure we don't clobber an existing output if os.path.exists(args.outDir): raise RuntimeError('Output path "%s" already exists; please move out of the way and try again' % args.outDir) # load the data volumes (EM image and labels, if any) print('[make_lmdb]: loading EM data file: %s' % args.emFileName) X = emlib.load_cube(args.emFileName, np.float32) if args.labelsFileName: print('[make_lmdb]: loading labels file: %s' % args.labelsFileName) Y = emlib.load_cube(args.labelsFileName, np.float32) Y = emlib.fix_class_labels(Y, eval(args.omitLabels)) assert(Y.shape == X.shape) else: print('[make_lmdb]: no labels file; assuming this is a test volume') Y = np.zeros(X.shape) # usually we expect fewer slices in Z than pixels in X or Y. # Make sure the dimensions look ok before proceeding. assert(X.shape[0] < X.shape[1]) assert(X.shape[0] < X.shape[2]) # Identify the subset of the data to use for training. # (default is to use it all) if len(args.slicesExpr): sliceIdx = eval(args.slicesExpr) X = X[sliceIdx, :, :] # python puts the z dimension first... Y = Y[sliceIdx, :, :] X = X.astype(np.uint8) # critical!! otherwise, Caffe just flails... print('[make_lmdb]: EM volume shape: %s' % str(X.shape)) print('[make_lmdb]: yAll is %s' % np.unique(Y)) print('[make_lmdb]: %0.2f%% pixels will be omitted' % (100.0*np.sum(Y==-1)/numel(Y))) print('[make_lmdb]: writing results to: %s' % args.outDir) print('') sys.stdout.flush() # Create the output database. # Multiply the actual size by a fudge factor to get a safe upper bound dbSize = (X.nbytes * args.tileSize * args.tileSize + Y.nbytes) * 10 env = lmdb.open(args.outDir, map_size=dbSize) # Extract all possible tiles. # This corresponds to extracting one "epoch" worth of tiles. tileId = 0 lastChatter = -1 tic = time.time() yCnt = np.zeros(sum(np.unique(Y) >= 0)) if np.any(Y > 0): # generates a balanced training data set (subsamples and shuffles) it = emlib.stratified_interior_pixel_generator(Y, tileRadius, nMiniBatch, omitLabels=[-1]) else: # enumerates all possible tiles in order (no shuffling) it = emlib.interior_pixel_generator(X, tileRadius, nMiniBatch) for Idx, epochPct in it: # respect upper bound on number of examples if tileId > args.maxNumExamples: print('[make_lmdb]: stopping at %d (max number of examples reached\n)' % (tileId-1)) break # Each mini-batch will be added to the database as a single transaction. with env.begin(write=True) as txn: # Translate indices Idx -> tiles Xi and labels yi. for jj in range(Idx.shape[0]): yi = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ] yi = int(yi) a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi = X[ Idx[jj,0], a:b, c:d ] assert(Xi.shape == (args.tileSize, args.tileSize)) datum = caffe.proto.caffe_pb2.Datum() datum.channels = 1 datum.height = Xi.shape[0] datum.width = Xi.shape[1] datum.data = Xi.tostring() # use tobytes() for newer numpy datum.label = yi strId = '{:08}'.format(tileId) txn.put(strId.encode('ascii'), datum.SerializeToString()) tileId += 1 yCnt[yi] += 1 # check early termination conditions if tileId > args.maxNumExamples: break #if np.floor(epochPct) > lastChatter: print('[make_lmdb] %% %0.2f done (%0.2f min; yCnt=%s)' % ((100*epochPct), (time.time() - tic)/60, str(yCnt))) lastChatter = epochPct
def train_one_epoch(solver, X, Y, trainInfo, batchDim, outDir='./', omitLabels=[], data_augment=None): """ Trains a CNN for a single epoch. batchDim := (nExamples, nFilters, width, height) """ # Pre-allocate some variables & storate. # tileRadius = int(batchDim[2] / 2) Xi = np.zeros(batchDim, dtype=np.float32) yi = np.zeros((batchDim[0], ), dtype=np.float32) yMax = np.max(Y).astype(np.int32) tic = time.time() it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], omitLabels=omitLabels) for Idx, epochPct in it: # Map the indices Idx -> tiles Xi and labels yi # # Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) # a few examples from the previous minibatch will be "recycled" here. # This is intentional (to keep batch sizes consistent even if data # set size is not a multiple of the minibatch size). # for jj in range(Idx.shape[0]): a = Idx[jj, 1] - tileRadius b = Idx[jj, 1] + tileRadius + 1 c = Idx[jj, 2] - tileRadius d = Idx[jj, 2] + tileRadius + 1 Xi[jj, 0, :, :] = X[Idx[jj, 0], a:b, c:d] yi[jj] = Y[Idx[jj, 0], Idx[jj, 1], Idx[jj, 2]] # label-preserving data transformation (synthetic data generation) if data_augment is not None: Xi = data_augment(Xi) #---------------------------------------- # one forward/backward pass and update weights #---------------------------------------- _tmp = time.time() solver.net.set_input_arrays(Xi, yi) out = solver.net.forward() solver.net.backward() # SGD with momentum for lIdx, layer in enumerate(solver.net.layers): for bIdx, blob in enumerate(layer.blobs): key = (lIdx, bIdx) V = trainInfo.V.get(key, 0.0) Vnext = (trainInfo.mu * V) - (trainInfo.alpha * blob.diff) blob.data[...] += Vnext trainInfo.V[key] = Vnext # (try to) extract some useful info from the net loss = out.get('loss', None) acc = out.get('acc', None) # accuracy is not required... # update run statistics trainInfo.cnnTime += time.time() - _tmp trainInfo.iter += 1 trainInfo.netTime += (time.time() - tic) tic = time.time() #---------------------------------------- # Some events occur on regular intervals. # Address these here. #---------------------------------------- if (trainInfo.iter % trainInfo.param.snapshot) == 0: fn = os.path.join(outDir, 'iter_%06d.caffemodel' % trainInfo.iter) solver.net.save(str(fn)) if trainInfo.isModeStep and ((trainInfo.iter % trainInfo.param.stepsize) == 0): trainInfo.alpha *= trainInfo.gamma if (trainInfo.iter % trainInfo.param.display) == 1: print "[emCNN]: completed iteration %d of %d (epoch=%0.2f);" % ( trainInfo.iter, trainInfo.param.max_iter, trainInfo.epoch + epochPct) print "[emCNN]: %0.2f min elapsed (%0.2f CNN min)" % ( trainInfo.netTime / 60., trainInfo.cnnTime / 60.) if loss: print "[emCNN]: loss=%0.2f" % loss if acc: print "[emCNN]: accuracy (train volume)=%0.2f" % acc sys.stdout.flush() if trainInfo.iter >= trainInfo.param.max_iter: break # we hit max_iter on a non-epoch boundary...all done. # all finished with this epoch print "[emCNN]: epoch complete." sys.stdout.flush() return loss, acc
def train_one_epoch(solverMD, X, Y, batchDim, outDir='./', omitLabels=[], data_augment=None): """ Trains a CNN for a single epoch. PARAMETERS: solverMD : an SGDSolverMemoryData object X : a data volume/tensor with dimensions (#slices, height, width) Y : a labels tensor with same size as X batchDim : the tuple (#classes, minibatchSize, height, width) outDir : output directory (e.g. for model snapshots) omitLabels : class labels to skip during training (or [] for none) data_agument : synthetic data augmentation function """ # Pre-allocate some variables & storage. # tileRadius = int(batchDim[2]/2) Xi = np.zeros(batchDim, dtype=np.float32) yi = np.zeros((batchDim[0],), dtype=np.float32) yMax = np.max(Y).astype(np.int32) lastChatter = -2 startTime = time.time() it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], omitLabels=omitLabels) for Idx, epochPct in it: # Map the indices Idx -> tiles Xi and labels yi # # Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) # a few examples from the previous minibatch will be "recycled" here. # This is intentional (to keep batch sizes consistent even if data # set size is not a multiple of the minibatch size). # for jj in range(Idx.shape[0]): a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ] yi[jj] = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ] # label-preserving data transformation (synthetic data generation) if data_augment is not None: Xi = data_augment(Xi) assert(not np.any(np.isnan(Xi))) assert(not np.any(np.isnan(yi))) #---------------------------------------- # one forward/backward pass and update weights #---------------------------------------- out = solverMD.step(Xi, yi) #---------------------------------------- # Some events occur on regular intervals. # Address these here. #---------------------------------------- if solverMD.is_time_for_snapshot(): fn = os.path.join(outDir, 'iter_%06d.caffemodel' % solverMD._iter) solverMD._solver.net.save(str(fn)) print "[emCNN]: Saved snapshot." if solverMD.is_training_complete(): break # we hit max_iter on a non-epoch boundary...all done. elapsed = (time.time() - startTime) / 60.0 if (lastChatter+2) < elapsed: # notify progress every 2 min lastChatter = elapsed print('[emCNN]: we are %0.2f%% complete with this epoch' % (100.*epochPct)) sys.stdout.flush() # all finished with this epoch print "[emCNN]: epoch complete." sys.stdout.flush()
def _training_loop(solver, X, Y, M, solverParam, batchDim, outDir, omitLabels=[], Xvalid=None, Yvalid=None, syn_func=None): """Main CNN training loop. """ assert(batchDim[2] == batchDim[3]) # tiles must be square # Some variables and storage that we'll use in the loop below # tileRadius = int(batchDim[2]/2) Xi = np.zeros(batchDim, dtype=np.float32) yi = np.zeros((batchDim[0],), dtype=np.float32) yMax = np.max(Y).astype(np.int32) losses = np.zeros((solverParam.max_iter,)) acc = np.zeros((solverParam.max_iter,)) currIter = 0 currEpoch = 0 # SGD parameters. SGD with momentum is of the form: # # V_{t+1} = \mu V_t - \alpha \nablaL(W_t) # W_{t+1} = W_t + V_{t+1} # # where W are the weights and V the previous update. # Ref: http://caffe.berkeleyvision.org/tutorial/solver.html # alpha = solverParam.base_lr # alpha := learning rate mu = solverParam.momentum # mu := momentum gamma = solverParam.gamma # gamma := step factor isModeStep = (solverParam.lr_policy == u'step') isTypeSGD = (solverParam.solver_type == solverParam.SolverType.Value('SGD')) Vall = {} # stores previous SGD steps (for all layers) if not (isModeStep and isTypeSGD): raise RuntimeError('Sorry - only support SGD "step" mode at the present') # TODO: weight decay # TODO: layer-specific weights cnnTime = 0.0 # time spent doing core CNN operations tic = time.time() while currIter < solverParam.max_iter: #-------------------------------------------------- # Each generator provides a single epoch's worth of data. # However, Caffe doesn't really recognize the notion of an epoch; instead, # they specify a number of training "iterations" (mini-batch evaluations, I assume). # So the inner loop below is for a single epoch, which we may terminate # early if the max # of iterations is reached. #-------------------------------------------------- currEpoch += 1 it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], mask=M, omitLabels=omitLabels) for Idx, epochPct in it: # Map the indices Idx -> tiles Xi and labels yi # # Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) a few examples # from the previous minibatch will be "recycled" here. This is intentional # (to keep batch sizes consistent even if data set size is not a multiple # of the minibatch size). # for jj in range(Idx.shape[0]): yi[jj] = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ] a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ] # label-preserving data transformation (synthetic data generation) if syn_func is not None: #Xi = _xform_minibatch(Xi) Xi = syn_func(Xi) #---------------------------------------- # one forward/backward pass and update weights # (SGD with momentum term) #---------------------------------------- _tmp = time.time() solver.net.set_input_arrays(Xi, yi) # XXX: could call preprocess() here? rv = solver.net.forward() solver.net.backward() for lIdx, layer in enumerate(solver.net.layers): for bIdx, blob in enumerate(layer.blobs): key = (lIdx, bIdx) V = Vall.get(key, 0.0) Vnext = mu*V - alpha * blob.diff blob.data[...] += Vnext Vall[key] = Vnext cnnTime += time.time() - _tmp # update running list of losses with the loss from this mini batch losses[currIter] = np.squeeze(rv['loss']) acc[currIter] = np.squeeze(rv['accuracy']) currIter += 1 #---------------------------------------- # Some events occur on mini-batch intervals. # Deal with those now. #---------------------------------------- if (currIter % solverParam.snapshot) == 0: fn = os.path.join(outDir, 'iter_%06d.caffemodel' % (currIter)) solver.net.save(str(fn)) if isModeStep and ((currIter % solverParam.stepsize) == 0): alpha *= gamma if (currIter % solverParam.display) == 1: elapsed = (time.time() - tic)/60. print "[train]: completed iteration %d (of %d; %0.2f min elapsed; %0.2f CNN min)" % (currIter, solverParam.max_iter, elapsed, cnnTime/60.) print "[train]: epoch: %d (%0.2f), loss: %0.3f, acc: %0.3f, learn rate: %0.3e" % (currEpoch, 100*epochPct, np.mean(losses[max(0,currIter-10):currIter]), np.mean(acc[max(0,currIter-10):currIter]), alpha) sys.stdout.flush() if currIter >= solverParam.max_iter: break # in case we hit max_iter on a non-epoch boundary #-------------------------------------------------- # After each training epoch is complete, if we have validation # data, evaluate it. # Note: this only requires forward passes through the network #-------------------------------------------------- if (Xvalid is not None) and (Xvalid.size != 0) and (Yvalid is not None) and (Yvalid.size != 0): # Mask out pixels whose label we don't care about. Mvalid = np.ones(Yvalid.shape, dtype=bool) for yIgnore in omitLabels: Mvalid[Yvalid==yIgnore] = False print "[train]: Evaluating on validation data (%d pixels)..." % np.sum(Mvalid) Confusion = np.zeros((yMax+1, yMax+1)) # confusion matrix it = emlib.interior_pixel_generator(Yvalid, tileRadius, batchDim[0], mask=Mvalid) for Idx, epochPct in it: # Extract subtiles from validation data set for jj in range(Idx.shape[0]): yi[jj] = Yvalid[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ] a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = Xvalid[ Idx[jj,0], a:b, c:d ] #---------------------------------------- # one forward pass; no backward pass #---------------------------------------- solver.net.set_input_arrays(Xi, yi) # XXX: could call preprocess() here? rv = solver.net.forward() # extract statistics Prob = np.squeeze(rv['prob']) # matrix of estimated probabilities for each object yHat = np.argmax(Prob,1) # estimated class is highest probability in vector for yTmp in range(yMax+1): # note: assumes class labels are in {0, 1,..,n_classes-1} bits = (yi.astype(np.int32) == yTmp) for jj in range(yMax+1): Confusion[yTmp,jj] += np.sum(yHat[bits]==jj) print '[train]: Validation results:' print ' %s' % str(Confusion) if yMax == 1: # Assume a binary classification problem where 0 is non-target and 1 is target. # # precision := TP / (TP + FP) # recall := TP / (TP + FN) # # Confusion Matrix: # yHat=0 yHat=1 # y=0 TN FP # y=1 FN TP # precision = (1.0*Confusion[1,1]) / np.sum(Confusion[:,1]) recall = (1.0*Confusion[1,1]) / np.sum(Confusion[1,:]) f1Score = (2.0 * precision * recall) / (precision + recall); print ' precision=%0.3f, recall=%0.3f' % (precision, recall) print ' F1=%0.3f' % f1Score else: print '[train]: Not using a validation data set' sys.stdout.flush() # ----- end one epoch ----- # complete print "[train]: all done!" return losses, acc
def train_one_epoch(solver, X, Y, trainInfo, batchDim, outDir='./', omitLabels=[], data_augment=None): """ Trains a CNN for a single epoch. batchDim := (nExamples, nFilters, width, height) """ # Pre-allocate some variables & storate. # tileRadius = int(batchDim[2]/2) Xi = np.zeros(batchDim, dtype=np.float32) yi = np.zeros((batchDim[0],), dtype=np.float32) yMax = np.max(Y).astype(np.int32) tic = time.time() it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], omitLabels=omitLabels) for Idx, epochPct in it: # Map the indices Idx -> tiles Xi and labels yi # # Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) # a few examples from the previous minibatch will be "recycled" here. # This is intentional (to keep batch sizes consistent even if data # set size is not a multiple of the minibatch size). # for jj in range(Idx.shape[0]): a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ] yi[jj] = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ] # label-preserving data transformation (synthetic data generation) if data_augment is not None: Xi = data_augment(Xi) #---------------------------------------- # one forward/backward pass and update weights #---------------------------------------- _tmp = time.time() solver.net.set_input_arrays(Xi, yi) out = solver.net.forward() solver.net.backward() # SGD with momentum for lIdx, layer in enumerate(solver.net.layers): for bIdx, blob in enumerate(layer.blobs): key = (lIdx, bIdx) V = trainInfo.V.get(key, 0.0) Vnext = (trainInfo.mu * V) - (trainInfo.alpha * blob.diff) blob.data[...] += Vnext trainInfo.V[key] = Vnext # (try to) extract some useful info from the net loss = out.get('loss', None) acc = out.get('acc', None) # accuracy is not required... # update run statistics trainInfo.cnnTime += time.time() - _tmp trainInfo.iter += 1 trainInfo.netTime += (time.time() - tic) tic = time.time() #---------------------------------------- # Some events occur on regular intervals. # Address these here. #---------------------------------------- if (trainInfo.iter % trainInfo.param.snapshot) == 0: fn = os.path.join(outDir, 'iter_%06d.caffemodel' % trainInfo.iter) solver.net.save(str(fn)) if trainInfo.isModeStep and ((trainInfo.iter % trainInfo.param.stepsize) ==0): trainInfo.alpha *= trainInfo.gamma if (trainInfo.iter % trainInfo.param.display) == 1: print "[emCNN]: completed iteration %d of %d (epoch=%0.2f);" % (trainInfo.iter, trainInfo.param.max_iter, trainInfo.epoch+epochPct) print "[emCNN]: %0.2f min elapsed (%0.2f CNN min)" % (trainInfo.netTime/60., trainInfo.cnnTime/60.) if loss: print "[emCNN]: loss=%0.2f" % loss if acc: print "[emCNN]: accuracy (train volume)=%0.2f" % acc sys.stdout.flush() if trainInfo.iter >= trainInfo.param.max_iter: break # we hit max_iter on a non-epoch boundary...all done. # all finished with this epoch print "[emCNN]: epoch complete." sys.stdout.flush() return loss, acc
def _training_loop(solver, X, Y, M, solverParam, batchDim, outDir): """Performs CNN training. """ assert(batchDim[2] == batchDim[3]) # tiles must be square # Some variables and storage that we'll use in the loop below # tileRadius = int(batchDim[2]/2) Xi = np.zeros(batchDim, dtype=np.float32) yi = np.zeros((batchDim[0],), dtype=np.float32) losses = np.zeros((solverParam.max_iter,)) acc = np.zeros((solverParam.max_iter,)) currIter = 0 currEpoch = 0 # SGD parameters. SGD with momentum is of the form: # # V_{t+1} = \mu V_t - \alpha \nablaL(W_t) # W_{t+1} = W_t + V_{t+1} # # where W are the weights and V the previous update. # Ref: http://caffe.berkeleyvision.org/tutorial/solver.html # alpha = solverParam.base_lr # alpha := learning rate mu = solverParam.momentum # mu := momentum gamma = solverParam.gamma # gamma := step factor isModeStep = (solverParam.lr_policy == u'step') isTypeSGD = (solverParam.solver_type == solverParam.SolverType.Value('SGD')) Vall = {} # stores previous SGD steps (for all layers) if not (isModeStep and isTypeSGD): raise RuntimeError('Sorry - only support SGD "step" mode at the present') # TODO: weight decay # TODO: layer-specific weights # TODO: evaluate performance on valid slices instead of train slices? # (complicated by the fact that pycaffe doesn't support test mode) cnnTime = 0.0 # time spent doing core CNN operations tic = time.time() while currIter < solverParam.max_iter: # Each generator provides a single epoch's worth of data. # However, Caffe doesn't really recognize the notion of an epoch; instead, # they specify a number of training "iterations" (mini-batch evaluations, I assume). # So the inner loop below is for a single epoch, which we may terminate # early if the max # of iterations is reached. currEpoch += 1 it = emlib.stratified_interior_pixel_generator(Y, tileRadius, batchDim[0], mask=M) for Idx, epochPct in it: # Map the indices Idx -> tiles Xi and labels yi # # Note: if Idx.shape[0] < batchDim[0] (last iteration of an epoch) a few examples # from the previous minibatch will be "recycled" here. This is intentional # (to keep batch sizes consistent even if data set size is not a multiple # of the minibatch size). # for jj in range(Idx.shape[0]): yi[jj] = Y[ Idx[jj,0], Idx[jj,1], Idx[jj,2] ] a = Idx[jj,1] - tileRadius b = Idx[jj,1] + tileRadius + 1 c = Idx[jj,2] - tileRadius d = Idx[jj,2] + tileRadius + 1 Xi[jj, 0, :, :] = X[ Idx[jj,0], a:b, c:d ] # label-preserving data transformation (synthetic data generation) Xi = _xform_minibatch(Xi) #---------------------------------------- # one forward/backward pass and update weights # (SGD with momentum term) #---------------------------------------- _tmp = time.time() solver.net.set_input_arrays(Xi, yi) # XXX: could call preprocess() here? rv = solver.net.forward() solver.net.backward() for lIdx, layer in enumerate(solver.net.layers): for bIdx, blob in enumerate(layer.blobs): key = (lIdx, bIdx) V = Vall.get(key, 0.0) Vnext = mu*V - alpha * blob.diff blob.data[...] += Vnext Vall[key] = Vnext cnnTime += time.time() - _tmp # update running list of losses with the loss from this mini batch losses[currIter] = np.squeeze(rv['loss']) acc[currIter] = np.squeeze(rv['accuracy']) currIter += 1 #---------------------------------------- # Some events occur on mini-batch intervals. # Deal with those now. #---------------------------------------- if (currIter % solverParam.snapshot) == 0: fn = os.path.join(outDir, 'iter_%05d.caffemodel' % (currIter)) solver.net.save(str(fn)) if isModeStep and ((currIter % solverParam.stepsize) == 0): alpha *= gamma if (currIter % solverParam.display) == 1: elapsed = (time.time() - tic)/60. print "[train]: completed iteration %d (of %d; %0.2f min elapsed; %0.2f CNN min)" % (currIter, solverParam.max_iter, elapsed, cnnTime/60.) print "[train]: epoch: %d (%0.2f), loss: %0.3f, acc: %0.3f, learn rate: %0.3e" % (currEpoch, 100*epochPct, np.mean(losses[max(0,currIter-10):currIter]), np.mean(acc[max(0,currIter-10):currIter]), alpha) sys.stdout.flush() if currIter >= solverParam.max_iter: break # in case we hit max_iter on a non-epoch boundary return losses, acc