def add_parameter_update_ops(model): brew.add_weight_decay(model, weight_decay) iter = brew.iter(model, "iter") lr = model.net.LearningRate( [iter], "lr", base_lr=base_learning_rate, policy="step", stepsize=stepsize, gamma=0.1, ) for param in model.GetParams(): param_grad = model.param_to_grad[param] param_momentum = model.param_init_net.ConstantFill( [param], param + '_momentum', value=0.0 ) # Update param_grad and param_momentum in place model.net.MomentumSGDUpdate( [param_grad, param_momentum, lr, param], [param_grad, param_momentum, param], # almost 100% but with room to grow momentum=0.9, # netsterov is a defenseman for the Montreal Canadiens, but # Nesterov Momentum works slightly better than standard momentum nesterov=1, )
def train(INIT_NET, PREDICT_NET, epochs, batch_size, device_opts) : data, label = get_data(batch_size) workspace.FeedBlob("data", data, device_option=device_opts) workspace.FeedBlob("label", label, device_option=device_opts) train_model= model_helper.ModelHelper(name="train_net") softmax = create_model(train_model, device_opts=device_opts) add_training_operators(softmax, train_model, device_opts=device_opts) with core.DeviceScope(device_opts): brew.add_weight_decay(train_model, 0.001) # any effect??? workspace.RunNetOnce(train_model.param_init_net) workspace.CreateNet(train_model.net) print '\ntraining for', epochs, 'epochs' for j in range(0, epochs): data, label = get_data(batch_size) workspace.FeedBlob("data", data, device_option=device_opts) workspace.FeedBlob("label", label, device_option=device_opts) workspace.RunNet(train_model.net, 10) # run for 10 times print str(j) + ': ' + str(workspace.FetchBlob("loss")) + ' - ' + str(workspace.FetchBlob("accuracy")) print 'training done' print '\nrunning test model' test_model= model_helper.ModelHelper(name="test_net", init_params=False) create_model(test_model, device_opts=device_opts) workspace.RunNetOnce(test_model.param_init_net) workspace.CreateNet(test_model.net, overwrite=True) data = np.zeros((1,1,30,30)).astype('float32') workspace.FeedBlob("data", data, device_option=device_opts) workspace.RunNet(test_model.net, 1) print "\nInput: zeros" print "Output:", workspace.FetchBlob("softmax") print "Output class:", np.argmax(workspace.FetchBlob("softmax")) data = np.ones((1,1,30,30)).astype('float32') workspace.FeedBlob("data", data, device_option=device_opts) workspace.RunNet(test_model.net, 1) print "\nInput: ones" print "Output:", workspace.FetchBlob("softmax") print "Output class:", np.argmax(workspace.FetchBlob("softmax")) print '\nsaving test model' save_net(INIT_NET, PREDICT_NET, test_model)
def add_parameter_update_ops(model): brew.add_weight_decay(model, args.weight_decay) ITER = brew.iter(model, "ITER") stepsz = int(30 * args.epoch_size / total_batch_size / num_shards) LR = model.net.LearningRate( [ITER], "LR", base_lr=args.base_learning_rate, policy="step", stepsize=stepsz, gamma=0.1, ) AddMomentumParameterUpdate(model, LR)
def add_parameter_update_ops_resnet(model, base_learning_rate, weight_decay): brew.add_weight_decay(model, weight_decay) iter = brew.iter(model, "iter") lr = model.net.LearningRate([iter], "lr", base_lr=base_learning_rate, policy="fixed", gamma=0.1) for param in model.GetParams(): param_grad = model.param_to_grad[param] param_momentum = model.param_init_net.ConstantFill( [param], param + '_momentum', value=0.0 ) model.net.MomentumSGDUpdate( [param_grad, param_momentum, lr, param], [param_grad, param_momentum, param], momentum=0.9, nesterov=1)
def train(INIT_NET, PREDICT_NET, epochs, batch_size, device_opts): data, gt_segmentation = get_data(batch_size) workspace.FeedBlob("data", data, device_option=device_opts) workspace.FeedBlob("gt_segmentation", gt_segmentation, device_option=device_opts) train_model = model_helper.ModelHelper(name="train_net", arg_scope={"order": "NHWC"}) output_segmentation = create_unet_model(train_model, device_opts=device_opts, is_test=0) add_training_operators(output_segmentation, train_model, device_opts=device_opts) with core.DeviceScope(device_opts): brew.add_weight_decay(train_model, 0.001) workspace.RunNetOnce(train_model.param_init_net) workspace.CreateNet(train_model.net) print '\ntraining for', epochs, 'epochs' for j in range(0, epochs): data, gt_segmentation = get_data(batch_size, 4) workspace.FeedBlob("data", data, device_option=device_opts) workspace.FeedBlob("gt_segmentation", gt_segmentation, device_option=device_opts) workspace.RunNet(train_model.net, 1) # run for 10 times print str(j) + ': ' + str(workspace.FetchBlob("avg_loss")) print 'training done' test_model = model_helper.ModelHelper(name="test_net", arg_scope={"order": "NHWC"}, init_params=False) create_unet_model(test_model, device_opts=device_opts, is_test=1) workspace.RunNetOnce(test_model.param_init_net) workspace.CreateNet(test_model.net, overwrite=True) print '\nsaving test model' save_net(INIT_NET, PREDICT_NET, test_model)
def AddParameterUpdate_ops(model): brew.add_weight_decay(model, weight_decay) iter = brew.iter(model, "iter") lr = model.net.LearningRate( [iter], "lr", base_lr=base_learning_rate, policy="step", stepsize=stepsize, gamma=0.1, ) for param in model.GetParams(): param_grad = model.param_to_grad[param] param_momentum = model.param_init_net.ConstantFill([param], param + "_momentum", value=0.0) model.net.MomentumSGDUpdate( [param_grad, param_momentum, lr, param], [param_grad, param_momentum, param], momentum=0.9, nesterov=1, )
def AddWeightDecay(self, weight_decay): return brew.add_weight_decay(self, weight_decay)
def train(self, num_epoch=1000, batch_size=64, context='gpu', eval_metric='accuracy', loss='cross_entropy', opt_type='adam', base_learning_rate=0.001, weight_decay=0.001, policy='fixed', stepsize=1, epsilon=1E-8, beta1=0.9, beta2=0.999, gamma=0.999, momentum=0.9): if context == 'cpu': device_opts = core.DeviceOption(caffe2_pb2.CPU, 0) print("CPU mode selected") elif context == 'gpu': device_opts = core.DeviceOption(caffe2_pb2.CUDA, 0) print("GPU mode selected") workspace.ResetWorkspace(self._model_dir_) arg_scope = {"order": "NCHW"} # == Training model == train_model = model_helper.ModelHelper(name="train_net", arg_scope=arg_scope) data, label, train_dataset_size = self.add_input( train_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'train_lmdb'), db_type='lmdb', device_opts=device_opts) predictions_ = self.create_model(train_model, data, device_opts=device_opts, is_test=False) self.add_training_operators(train_model, predictions_, label, device_opts, loss, opt_type, base_learning_rate, policy, stepsize, epsilon, beta1, beta2, gamma, momentum) if not loss == 'euclidean': self.add_accuracy(train_model, predictions_, label, device_opts, eval_metric) with core.DeviceScope(device_opts): brew.add_weight_decay(train_model, weight_decay) # Initialize and create the training network workspace.RunNetOnce(train_model.param_init_net) workspace.CreateNet(train_model.net, overwrite=True) # Main Training Loop iterations = self.get_total_num_iter(num_epoch, batch_size, train_dataset_size) epoch_as_iter = self.get_epoch_as_iter(num_epoch, batch_size, train_dataset_size) print("\n*** Starting Training for " + str(num_epoch) + " epochs = " + str(iterations) + " iterations ***") start_date = datetime.datetime.now() for i in range(iterations): workspace.RunNet(train_model.net) if i % 50 == 0 or i % epoch_as_iter == 0: if not loss == 'euclidean': print 'Iter ' + str(i) + ': ' + 'Loss ' + str( workspace.FetchBlob( "loss")) + ' - ' + 'Accuracy ' + str( workspace.FetchBlob('accuracy')) else: print 'Iter ' + str(i) + ': ' + 'Loss ' + str( workspace.FetchBlob("loss")) current_time = datetime.datetime.now() elapsed_time = current_time - start_date print 'Progress: ' + str(i) + '/' + str( iterations) + ', ' + 'Current time spent: ' + str( elapsed_time) current_time = datetime.datetime.now() elapsed_time = current_time - start_date print 'Progress: ' + str(iterations) + '/' + str( iterations) + ' Training done' + ', ' + 'Total time spent: ' + str( elapsed_time) print("\n*** Running Test model ***") # == Testing model. == test_model = model_helper.ModelHelper(name="test_net", arg_scope=arg_scope, init_params=False) data, label, test_dataset_size = self.add_input( test_model, batch_size=batch_size, db=os.path.join(self._data_dir_, 'test_lmdb'), db_type='lmdb', device_opts=device_opts) predictions_ = self.create_model(test_model, data, device_opts=device_opts, is_test=True) if not loss == 'euclidean': self.add_accuracy(test_model, predictions_, label, device_opts, eval_metric) workspace.RunNetOnce(test_model.param_init_net) workspace.CreateNet(test_model.net, overwrite=True) # Main Testing Loop test_accuracy = np.zeros(test_dataset_size / batch_size) start_date = datetime.datetime.now() for i in range(test_dataset_size / batch_size): # Run a forward pass of the net on the current batch workspace.RunNet(test_model.net) # Collect the batch accuracy from the workspace if not loss == 'euclidean': test_accuracy[i] = workspace.FetchBlob('accuracy') print 'Iter ' + str(i) + ': ' + 'Accuracy ' + str( workspace.FetchBlob("accuracy")) else: test_accuracy[i] = workspace.FetchBlob("loss") print 'Iter ' + str(i) + ': ' + 'Loss ' + str( workspace.FetchBlob("loss")) current_time = datetime.datetime.now() elapsed_time = current_time - start_date print 'Progress: ' + str(i) + '/' + str( test_dataset_size / batch_size) + ', ' + 'Current time spent: ' + str(elapsed_time) current_time = datetime.datetime.now() elapsed_time = current_time - start_date print 'Progress: ' + str(test_dataset_size / batch_size) + '/' + str( test_dataset_size / batch_size ) + ' Testing done' + ', ' + 'Total time spent: ' + str(elapsed_time) print('Test accuracy mean: {:.9f}'.format(test_accuracy.mean())) # == Deployment model. == # We simply need the main AddModel part. deploy_model = model_helper.ModelHelper(name="deploy_net", arg_scope=arg_scope, init_params=False) self.create_model(deploy_model, "data", device_opts, is_test=True) print("\n*** Saving deploy model ***") self.save_net(self._init_net_, self._predict_net_, deploy_model)