def test_net_transformer_function(self): devices = [1, 2, 3] def add_input_ops(model): model.param_init_net.UniformFill([], ["data"], shape=[32, 8]) def add_optimizer(model): optimizer.build_sgd(model, 0.1) def add_model_ops(model, loss_scale): fc1 = brew.fc(model, "data", "fc1", dim_in=8, dim_out=8) return [fc1] kwargs = { 'input_builder_fun': add_input_ops, 'forward_pass_builder_fun': add_model_ops, 'devices': devices, } # assert that the transformer is called for both train and test cases transform = Mock() kwargs['net_transformer_fun'] = transform model = model_helper.ModelHelper(name="r", init_params=False) data_parallel_model.Parallelize_CPU(model, **kwargs) self.assertTrue(transform.called) self.assertEqual(transform.call_count, 1) transform = Mock() kwargs['net_transformer_fun'] = transform kwargs['optimizer_builder_fun'] = add_optimizer model = model_helper.ModelHelper(name="r", init_params=True) data_parallel_model.Parallelize_CPU(model, **kwargs) self.assertTrue(transform.called) self.assertEqual(transform.call_count, 1)
def test_checkpoint_params(self): def add_input_ops(model): pass def add_model_ops(model, loss_scale): model.NHWC2NCHW("data", "data_nchw") model.Conv("data_nchw", 'conv1', 3, 64, weight_init=("MSRAFill", {}), kernel=7, stride=2, pad=3, no_bias=0) model.SpatialBN('conv1', 'conv1_spatbn_relu', 64, epsilon=1e-3) model.Relu('conv1_spatbn_relu', 'conv1_spatbn_relu') model.MaxPool('conv1_spatbn_relu', 'pool1', kernel=3, stride=2) model.FC('pool1', 'fc', dim_in=(64 * 56 * 56), dim_out=100) model.Sigmoid('fc', 'fc_sigm') model.Softmax('fc_sigm', 'softmax') model.LabelCrossEntropy(['softmax', 'label'], 'xent') loss = model.AveragedLoss('xent', 'loss') # Add a duplicate param init to ensure it does not cause issues model.param_init_net.ConstantFill([], ["fc_w"], shape=((64 * 56 * 56), 1000)) return [loss] def add_optimizer(model): optimizer.build_sgd(model, 0.1, policy="fixed", momentum=0.9) model = cnn.CNNModelHelper( order="NHWC", name="test", ) data_parallel_model.Parallelize_CPU( model, input_builder_fun=add_input_ops, forward_pass_builder_fun=add_model_ops, optimizer_builder_fun=add_optimizer, devices=[1, 2, 3], ) # Only gpu_1 params should be returned (gpu_1 is the first gpu) checkpoint_params = data_parallel_model.GetCheckpointParams(model) for p in model.GetParams("cpu_1/"): self.assertTrue(p in checkpoint_params) self.assertTrue(p + "_momentum" in checkpoint_params) for p in model.GetParams("cpu_2/"): self.assertFalse(p in checkpoint_params) self.assertTrue( core.BlobReference("cpu_1/fc_w_momentum") in checkpoint_params) for c in model.GetComputedParams("cpu_1/"): self.assertTrue(c in checkpoint_params) for c in model.GetComputedParams("cpu_2/"): self.assertFalse(c in checkpoint_params) self.assertFalse(core.BlobReference("cpu_1/data") in checkpoint_params) self.assertTrue( core.BlobReference("optimizer_iteration") in checkpoint_params)
def test_net_conversion_and_append_net(self): other = model_helper.ModelHelper() fc1 = brew.fc(other, "data", "other_fc1", dim_in=3 * 227 * 227, dim_out=10) fc2 = brew.fc(other, fc1, "other_fc2", dim_in=10, dim_out=10) brew.fc(other, fc2, "other_fc3", dim_in=10, dim_out=10) def add_input_ops(model): model.net.UniformFill([], ["data"], shape=[4, 227, 227, 3]) model.net.UniformFill([], ["label"], shape=[4]) def add_model_ops(model, loss_scale): model.NHWC2NCHW("data", "data_nchw") model.Conv("data_nchw", 'conv1', 3, 64, weight_init=("MSRAFill", {}), kernel=7, stride=2, pad=3, no_bias=0) model.SpatialBN('conv1', 'conv1_spatbn_relu', 64, epsilon=1e-3) model.Relu('conv1_spatbn_relu', 'conv1_spatbn_relu') model.MaxPool('conv1_spatbn_relu', 'pool1', kernel=3, stride=2) model.FC('pool1', 'fc', dim_in=(64 * 56 * 56), dim_out=10) # Append the net and param_init_net of the other model appendnet = data_parallel_model.ConvertNetForDevice(other.net) model.net.AppendNet(appendnet) model.param_init_net.AppendNet( data_parallel_model.ConvertNetForDevice(other.param_init_net)) model.Sigmoid('fc', 'fc_sigm') model.Softmax('fc_sigm', 'softmax') loss = model.AveragedLoss('softmax', 'loss') return [loss] def add_optimizer(model): optimizer.build_sgd(model, 0.1, policy="fixed", momentum=0.9) model = cnn.CNNModelHelper( order="NCHW", name="test", ) data_parallel_model.Parallelize_CPU( model, input_builder_fun=add_input_ops, forward_pass_builder_fun=add_model_ops, optimizer_builder_fun=add_optimizer, devices=range(4)) # Just create and run net and confirm no exception is thrown workspace.RunNetOnce(model.param_init_net) workspace.CreateNet(model.net) workspace.RunNet(model.net)
def run(comm_rank, comm_size, tmpdir): def add_input_ops(model): pass def add_model_ops(model, loss_scale): return [] def add_optimizer(model): pass workspace.ResetWorkspace() store_handler = "store_handler" workspace.RunOperatorOnce( core.CreateOperator( "FileStoreHandlerCreate", [], [store_handler], path=tmpdir)) rendezvous = dict( kv_handler=store_handler, shard_id=comm_rank, num_shards=comm_size, engine='GLOO', ) model = cnn.CNNModelHelper( order="NHWC", name="test", ) # Set network timeout to 2 seconds, and add a 3 seconds # sleep for 1 host. Make sure there is no timeout on the # second RunNet. data_parallel_model._DEFAULT_TIMEOUT_SEC=2 data_parallel_model.Parallelize_CPU( model, input_builder_fun=add_input_ops, forward_pass_builder_fun=add_model_ops, optimizer_builder_fun=add_optimizer, devices=[1, 2, 3], rendezvous=rendezvous, barrier_net_timeout_sec=5 ) data_parallel_model.RunInitNet(model) data_parallel_model.RunNet(model, 2) if comm_rank == 0: time.sleep(data_parallel_model._DEFAULT_TIMEOUT_SEC) data_parallel_model.RunNet(model, 2)
def run(comm_rank, comm_size, tmpdir): def add_input_ops(model): pass def add_model_ops(model, loss_scale): return [] def add_optimizer(model): pass store_handler = "store_handler" workspace.RunOperatorOnce( core.CreateOperator( "FileStoreHandlerCreate", [], [store_handler], path=tmpdir)) rendezvous = dict( kv_handler=store_handler, shard_id=comm_rank, num_shards=comm_size, engine='GLOO', ) model = cnn.CNNModelHelper( order="NHWC", name="test", ) data_parallel_model.Parallelize_CPU( model, input_builder_fun=add_input_ops, forward_pass_builder_fun=add_model_ops, optimizer_builder_fun=add_optimizer, devices=[1, 2, 3], rendezvous=rendezvous ) data_parallel_model.RunInitNet(model) for _ in range(2): data_parallel_model.Synchronize(model)
def Train_Model(args): # Get relative path name of current directory dir_path = os.path.dirname(os.path.realpath(__file__)) # create folder path for data sets current_folder = os.path.join(dir_path, '../../', 'datasets', 'blood-cells') data_folder = os.path.join(current_folder, 'dataset2-master') root_folder = current_folder db_missing = False # find the dataset folder if not os.path.exists(data_folder): os.makedirs(data_folder) print("Your data folder was not found!! This was generated: {}".format(data_folder)) # Look for existing database: lmdb if os.path.exists(os.path.join(data_folder,'blood_cells_train_lmdb')): print("lmdb train db found!") else: db_missing = True if os.path.exists(os.path.join(data_folder,'blood_cells_test_lmdb')): print("lmdb test db found!") else: db_missing = True if db_missing: print("DB is not found. Please fix file paths") sys.exit() # prepare the databases for training train_data_db = os.path.join(data_folder,'blood_cells_train_lmdb') test_data_db = os.path.join(data_folder, 'blood_cells_test_lmdb') arg_scope = { "order" : "NCHW", "use_cudnn" : args.gpu, } train_model = model_helper.ModelHelper(name="vgg19_train", arg_scope=arg_scope) reader = train_model.CreateDB( "train_reader", db= train_data_db, db_type= 'lmdb', ) # prepare precursor hyper-parameters devices = [] for i in range(args.shards): devices.append(i) train_data_count = 9893 batch_per_device = 16 total_batch_size = batch_per_device * len(devices) num_labels = 4 base_learning_rate = .001 * total_batch_size weight_decay = (5 * 10**(-4)) stepsize = int(2 * train_data_count / total_batch_size) def add_image_input_ops(model): ''' The image input operator loads image and label data from the reader and applies transformations to the images (random cropping, mirroring, ...). ''' data, label = brew.image_input( model, reader, ["data", "label"], batch_size=batch_per_device, # mean: to remove color values that are common # mean=128., # std is going to be modified randomly to influence the mean subtraction # std=128., # scale to rescale each image to a common size scale=224, # crop to the square each image to exact dimensions crop=224, # not running in test mode is_test=False, # mirroring of the images will occur randomly mirror=1 ) # prevent back-propagation: optional performance improvement; may not be observable at small scale data = model.StopGradient(data, data) def create_vgg_model_ops(model, loss_scale= 1.0): [softmax, loss] = vgg19.create_VGG19(model, "data", num_labels, "label") prefix = model.net.Proto().name loss = model.net.Scale(loss, prefix + "_loss", scale= loss_scale) brew.accuracy(model, [softmax, "label"], prefix + "_accuracy") return [loss] def add_parameter_update_ops(model): brew.add_weight_decay(model, weight_decay) iter = brew.iter(model, "iter") lr = model.net.LearningRate( [iter], "lr", base_lr=base_learning_rate, policy="step", stepsize=stepsize, gamma=0.1, ) for param in model.GetParams(): param_grad = model.param_to_grad[param] param_momentum = model.param_init_net.ConstantFill( [param], param + '_momentum', value=0.0 ) # Update param_grad and param_momentum in place model.net.MomentumSGDUpdate( [param_grad, param_momentum, lr, param], [param_grad, param_momentum, param], # almost 100% but with room to grow momentum=0.9, # netsterov is a defenseman for the Montreal Canadiens, but # Nesterov Momentum works slightly better than standard momentum nesterov=1, ) def accuracy(model): accuracy = [] prefix = model.net.Proto().name for device in model._devices: accuracy.append( np.asscalar(workspace.FetchBlob("{}_{}/{}_accuracy".format('gpu' if args.gpu else 'cpu', device, prefix)))) return np.average(accuracy) if args.gpu: dpm.Parallelize_GPU( train_model, input_builder_fun=add_image_input_ops, forward_pass_builder_fun=create_vgg_model_ops, param_update_builder_fun=add_parameter_update_ops, devices=devices, dynamic_memory_management=True, ) else: dpm.Parallelize_CPU( train_model, input_builder_fun=add_image_input_ops, forward_pass_builder_fun=create_vgg_model_ops, param_update_builder_fun=add_parameter_update_ops, devices=devices, optimize_gradient_memory=True, ) workspace.RunNetOnce(train_model.param_init_net) workspace.CreateNet(train_model.net) test_model = model_helper.ModelHelper(name="vgg_test", arg_scope=arg_scope, init_params=False) reader = test_model.CreateDB( "test_reader", db=test_data_db, db_type='lmdb', ) if args.gpu: # Validation is parallelized across devices as well dpm.Parallelize_GPU( test_model, input_builder_fun=add_image_input_ops, forward_pass_builder_fun=create_vgg_model_ops, param_update_builder_fun=None, devices=devices, ) else: dpm.Parallelize_CPU( test_model, input_builder_fun=add_image_input_ops, forward_pass_builder_fun=create_vgg_model_ops, param_update_builder_fun=None, devices=devices, ) workspace.RunNetOnce(test_model.param_init_net) workspace.CreateNet(test_model.net) # Start looping through epochs where we run the batches of images to cover the entire dataset # Usually you would want to run a lot more epochs to increase your model's accuracy num_epochs = 20 for epoch in range(num_epochs): # Split up the images evenly: total images / batch size num_iters = int(train_data_count / total_batch_size) for iter in range(num_iters): # Stopwatch start! t1 = time.time() # Run this iteration! workspace.RunNet(train_model.net.Proto().name) t2 = time.time() dt = t2 - t1 # Stopwatch stopped! How'd we do? print(( "Finished iteration {:>" + str(len(str(num_iters))) + "}/{}" + " (epoch {:>" + str(len(str(num_epochs))) + "}/{})" + " ({:.2f} images/sec)"). format(iter+1, num_iters, epoch+1, num_epochs, total_batch_size/dt)) # Get the average accuracy for the training model train_accuracy = accuracy(train_model) # Run the test model and assess accuracy test_accuracies = [] for _ in range(test_data_count / total_batch_size): # Run the test model workspace.RunNet(test_model.net.Proto().name) test_accuracies.append(accuracy(test_model)) test_accuracy = np.average(test_accuracies) print( "Train accuracy: {:.3f}, test accuracy: {:.3f}". format(train_accuracy, test_accuracy))