def add_parameter_update_ops(model):
brew.add_weight_decay(model, weight_decay)
iter = brew.iter(model, "iter")
lr = model.net.LearningRate(
[iter],
"lr",
base_lr=base_learning_rate,
policy="step",
stepsize=stepsize,
gamma=0.1,
)
for param in model.GetParams():
param_grad = model.param_to_grad[param]
param_momentum = model.param_init_net.ConstantFill(
[param], param + '_momentum', value=0.0
)
# Update param_grad and param_momentum in place
model.net.MomentumSGDUpdate(
[param_grad, param_momentum, lr, param],
[param_grad, param_momentum, param],
# almost 100% but with room to grow
momentum=0.9,
# netsterov is a defenseman for the Montreal Canadiens, but
# Nesterov Momentum works slightly better than standard momentum
nesterov=1,
)
def add_parameter_update_ops(model):
brew.add_weight_decay(model, weight_decay)
iter = brew.iter(model, "iter")
lr = model.net.LearningRate(
[iter],
"lr",
base_lr=base_learning_rate,
policy="step",
stepsize=stepsize,
gamma=0.1,
)
for param in model.GetParams():
param_grad = model.param_to_grad[param]
param_momentum = model.param_init_net.ConstantFill(
[param], param + '_momentum', value=0.0
)
# Update param_grad and param_momentum in place
model.net.MomentumSGDUpdate(
[param_grad, param_momentum, lr, param],
[param_grad, param_momentum, param],
# almost 100% but with room to grow
momentum=0.9,
# netsterov is a defenseman for the Montreal Canadiens, but
# Nesterov Momentum works slightly better than standard momentum
nesterov=1,
)
def train(INIT_NET, PREDICT_NET, epochs, batch_size, device_opts) :
data, label = get_data(batch_size)
workspace.FeedBlob("data", data, device_option=device_opts)
workspace.FeedBlob("label", label, device_option=device_opts)
train_model= model_helper.ModelHelper(name="train_net")
softmax = create_model(train_model, device_opts=device_opts)
add_training_operators(softmax, train_model, device_opts=device_opts)
with core.DeviceScope(device_opts):
brew.add_weight_decay(train_model, 0.001) # any effect???
workspace.RunNetOnce(train_model.param_init_net)
workspace.CreateNet(train_model.net)
print '\ntraining for', epochs, 'epochs'
for j in range(0, epochs):
data, label = get_data(batch_size)
workspace.FeedBlob("data", data, device_option=device_opts)
workspace.FeedBlob("label", label, device_option=device_opts)
workspace.RunNet(train_model.net, 10) # run for 10 times
print str(j) + ': ' + str(workspace.FetchBlob("loss")) + ' - ' + str(workspace.FetchBlob("accuracy"))
print 'training done'
print '\nrunning test model'
test_model= model_helper.ModelHelper(name="test_net", init_params=False)
create_model(test_model, device_opts=device_opts)
workspace.RunNetOnce(test_model.param_init_net)
workspace.CreateNet(test_model.net, overwrite=True)
data = np.zeros((1,1,30,30)).astype('float32')
workspace.FeedBlob("data", data, device_option=device_opts)
workspace.RunNet(test_model.net, 1)
print "\nInput: zeros"
print "Output:", workspace.FetchBlob("softmax")
print "Output class:", np.argmax(workspace.FetchBlob("softmax"))
data = np.ones((1,1,30,30)).astype('float32')
workspace.FeedBlob("data", data, device_option=device_opts)
workspace.RunNet(test_model.net, 1)
print "\nInput: ones"
print "Output:", workspace.FetchBlob("softmax")
print "Output class:", np.argmax(workspace.FetchBlob("softmax"))
print '\nsaving test model'
save_net(INIT_NET, PREDICT_NET, test_model)
def add_parameter_update_ops(model):
brew.add_weight_decay(model, args.weight_decay)
ITER = brew.iter(model, "ITER")
stepsz = int(30 * args.epoch_size / total_batch_size / num_shards)
LR = model.net.LearningRate(
[ITER],
"LR",
base_lr=args.base_learning_rate,
policy="step",
stepsize=stepsz,
gamma=0.1,
)
AddMomentumParameterUpdate(model, LR)
def add_parameter_update_ops_resnet(model, base_learning_rate, weight_decay):
brew.add_weight_decay(model, weight_decay)
iter = brew.iter(model, "iter")
lr = model.net.LearningRate([iter],
"lr",
base_lr=base_learning_rate,
policy="fixed",
gamma=0.1)
for param in model.GetParams():
param_grad = model.param_to_grad[param]
param_momentum = model.param_init_net.ConstantFill(
[param], param + '_momentum', value=0.0 )
model.net.MomentumSGDUpdate(
[param_grad, param_momentum, lr, param],
[param_grad, param_momentum, param],
momentum=0.9,
nesterov=1)
def train(INIT_NET, PREDICT_NET, epochs, batch_size, device_opts):
data, gt_segmentation = get_data(batch_size)
workspace.FeedBlob("data", data, device_option=device_opts)
workspace.FeedBlob("gt_segmentation",
gt_segmentation,
device_option=device_opts)
train_model = model_helper.ModelHelper(name="train_net",
arg_scope={"order": "NHWC"})
output_segmentation = create_unet_model(train_model,
device_opts=device_opts,
is_test=0)
add_training_operators(output_segmentation,
train_model,
device_opts=device_opts)
with core.DeviceScope(device_opts):
brew.add_weight_decay(train_model, 0.001)
workspace.RunNetOnce(train_model.param_init_net)
workspace.CreateNet(train_model.net)
print '\ntraining for', epochs, 'epochs'
for j in range(0, epochs):
data, gt_segmentation = get_data(batch_size, 4)
workspace.FeedBlob("data", data, device_option=device_opts)
workspace.FeedBlob("gt_segmentation",
gt_segmentation,
device_option=device_opts)
workspace.RunNet(train_model.net, 1) # run for 10 times
print str(j) + ': ' + str(workspace.FetchBlob("avg_loss"))
print 'training done'
test_model = model_helper.ModelHelper(name="test_net",
arg_scope={"order": "NHWC"},
init_params=False)
create_unet_model(test_model, device_opts=device_opts, is_test=1)
workspace.RunNetOnce(test_model.param_init_net)
workspace.CreateNet(test_model.net, overwrite=True)
print '\nsaving test model'
save_net(INIT_NET, PREDICT_NET, test_model)
def AddParameterUpdate_ops(model):
brew.add_weight_decay(model, weight_decay)
iter = brew.iter(model, "iter")
lr = model.net.LearningRate(
[iter],
"lr",
base_lr=base_learning_rate,
policy="step",
stepsize=stepsize,
gamma=0.1,
)
for param in model.GetParams():
param_grad = model.param_to_grad[param]
param_momentum = model.param_init_net.ConstantFill([param],
param + "_momentum",
value=0.0)
model.net.MomentumSGDUpdate(
[param_grad, param_momentum, lr, param],
[param_grad, param_momentum, param],
momentum=0.9,
nesterov=1,
)
def AddWeightDecay(self, weight_decay):
return brew.add_weight_decay(self, weight_decay)
def train(self,
num_epoch=1000,
batch_size=64,
context='gpu',
eval_metric='accuracy',
loss='cross_entropy',
opt_type='adam',
base_learning_rate=0.001,
weight_decay=0.001,
policy='fixed',
stepsize=1,
epsilon=1E-8,
beta1=0.9,
beta2=0.999,
gamma=0.999,
momentum=0.9):
if context == 'cpu':
device_opts = core.DeviceOption(caffe2_pb2.CPU, 0)
print("CPU mode selected")
elif context == 'gpu':
device_opts = core.DeviceOption(caffe2_pb2.CUDA, 0)
print("GPU mode selected")
workspace.ResetWorkspace(self._model_dir_)
arg_scope = {"order": "NCHW"}
# == Training model ==
train_model = model_helper.ModelHelper(name="train_net",
arg_scope=arg_scope)
data, label, train_dataset_size = self.add_input(
train_model,
batch_size=batch_size,
db=os.path.join(self._data_dir_, 'train_lmdb'),
db_type='lmdb',
device_opts=device_opts)
predictions_ = self.create_model(train_model,
data,
device_opts=device_opts,
is_test=False)
self.add_training_operators(train_model, predictions_, label,
device_opts, loss, opt_type,
base_learning_rate, policy, stepsize,
epsilon, beta1, beta2, gamma, momentum)
if not loss == 'euclidean':
self.add_accuracy(train_model, predictions_, label, device_opts,
eval_metric)
with core.DeviceScope(device_opts):
brew.add_weight_decay(train_model, weight_decay)
# Initialize and create the training network
workspace.RunNetOnce(train_model.param_init_net)
workspace.CreateNet(train_model.net, overwrite=True)
# Main Training Loop
iterations = self.get_total_num_iter(num_epoch, batch_size,
train_dataset_size)
epoch_as_iter = self.get_epoch_as_iter(num_epoch, batch_size,
train_dataset_size)
print("\n*** Starting Training for " + str(num_epoch) + " epochs = " +
str(iterations) + " iterations ***")
start_date = datetime.datetime.now()
for i in range(iterations):
workspace.RunNet(train_model.net)
if i % 50 == 0 or i % epoch_as_iter == 0:
if not loss == 'euclidean':
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(
workspace.FetchBlob(
"loss")) + ' - ' + 'Accuracy ' + str(
workspace.FetchBlob('accuracy'))
else:
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(
workspace.FetchBlob("loss"))
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(i) + '/' + str(
iterations) + ', ' + 'Current time spent: ' + str(
elapsed_time)
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(iterations) + '/' + str(
iterations) + ' Training done' + ', ' + 'Total time spent: ' + str(
elapsed_time)
print("\n*** Running Test model ***")
# == Testing model. ==
test_model = model_helper.ModelHelper(name="test_net",
arg_scope=arg_scope,
init_params=False)
data, label, test_dataset_size = self.add_input(
test_model,
batch_size=batch_size,
db=os.path.join(self._data_dir_, 'test_lmdb'),
db_type='lmdb',
device_opts=device_opts)
predictions_ = self.create_model(test_model,
data,
device_opts=device_opts,
is_test=True)
if not loss == 'euclidean':
self.add_accuracy(test_model, predictions_, label, device_opts,
eval_metric)
workspace.RunNetOnce(test_model.param_init_net)
workspace.CreateNet(test_model.net, overwrite=True)
# Main Testing Loop
test_accuracy = np.zeros(test_dataset_size / batch_size)
start_date = datetime.datetime.now()
for i in range(test_dataset_size / batch_size):
# Run a forward pass of the net on the current batch
workspace.RunNet(test_model.net)
# Collect the batch accuracy from the workspace
if not loss == 'euclidean':
test_accuracy[i] = workspace.FetchBlob('accuracy')
print 'Iter ' + str(i) + ': ' + 'Accuracy ' + str(
workspace.FetchBlob("accuracy"))
else:
test_accuracy[i] = workspace.FetchBlob("loss")
print 'Iter ' + str(i) + ': ' + 'Loss ' + str(
workspace.FetchBlob("loss"))
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(i) + '/' + str(
test_dataset_size /
batch_size) + ', ' + 'Current time spent: ' + str(elapsed_time)
current_time = datetime.datetime.now()
elapsed_time = current_time - start_date
print 'Progress: ' + str(test_dataset_size / batch_size) + '/' + str(
test_dataset_size / batch_size
) + ' Testing done' + ', ' + 'Total time spent: ' + str(elapsed_time)
print('Test accuracy mean: {:.9f}'.format(test_accuracy.mean()))
# == Deployment model. ==
# We simply need the main AddModel part.
deploy_model = model_helper.ModelHelper(name="deploy_net",
arg_scope=arg_scope,
init_params=False)
self.create_model(deploy_model, "data", device_opts, is_test=True)
print("\n*** Saving deploy model ***")
self.save_net(self._init_net_, self._predict_net_, deploy_model)
def AddWeightDecay(self, weight_decay):
return brew.add_weight_decay(self, weight_decay)