def train(network, trainer, od_minibatch_source, reader_dev, minibatch_size,
epochs_to_train, cfg):
input_map = {
od_minibatch_source.image_si: network['features'],
od_minibatch_source.roi_si: network['roi_input'],
od_minibatch_source.dims_si: network['dim_input']
}
# Train all minibatches
progress_printer = ProgressPrinter(tag='Training',
num_epochs=epochs_to_train)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < cfg[
"DATA"].NUM_TRAIN_IMAGES: # loop over minibatches in the epoch
data = od_minibatch_source.next_minibatch(min(
cfg.MB_SIZE, cfg["DATA"].NUM_TRAIN_IMAGES - sample_count),
input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
return create_faster_rcnn_eval_model(network['loss'], network['features'],
network['dim_input'], cfg)
def train_fast_rcnn(cfg):
# Train only if no model exists yet
model_path = cfg['MODEL_PATH']
if os.path.exists(model_path) and cfg["CNTK"].MAKE_MODE:
print("Loading existing model from %s" % model_path)
return load_model(model_path)
else:
# Input variables denoting features and labeled ground truth rois (as 5-tuples per roi)
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT,
cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=cfg["MODEL"].FEATURE_NODE_NAME)
roi_proposals = input_variable(
(cfg.NUM_ROI_PROPOSALS, 4),
dynamic_axes=[Axis.default_batch_axis()],
name="roi_proposals")
label_targets = input_variable(
(cfg.NUM_ROI_PROPOSALS, cfg["DATA"].NUM_CLASSES),
dynamic_axes=[Axis.default_batch_axis()])
bbox_targets = input_variable(
(cfg.NUM_ROI_PROPOSALS, 4 * cfg["DATA"].NUM_CLASSES),
dynamic_axes=[Axis.default_batch_axis()])
bbox_inside_weights = input_variable(
(cfg.NUM_ROI_PROPOSALS, 4 * cfg["DATA"].NUM_CLASSES),
dynamic_axes=[Axis.default_batch_axis()])
# Instantiate the Fast R-CNN prediction model and loss function
loss, pred_error = create_fast_rcnn_model(image_input, roi_proposals,
label_targets, bbox_targets,
bbox_inside_weights, cfg)
if isinstance(loss, cntk.Variable):
loss = combine([loss])
if cfg["CNTK"].DEBUG_OUTPUT:
print("Storing graphs and models to %s." % cfg.OUTPUT_PATH)
plot(
loss,
os.path.join(cfg.OUTPUT_PATH,
"graph_frcn_train." + cfg["CNTK"].GRAPH_TYPE))
# Set learning parameters
lr_factor = cfg["CNTK"].LR_FACTOR
lr_per_sample_scaled = [
x * lr_factor for x in cfg["CNTK"].LR_PER_SAMPLE
]
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
l2_reg_weight = cfg["CNTK"].L2_REG_WEIGHT
epochs_to_train = cfg["CNTK"].MAX_EPOCHS
print("Using base model: {}".format(cfg["MODEL"].BASE_MODEL))
print("lr_per_sample: {}".format(lr_per_sample_scaled))
# --- train ---
# Instantiate the learners and the trainer object
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
lr_schedule = learning_parameter_schedule_per_sample(
lr_per_sample_scaled)
learner = momentum_sgd(others,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight,
unit_gain=False,
use_mean_gradient=True)
bias_lr_per_sample = [
v * bias_lr_mult for v in cfg["CNTK"].LR_PER_SAMPLE
]
bias_lr_schedule = learning_parameter_schedule_per_sample(
bias_lr_per_sample)
bias_learner = momentum_sgd(biases,
bias_lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight,
unit_gain=False,
use_mean_gradient=True)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
if cfg.USE_PRECOMPUTED_PROPOSALS:
proposal_provider = ProposalProvider.fromfile(
cfg["DATA"].TRAIN_PRECOMPUTED_PROPOSALS_FILE,
cfg.NUM_ROI_PROPOSALS)
else:
proposal_provider = ProposalProvider.fromconfig(cfg)
od_minibatch_source = ObjectDetectionMinibatchSource(
cfg["DATA"].TRAIN_MAP_FILE,
cfg["DATA"].TRAIN_ROI_FILE,
max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE,
pad_width=cfg.IMAGE_WIDTH,
pad_height=cfg.IMAGE_HEIGHT,
pad_value=cfg["MODEL"].IMG_PAD_COLOR,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["DATA"].NUM_TRAIN_IMAGES,
num_classes=cfg["DATA"].NUM_CLASSES,
proposal_provider=proposal_provider,
provide_targets=True,
proposal_iou_threshold=cfg.BBOX_THRESH,
normalize_means=None
if not cfg.BBOX_NORMALIZE_TARGETS else cfg.BBOX_NORMALIZE_MEANS,
normalize_stds=None
if not cfg.BBOX_NORMALIZE_TARGETS else cfg.BBOX_NORMALIZE_STDS)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.proposals_si: roi_proposals,
od_minibatch_source.label_targets_si: label_targets,
od_minibatch_source.bbox_targets_si: bbox_targets,
od_minibatch_source.bbiw_si: bbox_inside_weights
}
progress_printer = ProgressPrinter(tag='Training',
num_epochs=epochs_to_train,
gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < cfg[
"DATA"].NUM_TRAIN_IMAGES: # loop over minibatches in the epoch
data = od_minibatch_source.next_minibatch(min(
cfg.MB_SIZE, cfg["DATA"].NUM_TRAIN_IMAGES - sample_count),
input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
continue
#print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
eval_model = create_fast_rcnn_eval_model(loss, image_input,
roi_proposals, cfg)
eval_model.save(cfg['MODEL_PATH'])
return eval_model
def train_model(image_input, roi_input, dims_input, loss, pred_error,
lr_per_sample, mm_schedule, l2_reg_weight, epochs_to_train, cfg,
rpn_rois_input=None, buffered_rpn_proposals=None):
if isinstance(loss, cntk.Variable):
loss = combine([loss])
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
if cfg["CNTK"].DEBUG_OUTPUT:
print("biases")
for p in biases: print(p)
print("others")
for p in others: print(p)
print("bias_lr_mult: {}".format(bias_lr_mult))
# Instantiate the learners and the trainer object
lr_schedule = learning_parameter_schedule_per_sample(lr_per_sample)
learner = momentum_sgd(others, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=True)
bias_lr_per_sample = [v * bias_lr_mult for v in lr_per_sample]
bias_lr_schedule = learning_parameter_schedule_per_sample(bias_lr_per_sample)
bias_learner = momentum_sgd(biases, bias_lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=True)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
if buffered_rpn_proposals is not None:
proposal_provider = ProposalProvider.fromlist(buffered_rpn_proposals, requires_scaling=False)
else:
proposal_provider = None
od_minibatch_source = ObjectDetectionMinibatchSource(
cfg["DATA"].TRAIN_MAP_FILE, cfg["DATA"].TRAIN_ROI_FILE,
num_classes=cfg["DATA"].NUM_CLASSES,
max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE,
pad_width=cfg.IMAGE_WIDTH,
pad_height=cfg.IMAGE_HEIGHT,
pad_value=cfg["MODEL"].IMG_PAD_COLOR,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["DATA"].NUM_TRAIN_IMAGES,
proposal_provider=proposal_provider)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.roi_si: roi_input,
}
if buffered_rpn_proposals is not None:
input_map[od_minibatch_source.proposals_si] = rpn_rois_input
else:
input_map[od_minibatch_source.dims_si] = dims_input
progress_printer = ProgressPrinter(tag='Training', num_epochs=epochs_to_train, gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < cfg["DATA"].NUM_TRAIN_IMAGES: # loop over minibatches in the epoch
data = od_minibatch_source.next_minibatch(min(cfg.MB_SIZE, cfg["DATA"].NUM_TRAIN_IMAGES-sample_count), input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
#progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
continue
#print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
def train(train_reader, valid_reader, vocab, i2w, s2smodel, max_epochs,
epoch_size):
# Note: We would like to set the signature of 's2smodel' (s2smodel.update_signature()), but that will cause
# an error since the training criterion uses a reduced sequence axis for the labels.
# This is because it removes the initial <s> symbol. Hence, we must leave the model
# with unspecified input shapes and axes.
# create the training wrapper for the s2smodel, as well as the criterion function
model_train = create_model_train(s2smodel)
criterion = create_criterion_function(model_train)
# also wire in a greedy decoder so that we can properly log progress on a validation example
# This is not used for the actual training process.
model_greedy = create_model_greedy(s2smodel)
# This does not need to be done in training generally though
# Instantiate the trainer object to drive the model training
minibatch_size = 72
lr = 0.001 if use_attention else 0.005 # TODO: can we use the same value for both?
learner = adam_sgd(
model_train.parameters,
lr=learning_rate_schedule([lr] * 2 + [lr / 2] * 3 + [lr / 4],
UnitType.sample, epoch_size),
momentum=momentum_as_time_constant_schedule(1100),
gradient_clipping_threshold_per_sample=2.3,
gradient_clipping_with_truncation=True)
trainer = Trainer(None, criterion, learner)
# Get minibatches of sequences to train with and perform model training
total_samples = 0
mbs = 0
eval_freq = 100
# print out some useful training information
log_number_of_parameters(model_train)
print()
progress_printer = ProgressPrinter(freq=30, tag='Training')
#progress_printer = ProgressPrinter(freq=30, tag='Training', log_to_file=model_path_stem + ".log") # use this to log to file
sparse_to_dense = create_sparse_to_dense(input_vocab_dim)
for epoch in range(max_epochs):
print("Saving model to '%s'" % model_path(epoch))
s2smodel.save(model_path(epoch))
while total_samples < (epoch + 1) * epoch_size:
# get next minibatch of training data
mb_train = train_reader.next_minibatch(minibatch_size)
#trainer.train_minibatch(mb_train[train_reader.streams.features], mb_train[train_reader.streams.labels])
trainer.train_minibatch({
criterion.arguments[0]:
mb_train[train_reader.streams.features],
criterion.arguments[1]:
mb_train[train_reader.streams.labels]
})
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
# every N MBs evaluate on a test sequence to visually show how we're doing
if mbs % eval_freq == 0:
mb_valid = valid_reader.next_minibatch(1)
# run an eval on the decoder output model (i.e. don't use the groundtruth)
e = model_greedy(mb_valid[valid_reader.streams.features])
print(
format_sequences(
sparse_to_dense(
mb_valid[valid_reader.streams.features]), i2w))
print("->")
print(format_sequences(e, i2w))
# debugging attention
if use_attention:
debug_attention(model_greedy,
mb_valid[valid_reader.streams.features])
total_samples += mb_train[train_reader.streams.labels].num_samples
mbs += 1
# log a summary of the stats for the epoch
progress_printer.epoch_summary(with_metric=True)
# done: save the final model
print("Saving final model to '%s'" % model_path(max_epochs))
s2smodel.save(model_path(max_epochs))
print("%d epochs complete." % max_epochs)
def train_model(image_input,
roi_input,
dims_input,
loss,
pred_error,
lr_per_sample,
mm_schedule,
l2_reg_weight,
epochs_to_train,
rpn_rois_input=None,
buffered_rpn_proposals=None):
if isinstance(loss, cntk.Variable):
loss = combine([loss])
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
if cfg["CNTK"].DEBUG_OUTPUT:
print("biases")
for p in biases:
print(p)
print("others")
for p in others:
print(p)
print("bias_lr_mult: {}".format(bias_lr_mult))
# Instantiate the learners and the trainer object
lr_schedule = learning_rate_schedule(lr_per_sample, unit=UnitType.sample)
learner = momentum_sgd(others,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight,
unit_gain=False,
use_mean_gradient=cfg["CNTK"].USE_MEAN_GRADIENT)
bias_lr_per_sample = [v * bias_lr_mult for v in lr_per_sample]
bias_lr_schedule = learning_rate_schedule(bias_lr_per_sample,
unit=UnitType.sample)
bias_learner = momentum_sgd(
biases,
bias_lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight,
unit_gain=False,
use_mean_gradient=cfg["CNTK"].USE_MEAN_GRADIENT)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
od_minibatch_source = ObjectDetectionMinibatchSource(
globalvars['train_map_file'],
globalvars['train_roi_file'],
max_annotations_per_image=cfg["CNTK"].INPUT_ROIS_PER_IMAGE,
pad_width=image_width,
pad_height=image_height,
pad_value=img_pad_value,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["CNTK"].NUM_TRAIN_IMAGES,
buffered_rpn_proposals=buffered_rpn_proposals)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.roi_si: roi_input,
od_minibatch_source.dims_si: dims_input
}
use_buffered_proposals = buffered_rpn_proposals is not None
progress_printer = ProgressPrinter(tag='Training',
num_epochs=epochs_to_train,
gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data, proposals = od_minibatch_source.next_minibatch_with_proposals(
min(mb_size, epoch_size - sample_count), input_map=input_map)
if use_buffered_proposals:
data[rpn_rois_input] = MinibatchData(
Value(batch=np.asarray(proposals, dtype=np.float32)), 1, 1,
False)
# remove dims input if no rpn is required to avoid warnings
del data[[k for k in data if '[6]' in str(k)][0]]
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
def train_and_evaluate(reader_train, reader_test, max_epochs, model_func):
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# Normalize the input
feature_scale = 1.0 / 256.0
input_var_norm = element_times(feature_scale, input_var)
# apply model to input
z = model_func(input_var_norm, out_dims=num_classes)
#
# Training action
#
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# training config
epoch_size = 20000
minibatch_size = 64
# Set training parameters
lr_per_minibatch = learning_rate_schedule([0.01]*10 + [0.003]*10 + [0.001], UnitType.minibatch, epoch_size)
momentum_time_constant = momentum_as_time_constant_schedule(-minibatch_size/np.log(0.9))
l2_reg_weight = 0.001
# trainer object
progress_printer = ProgressPrinter(0)
learner = momentum_sgd(z.parameters,
lr = lr_per_minibatch, momentum = momentum_time_constant,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(z, (ce, pe), [learner], [progress_printer])
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z) ; print()
#progress_printer = ProgressPrinter(tag='Training')
# perform model training
stop_run=False
batch_index = 0
plot_data = {'batchindex':[], 'loss':[], 'error':[]}
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(min(minibatch_size, epoch_size - sample_count), input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += data[label_var].num_samples # count samples processed so far
# For visualization...
plot_data['batchindex'].append(batch_index)
plot_data['loss'].append(trainer.previous_minibatch_loss_average)
plot_data['error'].append(trainer.previous_minibatch_evaluation_average)
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
batch_index += 1
if trainer.previous_minibatch_evaluation_average < 0.025:
stop_run=True
break
if stop_run:
break
progress_printer.epoch_summary(with_metric=True)
#trainer.save_checkpoint(model_temp_file)
#
# Evaluation action
#
epoch_size = 6600
minibatch_size = 32
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
input_map = {
input_var: reader_test.streams.features,
label_var: reader_test.streams.labels
}
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch, input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.1f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom))
print("")
# Visualize training result:
window_width = 32
loss_cumsum = np.cumsum(np.insert(plot_data['loss'], 0, 0))
error_cumsum = np.cumsum(np.insert(plot_data['error'], 0, 0))
# Moving average.
plot_data['batchindex'] = np.insert(plot_data['batchindex'], 0, 0)[window_width:]
plot_data['avg_loss'] = (loss_cumsum[window_width:] - loss_cumsum[:-window_width]) / window_width
plot_data['avg_error'] = (error_cumsum[window_width:] - error_cumsum[:-window_width]) / window_width
plt.figure(1)
plt.subplot(211)
plt.plot(plot_data["batchindex"], plot_data["avg_loss"], 'b--')
plt.xlabel('Minibatch number')
plt.ylabel('Loss')
plt.title('Minibatch run vs. Training loss ')
plt.show()
plt.subplot(212)
plt.plot(plot_data["batchindex"], plot_data["avg_error"], 'r--')
plt.xlabel('Minibatch number')
plt.ylabel('Label Prediction Error')
plt.title('Minibatch run vs. Label Prediction Error ')
plt.show()
return softmax(z)
def train_fast_rcnn(cfg):
# Train only if no model exists yet
model_path = cfg['MODEL_PATH']
if os.path.exists(model_path) and cfg["CNTK"].MAKE_MODE:
print("Loading existing model from %s" % model_path)
return load_model(model_path)
else:
# Input variables denoting features and labeled ground truth rois (as 5-tuples per roi)
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=cfg["MODEL"].FEATURE_NODE_NAME)
roi_proposals = input_variable((cfg.NUM_ROI_PROPOSALS, 4), dynamic_axes=[Axis.default_batch_axis()], name = "roi_proposals")
label_targets = input_variable((cfg.NUM_ROI_PROPOSALS, cfg["DATA"].NUM_CLASSES), dynamic_axes=[Axis.default_batch_axis()])
bbox_targets = input_variable((cfg.NUM_ROI_PROPOSALS, 4*cfg["DATA"].NUM_CLASSES), dynamic_axes=[Axis.default_batch_axis()])
bbox_inside_weights = input_variable((cfg.NUM_ROI_PROPOSALS, 4*cfg["DATA"].NUM_CLASSES), dynamic_axes=[Axis.default_batch_axis()])
# Instantiate the Fast R-CNN prediction model and loss function
loss, pred_error = create_fast_rcnn_model(image_input, roi_proposals, label_targets, bbox_targets, bbox_inside_weights, cfg)
if isinstance(loss, cntk.Variable):
loss = combine([loss])
if cfg["CNTK"].DEBUG_OUTPUT:
print("Storing graphs and models to %s." % cfg.OUTPUT_PATH)
plot(loss, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train." + cfg["CNTK"].GRAPH_TYPE))
# Set learning parameters
lr_factor = cfg["CNTK"].LR_FACTOR
lr_per_sample_scaled = [x * lr_factor for x in cfg["CNTK"].LR_PER_SAMPLE]
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
l2_reg_weight = cfg["CNTK"].L2_REG_WEIGHT
epochs_to_train = cfg["CNTK"].MAX_EPOCHS
print("Using base model: {}".format(cfg["MODEL"].BASE_MODEL))
print("lr_per_sample: {}".format(lr_per_sample_scaled))
# --- train ---
# Instantiate the learners and the trainer object
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
lr_schedule = learning_rate_schedule(lr_per_sample_scaled, unit=UnitType.sample)
learner = momentum_sgd(others, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight, unit_gain=False, use_mean_gradient=True)
bias_lr_per_sample = [v * bias_lr_mult for v in cfg["CNTK"].LR_PER_SAMPLE]
bias_lr_schedule = learning_rate_schedule(bias_lr_per_sample, unit=UnitType.sample)
bias_learner = momentum_sgd(biases, bias_lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight, unit_gain=False, use_mean_gradient=True)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
if cfg.USE_PRECOMPUTED_PROPOSALS:
proposal_provider = ProposalProvider.fromfile(cfg["DATA"].TRAIN_PRECOMPUTED_PROPOSALS_FILE, cfg.NUM_ROI_PROPOSALS)
else:
proposal_provider = ProposalProvider.fromconfig(cfg)
od_minibatch_source = ObjectDetectionMinibatchSource(
cfg["DATA"].TRAIN_MAP_FILE, cfg["DATA"].TRAIN_ROI_FILE,
max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE,
pad_width=cfg.IMAGE_WIDTH,
pad_height=cfg.IMAGE_HEIGHT,
pad_value=cfg["MODEL"].IMG_PAD_COLOR,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["DATA"].NUM_TRAIN_IMAGES,
num_classes=cfg["DATA"].NUM_CLASSES,
proposal_provider=proposal_provider,
provide_targets=True,
proposal_iou_threshold = cfg.BBOX_THRESH,
normalize_means = None if not cfg.BBOX_NORMALIZE_TARGETS else cfg.BBOX_NORMALIZE_MEANS,
normalize_stds = None if not cfg.BBOX_NORMALIZE_TARGETS else cfg.BBOX_NORMALIZE_STDS)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.proposals_si: roi_proposals,
od_minibatch_source.label_targets_si: label_targets,
od_minibatch_source.bbox_targets_si: bbox_targets,
od_minibatch_source.bbiw_si: bbox_inside_weights
}
progress_printer = ProgressPrinter(tag='Training', num_epochs=epochs_to_train, gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < cfg["DATA"].NUM_TRAIN_IMAGES: # loop over minibatches in the epoch
data = od_minibatch_source.next_minibatch(min(cfg.MB_SIZE, cfg["DATA"].NUM_TRAIN_IMAGES - sample_count), input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
eval_model = create_fast_rcnn_eval_model(loss, image_input, roi_proposals, cfg)
eval_model.save(cfg['MODEL_PATH'])
return eval_model
label_var].num_samples # count samples processed so far
# For visualization...
plot_data['batchindex'].append(batch_index)
plot_data['loss'].append(trainer.previous_minibatch_loss_average)
plot_data['error'].append(
trainer.previous_minibatch_evaluation_average)
progress_printer.update_with_trainer(trainer,
with_metric=True) # log progress
batch_index += 1
ev_avg += trainer.previous_minibatch_evaluation_average
i_count += 1
# if ev_avg/i_count < 0.02:
# break
progress_printer.epoch_summary(with_metric=True)
# print(epoch,' : ',ev_avg/i_count)
try:
trainer.save_checkpoint(model_temp_file)
except:
print('Problem with saving temp model')
#
# Evaluation action
#
epoch_size = 2200 #4000 #5000
minibatch_size = 32
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
def train_model(image_input, roi_input, dims_input, loss, pred_error,
lr_per_sample, mm_schedule, l2_reg_weight, epochs_to_train, cfg,
rpn_rois_input=None, buffered_rpn_proposals=None):
if isinstance(loss, cntk.Variable):
loss = combine([loss])
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
if cfg["CNTK"].DEBUG_OUTPUT:
print("biases")
for p in biases: print(p)
print("others")
for p in others: print(p)
print("bias_lr_mult: {}".format(bias_lr_mult))
# Instantiate the learners and the trainer object
lr_schedule = learning_parameter_schedule_per_sample(lr_per_sample)
learner = momentum_sgd(others, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=True)
bias_lr_per_sample = [v * bias_lr_mult for v in lr_per_sample]
bias_lr_schedule = learning_parameter_schedule_per_sample(bias_lr_per_sample)
bias_learner = momentum_sgd(biases, bias_lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=True)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
if buffered_rpn_proposals is not None:
proposal_provider = ProposalProvider.fromlist(buffered_rpn_proposals, requires_scaling=False)
else:
proposal_provider = None
od_minibatch_source = ObjectDetectionMinibatchSource(
cfg["DATA"].TRAIN_MAP_FILE, cfg["DATA"].TRAIN_ROI_FILE,
num_classes=cfg["DATA"].NUM_CLASSES,
max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE,
pad_width=cfg.IMAGE_WIDTH,
pad_height=cfg.IMAGE_HEIGHT,
pad_value=cfg["MODEL"].IMG_PAD_COLOR,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["DATA"].NUM_TRAIN_IMAGES,
proposal_provider=proposal_provider)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.roi_si: roi_input,
}
if buffered_rpn_proposals is not None:
input_map[od_minibatch_source.proposals_si] = rpn_rois_input
else:
input_map[od_minibatch_source.dims_si] = dims_input
progress_printer = ProgressPrinter(tag='Training', num_epochs=epochs_to_train, gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < cfg["DATA"].NUM_TRAIN_IMAGES: # loop over minibatches in the epoch
data = od_minibatch_source.next_minibatch(min(cfg.MB_SIZE, cfg["DATA"].NUM_TRAIN_IMAGES-sample_count), input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
def train_model(image_input, roi_input, dims_input, loss, pred_error,
lr_per_sample, mm_schedule, l2_reg_weight, epochs_to_train,
rpn_rois_input=None, buffered_rpn_proposals=None):
if isinstance(loss, cntk.Variable):
loss = combine([loss])
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
if cfg["CNTK"].DEBUG_OUTPUT:
print("biases")
for p in biases: print(p)
print("others")
for p in others: print(p)
print("bias_lr_mult: {}".format(bias_lr_mult))
# Instantiate the learners and the trainer object
lr_schedule = learning_rate_schedule(lr_per_sample, unit=UnitType.sample)
learner = momentum_sgd(others, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=cfg["CNTK"].USE_MEAN_GRADIENT)
bias_lr_per_sample = [v * bias_lr_mult for v in lr_per_sample]
bias_lr_schedule = learning_rate_schedule(bias_lr_per_sample, unit=UnitType.sample)
bias_learner = momentum_sgd(biases, bias_lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=cfg["CNTK"].USE_MEAN_GRADIENT)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
od_minibatch_source = ObjectDetectionMinibatchSource(
globalvars['train_map_file'], globalvars['train_roi_file'],
max_annotations_per_image=cfg["CNTK"].INPUT_ROIS_PER_IMAGE,
pad_width=image_width, pad_height=image_height, pad_value=img_pad_value,
randomize=True, use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["CNTK"].NUM_TRAIN_IMAGES,
buffered_rpn_proposals=buffered_rpn_proposals)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.roi_si: roi_input,
od_minibatch_source.dims_si: dims_input
}
use_buffered_proposals = buffered_rpn_proposals is not None
progress_printer = ProgressPrinter(tag='Training', num_epochs=epochs_to_train, gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data, proposals = od_minibatch_source.next_minibatch_with_proposals(min(mb_size, epoch_size-sample_count), input_map=input_map)
if use_buffered_proposals:
data[rpn_rois_input] = MinibatchData(Value(batch=np.asarray(proposals, dtype=np.float32)), 1, 1, False)
# remove dims input if no rpn is required to avoid warnings
del data[[k for k in data if '[6]' in str(k)][0]]
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
def train(train_reader, valid_reader, vocab, i2w, s2smodel, max_epochs, epoch_size):
# Note: We would like to set the signature of 's2smodel' (s2smodel.update_signature()), but that will cause
# an error since the training criterion uses a reduced sequence axis for the labels.
# This is because it removes the initial <s> symbol. Hence, we must leave the model
# with unspecified input shapes and axes.
# create the training wrapper for the s2smodel, as well as the criterion function
model_train = create_model_train(s2smodel)
criterion = create_criterion_function(model_train)
# also wire in a greedy decoder so that we can properly log progress on a validation example
# This is not used for the actual training process.
model_greedy = create_model_greedy(s2smodel)
# This does not need to be done in training generally though
# Instantiate the trainer object to drive the model training
minibatch_size = 72
lr = 0.001 if use_attention else 0.005 # TODO: can we use the same value for both?
learner = fsadagrad(model_train.parameters,
lr = learning_rate_schedule([lr]*2+[lr/2]*3+[lr/4], UnitType.sample, epoch_size),
momentum = momentum_as_time_constant_schedule(1100),
gradient_clipping_threshold_per_sample=2.3,
gradient_clipping_with_truncation=True)
trainer = Trainer(None, criterion, learner)
# Get minibatches of sequences to train with and perform model training
total_samples = 0
mbs = 0
eval_freq = 100
# print out some useful training information
log_number_of_parameters(model_train) ; print()
progress_printer = ProgressPrinter(freq=30, tag='Training')
#progress_printer = ProgressPrinter(freq=30, tag='Training', log_to_file=model_path_stem + ".log") # use this to log to file
sparse_to_dense = create_sparse_to_dense(input_vocab_dim)
for epoch in range(max_epochs):
print("Saving model to '%s'" % model_path(epoch))
s2smodel.save(model_path(epoch))
while total_samples < (epoch+1) * epoch_size:
# get next minibatch of training data
mb_train = train_reader.next_minibatch(minibatch_size)
#trainer.train_minibatch(mb_train[train_reader.streams.features], mb_train[train_reader.streams.labels])
trainer.train_minibatch({criterion.arguments[0]: mb_train[train_reader.streams.features], criterion.arguments[1]: mb_train[train_reader.streams.labels]})
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
# every N MBs evaluate on a test sequence to visually show how we're doing
if mbs % eval_freq == 0:
mb_valid = valid_reader.next_minibatch(1)
# run an eval on the decoder output model (i.e. don't use the groundtruth)
e = model_greedy(mb_valid[valid_reader.streams.features])
print(format_sequences(sparse_to_dense(mb_valid[valid_reader.streams.features]), i2w))
print("->")
print(format_sequences(e, i2w))
# debugging attention
if use_attention:
debug_attention(model_greedy, mb_valid[valid_reader.streams.features])
total_samples += mb_train[train_reader.streams.labels].num_samples
mbs += 1
# log a summary of the stats for the epoch
progress_printer.epoch_summary(with_metric=True)
# done: save the final model
print("Saving final model to '%s'" % model_path(max_epochs))
s2smodel.save(model_path(max_epochs))
print("%d epochs complete." % max_epochs)