def train_fast_rcnn(debug_output=False):
if debug_output:
print("Storing graphs and intermediate models to %s." % os.path.join(abs_path, "Output"))
# Create the minibatch source
minibatch_source = create_mb_source(image_height, image_width, num_channels,
num_classes, num_rois, base_path, "train")
# Input variables denoting features, rois and label data
image_input = input_variable((num_channels, image_height, image_width))
roi_input = input_variable((num_rois, 4))
label_input = input_variable((num_rois, num_classes))
# define mapping from reader streams to network inputs
input_map = {
image_input: minibatch_source[features_stream_name],
roi_input: minibatch_source[roi_stream_name],
label_input: minibatch_source[label_stream_name]
}
# Instantiate the Fast R-CNN prediction model and loss function
frcn_output = frcn_predictor(image_input, roi_input, num_classes)
ce = cross_entropy_with_softmax(frcn_output, label_input, axis=1)
pe = classification_error(frcn_output, label_input, axis=1)
if debug_output:
plot(frcn_output, os.path.join(abs_path, "Output", "graph_frcn.png"))
# Set learning parameters
l2_reg_weight = 0.0005
lr_per_sample = [0.00001] * 10 + [0.000001] * 5 + [0.0000001]
lr_schedule = learning_rate_schedule(lr_per_sample, unit=UnitType.sample)
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant)
# Instantiate the trainer object
learner = momentum_sgd(frcn_output.parameters, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight)
trainer = Trainer(frcn_output, (ce, pe), learner)
# Get minibatches of images and perform model training
print("Training Fast R-CNN model for %s epochs." % max_epochs)
log_number_of_parameters(frcn_output)
progress_printer = ProgressPrinter(tag='Training', num_epochs=max_epochs)
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = minibatch_source.next_minibatch(min(mb_size, epoch_size-sample_count), input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
if debug_output:
frcn_output.save(os.path.join(abs_path, "Output", "frcn_py_%s.model" % (epoch+1)))
return frcn_output
def train_model(base_model_file, feature_node_name, last_hidden_node_name,
image_width, image_height, num_channels, num_classes, train_map_file,
num_epochs, max_images=-1, freeze=False):
epoch_size = sum(1 for line in open(train_map_file))
if max_images > 0:
epoch_size = min(epoch_size, max_images)
# Create the minibatch source and input variables
minibatch_source = create_mb_source(train_map_file, image_width, image_height, num_channels, num_classes)
image_input = input_variable((num_channels, image_height, image_width))
label_input = input_variable(num_classes)
# Define mapping from reader streams to network inputs
input_map = {
image_input: minibatch_source[features_stream_name],
label_input: minibatch_source[label_stream_name]
}
# Instantiate the transfer learning model and loss function
tl_model = create_model(base_model_file, feature_node_name, last_hidden_node_name, num_classes, image_input, freeze)
ce = cross_entropy_with_softmax(tl_model, label_input)
pe = classification_error(tl_model, label_input)
# Instantiate the trainer object
lr_schedule = learning_rate_schedule(lr_per_mb, unit=UnitType.minibatch)
mm_schedule = momentum_schedule(momentum_per_mb)
learner = momentum_sgd(tl_model.parameters, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight)
trainer = Trainer(tl_model, (ce, pe), learner)
# Get minibatches of images and perform model training
print("Training transfer learning model for {0} epochs (epoch_size = {1}).".format(num_epochs, epoch_size))
log_number_of_parameters(tl_model)
progress_printer = ProgressPrinter(tag='Training', num_epochs=num_epochs)
for epoch in range(num_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = minibatch_source.next_minibatch(min(mb_size, epoch_size-sample_count), input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
if sample_count % (100 * mb_size) == 0:
print ("Processed {0} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
return tl_model
def main():
# Ensure we always get the same amount of randomness
np.random.seed(0)
global minibatch_size, skip_window
if len(sys.argv) < 2:
print(
'Insufficient number of arguments. For running the example case, run: $ python word2vec.py runexample'
)
exit(1)
filename = sys.argv[1]
process_text(filename)
inp, label, trainer = train(emb_size, vocab_size)
pp = ProgressPrinter(50)
for _epoch in range(num_epochs):
i = 0
while curr_epoch == _epoch:
features, labels = generate_batch(minibatch_size, skip_window)
features = get_one_hot(features)
labels = get_one_hot(labels)
trainer.train_minibatch({inp: features, label: labels})
pp.update_with_trainer(trainer)
i += 1
if i % 200 == 0:
print('Saving Embeddings..')
with open(embpickle, 'wb') as handle:
pickle.dump(embeddings.value, handle)
pp.epoch_summary()
test_features, test_labels = generate_batch(minibatch_size, skip_window)
test_features = get_one_hot(test_features)
test_labels = get_one_hot(test_labels)
avg_error = trainer.test_minibatch({
inp: test_features,
label: test_labels
})
print('Avg. Error on Test Set: ', avg_error)
def ffnet():
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
input = input_variable((input_dim), np.float32)
label = input_variable((num_output_classes), np.float32)
# Instantiate the feedforward classification model
netout = fully_connected_classifier_net(input, num_output_classes,
hidden_layers_dim,
num_hidden_layers, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
# Instantiate the trainer object to drive the model training
trainer = Trainer(netout, ce, pe, [sgd(netout.parameters, lr=0.005)])
# Get minibatches of training data and perform model training
minibatch_size = 25
pp = ProgressPrinter(128)
for i in range(1024):
features, labels = generate_random_data(minibatch_size, input_dim,
num_output_classes)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({input: features, label: labels})
pp.update_with_trainer(trainer)
pp.epoch_summary()
test_features, test_labels = generate_random_data(minibatch_size,
input_dim,
num_output_classes)
avg_error = trainer.test_minibatch({
input: test_features,
label: test_labels
})
return avg_error
def ffnet():
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
input = input_variable((input_dim), np.float32)
label = input_variable((num_output_classes), np.float32)
# Instantiate the feedforward classification model
netout = fully_connected_classifier_net(
input, num_output_classes, hidden_layers_dim, num_hidden_layers, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
lr_per_minibatch=learning_rate_schedule(0.5, UnitType.minibatch)
# Instantiate the trainer object to drive the model training
trainer = Trainer(netout, ce, pe, sgd(netout.parameters, lr=lr_per_minibatch))
# Get minibatches of training data and perform model training
minibatch_size = 25
pp = ProgressPrinter(128)
for i in range(1024):
features, labels = generate_random_data(
minibatch_size, input_dim, num_output_classes)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({input: features, label: labels})
pp.update_with_trainer(trainer)
pp.epoch_summary()
test_features, test_labels = generate_random_data(
minibatch_size, input_dim, num_output_classes)
avg_error = trainer.test_minibatch(
{input: test_features, label: test_labels})
return avg_error
if origlabels[t, 0] < vocab_size and origlabels[t, 0] >= 0:
labels[t, origlabels[t, 0]] = 1.0
return labels
#Testing & training
build_dataset()
inp, label, trainer = train(emb_size, vocab_size)
print('Model Creation Done.')
pp = ProgressPrinter(50)
for _epoch in range(num_epochs):
i = 0
while curr_epoch == _epoch:
features, labels = generate_batch(minibatch_size, skip_window)
features = get_one_hot(features)
labels = get_one_hot(labels)
trainer.train_minibatch({inp: features, label: labels})
pp.update_with_trainer(trainer)
i += 1
if i % 200 == 0:
print('Saving Embeddings..')
with open(embpickle, 'wb') as handle:
pickle.dump(embeddings.value, handle)
pp.epoch_summary()
test_features, test_labels = generate_batch(minibatch_size, skip_window)
test_features = get_one_hot(test_features)
test_labels = get_one_hot(test_labels)
avg_error = trainer.test_minibatch({inp: test_features, label: test_labels})
print('Avg. Error on Test Set: ', avg_error)
def train_model(base_model_file,
feature_node_name,
last_hidden_node_name,
image_width,
image_height,
num_channels,
num_classes,
train_map_file,
num_epochs,
max_images=-1,
freeze=False):
epoch_size = sum(1 for line in open(train_map_file))
if max_images > 0:
epoch_size = min(epoch_size, max_images)
# Create the minibatch source and input variables
minibatch_source = create_mb_source(train_map_file, image_width,
image_height, num_channels,
num_classes)
image_input = input_variable((num_channels, image_height, image_width))
label_input = input_variable(num_classes)
# Define mapping from reader streams to network inputs
input_map = {
image_input: minibatch_source[features_stream_name],
label_input: minibatch_source[label_stream_name]
}
# Instantiate the transfer learning model and loss function
tl_model = create_model(base_model_file, feature_node_name,
last_hidden_node_name, num_classes, image_input,
freeze)
ce = cross_entropy_with_softmax(tl_model, label_input)
pe = classification_error(tl_model, label_input)
# Instantiate the trainer object
lr_schedule = learning_rate_schedule(lr_per_mb, unit=UnitType.minibatch)
mm_schedule = momentum_schedule(momentum_per_mb)
learner = momentum_sgd(tl_model.parameters,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(tl_model, (ce, pe), learner)
# Get minibatches of images and perform model training
print(
"Training transfer learning model for {0} epochs (epoch_size = {1}).".
format(num_epochs, epoch_size))
log_number_of_parameters(tl_model)
progress_printer = ProgressPrinter(tag='Training', num_epochs=num_epochs)
for epoch in range(num_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = minibatch_source.next_minibatch(min(
mb_size, epoch_size - sample_count),
input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
if sample_count % (100 * mb_size) == 0:
print("Processed {0} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
return tl_model
def train(train_reader, valid_reader, vocab, i2w, s2smodel, max_epochs, epoch_size):
# Note: We would like to set the signature of 's2smodel' (s2smodel.update_signature()), but that will cause
# an error since the training criterion uses a reduced sequence axis for the labels.
# This is because it removes the initial <s> symbol. Hence, we must leave the model
# with unspecified input shapes and axes.
# create the training wrapper for the s2smodel, as well as the criterion function
model_train = create_model_train(s2smodel)
criterion = create_criterion_function(model_train)
# also wire in a greedy decoder so that we can properly log progress on a validation example
# This is not used for the actual training process.
model_greedy = create_model_greedy(s2smodel)
# This does not need to be done in training generally though
# Instantiate the trainer object to drive the model training
minibatch_size = 72
lr = 0.001 if use_attention else 0.005 # TODO: can we use the same value for both?
learner = adam_sgd(model_train.parameters,
lr = learning_rate_schedule([lr]*2+[lr/2]*3+[lr/4], UnitType.sample, epoch_size),
momentum = momentum_as_time_constant_schedule(1100),
gradient_clipping_threshold_per_sample=2.3,
gradient_clipping_with_truncation=True)
trainer = Trainer(None, criterion, learner)
# Get minibatches of sequences to train with and perform model training
total_samples = 0
mbs = 0
eval_freq = 100
# print out some useful training information
log_number_of_parameters(model_train) ; print()
progress_printer = ProgressPrinter(freq=30, tag='Training')
#progress_printer = ProgressPrinter(freq=30, tag='Training', log_to_file=model_path_stem + ".log") # use this to log to file
sparse_to_dense = create_sparse_to_dense(input_vocab_dim)
for epoch in range(max_epochs):
print("Saving model to '%s'" % model_path(epoch))
s2smodel.save(model_path(epoch))
while total_samples < (epoch+1) * epoch_size:
# get next minibatch of training data
mb_train = train_reader.next_minibatch(minibatch_size)
#trainer.train_minibatch(mb_train[train_reader.streams.features], mb_train[train_reader.streams.labels])
trainer.train_minibatch({criterion.arguments[0]: mb_train[train_reader.streams.features], criterion.arguments[1]: mb_train[train_reader.streams.labels]})
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
# every N MBs evaluate on a test sequence to visually show how we're doing
if mbs % eval_freq == 0:
mb_valid = valid_reader.next_minibatch(1)
# run an eval on the decoder output model (i.e. don't use the groundtruth)
e = model_greedy(mb_valid[valid_reader.streams.features])
print(format_sequences(sparse_to_dense(mb_valid[valid_reader.streams.features]), i2w))
print("->")
print(format_sequences(e, i2w))
# debugging attention
if use_attention:
debug_attention(model_greedy, mb_valid[valid_reader.streams.features])
total_samples += mb_train[train_reader.streams.labels].num_samples
mbs += 1
# log a summary of the stats for the epoch
progress_printer.epoch_summary(with_metric=True)
# done: save the final model
print("Saving final model to '%s'" % model_path(max_epochs))
s2smodel.save(model_path(max_epochs))
print("%d epochs complete." % max_epochs)
def train(train_reader, valid_reader, vocab, i2w, s2smodel, max_epochs,
epoch_size):
# Note: We would like to set the signature of 's2smodel' (s2smodel.update_signature()), but that will cause
# an error since the training criterion uses a reduced sequence axis for the labels.
# This is because it removes the initial <s> symbol. Hence, we must leave the model
# with unspecified input shapes and axes.
# create the training wrapper for the s2smodel, as well as the criterion function
model_train = create_model_train(s2smodel)
criterion = create_criterion_function(model_train)
# also wire in a greedy decoder so that we can properly log progress on a validation example
# This is not used for the actual training process.
model_greedy = create_model_greedy(s2smodel)
# This does not need to be done in training generally though
# Instantiate the trainer object to drive the model training
minibatch_size = 72
lr = 0.001 if use_attention else 0.005 # TODO: can we use the same value for both?
learner = adam_sgd(
model_train.parameters,
lr=learning_rate_schedule([lr] * 2 + [lr / 2] * 3 + [lr / 4],
UnitType.sample, epoch_size),
momentum=momentum_as_time_constant_schedule(1100),
gradient_clipping_threshold_per_sample=2.3,
gradient_clipping_with_truncation=True)
trainer = Trainer(None, criterion, learner)
# Get minibatches of sequences to train with and perform model training
total_samples = 0
mbs = 0
eval_freq = 100
# print out some useful training information
log_number_of_parameters(model_train)
print()
progress_printer = ProgressPrinter(freq=30, tag='Training')
#progress_printer = ProgressPrinter(freq=30, tag='Training', log_to_file=model_path_stem + ".log") # use this to log to file
sparse_to_dense = create_sparse_to_dense(input_vocab_dim)
for epoch in range(max_epochs):
print("Saving model to '%s'" % model_path(epoch))
s2smodel.save(model_path(epoch))
while total_samples < (epoch + 1) * epoch_size:
# get next minibatch of training data
mb_train = train_reader.next_minibatch(minibatch_size)
#trainer.train_minibatch(mb_train[train_reader.streams.features], mb_train[train_reader.streams.labels])
trainer.train_minibatch({
criterion.arguments[0]:
mb_train[train_reader.streams.features],
criterion.arguments[1]:
mb_train[train_reader.streams.labels]
})
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
# every N MBs evaluate on a test sequence to visually show how we're doing
if mbs % eval_freq == 0:
mb_valid = valid_reader.next_minibatch(1)
# run an eval on the decoder output model (i.e. don't use the groundtruth)
e = model_greedy(mb_valid[valid_reader.streams.features])
print(
format_sequences(
sparse_to_dense(
mb_valid[valid_reader.streams.features]), i2w))
print("->")
print(format_sequences(e, i2w))
# debugging attention
if use_attention:
debug_attention(model_greedy,
mb_valid[valid_reader.streams.features])
total_samples += mb_train[train_reader.streams.labels].num_samples
mbs += 1
# log a summary of the stats for the epoch
progress_printer.epoch_summary(with_metric=True)
# done: save the final model
print("Saving final model to '%s'" % model_path(max_epochs))
s2smodel.save(model_path(max_epochs))
print("%d epochs complete." % max_epochs)
