def train(reader, model, loss_function, error_function, input_map, num_sweeps_to_train_with = 10, num_samples_per_sweep = 6000, minibatch_size = 64, learning_rate = 0.2):
# Instantiate the trainer object to drive the model training
lr_schedule = C.learning_parameter_schedule(learning_rate)
learner = C.sgd(model.parameters, lr_schedule)
# Print progress
progress_printer_stdout = ProgressPrinter(freq=minibatch_size)
# Instantiate trainer
trainer = C.Trainer(model, (loss_function, error_function), [learner], progress_writers=progress_printer_stdout)
# Start a timer
start = time.time()
aggregate_metric = 0
total_samples = 0
num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size
for i in range(0, int(num_minibatches_to_train)):
# Read a mini batch from the training data file
data = reader.next_minibatch(minibatch_size, input_map=input_map)
trainer.train_minibatch(data)
samples = trainer.previous_minibatch_sample_count
aggregate_metric += trainer.previous_minibatch_evaluation_average * samples
total_samples += samples
# Print training time
print("Training took {:.1f} sec".format(time.time() - start))
print("Average error: {0:.2f}%".format((aggregate_metric * 100.0) / (total_samples)))
return trainer
def evaluate_decoding(reader, s2smodel, i2w):
model_decoding = create_model_greedy(
s2smodel) # wrap the greedy decoder around the model
progress_printer = ProgressPrinter(tag='Evaluation')
sparse_to_dense = create_sparse_to_dense(input_vocab_dim)
minibatch_size = 1024
num_total = 0
num_wrong = 0
while True:
mb = reader.next_minibatch(minibatch_size)
if not mb: # finish when end of test set reached
break
e = model_decoding(mb[reader.streams.features])
outputs = format_sequences(e, i2w)
labels = format_sequences(sparse_to_dense(mb[reader.streams.labels]),
i2w)
# prepend sentence start for comparison
outputs = ["<s> " + output for output in outputs]
num_total += len(outputs)
num_wrong += sum(
[label != output for output, label in zip(outputs, labels)])
rate = num_wrong / num_total
print("string error rate of {:.1f}% in {} samples".format(
100 * rate, num_total))
return rate
def bn_inception_train_and_eval(train_data,
test_data,
mean_data,
minibatch_size=32,
epoch_size=1281167,
max_epochs=300,
restore=True,
log_to_file=None,
num_mbs_per_log=100,
gen_heartbeat=False,
profiler_dir=None,
testing_parameters=(5000, 32)):
_cntk_py.set_computation_network_trace_level(1)
progress_printer = ProgressPrinter(freq=num_mbs_per_log,
tag='Training',
log_to_file=log_to_file,
gen_heartbeat=gen_heartbeat,
num_epochs=max_epochs)
network = create_bn_inception()
trainer = create_trainer(network, epoch_size, max_epochs, minibatch_size)
train_source = create_image_mb_source(train_data,
mean_data,
True,
total_number_of_samples=max_epochs *
epoch_size)
test_source = create_image_mb_source(
test_data, mean_data, False, total_number_of_samples=FULL_DATA_SWEEP)
return train_and_test(network, trainer, train_source, test_source,
progress_printer, max_epochs, minibatch_size,
epoch_size, restore, profiler_dir,
testing_parameters)
def bn_inception_train_and_eval(train_data, test_data, mean_data, num_quantization_bits=32, epoch_size=1281167, max_epochs=300, minibatch_size=None,
restore=True, log_to_file=None, num_mbs_per_log=100, gen_heartbeat=False, scale_up=False, profiling=False):
_cntk_py.set_computation_network_trace_level(0)
# NOTE: scaling up minibatch_size increases sample throughput. In 8-GPU machine,
# ResNet110 samples-per-second is ~7x of single GPU, comparing to ~3x without scaling
# up. However, bigger minibatch size on the same number of samples means less updates,
# thus leads to higher training error. This is a trade-off of speed and accuracy
if minibatch_size is None:
mb_size = 32 * (Communicator.num_workers() if scale_up else 1)
else:
mb_size = minibatch_size
progress_printer = ProgressPrinter(
freq=num_mbs_per_log,
tag='Training',
log_to_file=log_to_file,
rank=Communicator.rank(),
gen_heartbeat=gen_heartbeat,
num_epochs=max_epochs)
network = create_bn_inception()
trainer = create_trainer(network, epoch_size, max_epochs, mb_size, num_quantization_bits, progress_printer)
train_source = create_image_mb_source(train_data, mean_data, True, total_number_of_samples=max_epochs * epoch_size)
test_source = create_image_mb_source(test_data, mean_data, False, total_number_of_samples=FULL_DATA_SWEEP)
train_and_test(network, trainer, train_source, test_source, mb_size, epoch_size, restore, profiling)
def ffnet(learner, trainer=None):
inputs = 5
outputs = 3
layers = 2
hidden_dimension = 3
if trainer is None:
# input variables denoting the features and label data
features = C.input_variable((inputs), np.float32)
label = C.input_variable((outputs), np.float32)
# Instantiate the feedforward classification model
my_model = Sequential([
Dense(hidden_dimension,
activation=C.sigmoid,
init=C.glorot_uniform(seed=98052)),
Dense(outputs, init=C.glorot_uniform(seed=98052))
])
z = my_model(features)
ce = C.cross_entropy_with_softmax(z, label)
pe = C.classification_error(z, label)
# Instantiate the trainer object to drive the model training
progress_printer = ProgressPrinter(0)
trainer = C.Trainer(z, (ce, pe), [learner(z.parameters)],
[progress_printer])
else:
features = trainer.loss_function.arguments[0]
label = trainer.loss_function.arguments[1]
# Get minibatches of training data and perform model training
minibatch_size = 25
num_minibatches_to_train = 100
aggregate_loss = 0.0
for i in range(num_minibatches_to_train):
train_features, labels = generate_random_data(minibatch_size, inputs,
outputs)
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
trainer.train_minibatch({features: train_features, label: labels})
sample_count = trainer.previous_minibatch_sample_count
aggregate_loss += trainer.previous_minibatch_loss_average * sample_count
last_avg_error = aggregate_loss / trainer.total_number_of_samples_seen
test_features, test_labels = generate_random_data(minibatch_size, inputs,
outputs)
avg_error = trainer.test_minibatch({
features: test_features,
label: test_labels
})
print(' error rate on an unseen minibatch: {}'.format(avg_error))
return last_avg_error, avg_error, trainer
def finalize_network(reader, model_details, max_amount_of_epochs,
samples_per_epoch, samples_per_minibatch,
pixel_dimensions, classes, learning_rate):
features = input_variable(shape=(pixel_dimensions['depth'],
pixel_dimensions['height'],
pixel_dimensions['width']))
label = input_variable(shape=len(classes))
# speeds up training
normalized_features = element_times(1.0 / 256.0, features)
model = create_tf_model(model_details,
num_classes=len(classes),
input_features=normalized_features,
freeze=True)
loss = cross_entropy_with_softmax(model, label)
metric = classification_error(model, label)
learner = momentum_sgd(parameters=model.parameters,
lr=learning_rate_schedule(learning_rate,
UnitType.minibatch),
momentum=0.9,
l2_regularization_weight=0.0005)
reporter = ProgressPrinter(tag='training', num_epochs=max_amount_of_epochs)
trainer = Trainer(model=model,
criterion=(loss, metric),
parameter_learners=[learner],
progress_writers=[reporter])
log_number_of_parameters(model)
map_input_to_streams_train = {
features: reader.streams.features,
label: reader.streams.labels
}
training_session(trainer=trainer,
mb_source=reader,
model_inputs_to_streams=map_input_to_streams_train,
mb_size=samples_per_minibatch,
progress_frequency=samples_per_epoch,
checkpoint_config=CheckpointConfig(
frequency=samples_per_epoch,
filename=os.path.join("./checkpoints",
"ConvNet_Lego_VisiOn"),
restore=True)).train()
network = {'features': features, 'label': label, 'model': softmax(model)}
model_name = f"CNN-3200-224-resnet-18.model"
export_path = os.path.abspath(
os.path.join("..", "..", "Final models", "CNN", model_name))
model.save(export_path)
return network
def train_fast_rcnn(debug_output=False):
if debug_output:
print("Storing graphs and intermediate models to %s." % os.path.join(abs_path, "Output"))
# Create the minibatch source
minibatch_source = create_mb_source(image_height, image_width, num_channels,
num_classes, num_rois, base_path, "train")
# Input variables denoting features, rois and label data
image_input = input((num_channels, image_height, image_width))
roi_input = input((num_rois, 4))
label_input = input((num_rois, num_classes))
# define mapping from reader streams to network inputs
input_map = {
image_input: minibatch_source.streams.features,
roi_input: minibatch_source.streams.rois,
label_input: minibatch_source.streams.roiLabels
}
# Instantiate the Fast R-CNN prediction model and loss function
frcn_output = frcn_predictor(image_input, roi_input, num_classes)
ce = cross_entropy_with_softmax(frcn_output, label_input, axis=1)
pe = classification_error(frcn_output, label_input, axis=1)
if debug_output:
plot(frcn_output, os.path.join(abs_path, "Output", "graph_frcn.png"))
# Set learning parameters
l2_reg_weight = 0.0005
lr_per_sample = [0.00001] * 10 + [0.000001] * 5 + [0.0000001]
lr_schedule = learning_rate_schedule(lr_per_sample, unit=UnitType.sample)
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant)
# Instantiate the trainer object
learner = momentum_sgd(frcn_output.parameters, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight)
progress_printer = ProgressPrinter(tag='Training', num_epochs=max_epochs)
trainer = Trainer(frcn_output, (ce, pe), learner, progress_printer)
# Get minibatches of images and perform model training
print("Training Fast R-CNN model for %s epochs." % max_epochs)
log_number_of_parameters(frcn_output)
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = minibatch_source.next_minibatch(min(mb_size, epoch_size-sample_count), input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
trainer.summarize_training_progress()
if debug_output:
frcn_output.save(os.path.join(abs_path, "Output", "frcn_py_%s.model" % (epoch+1)))
return frcn_output
def train(network, trainer, od_minibatch_source, reader_dev, minibatch_size,
epochs_to_train, cfg):
input_map = {
od_minibatch_source.image_si: network['features'],
od_minibatch_source.roi_si: network['roi_input'],
od_minibatch_source.dims_si: network['dim_input']
}
# Train all minibatches
progress_printer = ProgressPrinter(tag='Training',
num_epochs=epochs_to_train)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < cfg[
"DATA"].NUM_TRAIN_IMAGES: # loop over minibatches in the epoch
data = od_minibatch_source.next_minibatch(min(
cfg.MB_SIZE, cfg["DATA"].NUM_TRAIN_IMAGES - sample_count),
input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
return create_faster_rcnn_eval_model(network['loss'], network['features'],
network['dim_input'], cfg)
def ffnet():
inputs = 3
outputs = 3
layers = 2
hidden_dimension = 3
# input variables denoting the features and label data
features = C.input((inputs), np.float32)
label = C.input((outputs), np.float32)
# Instantiate the feedforward classification model
my_model = Sequential(
[Dense(hidden_dimension, activation=C.sigmoid),
Dense(outputs)])
z = my_model(features)
ce = C.cross_entropy_with_softmax(z, label)
pe = C.classification_error(z, label)
# Instantiate the trainer object to drive the model training
lr_per_minibatch = learning_rate_schedule(0.125, UnitType.minibatch)
progress_printer = ProgressPrinter(0)
trainer = C.Trainer(z, (ce, pe), [
sgd(z.parameters,
lr=lr_per_minibatch,
gaussian_noise_injection_std_dev=0.01)
], [progress_printer])
# Get minibatches of training data and perform model training
minibatch_size = 25
num_minibatches_to_train = 100
aggregate_loss = 0.0
for i in range(num_minibatches_to_train):
train_features, labels = generate_random_data(minibatch_size, inputs,
outputs)
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
trainer.train_minibatch({features: train_features, label: labels})
sample_count = trainer.previous_minibatch_sample_count
aggregate_loss += trainer.previous_minibatch_loss_average * sample_count
last_avg_error = aggregate_loss / trainer.total_number_of_samples_seen
test_features, test_labels = generate_random_data(minibatch_size, inputs,
outputs)
avg_error = trainer.test_minibatch({
features: test_features,
label: test_labels
})
print(' error rate on an unseen minibatch: {}'.format(avg_error))
return last_avg_error, avg_error
def train_sequence_classifier():
input_dim = 2000
cell_dim = 25
hidden_dim = 25
embedding_dim = 50
num_output_classes = 5
# Input variables denoting the features and label data
features = sequence.input(shape=input_dim, is_sparse=True)
label = input(num_output_classes)
# Instantiate the sequence classification model
classifier_output = LSTM_sequence_classifier_net(
features, num_output_classes, embedding_dim, hidden_dim, cell_dim)
ce = cross_entropy_with_softmax(classifier_output, label)
pe = classification_error(classifier_output, label)
rel_path = r"../../../../Tests/EndToEndTests/Text/SequenceClassification/Data/Train.ctf"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
reader = create_reader(path, True, input_dim, num_output_classes)
input_map = {
features : reader.streams.features,
label : reader.streams.labels
}
lr_per_sample = learning_rate_schedule(0.0005, UnitType.sample)
# Instantiate the trainer object to drive the model training
progress_printer = ProgressPrinter(10)
trainer = Trainer(classifier_output, (ce, pe),
sgd(classifier_output.parameters, lr=lr_per_sample),
progress_printer)
# Get minibatches of sequences to train with and perform model training
minibatch_size = 200
for i in range(251):
mb = reader.next_minibatch(minibatch_size, input_map=input_map)
trainer.train_minibatch(mb)
import copy
evaluation_average = copy.copy(
trainer.previous_minibatch_evaluation_average)
loss_average = copy.copy(trainer.previous_minibatch_loss_average)
return evaluation_average, loss_average
def convnet_cifar10(train_source,
test_source,
epoch_size,
num_convolution_layers=2,
minibatch_size=64,
max_epochs=30,
log_file=None,
tboard_log_dir='.',
results_path=_MODEL_PATH):
_cntk_py.set_computation_network_trace_level(0)
logger.info("""Running network with:
{num_convolution_layers} convolution layers
{minibatch_size} minibatch size
for {max_epochs} epochs""".format(
num_convolution_layers=num_convolution_layers,
minibatch_size=minibatch_size,
max_epochs=max_epochs))
network = create_network(num_convolution_layers)
progress_printer = ProgressPrinter(tag='Training',
log_to_file=log_file,
rank=cntk.Communicator.rank(),
num_epochs=max_epochs)
tensorboard_writer = TensorBoardProgressWriter(freq=10,
log_dir=tboard_log_dir,
model=network['output'])
trainer = create_trainer(network, minibatch_size, epoch_size,
[progress_printer, tensorboard_writer])
cv_config = CrossValidationConfig(
minibatch_source=test_source,
minibatch_size=16,
callback=create_results_callback(
os.path.join(results_path, "model_results.json"),
num_convolution_layers=num_convolution_layers,
minibatch_size=minibatch_size,
max_epochs=max_epochs))
train_and_test(network,
trainer,
train_source,
test_source,
minibatch_size,
epoch_size,
restore=False,
cv_config=cv_config)
network['output'].save(os.path.join(results_path, _MODEL_NAME))
def ffnet(optimizer, num_minibatches_to_train, learning_rate_func, lr_args,
learner_kwargs):
inputs = 2
outputs = 2
hidden_dimension = 50
# input variables denoting the features and label data
features = C.input_variable((inputs), np.float32)
label = C.input_variable((outputs), np.float32)
# Instantiate the feedforward classification model
my_model = Sequential([
Dense(hidden_dimension,
activation=C.sigmoid,
init=C.glorot_uniform(seed=SEED)),
Dense(outputs, init=C.glorot_uniform(seed=SEED))
])
z = my_model(features)
ce = C.cross_entropy_with_softmax(z, label)
pe = C.classification_error(z, label)
# Instantiate the trainer object to drive the model training
lr = learning_rate_func(0.125, *lr_args)
progress_printer = ProgressPrinter(0)
learner = optimizer(z.parameters, lr) if optimizer != sgd else sgd(
z.parameters, lr, **learner_kwargs)
trainer = C.Trainer(z, (ce, pe), [learner], progress_printer)
# Get minibatches of training data and perform model training
minibatch_size = 25
for i in range(num_minibatches_to_train):
train_features, labels = generate_random_data(minibatch_size, inputs,
outputs)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({features: train_features, label: labels})
test_features, test_labels = generate_random_data(minibatch_size, inputs,
outputs)
avg_error = trainer.test_minibatch({
features: test_features,
label: test_labels
})
print(' error rate on an unseen minibatch: {}'.format(avg_error))
return z.parameters
def train_model(base_model_file, feature_node_name, last_hidden_node_name,
image_width, image_height, num_channels, num_classes, train_map_file,
num_epochs, max_images=-1, freeze=False):
epoch_size = sum(1 for line in open(train_map_file))
if max_images > 0:
epoch_size = min(epoch_size, max_images)
# Create the minibatch source and input variables
minibatch_source = create_mb_source(train_map_file, image_width, image_height, num_channels, num_classes)
image_input = C.input_variable((num_channels, image_height, image_width))
label_input = C.input_variable(num_classes)
# Define mapping from reader streams to network inputs
input_map = {
image_input: minibatch_source[features_stream_name],
label_input: minibatch_source[label_stream_name]
}
# Instantiate the transfer learning model and loss function
tl_model = create_model(base_model_file, feature_node_name, last_hidden_node_name, num_classes, image_input, freeze)
ce = cross_entropy_with_softmax(tl_model, label_input)
pe = classification_error(tl_model, label_input)
# Instantiate the trainer object
lr_schedule = learning_rate_schedule(lr_per_mb, unit=UnitType.minibatch)
mm_schedule = momentum_schedule(momentum_per_mb)
learner = momentum_sgd(tl_model.parameters, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight)
progress_printer = ProgressPrinter(tag='Training', num_epochs=num_epochs)
trainer = Trainer(tl_model, (ce, pe), learner, progress_printer)
# Get minibatches of images and perform model training
print("Training transfer learning model for {0} epochs (epoch_size = {1}).".format(num_epochs, epoch_size))
log_number_of_parameters(tl_model)
for epoch in range(num_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = minibatch_source.next_minibatch(min(mb_size, epoch_size-sample_count), input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
if sample_count % (100 * mb_size) == 0:
print("Processed {0} samples".format(sample_count))
trainer.summarize_training_progress()
return tl_model
def ffnet():
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
feature = input_variable((input_dim), np.float32)
label = input_variable((num_output_classes), np.float32)
netout = Sequential([
For(range(num_hidden_layers),
lambda i: Dense(hidden_layers_dim, activation=sigmoid)),
Dense(num_output_classes)
])(feature)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
lr_per_minibatch = learning_parameter_schedule(0.5)
# Instantiate the trainer object to drive the model training
learner = sgd(netout.parameters, lr=lr_per_minibatch)
progress_printer = ProgressPrinter(128)
trainer = Trainer(netout, (ce, pe), learner, progress_printer)
# Get minibatches of training data and perform model training
minibatch_size = 25
for i in range(1024):
features, labels = generate_random_data(minibatch_size, input_dim,
num_output_classes)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({feature: features, label: labels})
trainer.summarize_training_progress()
test_features, test_labels = generate_random_data(minibatch_size,
input_dim,
num_output_classes)
avg_error = trainer.test_minibatch({
feature: test_features,
label: test_labels
})
return avg_error
def ffnet():
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
input = input_variable((input_dim), np.float32)
label = input_variable((num_output_classes), np.float32)
# Instantiate the feedforward classification model
netout = fully_connected_classifier_net(input, num_output_classes,
hidden_layers_dim,
num_hidden_layers, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
lr_per_minibatch = learning_rate_schedule(0.5, UnitType.minibatch)
# Instantiate the trainer object to drive the model training
learner = sgd(netout.parameters, lr=lr_per_minibatch)
progress_printer = ProgressPrinter(128)
trainer = Trainer(netout, (ce, pe), learner, progress_printer)
# Get minibatches of training data and perform model training
minibatch_size = 25
for i in range(1024):
features, labels = generate_random_data(minibatch_size, input_dim,
num_output_classes)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({input: features, label: labels})
trainer.summarize_training_progress()
test_features, test_labels = generate_random_data(minibatch_size,
input_dim,
num_output_classes)
avg_error = trainer.test_minibatch({
input: test_features,
label: test_labels
})
return avg_error
def simple_mnist(tensorboard_logdir=None):
input_dim = 784
num_output_classes = 10
num_hidden_layers = 2
hidden_layers_dim = 200
# Input variables denoting the features and label data
feature = C.input_variable(input_dim, np.float32)
label = C.input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), feature)
z = Sequential([
For(range(num_hidden_layers),
lambda i: Dense(hidden_layers_dim, activation=relu)),
Dense(num_output_classes)
])(scaled_input)
ce = cross_entropy_with_softmax(z, label)
pe = classification_error(z, label)
data_dir = os.path.dirname(os.path.abspath(__file__))
path = os.path.join(data_dir, 'Train-28x28_cntk_text.txt')
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
feature: reader_train.streams.features,
label: reader_train.streams.labels
}
# Training config
minibatch_size = 64
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 10
# Instantiate progress writers.
# training_progress_output_freq = 100
progress_writers = [
ProgressPrinter(
# freq=training_progress_output_freq,
tag='Training',
num_epochs=num_sweeps_to_train_with)
]
if tensorboard_logdir is not None:
progress_writers.append(
TensorBoardProgressWriter(freq=10,
log_dir=tensorboard_logdir,
model=z))
# Instantiate the trainer object to drive the model training
lr = 0.001
trainer = Trainer(z, (ce, pe), sgd(z.parameters, lr), progress_writers)
training_session(trainer=trainer,
mb_source=reader_train,
mb_size=minibatch_size,
model_inputs_to_streams=input_map,
max_samples=num_samples_per_sweep *
num_sweeps_to_train_with,
progress_frequency=num_samples_per_sweep).train()
# Load test data
path = os.path.normpath(os.path.join(data_dir, "Test-28x28_cntk_text.txt"))
check_path(path)
reader_test = create_reader(path, False, input_dim, num_output_classes)
input_map = {
feature: reader_test.streams.features,
label: reader_test.streams.labels
}
# Test data for trained model
C.debugging.start_profiler()
C.debugging.enable_profiler()
C.debugging.set_node_timing(True)
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size,
input_map=input_map)
eval_error = trainer.test_minibatch(mb)
test_result = test_result + eval_error
C.debugging.stop_profiler()
trainer.print_node_timing()
# Average of evaluation errors of all test minibatches
return test_result * 100 / num_minibatches_to_test
def train(train_reader, valid_reader, vocab, i2w, s2smodel, max_epochs, epoch_size):
# Note: We would like to set the signature of 's2smodel' (s2smodel.update_signature()), but that will cause
# an error since the training criterion uses a reduced sequence axis for the labels.
# This is because it removes the initial <s> symbol. Hence, we must leave the model
# with unspecified input shapes and axes.
# create the training wrapper for the s2smodel, as well as the criterion function
model_train = create_model_train(s2smodel)
criterion = create_criterion_function(model_train)
# also wire in a greedy decoder so that we can properly log progress on a validation example
# This is not used for the actual training process.
model_greedy = create_model_greedy(s2smodel)
# This does not need to be done in training generally though
# Instantiate the trainer object to drive the model training
minibatch_size = 72
lr = 0.001 if use_attention else 0.005 # TODO: can we use the same value for both?
learner = fsadagrad(model_train.parameters,
lr = learning_rate_schedule([lr]*2+[lr/2]*3+[lr/4], UnitType.sample, epoch_size),
momentum = momentum_as_time_constant_schedule(1100),
gradient_clipping_threshold_per_sample=2.3,
gradient_clipping_with_truncation=True)
trainer = Trainer(None, criterion, learner)
# Get minibatches of sequences to train with and perform model training
total_samples = 0
mbs = 0
eval_freq = 100
# print out some useful training information
log_number_of_parameters(model_train) ; print()
progress_printer = ProgressPrinter(freq=30, tag='Training')
#progress_printer = ProgressPrinter(freq=30, tag='Training', log_to_file=model_path_stem + ".log") # use this to log to file
sparse_to_dense = create_sparse_to_dense(input_vocab_dim)
for epoch in range(max_epochs):
print("Saving model to '%s'" % model_path(epoch))
s2smodel.save(model_path(epoch))
while total_samples < (epoch+1) * epoch_size:
# get next minibatch of training data
mb_train = train_reader.next_minibatch(minibatch_size)
#trainer.train_minibatch(mb_train[train_reader.streams.features], mb_train[train_reader.streams.labels])
trainer.train_minibatch({criterion.arguments[0]: mb_train[train_reader.streams.features], criterion.arguments[1]: mb_train[train_reader.streams.labels]})
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
# every N MBs evaluate on a test sequence to visually show how we're doing
if mbs % eval_freq == 0:
mb_valid = valid_reader.next_minibatch(1)
# run an eval on the decoder output model (i.e. don't use the groundtruth)
e = model_greedy(mb_valid[valid_reader.streams.features])
print(format_sequences(sparse_to_dense(mb_valid[valid_reader.streams.features]), i2w))
print("->")
print(format_sequences(e, i2w))
# debugging attention
if use_attention:
debug_attention(model_greedy, mb_valid[valid_reader.streams.features])
total_samples += mb_train[train_reader.streams.labels].num_samples
mbs += 1
# log a summary of the stats for the epoch
progress_printer.epoch_summary(with_metric=True)
# done: save the final model
print("Saving final model to '%s'" % model_path(max_epochs))
s2smodel.save(model_path(max_epochs))
print("%d epochs complete." % max_epochs)
def train(train_reader, valid_reader, vocab, i2w, s2smodel, max_epochs,
epoch_size):
# Note: We would like to set the signature of 's2smodel' (s2smodel.update_signature()), but that will cause
# an error since the training criterion uses a reduced sequence axis for the labels.
# This is because it removes the initial <s> symbol. Hence, we must leave the model
# with unspecified input shapes and axes.
# create the training wrapper for the s2smodel, as well as the criterion function
model_train = create_model_train(s2smodel)
criterion = create_criterion_function(model_train)
# also wire in a greedy decoder so that we can properly log progress on a validation example
# This is not used for the actual training process.
model_greedy = create_model_greedy(s2smodel)
# This does not need to be done in training generally though
# Instantiate the trainer object to drive the model training
minibatch_size = 72
lr = 0.001 if use_attention else 0.005 # TODO: can we use the same value for both?
learner = adam_sgd(
model_train.parameters,
lr=learning_rate_schedule([lr] * 2 + [lr / 2] * 3 + [lr / 4],
UnitType.sample, epoch_size),
momentum=momentum_as_time_constant_schedule(1100),
gradient_clipping_threshold_per_sample=2.3,
gradient_clipping_with_truncation=True)
trainer = Trainer(None, criterion, learner)
# Get minibatches of sequences to train with and perform model training
total_samples = 0
mbs = 0
eval_freq = 100
# print out some useful training information
log_number_of_parameters(model_train)
print()
progress_printer = ProgressPrinter(freq=30, tag='Training')
#progress_printer = ProgressPrinter(freq=30, tag='Training', log_to_file=model_path_stem + ".log") # use this to log to file
sparse_to_dense = create_sparse_to_dense(input_vocab_dim)
for epoch in range(max_epochs):
print("Saving model to '%s'" % model_path(epoch))
s2smodel.save(model_path(epoch))
while total_samples < (epoch + 1) * epoch_size:
# get next minibatch of training data
mb_train = train_reader.next_minibatch(minibatch_size)
#trainer.train_minibatch(mb_train[train_reader.streams.features], mb_train[train_reader.streams.labels])
trainer.train_minibatch({
criterion.arguments[0]:
mb_train[train_reader.streams.features],
criterion.arguments[1]:
mb_train[train_reader.streams.labels]
})
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
# every N MBs evaluate on a test sequence to visually show how we're doing
if mbs % eval_freq == 0:
mb_valid = valid_reader.next_minibatch(1)
# run an eval on the decoder output model (i.e. don't use the groundtruth)
e = model_greedy(mb_valid[valid_reader.streams.features])
print(
format_sequences(
sparse_to_dense(
mb_valid[valid_reader.streams.features]), i2w))
print("->")
print(format_sequences(e, i2w))
# debugging attention
if use_attention:
debug_attention(model_greedy,
mb_valid[valid_reader.streams.features])
total_samples += mb_train[train_reader.streams.labels].num_samples
mbs += 1
# log a summary of the stats for the epoch
progress_printer.epoch_summary(with_metric=True)
# done: save the final model
print("Saving final model to '%s'" % model_path(max_epochs))
s2smodel.save(model_path(max_epochs))
print("%d epochs complete." % max_epochs)
"""
loss = squared_error(netout, label)
evaluation = squared_error(netout, label)
lr_per_minibatch=learning_rate_schedule(learning_rate, UnitType.minibatch)
"""
Instantiate the trainer object to drive the model training
See: https://www.cntk.ai/pythondocs/cntk.learners.html
"""
learner = sgd(netout.parameters, lr=lr_per_minibatch)
# Other learners to try
#learner = momentum_sgd(netout.parameters, lr=lr_per_minibatch, momentum = momentum_schedule(0.9))
#learner = adagrad(netout.parameters, lr=lr_per_minibatch)
progress_printer = ProgressPrinter(minibatch_size)
"""
Instantiate the trainer
See: https://www.cntk.ai/pythondocs/cntk.train.html#module-cntk.train.trainer
"""
trainer = Trainer(netout, (loss, evaluation), learner, progress_printer)
#%%
"""
Run training
"""
plotdata = {"loss":[]}
for i in range(10000):
data = training_reader.next_minibatch(minibatch_size, input_map = input_map)
"""
z_sm = cntk.softmax(z)
ce = cross_entropy_with_softmax(z, label_sequence)
errs = classification_error(z, label_sequence)
lr_per_sample = learning_rate_schedule(0.001, UnitType.sample)
momentum_time_constant = momentum_as_time_constant_schedule(1100)
clipping_threshold_per_sample = 5.0
gradient_clipping_with_truncation = True
learner = momentum_sgd(
z.parameters,
lr_per_sample,
momentum_time_constant,
gradient_clipping_threshold_per_sample=clipping_threshold_per_sample,
gradient_clipping_with_truncation=gradient_clipping_with_truncation)
progress_printer = ProgressPrinter(freq=100, tag='Training')
trainer = Trainer(z, (ce, errs), learner, progress_printer)
log_number_of_parameters(z)
def sample(net, prime_text='', use_hardmax=True, length=100, temperature=1.0):
# Применяем температуру: T < 1 - сглаживание; T=1.0 - без изменений; T > 1 - выделение пиков
def apply_temp(p):
p = np.power(p, (temperature))
# повторно нормализуем
return (p / np.sum(p))
def sample_word(p):
if use_hardmax:
def train_model(image_input,
roi_input,
dims_input,
loss,
pred_error,
lr_per_sample,
mm_schedule,
l2_reg_weight,
epochs_to_train,
rpn_rois_input=None,
buffered_rpn_proposals=None):
if isinstance(loss, cntk.Variable):
loss = combine([loss])
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
if cfg["CNTK"].DEBUG_OUTPUT:
print("biases")
for p in biases:
print(p)
print("others")
for p in others:
print(p)
print("bias_lr_mult: {}".format(bias_lr_mult))
# Instantiate the learners and the trainer object
lr_schedule = learning_rate_schedule(lr_per_sample, unit=UnitType.sample)
learner = momentum_sgd(others,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight,
unit_gain=False,
use_mean_gradient=cfg["CNTK"].USE_MEAN_GRADIENT)
bias_lr_per_sample = [v * bias_lr_mult for v in lr_per_sample]
bias_lr_schedule = learning_rate_schedule(bias_lr_per_sample,
unit=UnitType.sample)
bias_learner = momentum_sgd(
biases,
bias_lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight,
unit_gain=False,
use_mean_gradient=cfg["CNTK"].USE_MEAN_GRADIENT)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
od_minibatch_source = ObjectDetectionMinibatchSource(
globalvars['train_map_file'],
globalvars['train_roi_file'],
max_annotations_per_image=cfg["CNTK"].INPUT_ROIS_PER_IMAGE,
pad_width=image_width,
pad_height=image_height,
pad_value=img_pad_value,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["CNTK"].NUM_TRAIN_IMAGES,
buffered_rpn_proposals=buffered_rpn_proposals)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.roi_si: roi_input,
od_minibatch_source.dims_si: dims_input
}
use_buffered_proposals = buffered_rpn_proposals is not None
progress_printer = ProgressPrinter(tag='Training',
num_epochs=epochs_to_train,
gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data, proposals = od_minibatch_source.next_minibatch_with_proposals(
min(mb_size, epoch_size - sample_count), input_map=input_map)
if use_buffered_proposals:
data[rpn_rois_input] = MinibatchData(
Value(batch=np.asarray(proposals, dtype=np.float32)), 1, 1,
False)
# remove dims input if no rpn is required to avoid warnings
del data[[k for k in data if '[6]' in str(k)][0]]
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
def train(reader_train, reader_test, samples_per_epoch, max_amount_of_epochs,
samples_per_minibatch, dimensions, classes, learning_rate,
output_directory, with_tf):
features = input_variable(shape=(dimensions['depth'], dimensions['height'],
dimensions['width']))
label = input_variable(shape=len(classes))
# speeds up training
normalized_features = element_times(1.0 / 256.0, features)
if with_tf:
base_model = {
'model_file':
os.path.join("..", "..", "Pretrained Models/ResNet_18.model"),
'feature_node_name':
'features',
'last_hidden_node_name':
'z.x',
'image_dims': (3, 224, 224)
}
model = create_tf_model(base_model,
num_classes=len(classes),
input_features=normalized_features,
freeze=True)
else:
model = create_model(feature_dimensions=normalized_features,
classes=classes)
loss = cross_entropy_with_softmax(model, label)
metric = classification_error(model, label)
learner = momentum_sgd(parameters=model.parameters,
lr=learning_rate_schedule(learning_rate,
UnitType.minibatch),
momentum=0.9,
l2_regularization_weight=0.0005)
reporter = ProgressPrinter(tag='training', num_epochs=max_amount_of_epochs)
trainer = Trainer(model=model,
criterion=(loss, metric),
parameter_learners=[learner],
progress_writers=[reporter])
log_number_of_parameters(model)
map_input_to_streams_train = {
features: reader_train.streams.features,
label: reader_train.streams.labels
}
map_input_to_streams_test = {
features: reader_test.streams.features,
label: reader_test.streams.labels
}
training_session(
trainer=trainer,
mb_source=reader_train,
model_inputs_to_streams=map_input_to_streams_train,
mb_size=samples_per_minibatch,
progress_frequency=samples_per_epoch,
checkpoint_config=CheckpointConfig(frequency=samples_per_epoch,
filename=os.path.join(
output_directory,
"ConvNet_Lego_VisiOn"),
restore=False),
test_config=TestConfig(
reader_test,
minibatch_size=samples_per_minibatch,
model_inputs_to_streams=map_input_to_streams_test)).train()
network = {'features': features, 'label': label, 'model': softmax(model)}
return network
def train_fast_rcnn(cfg):
# Train only if no model exists yet
model_path = cfg['MODEL_PATH']
if os.path.exists(model_path) and cfg["CNTK"].MAKE_MODE:
print("Loading existing model from %s" % model_path)
return load_model(model_path)
else:
# Input variables denoting features and labeled ground truth rois (as 5-tuples per roi)
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT,
cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=cfg["MODEL"].FEATURE_NODE_NAME)
roi_proposals = input_variable(
(cfg.NUM_ROI_PROPOSALS, 4),
dynamic_axes=[Axis.default_batch_axis()],
name="roi_proposals")
label_targets = input_variable(
(cfg.NUM_ROI_PROPOSALS, cfg["DATA"].NUM_CLASSES),
dynamic_axes=[Axis.default_batch_axis()])
bbox_targets = input_variable(
(cfg.NUM_ROI_PROPOSALS, 4 * cfg["DATA"].NUM_CLASSES),
dynamic_axes=[Axis.default_batch_axis()])
bbox_inside_weights = input_variable(
(cfg.NUM_ROI_PROPOSALS, 4 * cfg["DATA"].NUM_CLASSES),
dynamic_axes=[Axis.default_batch_axis()])
# Instantiate the Fast R-CNN prediction model and loss function
loss, pred_error = create_fast_rcnn_model(image_input, roi_proposals,
label_targets, bbox_targets,
bbox_inside_weights, cfg)
if isinstance(loss, cntk.Variable):
loss = combine([loss])
if cfg["CNTK"].DEBUG_OUTPUT:
print("Storing graphs and models to %s." % cfg.OUTPUT_PATH)
plot(
loss,
os.path.join(cfg.OUTPUT_PATH,
"graph_frcn_train." + cfg["CNTK"].GRAPH_TYPE))
# Set learning parameters
lr_factor = cfg["CNTK"].LR_FACTOR
lr_per_sample_scaled = [
x * lr_factor for x in cfg["CNTK"].LR_PER_SAMPLE
]
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
l2_reg_weight = cfg["CNTK"].L2_REG_WEIGHT
epochs_to_train = cfg["CNTK"].MAX_EPOCHS
print("Using base model: {}".format(cfg["MODEL"].BASE_MODEL))
print("lr_per_sample: {}".format(lr_per_sample_scaled))
# --- train ---
# Instantiate the learners and the trainer object
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
lr_schedule = learning_parameter_schedule_per_sample(
lr_per_sample_scaled)
learner = momentum_sgd(others,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight,
unit_gain=False,
use_mean_gradient=True)
bias_lr_per_sample = [
v * bias_lr_mult for v in cfg["CNTK"].LR_PER_SAMPLE
]
bias_lr_schedule = learning_parameter_schedule_per_sample(
bias_lr_per_sample)
bias_learner = momentum_sgd(biases,
bias_lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight,
unit_gain=False,
use_mean_gradient=True)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
if cfg.USE_PRECOMPUTED_PROPOSALS:
proposal_provider = ProposalProvider.fromfile(
cfg["DATA"].TRAIN_PRECOMPUTED_PROPOSALS_FILE,
cfg.NUM_ROI_PROPOSALS)
else:
proposal_provider = ProposalProvider.fromconfig(cfg)
od_minibatch_source = ObjectDetectionMinibatchSource(
cfg["DATA"].TRAIN_MAP_FILE,
cfg["DATA"].TRAIN_ROI_FILE,
max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE,
pad_width=cfg.IMAGE_WIDTH,
pad_height=cfg.IMAGE_HEIGHT,
pad_value=cfg["MODEL"].IMG_PAD_COLOR,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["DATA"].NUM_TRAIN_IMAGES,
num_classes=cfg["DATA"].NUM_CLASSES,
proposal_provider=proposal_provider,
provide_targets=True,
proposal_iou_threshold=cfg.BBOX_THRESH,
normalize_means=None
if not cfg.BBOX_NORMALIZE_TARGETS else cfg.BBOX_NORMALIZE_MEANS,
normalize_stds=None
if not cfg.BBOX_NORMALIZE_TARGETS else cfg.BBOX_NORMALIZE_STDS)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.proposals_si: roi_proposals,
od_minibatch_source.label_targets_si: label_targets,
od_minibatch_source.bbox_targets_si: bbox_targets,
od_minibatch_source.bbiw_si: bbox_inside_weights
}
progress_printer = ProgressPrinter(tag='Training',
num_epochs=epochs_to_train,
gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < cfg[
"DATA"].NUM_TRAIN_IMAGES: # loop over minibatches in the epoch
data = od_minibatch_source.next_minibatch(min(
cfg.MB_SIZE, cfg["DATA"].NUM_TRAIN_IMAGES - sample_count),
input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
continue
#print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
eval_model = create_fast_rcnn_eval_model(loss, image_input,
roi_proposals, cfg)
eval_model.save(cfg['MODEL_PATH'])
return eval_model
targets_ph = C.input_variable(shape=1)
model = C.layers.Sequential([
C.layers.Recurrence(IndRNN(HIDDEN_DIM, 2).build()),
C.layers.Recurrence(IndRNN(1, HIDDEN_DIM).build()), C.sequence.last
])
output = model(input_ph)
loss = C.losses.squared_error(output, targets_ph)
comp = C.combine(output, loss)
lrs = [(1, 0.02), (300, 0.002), (600, 0.0001)]
lr_schedule = C.learners.learning_parameter_schedule(lrs)
learner = C.learners.adam(loss.parameters, lr_schedule, 0.9)
trainer = C.Trainer(output, loss, learner, ProgressPrinter(5))
input, target = get_batch()
for _ in range(1000):
trainer.train_minibatch({input_ph: input, targets_ph: target})
res = output.eval({input_ph: input})
print('predict:{}, target:{}'.format(res, target))
# just use lstm
model2 = C.layers.Sequential([
C.layers.Recurrence(C.layers.LSTM(HIDDEN_DIM)), C.sequence.last,
C.layers.Dense(1, activation=C.relu)
])
output = model2(input_ph)
def train_model(cntkModel, params, input_map):
log = logging.getLogger("neuralnets1.utils.train_model")
mb_size = params['mb_size']
num_epochs = params['num_epochs']
epoch_size_train = params['epoch_size_train']
epoch_size_test = params['epoch_size_test']
minibatch_source_train = params['train_mbs']
minibatch_source_valid = params['valid_mbs']
#minibatch_source_test = params['test_mbs'] ;
# Instantiate the trainer object
#lr_schedule = learning_rate_schedule(params['learn_rate'], unit=UnitType.minibatch)
lr_per_minibatch = learning_parameter_schedule(params['learn_rate'],
minibatch_size=mb_size,
epoch_size=epoch_size_train)
mm_schedule = momentum_schedule(params['beta_momentum_gd'])
learner = momentum_sgd(cntkModel.parameters,
lr_per_minibatch,
mm_schedule,
l2_regularization_weight=params['l2_reg_weight'])
progress_writers = [ProgressPrinter(tag='Training', num_epochs=num_epochs)]
trainer = Trainer(cntkModel, (params['ce'], params['pe']), learner,
progress_writers)
# Run training epochs
log.info(
'Training transfer learning model for %s epochs (epoch_size_train = %s ) .'
% (num_epochs, epoch_size_train))
# print("Training transfer learning model for {0} epochs (epoch_size_train = {1}).".format(num_epochs, epoch_size_train))
errsVal = []
errsTrain = []
log_number_of_parameters(cntkModel)
for epoch in range(num_epochs):
err_numer = 0
sample_counts = 0
while sample_counts < epoch_size_train: # Loop over minibatches in the epoch
sample_count = min(mb_size, epoch_size_train - sample_counts)
data = minibatch_source_train.next_minibatch(sample_count,
input_map=input_map)
trainer.train_minibatch(data) # Update model with it
sample_counts += sample_count # Count samples processed so far
err_numer += trainer.previous_minibatch_evaluation_average * sample_count
if sample_counts % (100 * mb_size) == 0:
log.info("Training: processed %s samples" % sample_counts)
# Compute accuracy on training and test sets
errsTrain.append(err_numer / float(sample_counts))
trainer.summarize_training_progress()
errsVal.append(
cntkComputeTestError(trainer, minibatch_source_valid, mb_size,
epoch_size_test, input_map))
trainer.summarize_test_progress()
# Plot training progress
plt.plot(errsTrain, 'b-', errsVal, 'g-')
plt.xlabel('Epoch number')
plt.ylabel('Error')
plt.title('Training error (blue), validation error (green)')
plt.draw()
return cntkModel
def train_lm(training_file, epochs, max_num_minibatches):
# load the data and vocab
data, char_to_ix, ix_to_char, data_size, vocab_dim = load_data_and_vocab(
training_file)
# Model the source and target inputs to the model
input_sequence, label_sequence = create_inputs(vocab_dim)
# create the model
model = create_model(vocab_dim)
# and apply it to the input sequence
z = model(input_sequence)
# setup the criterions (loss and metric)
ce = cross_entropy_with_softmax(z, label_sequence)
errs = classification_error(z, label_sequence)
# Instantiate the trainer object to drive the model training
lr_per_sample = learning_parameter_schedule_per_sample(0.001)
momentum_schedule = momentum_schedule_per_sample(0.9990913221888589)
clipping_threshold_per_sample = 5.0
gradient_clipping_with_truncation = True
learner = momentum_sgd(
z.parameters,
lr_per_sample,
momentum_schedule,
gradient_clipping_threshold_per_sample=clipping_threshold_per_sample,
gradient_clipping_with_truncation=gradient_clipping_with_truncation)
progress_printer = ProgressPrinter(freq=100, tag='Training')
trainer = Trainer(z, (ce, errs), learner, progress_printer)
sample_freq = 1000
minibatches_per_epoch = min(data_size // minibatch_size,
max_num_minibatches // epochs)
# print out some useful training information
log_number_of_parameters(z)
print("Running %d epochs with %d minibatches per epoch" %
(epochs, minibatches_per_epoch))
print()
for e in range(0, epochs):
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
# If it's the start of the data, we specify that we are looking at a new sequence (True)
mask = [True]
for b in range(0, minibatches_per_epoch):
# get the data
features, labels = get_data(b, minibatch_size, data, char_to_ix,
vocab_dim)
arguments = ({
input_sequence: features,
label_sequence: labels
}, mask)
mask = [False]
trainer.train_minibatch(arguments)
global_minibatch = e * minibatches_per_epoch + b
if global_minibatch % sample_freq == 0:
print(sample(z, ix_to_char, vocab_dim, char_to_ix))
model_filename = "models/shakespeare_epoch%d.dnn" % (e + 1)
z.save(model_filename)
print("Saved model to '%s'" % model_filename)
def simple_mnist(tensorboard_logdir=None):
input_dim = 19
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 1024
# Input variables denoting the features and label data
feature = C.input_variable(input_dim, np.float32)
label = C.input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
# scaled_input = element_times(constant(0.00390625), feature)
z = Sequential([For(range(num_hidden_layers), lambda i: Dense(hidden_layers_dim, activation=relu)),
Dense(num_output_classes)])(feature)
ce = cross_entropy_with_softmax(z, label)
pe = classification_error(z, label)
data_dir = r"."
path = os.path.normpath(os.path.join(data_dir, "train.ctf"))
check_path(path)
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
feature : reader_train.streams.features,
label : reader_train.streams.labels
}
# Training config
minibatch_size = 512
num_samples_per_sweep = 1825000
num_sweeps_to_train_with = 100
# Instantiate progress writers.
progress_writers = [ProgressPrinter(
tag='Training',
num_epochs=num_sweeps_to_train_with)]
if tensorboard_logdir is not None:
tensorboard_writer = TensorBoardProgressWriter(freq=10, log_dir=tensorboard_logdir, model=z)
progress_writers.append(tensorboard_writer)
# Instantiate the trainer object to drive the model training
lr = learning_parameter_schedule_per_sample(0.001)
learner = create_learner(model=z)
trainer = Trainer(z, (ce, pe), learner, progress_writers)
num_minibatches_to_train = int(num_samples_per_sweep / minibatch_size * num_sweeps_to_train_with)
model_dir = "model"
for i in range(num_minibatches_to_train):
mb = reader_train.next_minibatch(minibatch_size, input_map=input_map)
trainer.train_minibatch(mb)
freq = int(num_samples_per_sweep / minibatch_size)
if i > 0 and i % freq == 0:
timestamp = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")
current_trainer_cp = os.path.join(model_dir, timestamp + "_epoch_" + str(freq) + ".trainer")
trainer.save_checkpoint(current_trainer_cp)
train_error = get_error_rate(os.path.join(data_dir, "train_subset.ctf"), input_map, input_dim,
num_output_classes, trainer)
valid_error = get_error_rate(os.path.join(data_dir, "validation.ctf"), input_map, input_dim, num_output_classes,
trainer)
if train_error > 0:
tensorboard_writer.write_value("train_error", train_error, i)
if valid_error > 0:
tensorboard_writer.write_value("valid_error", valid_error, i)
feat_path = os.path.normpath(os.path.join(data_dir, "test.ctf"))
return get_error_rate(feat_path, input_map, input_dim, num_output_classes, trainer)
def train_fast_rcnn(cfg):
# Train only if no model exists yet
model_path = cfg['MODEL_PATH']
if os.path.exists(model_path) and cfg["CNTK"].MAKE_MODE:
print("Loading existing model from %s" % model_path)
return load_model(model_path)
else:
# Input variables denoting features and labeled ground truth rois (as 5-tuples per roi)
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=cfg["MODEL"].FEATURE_NODE_NAME)
roi_proposals = input_variable((cfg.NUM_ROI_PROPOSALS, 4), dynamic_axes=[Axis.default_batch_axis()], name = "roi_proposals")
label_targets = input_variable((cfg.NUM_ROI_PROPOSALS, cfg["DATA"].NUM_CLASSES), dynamic_axes=[Axis.default_batch_axis()])
bbox_targets = input_variable((cfg.NUM_ROI_PROPOSALS, 4*cfg["DATA"].NUM_CLASSES), dynamic_axes=[Axis.default_batch_axis()])
bbox_inside_weights = input_variable((cfg.NUM_ROI_PROPOSALS, 4*cfg["DATA"].NUM_CLASSES), dynamic_axes=[Axis.default_batch_axis()])
# Instantiate the Fast R-CNN prediction model and loss function
loss, pred_error = create_fast_rcnn_model(image_input, roi_proposals, label_targets, bbox_targets, bbox_inside_weights, cfg)
if isinstance(loss, cntk.Variable):
loss = combine([loss])
if cfg["CNTK"].DEBUG_OUTPUT:
print("Storing graphs and models to %s." % cfg.OUTPUT_PATH)
plot(loss, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train." + cfg["CNTK"].GRAPH_TYPE))
# Set learning parameters
lr_factor = cfg["CNTK"].LR_FACTOR
lr_per_sample_scaled = [x * lr_factor for x in cfg["CNTK"].LR_PER_SAMPLE]
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
l2_reg_weight = cfg["CNTK"].L2_REG_WEIGHT
epochs_to_train = cfg["CNTK"].MAX_EPOCHS
print("Using base model: {}".format(cfg["MODEL"].BASE_MODEL))
print("lr_per_sample: {}".format(lr_per_sample_scaled))
# --- train ---
# Instantiate the learners and the trainer object
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
lr_schedule = learning_rate_schedule(lr_per_sample_scaled, unit=UnitType.sample)
learner = momentum_sgd(others, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight, unit_gain=False, use_mean_gradient=True)
bias_lr_per_sample = [v * bias_lr_mult for v in cfg["CNTK"].LR_PER_SAMPLE]
bias_lr_schedule = learning_rate_schedule(bias_lr_per_sample, unit=UnitType.sample)
bias_learner = momentum_sgd(biases, bias_lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight, unit_gain=False, use_mean_gradient=True)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
if cfg.USE_PRECOMPUTED_PROPOSALS:
proposal_provider = ProposalProvider.fromfile(cfg["DATA"].TRAIN_PRECOMPUTED_PROPOSALS_FILE, cfg.NUM_ROI_PROPOSALS)
else:
proposal_provider = ProposalProvider.fromconfig(cfg)
od_minibatch_source = ObjectDetectionMinibatchSource(
cfg["DATA"].TRAIN_MAP_FILE, cfg["DATA"].TRAIN_ROI_FILE,
max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE,
pad_width=cfg.IMAGE_WIDTH,
pad_height=cfg.IMAGE_HEIGHT,
pad_value=cfg["MODEL"].IMG_PAD_COLOR,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["DATA"].NUM_TRAIN_IMAGES,
num_classes=cfg["DATA"].NUM_CLASSES,
proposal_provider=proposal_provider,
provide_targets=True,
proposal_iou_threshold = cfg.BBOX_THRESH,
normalize_means = None if not cfg.BBOX_NORMALIZE_TARGETS else cfg.BBOX_NORMALIZE_MEANS,
normalize_stds = None if not cfg.BBOX_NORMALIZE_TARGETS else cfg.BBOX_NORMALIZE_STDS)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.proposals_si: roi_proposals,
od_minibatch_source.label_targets_si: label_targets,
od_minibatch_source.bbox_targets_si: bbox_targets,
od_minibatch_source.bbiw_si: bbox_inside_weights
}
progress_printer = ProgressPrinter(tag='Training', num_epochs=epochs_to_train, gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < cfg["DATA"].NUM_TRAIN_IMAGES: # loop over minibatches in the epoch
data = od_minibatch_source.next_minibatch(min(cfg.MB_SIZE, cfg["DATA"].NUM_TRAIN_IMAGES - sample_count), input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
eval_model = create_fast_rcnn_eval_model(loss, image_input, roi_proposals, cfg)
eval_model.save(cfg['MODEL_PATH'])
return eval_model
def simple_mnist(tensorboard_logdir=None):
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
feature = input(input_dim, np.float32)
label = input(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), feature)
z = fully_connected_classifier_net(scaled_input, num_output_classes,
hidden_layers_dim, num_hidden_layers,
relu)
ce = cross_entropy_with_softmax(z, label)
pe = classification_error(z, label)
data_dir = os.path.join(abs_path, "..", "..", "..", "DataSets", "MNIST")
path = os.path.normpath(os.path.join(data_dir,
"Train-28x28_cntk_text.txt"))
check_path(path)
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
feature: reader_train.streams.features,
label: reader_train.streams.labels
}
# Training config
minibatch_size = 64
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 10
# Instantiate progress writers.
#training_progress_output_freq = 100
progress_writers = [
ProgressPrinter(
#freq=training_progress_output_freq,
tag='Training',
num_epochs=num_sweeps_to_train_with)
]
if tensorboard_logdir is not None:
progress_writers.append(
TensorBoardProgressWriter(freq=10,
log_dir=tensorboard_logdir,
model=z))
# Instantiate the trainer object to drive the model training
trainer = Trainer(z, (ce, pe), adadelta(z.parameters), progress_writers)
training_session(trainer=trainer,
mb_source=reader_train,
mb_size=minibatch_size,
var_to_stream=input_map,
max_samples=num_samples_per_sweep *
num_sweeps_to_train_with,
progress_frequency=num_samples_per_sweep).train()
# Load test data
path = os.path.normpath(os.path.join(data_dir, "Test-28x28_cntk_text.txt"))
check_path(path)
reader_test = create_reader(path, False, input_dim, num_output_classes)
input_map = {
feature: reader_test.streams.features,
label: reader_test.streams.labels
}
# Test data for trained model
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size,
input_map=input_map)
eval_error = trainer.test_minibatch(mb)
test_result = test_result + eval_error
# Average of evaluation errors of all test minibatches
return test_result / num_minibatches_to_test
def __train_cntk(self, path_to_folder: str, model_definition, epochs: int,
output_model_path: str, classes, minibatch_size: int):
import cntk
from cntk.learners import learning_parameter_schedule
from cntk.ops import input_variable
from cntk.io import MinibatchSource, ImageDeserializer, StreamDefs, StreamDef, MinibatchData, UserDeserializer
import cntk.io.transforms as xforms
from cntk.layers import default_options, Dense, Sequential, Activation, Embedding, Convolution2D, MaxPooling, Stabilizer, Convolution, Dropout, BatchNormalization
from cntk.ops.functions import CloneMethod
from cntk.logging import ProgressPrinter
from cntk.losses import cross_entropy_with_softmax
from cntk import classification_error, softmax, relu, ModelFormat, element_times, momentum_schedule, momentum_sgd
import pandas as pd
path_to_folder = path_to_folder.rstrip('/')
map_file_train = path_to_folder + "/train_map.txt"
map_file_test = path_to_folder + "/test_map.txt"
classes_set = set()
num_train = 0
num_test = 0
num_channels = 3
class TrackDataset(UserDeserializer):
def __init__(self, map_file, streams, chunksize=100):
super(TrackDataset, self).__init__()
self._batch_size = chunksize
self.dataframes = pd.read_csv(map_file,
sep='\t',
dtype=str,
header=None,
names=["features", "labels"])
self._streams = [
cntk.io.StreamInformation(s['name'], i, 'dense',
np.float32, s['shape'])
for i, s in enumerate(streams)
]
self._num_chunks = int(
math.ceil(len(self.dataframes) / chunksize))
def _scale_image(self, image, width=224, height=168):
try:
return image.resize((width, height), Image.LINEAR)
except:
raise Exception('scale_image error')
def stream_infos(self):
return self._streams
def num_chunks(self):
return self._num_chunks
def get_chunk(self, chunk_id):
images = []
labels = []
maximum = (chunk_id + 1) * self._batch_size
if (maximum > len(self.dataframes)):
maximum = len(self.dataframes)
for i in range(chunk_id * self._batch_size, maximum):
img_name = self.dataframes.iloc[i, 0]
image = Image.open(img_name)
cl = self.dataframes.iloc[i, 1:].values[0]
image = self._scale_image(image)
image = np.moveaxis((np.array(image).astype('float32')),
-1, 0)
image -= np.mean(image, keepdims=True)
image /= (np.std(image, keepdims=True) + 1e-6)
images.append(image)
yv = np.zeros(num_classes)
yv[classes.index(cl)] = 1
labels.append(yv)
result = {}
features = np.array(images)
lab = np.array(labels).astype('float32')
result[self._streams[0].m_name] = features
result[self._streams[1].m_name] = lab
return result
try:
with open(map_file_train) as f:
csv_reader = csv.reader(f, delimiter='\t')
for row in csv_reader:
cmd = row[1]
classes_set.add(cmd)
num_train = num_train + 1
except Exception as e:
raise Exception(
"No train_map.txt file found in path " + path_to_folder +
". Did you create a dataset using create_balanced_dataset()?")
num_classes = len(classes)
with open(map_file_test) as f:
for num_test, l in enumerate(f):
pass
# transforms = [
# xforms.scale(width=self.__image_width, height=self.__image_height, channels=num_channels, interpolations='linear'),
# xforms.mean(mean_file)
# ]
dataset_train = TrackDataset(map_file=map_file_train,
streams=[
dict(name='features',
shape=(num_channels,
self.__image_height,
self.__image_width)),
dict(name='labels',
shape=(num_classes, ))
])
reader_train = MinibatchSource([dataset_train], randomize=True)
# a = dataset_train.num_chunks()
dataset_test = TrackDataset(map_file=map_file_test,
streams=[
dict(name='features',
shape=(num_channels,
self.__image_height,
self.__image_width)),
dict(name='labels',
shape=(num_classes, ))
])
reader_test = MinibatchSource([dataset_test], randomize=True)
# ImageDeserializer loads images in the BGR format, not RGB
# reader_train = MinibatchSource(ImageDeserializer(map_file_train, StreamDefs(
# features = StreamDef(field='image', transforms=transforms),
# labels = StreamDef(field='label', shape=num_classes)
# )))
# reader_test = MinibatchSource(ImageDeserializer(map_file_test, StreamDefs(
# features = StreamDef(field='image', transforms=transforms),
# labels = StreamDef(field='label', shape=num_classes)
# )))
# mb = reader_train.next_minibatch(10)
input_var = input_variable(
(num_channels, self.__image_height, self.__image_width))
label_var = input_variable((num_classes))
model = model_definition(input_var)
ce = cross_entropy_with_softmax(model, label_var)
pe = classification_error(model, label_var)
epoch_size = num_train
lr_per_minibatch = learning_parameter_schedule([0.01] * 10 +
[0.003] * 10 + [0.001],
epoch_size=epoch_size)
momentums = momentum_schedule(0.9, minibatch_size=minibatch_size)
l2_reg_weight = 0.001
learner = momentum_sgd(model.parameters,
lr=lr_per_minibatch,
momentum=momentums,
l2_regularization_weight=l2_reg_weight)
progress_printer = ProgressPrinter(tag='Training', num_epochs=epochs)
trainer = cntk.train.Trainer(model, (ce, pe), [learner],
[progress_printer])
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
print("Training started")
batch_index = 0
plot_data = {'batchindex': [], 'loss': [], 'error': []}
for epoch in range(epochs):
sample_count = 0
while sample_count < epoch_size:
data: MinibatchSource = reader_train.next_minibatch(
min(minibatch_size, epoch_size - sample_count),
input_map=input_map)
trainer.train_minibatch(data)
sample_count += data[label_var].num_samples
batch_index += 1
plot_data['batchindex'].append(batch_index)
plot_data['loss'].append(
trainer.previous_minibatch_loss_average)
plot_data['error'].append(
trainer.previous_minibatch_evaluation_average)
trainer.summarize_training_progress()
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
epoch_size = num_test
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
data = reader_test.next_minibatch(current_minibatch,
input_map=input_map)
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.1f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom,
metric_denom))
print("")
model.save(output_model_path, format=ModelFormat.ONNX)
def train_model(image_input, roi_input, dims_input, loss, pred_error,
lr_per_sample, mm_schedule, l2_reg_weight, epochs_to_train, cfg,
rpn_rois_input=None, buffered_rpn_proposals=None):
if isinstance(loss, cntk.Variable):
loss = combine([loss])
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
if cfg["CNTK"].DEBUG_OUTPUT:
print("biases")
for p in biases: print(p)
print("others")
for p in others: print(p)
print("bias_lr_mult: {}".format(bias_lr_mult))
# Instantiate the learners and the trainer object
lr_schedule = learning_parameter_schedule_per_sample(lr_per_sample)
learner = momentum_sgd(others, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=True)
bias_lr_per_sample = [v * bias_lr_mult for v in lr_per_sample]
bias_lr_schedule = learning_parameter_schedule_per_sample(bias_lr_per_sample)
bias_learner = momentum_sgd(biases, bias_lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=True)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
if buffered_rpn_proposals is not None:
proposal_provider = ProposalProvider.fromlist(buffered_rpn_proposals, requires_scaling=False)
else:
proposal_provider = None
od_minibatch_source = ObjectDetectionMinibatchSource(
cfg["DATA"].TRAIN_MAP_FILE, cfg["DATA"].TRAIN_ROI_FILE,
num_classes=cfg["DATA"].NUM_CLASSES,
max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE,
pad_width=cfg.IMAGE_WIDTH,
pad_height=cfg.IMAGE_HEIGHT,
pad_value=cfg["MODEL"].IMG_PAD_COLOR,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["DATA"].NUM_TRAIN_IMAGES,
proposal_provider=proposal_provider)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.roi_si: roi_input,
}
if buffered_rpn_proposals is not None:
input_map[od_minibatch_source.proposals_si] = rpn_rois_input
else:
input_map[od_minibatch_source.dims_si] = dims_input
progress_printer = ProgressPrinter(tag='Training', num_epochs=epochs_to_train, gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < cfg["DATA"].NUM_TRAIN_IMAGES: # loop over minibatches in the epoch
data = od_minibatch_source.next_minibatch(min(cfg.MB_SIZE, cfg["DATA"].NUM_TRAIN_IMAGES-sample_count), input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
#progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
continue
#print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
def train_and_evaluate(reader_train, reader_test, max_epochs, model_func):
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# Normalize the input
feature_scale = 1.0 / 256.0
input_var_norm = element_times(feature_scale, input_var)
# apply model to input
z = model_func(input_var_norm, out_dims=num_classes)
#
# Training action
#
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# training config
epoch_size = 20000
minibatch_size = 64
# Set training parameters
lr_per_minibatch = learning_rate_schedule([0.01]*10 + [0.003]*10 + [0.001], UnitType.minibatch, epoch_size)
momentum_time_constant = momentum_as_time_constant_schedule(-minibatch_size/np.log(0.9))
l2_reg_weight = 0.001
# trainer object
progress_printer = ProgressPrinter(0)
learner = momentum_sgd(z.parameters,
lr = lr_per_minibatch, momentum = momentum_time_constant,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(z, (ce, pe), [learner], [progress_printer])
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z) ; print()
#progress_printer = ProgressPrinter(tag='Training')
# perform model training
stop_run=False
batch_index = 0
plot_data = {'batchindex':[], 'loss':[], 'error':[]}
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(min(minibatch_size, epoch_size - sample_count), input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += data[label_var].num_samples # count samples processed so far
# For visualization...
plot_data['batchindex'].append(batch_index)
plot_data['loss'].append(trainer.previous_minibatch_loss_average)
plot_data['error'].append(trainer.previous_minibatch_evaluation_average)
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
batch_index += 1
if trainer.previous_minibatch_evaluation_average < 0.025:
stop_run=True
break
if stop_run:
break
progress_printer.epoch_summary(with_metric=True)
#trainer.save_checkpoint(model_temp_file)
#
# Evaluation action
#
epoch_size = 6600
minibatch_size = 32
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
input_map = {
input_var: reader_test.streams.features,
label_var: reader_test.streams.labels
}
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch, input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.1f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom))
print("")
# Visualize training result:
window_width = 32
loss_cumsum = np.cumsum(np.insert(plot_data['loss'], 0, 0))
error_cumsum = np.cumsum(np.insert(plot_data['error'], 0, 0))
# Moving average.
plot_data['batchindex'] = np.insert(plot_data['batchindex'], 0, 0)[window_width:]
plot_data['avg_loss'] = (loss_cumsum[window_width:] - loss_cumsum[:-window_width]) / window_width
plot_data['avg_error'] = (error_cumsum[window_width:] - error_cumsum[:-window_width]) / window_width
plt.figure(1)
plt.subplot(211)
plt.plot(plot_data["batchindex"], plot_data["avg_loss"], 'b--')
plt.xlabel('Minibatch number')
plt.ylabel('Loss')
plt.title('Minibatch run vs. Training loss ')
plt.show()
plt.subplot(212)
plt.plot(plot_data["batchindex"], plot_data["avg_error"], 'r--')
plt.xlabel('Minibatch number')
plt.ylabel('Label Prediction Error')
plt.title('Minibatch run vs. Label Prediction Error ')
plt.show()
return softmax(z)
def train_model(image_input, roi_input, dims_input, loss, pred_error,
lr_per_sample, mm_schedule, l2_reg_weight, epochs_to_train,
rpn_rois_input=None, buffered_rpn_proposals=None):
if isinstance(loss, cntk.Variable):
loss = combine([loss])
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
if cfg["CNTK"].DEBUG_OUTPUT:
print("biases")
for p in biases: print(p)
print("others")
for p in others: print(p)
print("bias_lr_mult: {}".format(bias_lr_mult))
# Instantiate the learners and the trainer object
lr_schedule = learning_rate_schedule(lr_per_sample, unit=UnitType.sample)
learner = momentum_sgd(others, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=cfg["CNTK"].USE_MEAN_GRADIENT)
bias_lr_per_sample = [v * bias_lr_mult for v in lr_per_sample]
bias_lr_schedule = learning_rate_schedule(bias_lr_per_sample, unit=UnitType.sample)
bias_learner = momentum_sgd(biases, bias_lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=cfg["CNTK"].USE_MEAN_GRADIENT)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
od_minibatch_source = ObjectDetectionMinibatchSource(
globalvars['train_map_file'], globalvars['train_roi_file'],
max_annotations_per_image=cfg["CNTK"].INPUT_ROIS_PER_IMAGE,
pad_width=image_width, pad_height=image_height, pad_value=img_pad_value,
randomize=True, use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["CNTK"].NUM_TRAIN_IMAGES,
buffered_rpn_proposals=buffered_rpn_proposals)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.roi_si: roi_input,
od_minibatch_source.dims_si: dims_input
}
use_buffered_proposals = buffered_rpn_proposals is not None
progress_printer = ProgressPrinter(tag='Training', num_epochs=epochs_to_train, gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data, proposals = od_minibatch_source.next_minibatch_with_proposals(min(mb_size, epoch_size-sample_count), input_map=input_map)
if use_buffered_proposals:
data[rpn_rois_input] = MinibatchData(Value(batch=np.asarray(proposals, dtype=np.float32)), 1, 1, False)
# remove dims input if no rpn is required to avoid warnings
del data[[k for k in data if '[6]' in str(k)][0]]
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
def train_model(image_input, roi_input, dims_input, loss, pred_error,
lr_per_sample, mm_schedule, l2_reg_weight, epochs_to_train, cfg,
rpn_rois_input=None, buffered_rpn_proposals=None):
if isinstance(loss, cntk.Variable):
loss = combine([loss])
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
if cfg["CNTK"].DEBUG_OUTPUT:
print("biases")
for p in biases: print(p)
print("others")
for p in others: print(p)
print("bias_lr_mult: {}".format(bias_lr_mult))
# Instantiate the learners and the trainer object
lr_schedule = learning_parameter_schedule_per_sample(lr_per_sample)
learner = momentum_sgd(others, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=True)
bias_lr_per_sample = [v * bias_lr_mult for v in lr_per_sample]
bias_lr_schedule = learning_parameter_schedule_per_sample(bias_lr_per_sample)
bias_learner = momentum_sgd(biases, bias_lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight,
unit_gain=False, use_mean_gradient=True)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
if buffered_rpn_proposals is not None:
proposal_provider = ProposalProvider.fromlist(buffered_rpn_proposals, requires_scaling=False)
else:
proposal_provider = None
od_minibatch_source = ObjectDetectionMinibatchSource(
cfg["DATA"].TRAIN_MAP_FILE, cfg["DATA"].TRAIN_ROI_FILE,
num_classes=cfg["DATA"].NUM_CLASSES,
max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE,
pad_width=cfg.IMAGE_WIDTH,
pad_height=cfg.IMAGE_HEIGHT,
pad_value=cfg["MODEL"].IMG_PAD_COLOR,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["DATA"].NUM_TRAIN_IMAGES,
proposal_provider=proposal_provider)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.roi_si: roi_input,
}
if buffered_rpn_proposals is not None:
input_map[od_minibatch_source.proposals_si] = rpn_rois_input
else:
input_map[od_minibatch_source.dims_si] = dims_input
progress_printer = ProgressPrinter(tag='Training', num_epochs=epochs_to_train, gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < cfg["DATA"].NUM_TRAIN_IMAGES: # loop over minibatches in the epoch
data = od_minibatch_source.next_minibatch(min(cfg.MB_SIZE, cfg["DATA"].NUM_TRAIN_IMAGES-sample_count), input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
