def entrenar(checkpoint, entrRuedas, entrOperaciones, input_dim, num_output_classes, testRuedas, testOperaciones):
minibatch_size = 100;
epocs=900;
minibatchIteraciones = int(len(entrOperaciones) / minibatch_size);
# Input variables denoting the features and label data
feature = input((input_dim), np.float32)
label = input((num_output_classes), np.float32)
netout = crearRed(input_dim, num_output_classes, feature);
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
lr_per_minibatch=learning_rate_schedule(0.25, UnitType.minibatch)
# Instantiate the trainer object to drive the model training
learner = sgd(netout.parameters, lr=lr_per_minibatch)
progress_printer = ProgressPrinter(log_to_file=checkpoint+".log", num_epochs=epocs);
trainer = Trainer(netout, (ce, pe), learner, progress_printer)
if os.path.isfile(checkpoint):
trainer.restore_from_checkpoint(checkpoint);
npentrRuedas = np.array(entrRuedas).astype(np.float32);
npentrOperaciones = np.array(entrOperaciones).astype(np.float32);
#iteramos una vez por cada "epoc"
for i in range(0, epocs):
p = np.random.permutation(len(entrRuedas));
npentrOperaciones = npentrOperaciones[p];
npentrRuedas = npentrRuedas[p];
#ahora partimos los datos en "minibatches" y entrenamos
for j in range(0, minibatchIteraciones):
features = npentrRuedas[j*minibatch_size:(j+1)*minibatch_size];
labels = npentrOperaciones[j*minibatch_size:(j+1)*minibatch_size];
trainer.train_minibatch({feature: features, label: labels});
trainer.summarize_training_progress()
trainer.save_checkpoint(checkpoint);
minibatchIteraciones = int(len(testOperaciones) / minibatch_size);
avg_error = 0;
for j in range(0, minibatchIteraciones):
test_features = np.array(testRuedas[j*minibatch_size:(j+1)*minibatch_size]).astype(np.float32);
test_labels = np.array(testOperaciones[j*minibatch_size:(j+1)*minibatch_size]).astype(np.float32);
#test_features = np.array( entrRuedas[0:minibatch_size]).astype(np.float32);
#test_labels = np.array(entrOperaciones[0:minibatch_size]).astype(np.float32);
avg_error = avg_error + ( trainer.test_minibatch(
{feature: test_features, label: test_labels}) / minibatchIteraciones)
return avg_error
def ffnet(debug_output=False):
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
input = input_variable((input_dim), np.float32)
label = input_variable((num_output_classes), np.float32)
# Instantiate the feedforward classification model
netout = fully_connected_classifier_net(input, num_output_classes,
hidden_layers_dim,
num_hidden_layers, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
# Instantiate the trainer object to drive the model training
trainer = Trainer(netout, ce, pe, [sgd(netout.parameters(), lr=0.02)])
# Get minibatches of training data and perform model training
minibatch_size = 25
num_samples_per_sweep = 10000
num_sweeps_to_train_with = 2
num_minibatches_to_train = (num_samples_per_sweep *
num_sweeps_to_train_with) / minibatch_size
training_progress_output_freq = 60
if debug_output:
training_progress_output_freq = training_progress_output_freq / 3
for i in range(0, int(num_minibatches_to_train)):
features, labels = generate_random_data(minibatch_size, input_dim,
num_output_classes)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({input: features, label: labels})
print_training_progress(trainer, i, training_progress_output_freq)
test_features, test_labels = generate_random_data(minibatch_size,
input_dim,
num_output_classes)
avg_error = trainer.test_minibatch({
input: test_features,
label: test_labels
})
return avg_error
def ffnet():
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
feature = input_variable((input_dim), np.float32)
label = input_variable((num_output_classes), np.float32)
netout = Sequential([
For(range(num_hidden_layers),
lambda i: Dense(hidden_layers_dim, activation=sigmoid)),
Dense(num_output_classes)
])(feature)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
lr_per_minibatch = learning_parameter_schedule(0.5)
# Instantiate the trainer object to drive the model training
learner = sgd(netout.parameters, lr=lr_per_minibatch)
progress_printer = ProgressPrinter(128)
trainer = Trainer(netout, (ce, pe), learner, progress_printer)
# Get minibatches of training data and perform model training
minibatch_size = 25
for i in range(1024):
features, labels = generate_random_data(minibatch_size, input_dim,
num_output_classes)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({feature: features, label: labels})
trainer.summarize_training_progress()
test_features, test_labels = generate_random_data(minibatch_size,
input_dim,
num_output_classes)
avg_error = trainer.test_minibatch({
feature: test_features,
label: test_labels
})
return avg_error
def ffnet(debug_output=False):
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
input = input_variable((input_dim), np.float32)
label = input_variable((num_output_classes), np.float32)
# Instantiate the feedforward classification model
netout = fully_connected_classifier_net(
input, num_output_classes, hidden_layers_dim, num_hidden_layers, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
# Instantiate the trainer object to drive the model training
trainer = Trainer(netout, ce, pe, [sgd(netout.parameters(), lr=0.02)])
# Get minibatches of training data and perform model training
minibatch_size = 25
num_samples_per_sweep = 10000
num_sweeps_to_train_with = 2
num_minibatches_to_train = (
num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size
training_progress_output_freq = 60
if debug_output:
training_progress_output_freq = training_progress_output_freq/3
for i in range(0, int(num_minibatches_to_train)):
features, labels = generate_random_data(
minibatch_size, input_dim, num_output_classes)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({input: features, label: labels})
print_training_progress(trainer, i, training_progress_output_freq)
test_features, test_labels = generate_random_data(
minibatch_size, input_dim, num_output_classes)
avg_error = trainer.test_minibatch(
{input: test_features, label: test_labels})
return avg_error
def ffnet():
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
input = input_variable((input_dim), np.float32)
label = input_variable((num_output_classes), np.float32)
# Instantiate the feedforward classification model
netout = fully_connected_classifier_net(input, num_output_classes,
hidden_layers_dim,
num_hidden_layers, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
lr_per_minibatch = learning_rate_schedule(0.5, UnitType.minibatch)
# Instantiate the trainer object to drive the model training
learner = sgd(netout.parameters, lr=lr_per_minibatch)
progress_printer = ProgressPrinter(128)
trainer = Trainer(netout, (ce, pe), learner, progress_printer)
# Get minibatches of training data and perform model training
minibatch_size = 25
for i in range(1024):
features, labels = generate_random_data(minibatch_size, input_dim,
num_output_classes)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({input: features, label: labels})
trainer.summarize_training_progress()
test_features, test_labels = generate_random_data(minibatch_size,
input_dim,
num_output_classes)
avg_error = trainer.test_minibatch({
input: test_features,
label: test_labels
})
return avg_error
def ffnet(data, labels):
input_dim = 800
num_output_classes = 3
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
feature = input((input_dim), np.float32)
label = input((num_output_classes), np.float32)
netout = Sequential([For(range(num_hidden_layers), lambda i: Dense(hidden_layers_dim, activation=sigmoid)),
Dense(num_output_classes)])(feature)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
lr_per_minibatch=learning_rate_schedule(0.5, UnitType.minibatch)
# Instantiate the trainer object to drive the model training
learner = sgd(netout.parameters, lr=lr_per_minibatch)
progress_printer = ProgressPrinter(128)
trainer = Trainer(netout, (ce, pe), learner, progress_printer)
# Get minibatches of training data and perform model training
minibatch_size = 25
features, labels = generate_stock_data(minibatch_size);
for i in range(1024):
# features, labels = generate_random_data(
# minibatch_size, input_dim, num_output_classes)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({feature: features, label: labels})
trainer.summarize_training_progress()
test_features, test_labels = generate_random_data(
minibatch_size, input_dim, num_output_classes)
avg_error = trainer.test_minibatch(
{feature: test_features, label: test_labels})
return avg_error
def ffnet():
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
input = input_variable((input_dim), np.float32)
label = input_variable((num_output_classes), np.float32)
# Instantiate the feedforward classification model
netout = fully_connected_classifier_net(
input, num_output_classes, hidden_layers_dim, num_hidden_layers, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
lr_per_minibatch=learning_rate_schedule(0.5, UnitType.minibatch)
# Instantiate the trainer object to drive the model training
learner = sgd(netout.parameters, lr=lr_per_minibatch)
progress_printer = ProgressPrinter(128)
trainer = Trainer(netout, (ce, pe), learner, progress_printer)
# Get minibatches of training data and perform model training
minibatch_size = 25
for i in range(1024):
features, labels = generate_random_data(
minibatch_size, input_dim, num_output_classes)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({input: features, label: labels})
trainer.summarize_training_progress()
test_features, test_labels = generate_random_data(
minibatch_size, input_dim, num_output_classes)
avg_error = trainer.test_minibatch(
{input: test_features, label: test_labels})
return avg_error
def train_and_evaluate(reader_train, reader_test, network_name, epoch_size, max_epochs, profiler_dir=None,
model_dir=None, log_dir=None, tensorboard_logdir=None, gen_heartbeat=False):
set_computation_network_trace_level(0)
# Input variables denoting the features and label data
input_var = C.input_variable((num_channels, image_height, image_width), name='features')
label_var = C.input_variable((num_classes))
# create model, and configure learning parameters
if network_name == 'resnet20':
z = create_cifar10_model(input_var, 3, num_classes)
lr_per_mb = [1.0]*80+[0.1]*40+[0.01]
elif network_name == 'resnet110':
z = create_cifar10_model(input_var, 18, num_classes)
lr_per_mb = [0.1]*1+[1.0]*80+[0.1]*40+[0.01]
else:
raise RuntimeError("Unknown model name!")
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# shared training parameters
minibatch_size = 128
momentum_time_constant = -minibatch_size/np.log(0.9)
l2_reg_weight = 0.0001
# Set learning parameters
lr_per_sample = [lr/minibatch_size for lr in lr_per_mb]
lr_schedule = learning_rate_schedule(lr_per_sample, epoch_size=epoch_size, unit=UnitType.sample)
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant)
# progress writers
progress_writers = [ProgressPrinter(tag='Training', log_to_file=log_dir, num_epochs=max_epochs, gen_heartbeat=gen_heartbeat)]
tensorboard_writer = None
if tensorboard_logdir is not None:
tensorboard_writer = TensorBoardProgressWriter(freq=10, log_dir=tensorboard_logdir, model=z)
progress_writers.append(tensorboard_writer)
# trainer object
learner = momentum_sgd(z.parameters, lr_schedule, mm_schedule,
l2_regularization_weight = l2_reg_weight)
trainer = Trainer(z, (ce, pe), learner, progress_writers)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z) ; print()
# perform model training
if profiler_dir:
start_profiler(profiler_dir, True)
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(min(minibatch_size, epoch_size-sample_count), input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
trainer.summarize_training_progress()
# Log mean of each parameter tensor, so that we can confirm that the parameters change indeed.
if tensorboard_writer:
for parameter in z.parameters:
tensorboard_writer.write_value(parameter.uid + "/mean", reduce_mean(parameter).eval(), epoch)
if model_dir:
z.save(os.path.join(model_dir, network_name + "_{}.dnn".format(epoch)))
enable_profiler() # begin to collect profiler data after first epoch
if profiler_dir:
stop_profiler()
# Evaluation parameters
test_epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
while sample_count < test_epoch_size:
current_minibatch = min(minibatch_size, test_epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch, input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
print("")
trainer.summarize_test_progress()
print("")
return metric_numer/metric_denom
def simple_mnist():
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
features = input_variable(input_dim, np.float32)
label = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), features)
netout = fully_connected_classifier_net(
scaled_input, num_output_classes, hidden_layers_dim, num_hidden_layers, relu)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
try:
rel_path = os.path.join(os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'],
*"Image/MNIST/v0/Train-28x28_cntk_text.txt".split("/"))
except KeyError:
rel_path = os.path.join(*"../Image/DataSets/MNIST/Train-28x28_cntk_text.txt".split("/"))
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
features: reader_train.streams.features,
label: reader_train.streams.labels
}
# Instantiate progress writers.
logdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "mnist_log")
tensorboard_writer = TensorBoardProgressWriter(freq=1, log_dir=logdir, model=netout)
progress_printer = ProgressPrinter(freq=10, tag='Training')
# Instantiate the trainer object to drive the model training
lr_per_minibatch = learning_rate_schedule(0.2, UnitType.minibatch)
learner = sgd(netout.parameters, lr=lr_per_minibatch)
trainer = Trainer(netout, (ce, pe), learner, [tensorboard_writer, progress_printer])
# Get minibatches of images to train with and perform model training
minibatch_size = 64
num_samples_per_sweep = 6000
num_sweeps_to_train_with = 2
num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size
for minibatch_idx in range(0, int(num_minibatches_to_train)):
trainer.train_minibatch(reader_train.next_minibatch(minibatch_size, input_map=input_map))
# Log max/min/mean of each parameter tensor, so that we can confirm that the parameters change indeed.
# Don't want to do that very often though, otherwise will spend too much time computing min/max/mean.
if minibatch_idx % 10 == 9:
for p in netout.parameters:
tensorboard_writer.write_value(p.uid + "/max", reduce_max(p).eval(), minibatch_idx)
tensorboard_writer.write_value(p.uid + "/min", reduce_min(p).eval(), minibatch_idx)
tensorboard_writer.write_value(p.uid + "/mean", reduce_mean(p).eval(), minibatch_idx)
trainer.summarize_training_progress()
# Load test data
try:
rel_path = os.path.join(os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'],
*"Image/MNIST/v0/Test-28x28_cntk_text.txt".split("/"))
except KeyError:
rel_path = os.path.join(*"../Image/DataSets/MNIST/Test-28x28_cntk_text.txt".split("/"))
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
reader_test = create_reader(path, False, input_dim, num_output_classes)
input_map = {
features: reader_test.streams.features,
label: reader_test.streams.labels
}
# Test data for trained model
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size, input_map=input_map)
test_result += trainer.test_minibatch(mb)
# Average of evaluation errors of all test minibatches
trainer.summarize_test_progress()
return test_result / num_minibatches_to_test
def train_and_evaluate(reader_train, reader_test, network_name, max_epochs):
set_computation_network_trace_level(0)
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# create model, and configure learning parameters
if network_name == 'resnet20':
z = create_cifar10_model(input_var, 3, num_classes)
lr_per_mb = [1.0] * 80 + [0.1] * 40 + [0.01]
elif network_name == 'resnet110':
z = create_cifar10_model(input_var, 18, num_classes)
lr_per_mb = [0.1] * 1 + [1.0] * 80 + [0.1] * 40 + [0.01]
else:
return RuntimeError("Unknown model name!")
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# shared training parameters
epoch_size = 50000 # for now we manually specify epoch size
minibatch_size = 128
momentum_time_constant = -minibatch_size / np.log(0.9)
l2_reg_weight = 0.0001
# Set learning parameters
lr_per_sample = [lr / minibatch_size for lr in lr_per_mb]
lr_schedule = learning_rate_schedule(lr_per_sample,
epoch_size=epoch_size,
unit=UnitType.sample)
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant)
# trainer object
learner = momentum_sgd(z.parameters,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(z, ce, pe, learner)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z)
print()
progress_printer = ProgressPrinter(tag='Training')
# perform model training
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(
min(minibatch_size, epoch_size - sample_count),
input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
persist.save_model(
z, os.path.join(model_path,
network_name + "_{}.dnn".format(epoch)))
# Evaluation parameters
epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch,
input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom,
metric_denom))
print("")
return metric_numer / metric_denom
def simple_mnist(debug_output=False):
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
input = input_variable(input_dim, np.float32)
label = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant((), 0.00390625), input)
netout = fully_connected_classifier_net(
scaled_input, num_output_classes, hidden_layers_dim, num_hidden_layers, sigmoid
)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
try:
rel_path = os.path.join(
os.environ["CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY"],
*"Image/MNIST/v0/Train-28x28_cntk_text.txt".split("/")
)
except KeyError:
rel_path = os.path.join(*"../../../../Examples/Image/Datasets/MNIST/Train-28x28_cntk_text.txt".split("/"))
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
feature_stream_name = "features"
labels_stream_name = "labels"
mb_source = text_format_minibatch_source(
path,
[
StreamConfiguration(feature_stream_name, input_dim),
StreamConfiguration(labels_stream_name, num_output_classes),
],
)
features_si = mb_source[feature_stream_name]
labels_si = mb_source[labels_stream_name]
# Instantiate the trainer object to drive the model training
trainer = Trainer(netout, ce, pe, [sgd(netout.parameters(), lr=0.003125)])
# Get minibatches of images to train with and perform model training
minibatch_size = 32
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 1
num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size
training_progress_output_freq = 80
if debug_output:
training_progress_output_freq = training_progress_output_freq / 4
for i in range(0, int(num_minibatches_to_train)):
mb = mb_source.get_next_minibatch(minibatch_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {input: mb[features_si], label: mb[labels_si]}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
# Load test data
try:
rel_path = os.path.join(
os.environ["CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY"], *"Image/MNIST/v0/Test-28x28_cntk_text.txt".split("/")
)
except KeyError:
rel_path = os.path.join(*"../../../../Examples/Image/Datasets/MNIST/Test-28x28_cntk_text.txt".split("/"))
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
test_mb_source = text_format_minibatch_source(
path,
[
StreamConfiguration(feature_stream_name, input_dim),
StreamConfiguration(labels_stream_name, num_output_classes),
],
randomize=False,
)
features_si = test_mb_source[feature_stream_name]
labels_si = test_mb_source[labels_stream_name]
# Test data for trained model
test_minibatch_size = 512
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = test_mb_source.get_next_minibatch(test_minibatch_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be tested with
arguments = {input: mb[features_si], label: mb[labels_si]}
eval_error = trainer.test_minibatch(arguments)
test_result = test_result + eval_error
# Average of evaluation errors of all test minibatches
return test_result / num_minibatches_to_test
Xs:
X_train[i * minibatch_size:(i + 1) * minibatch_size],
ys:
y_train[i * minibatch_size:(i + 1) * minibatch_size]
})
#trainer.train_minibatch({Xs:X_train, ys: y_train})
if (epoch % SHOW_FREQ == 0):
cur_loss = get_train_loss(trainer)
acc = get_train_eval_criterion(trainer)
print("{}/{}, loss = {}, acc = {}".format(epoch, EPOCH, cur_loss,
1 - acc))
#print("{}/{}, loss = {}".format(epoch, EPOCH, trainer.test_minibatch({Xs : X_train[:20], ys : y_train[:20]}) ))
if (epoch % TEST_FREQ == 0):
print(1 - trainer.test_minibatch({Xs: X_test1, ys: y_test1}))
acc2 = 1 - trainer.test_minibatch({Xs: X_test2, ys: y_test2})
print(acc2)
acc_all.append(acc2)
if acc2 > max_acc2:
max_acc2 = acc2
TtrainE = datetime.datetime.now()
plt.figure(figsize=(15, 9))
plt.plot(np.array(range(len(acc_all))) * 20, acc_all, linewidth=1.5)
plt.xlabel('epoch')
plt.ylabel('test acc')
plt.show()
print("Best acc2 = {}".format(max_acc2))
print("Compiling time: {}".format(TcmpE - TcmpS))
def cifar_resnet_distributed(data_path, run_test, num_epochs, communicator=None, save_model_filename=None, load_model_filename=None, debug_output=False):
image_height = 32
image_width = 32
num_channels = 3
num_classes = 10
feats_stream_name = 'features'
labels_stream_name = 'labels'
minibatch_source = create_reader(os.path.join(data_path, 'train_map.txt'), os.path.join(data_path, 'CIFAR-10_mean.xml'), True,
distributed_communicator = communicator)
features_si = minibatch_source[feats_stream_name]
labels_si = minibatch_source[labels_stream_name]
# Instantiate the resnet classification model, or load from file
if load_model_filename:
print("Loading model:", load_model_filename)
classifier_output = persist.load_model(load_model_filename)
image_input = classifier_output.arguments[0]
else:
image_input = input_variable(
(num_channels, image_height, image_width), features_si.m_element_type)
classifier_output = create_resnet_model(image_input, num_classes)
# Input variables denoting the features and label data
label_var = input_variable((num_classes), features_si.m_element_type)
ce = cross_entropy_with_softmax(classifier_output, label_var)
pe = classification_error(classifier_output, label_var)
# Instantiate the trainer object to drive the model training
mb_size = 128
num_mb_per_epoch = 100
num_mbs = num_mb_per_epoch * num_epochs
lr_per_sample = [1/mb_size]*80+[0.1/mb_size]*40+[0.01/mb_size]
lr_schedule = learning_rate_schedule(lr_per_sample, units = mb_size * num_mb_per_epoch)
momentum_time_constant = -mb_size/np.log(0.9)
# create data parallel distributed trainer if needed
dist_trainer = distributed.data_parallel_distributed_trainer(communicator, False) if communicator else None
# Instantiate the trainer object to drive the model training
trainer = Trainer(classifier_output, ce, pe,
[momentum_sgd(classifier_output.parameters, lr_schedule, momentum_time_constant, l2_regularization_weight=0.0001)],
distributed_trainer = dist_trainer)
# Get minibatches of images to train with and perform model training
training_progress_output_freq = 100 if communicator else 20
if debug_output:
training_progress_output_freq = training_progress_output_freq/4
for i in range(0, num_mbs):
# NOTE: depends on network, the mb_size can be changed dynamically here
mb = minibatch_source.next_minibatch(mb_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {
image_input: mb[features_si],
label_var: mb[labels_si]
}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
if save_model_filename:
print("Saving model:", save_model_filename)
persist.save_model(classifier_output, save_model_filename)
if run_test:
test_minibatch_source = create_reader(os.path.join(data_path, 'test_map.txt'), os.path.join(data_path, 'CIFAR-10_mean.xml'), False)
features_si = test_minibatch_source[feats_stream_name]
labels_si = test_minibatch_source[labels_stream_name]
mb_size = 128
num_mbs = 100
total_error = 0.0
for i in range(0, num_mbs):
mb = test_minibatch_source.next_minibatch(mb_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {
image_input: mb[features_si],
label_var: mb[labels_si]
}
error = trainer.test_minibatch(arguments)
total_error += error
return total_error / num_mbs
else:
return 0
sample_count = 0
minibatch_index = 0
input_map = {
input_var: reader_test.streams.features,
label_var: reader_test.streams.labels
}
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch, input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.1f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom, metric_denom))
print("")
# Visualize training result:
window_width = 32
loss_cumsum = np.cumsum(np.insert(plot_data['loss'], 0, 0))
error_cumsum = np.cumsum(np.insert(plot_data['error'], 0, 0))
def simple_mnist(debug_output=False):
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
input = input_variable(input_dim, np.float32)
label = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), input)
z = fully_connected_classifier_net(
scaled_input, num_output_classes, hidden_layers_dim, num_hidden_layers, relu)
ce = cross_entropy_with_softmax(z, label)
pe = classification_error(z, label)
data_dir = os.path.join(abs_path, "..", "..", "..", "DataSets", "MNIST")
path = os.path.normpath(os.path.join(data_dir, "Train-28x28_cntk_text.txt"))
check_path(path)
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
input : reader_train.streams.features,
label : reader_train.streams.labels
}
lr_per_minibatch=learning_rate_schedule(0.2, UnitType.minibatch)
# Instantiate the trainer object to drive the model training
trainer = Trainer(z, ce, pe, sgd(z.parameters, lr=lr_per_minibatch))
# Get minibatches of images to train with and perform model training
minibatch_size = 64
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 10
num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size
training_progress_output_freq = 500
if debug_output:
training_progress_output_freq = training_progress_output_freq/4
for i in range(0, int(num_minibatches_to_train)):
mb = reader_train.next_minibatch(minibatch_size, input_map=input_map)
trainer.train_minibatch(mb)
print_training_progress(trainer, i, training_progress_output_freq)
# Load test data
path = os.path.normpath(os.path.join(data_dir, "Test-28x28_cntk_text.txt"))
check_path(path)
reader_test = create_reader(path, False, input_dim, num_output_classes)
input_map = {
input : reader_test.streams.features,
label : reader_test.streams.labels
}
# Test data for trained model
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size, input_map=input_map)
eval_error = trainer.test_minibatch(mb)
test_result = test_result + eval_error
# Average of evaluation errors of all test minibatches
return test_result / num_minibatches_to_test
def convnet_mnist(debug_output=False):
image_height = 28
image_width = 28
num_channels = 1
input_dim = image_height * image_width * num_channels
num_output_classes = 10
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width), np.float32)
label_var = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), input_var)
with default_options (activation=relu, pad=False):
conv1 = Convolution((5,5), 32, pad=True)(scaled_input)
pool1 = MaxPooling((3,3), (2,2))(conv1)
conv2 = Convolution((3,3), 48)(pool1)
pool2 = MaxPooling((3,3), (2,2))(conv2)
conv3 = Convolution((3,3), 64)(pool2)
f4 = Dense(96)(conv3)
drop4 = Dropout(0.5)(f4)
z = Dense(num_output_classes, activation=None)(drop4)
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
reader_train = create_reader(os.path.join(data_path, 'Train-28x28_cntk_text.txt'), True, input_dim, num_output_classes)
# training config
epoch_size = 60000 # for now we manually specify epoch size
minibatch_size = 128
# Set learning parameters
lr_per_sample = [0.001]*10+[0.0005]*10+[0.0001]
lr_schedule = learning_rate_schedule(lr_per_sample, epoch_size)
momentum_time_constant = [0]*5+[1024]
# Instantiate the trainer object to drive the model training
learner = momentum_sgd(z.parameters, lr_schedule, momentum_time_constant)
trainer = Trainer(z, ce, pe, learner)
# define mapping from reader streams to network inputs
input_map = {
input_var : reader_train.streams.features,
label_var : reader_train.streams.labels
}
log_number_of_parameters(z) ; print()
progress_printer = ProgressPrinter(tag='Training')
# Get minibatches of images to train with and perform model training
max_epochs = 40
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(min(minibatch_size, epoch_size - sample_count), input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += data[label_var].num_samples # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
persist.save_model(z, os.path.join(model_path, "ConvNet_MNIST_{}.dnn".format(epoch)))
# Load test data
reader_test = create_reader(os.path.join(data_path, 'Test-28x28_cntk_text.txt'), False, input_dim, num_output_classes)
input_map = {
input_var : reader_test.streams.features,
label_var : reader_test.streams.labels
}
# Test data for trained model
epoch_size = 10000
minibatch_size = 1024
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch, input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom))
print("")
return metric_numer/metric_denom
def sequence_to_sequence_translator(debug_output=False):
input_vocab_dim = 69
label_vocab_dim = 69
hidden_dim = 512
num_layers = 2
# Source and target inputs to the model
batch_axis = Axis.default_batch_axis()
input_seq_axis = Axis('inputAxis')
label_seq_axis = Axis('labelAxis')
input_dynamic_axes = [batch_axis, input_seq_axis]
raw_input = input_variable(shape=(input_vocab_dim),
dynamic_axes=input_dynamic_axes)
label_dynamic_axes = [batch_axis, label_seq_axis]
raw_labels = input_variable(shape=(label_vocab_dim),
dynamic_axes=label_dynamic_axes)
# Instantiate the sequence to sequence translation model
input_sequence = raw_input
# Drop the sentence start token from the label, for decoder training
label_sequence = slice(raw_labels, label_seq_axis, 1, 0)
label_sentence_start = sequence.first(raw_labels)
is_first_label = sequence.is_first(label_sequence)
label_sentence_start_scattered = sequence.scatter(label_sentence_start,
is_first_label)
# Encoder
encoder_outputH = stabilize(input_sequence)
for i in range(0, num_layers):
(encoder_outputH,
encoder_outputC) = LSTMP_component_with_self_stabilization(
encoder_outputH.output(), hidden_dim, hidden_dim, future_value,
future_value)
thought_vectorH = sequence.first(encoder_outputH)
thought_vectorC = sequence.first(encoder_outputC)
thought_vector_broadcastH = sequence.broadcast_as(thought_vectorH,
label_sequence)
thought_vector_broadcastC = sequence.broadcast_as(thought_vectorC,
label_sequence)
# Decoder
decoder_history_from_ground_truth = label_sequence
decoder_input = element_select(
is_first_label, label_sentence_start_scattered,
past_value(decoder_history_from_ground_truth))
decoder_outputH = stabilize(decoder_input)
for i in range(0, num_layers):
if (i > 0):
recurrence_hookH = past_value
recurrence_hookC = past_value
else:
isFirst = sequence.is_first(label_sequence)
recurrence_hookH = lambda operand: element_select(
isFirst, thought_vector_broadcastH, past_value(operand))
recurrence_hookC = lambda operand: element_select(
isFirst, thought_vector_broadcastC, past_value(operand))
(decoder_outputH,
encoder_outputC) = LSTMP_component_with_self_stabilization(
decoder_outputH.output(), hidden_dim, hidden_dim,
recurrence_hookH, recurrence_hookC)
decoder_output = decoder_outputH
decoder_dim = hidden_dim
# Softmax output layer
z = linear_layer(stabilize(decoder_output), label_vocab_dim)
ce = cross_entropy_with_softmax(z, label_sequence)
errs = classification_error(z, label_sequence)
# Instantiate the trainer object to drive the model training
lr = 0.007
momentum_time_constant = 1100
momentum_per_sample = momentums_per_sample(
math.exp(-1.0 / momentum_time_constant))
clipping_threshold_per_sample = 2.3
gradient_clipping_with_truncation = True
trainer = Trainer(z, ce, errs, [
momentum_sgd(z.parameters(), lr, momentum_per_sample,
clipping_threshold_per_sample,
gradient_clipping_with_truncation)
])
rel_path = r"../../../../Examples/SequenceToSequence/CMUDict/Data/cmudict-0.7b.train-dev-20-21.ctf"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
feature_stream_name = 'features'
labels_stream_name = 'labels'
mb_source = text_format_minibatch_source(path, [
StreamConfiguration(feature_stream_name, input_vocab_dim, True, 'S0'),
StreamConfiguration(labels_stream_name, label_vocab_dim, True, 'S1')
], 10000)
features_si = mb_source[feature_stream_name]
labels_si = mb_source[labels_stream_name]
# Get minibatches of sequences to train with and perform model training
minibatch_size = 72
training_progress_output_freq = 30
if debug_output:
training_progress_output_freq = training_progress_output_freq / 3
while True:
mb = mb_source.get_next_minibatch(minibatch_size)
if len(mb) == 0:
break
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {raw_input: mb[features_si], raw_labels: mb[labels_si]}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
i += 1
rel_path = r"../../../../Examples/SequenceToSequence/CMUDict/Data/cmudict-0.7b.test.ctf"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
test_mb_source = text_format_minibatch_source(path, [
StreamConfiguration(feature_stream_name, input_vocab_dim, True, 'S0'),
StreamConfiguration(labels_stream_name, label_vocab_dim, True, 'S1')
], 10000, False)
features_si = test_mb_source[feature_stream_name]
labels_si = test_mb_source[labels_stream_name]
# choose this to be big enough for the longest sentence
train_minibatch_size = 1024
# Get minibatches of sequences to test and perform testing
i = 0
total_error = 0.0
while True:
mb = test_mb_source.get_next_minibatch(train_minibatch_size)
if len(mb) == 0:
break
# Specify the mapping of input variables in the model to actual
# minibatch data to be tested with
arguments = {raw_input: mb[features_si], raw_labels: mb[labels_si]}
mb_error = trainer.test_minibatch(arguments)
total_error += mb_error
if debug_output:
print("Minibatch {}, Error {} ".format(i, mb_error))
i += 1
# Average of evaluation errors of all test minibatches
return total_error / i
def train_and_evaluate(reader_train,
reader_test,
epoch_size,
max_epochs,
profiler_dir=None,
model_dir=None,
log_dir=None,
tensorboard_logdir=None,
gen_heartbeat=False,
fp16=False):
set_computation_network_trace_level(0)
# Input variables denoting the features and label data
input_var = C.input_variable((num_channels, image_height, image_width),
name='features')
label_var = C.input_variable((num_classes))
dtype = np.float16 if fp16 else np.float32
if fp16:
graph_input = C.cast(input_var, dtype=np.float16)
graph_label = C.cast(label_var, dtype=np.float16)
else:
graph_input = input_var
graph_label = label_var
z = create_cifar10_model(graph_input, 3, num_classes)
lr_per_sample = [0.00015625] * 20 + [0.00046875] * 20 + [
0.00015625
] * 20 + [0.000046875] * 20 + [0.000015625]
ce = cross_entropy_with_softmax(z, graph_label)
pe = classification_error(z, graph_label)
if fp16:
ce = C.cast(ce, dtype=np.float32)
pe = C.cast(pe, dtype=np.float32)
# shared training parameters
minibatch_size = 128
l2_reg_weight = 0.0001
# Set learning parameters
lr_schedule = C.learning_parameter_schedule_per_sample(
lr_per_sample, epoch_size=epoch_size)
mm_schedule = momentum_schedule(0.9, minibatch_size)
# progress writers
progress_writers = [
ProgressPrinter(tag='Training',
log_to_file=log_dir,
num_epochs=max_epochs,
gen_heartbeat=gen_heartbeat)
]
tensorboard_writer = None
if tensorboard_logdir is not None:
tensorboard_writer = TensorBoardProgressWriter(
freq=10, log_dir=tensorboard_logdir, model=z)
progress_writers.append(tensorboard_writer)
# trainer object
learner = momentum_sgd(z.parameters,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(z, (ce, pe), learner, progress_writers)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z)
print()
# perform model training
if profiler_dir:
start_profiler(profiler_dir, True)
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(
min(minibatch_size, epoch_size - sample_count),
input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
trainer.summarize_training_progress()
# Log mean of each parameter tensor, so that we can confirm that the parameters change indeed.
if tensorboard_writer:
for parameter in z.parameters:
tensorboard_writer.write_value(parameter.uid + "/mean",
reduce_mean(parameter).eval(),
epoch)
if model_dir:
z.save(os.path.join(model_dir, model_name))
enable_profiler() # begin to collect profiler data after first epoch
if profiler_dir:
stop_profiler()
# Evaluation parameters
test_epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
while sample_count < test_epoch_size:
current_minibatch = min(minibatch_size, test_epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch,
input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
print("")
trainer.summarize_test_progress()
print("")
return metric_numer / metric_denom
plt.ylabel('Loss')
plt.title('Minibatch run vs. Training loss')
plt.subplot(212)
plt.plot(plotdata["batchsize"], plotdata["avgerror"], 'r--')
plt.xlabel('Minibatch number')
plt.ylabel('Label Prediction Error')
plt.title('Minibatch run vs. Label Prediction Error')
plt.show()
test_minibatch_size = 25
features, labels = generate_random_data_sample(test_minibatch_size,
input_dim,
num_output_classes)
error = trainer.test_minibatch({input: features, label: labels})
print("Error test: " + str(error))
out = softmax(z)
result = out.eval({input: features})
print("Label :", np.argmax(labels[:25], axis=1))
print("Predicted:", np.argmax(result[0, :25, :], axis=1))
# Model parameters
print(mydict['b'].value)
bias_vector = mydict['b'].value
weight_matrix = mydict['w'].value
# given this is a 2 class
def cifar_resnet(base_path, debug_output=False):
image_height = 32
image_width = 32
num_channels = 3
num_classes = 10
feats_stream_name = 'features'
labels_stream_name = 'labels'
minibatch_source = create_mb_source(feats_stream_name, labels_stream_name,
image_height, image_width, num_channels, num_classes, base_path)
features_si = minibatch_source[feats_stream_name]
labels_si = minibatch_source[labels_stream_name]
# Input variables denoting the features and label data
image_input = input_variable(
(num_channels, image_height, image_width), features_si.m_element_type)
label_var = input_variable((num_classes), features_si.m_element_type)
# Instantiate the resnet classification model
classifier_output = resnet_classifer(image_input, num_classes)
ce = cross_entropy_with_softmax(classifier_output, label_var)
pe = classification_error(classifier_output, label_var)
# Instantiate the trainer object to drive the model training
trainer = Trainer(classifier_output, ce, pe,
[sgd(classifier_output.parameters(), lr=0.0078125)])
# Get minibatches of images to train with and perform model training
mb_size = 32
training_progress_output_freq = 60
num_mbs = 1000
if debug_output:
training_progress_output_freq = training_progress_output_freq/3
for i in range(0, num_mbs):
mb = minibatch_source.get_next_minibatch(mb_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {
image_input: mb[features_si],
label_var: mb[labels_si]
}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
test_minibatch_source = create_test_mb_source(feats_stream_name, labels_stream_name,
image_height, image_width, num_channels, num_classes, base_path)
features_si = test_minibatch_source[feats_stream_name]
labels_si = test_minibatch_source[labels_stream_name]
mb_size = 64
num_mbs = 300
total_error = 0.0
for i in range(0, num_mbs):
mb = test_minibatch_source.get_next_minibatch(mb_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {
image_input: mb[features_si],
label_var: mb[labels_si]
}
error = trainer.test_minibatch(arguments)
total_error += error
return total_error / num_mbs
lr_schedule = learning_rate_schedule(learning_rate, UnitType.minibatch)
learner = sgd(z.parameters, lr_schedule)
trainer = Trainer(z, (loss, error), [learner])
# training
num_sweeps = 13
minibatch_size = 50
num_train_samples = 60000
num_test_samples = 10000
for i in range(num_sweeps):
for j in range(0, num_train_samples, minibatch_size):
trainer.train_minibatch({
input: train_features[j:j + minibatch_size, :],
label: train_labels[j:j + minibatch_size, :]
})
train_error = get_train_eval_criterion(trainer)
test_error = 0.
for j in range(0, num_test_samples, minibatch_size):
test_data = {
input: test_features[j:j + minibatch_size, :],
label: test_labels[j:j + minibatch_size, :]
}
test_error = test_error + trainer.test_minibatch(test_data)
test_error = test_error / (num_test_samples / minibatch_size)
print('sweep {0} train error: {1:.4f} test error: {2:.4f}'.format(
i, train_error, test_error),
flush=True)
def simple_mnist(tensorboard_logdir=None):
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
input = input_variable(input_dim, np.float32)
label = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), input)
z = fully_connected_classifier_net(
scaled_input, num_output_classes, hidden_layers_dim, num_hidden_layers, relu)
ce = cross_entropy_with_softmax(z, label)
pe = classification_error(z, label)
data_dir = os.path.join(abs_path, "..", "..", "..", "DataSets", "MNIST")
path = os.path.normpath(os.path.join(data_dir, "Train-28x28_cntk_text.txt"))
check_path(path)
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
input : reader_train.streams.features,
label : reader_train.streams.labels
}
# Training config
minibatch_size = 64
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 10
# Instantiate progress writers.
#training_progress_output_freq = 100
progress_writers = [ProgressPrinter(
#freq=training_progress_output_freq,
tag='Training',
num_epochs=num_sweeps_to_train_with)]
if tensorboard_logdir is not None:
progress_writers.append(TensorBoardProgressWriter(freq=10, log_dir=tensorboard_logdir, model=z))
# Instantiate the trainer object to drive the model training
lr_per_minibatch = learning_rate_schedule(0.2, UnitType.minibatch)
trainer = Trainer(z, (ce, pe), sgd(z.parameters, lr=lr_per_minibatch), progress_writers)
training_session(
trainer=trainer,
mb_source = reader_train,
mb_size = minibatch_size,
var_to_stream = input_map,
max_samples = num_samples_per_sweep * num_sweeps_to_train_with,
progress_frequency=num_samples_per_sweep
).train()
# Load test data
path = os.path.normpath(os.path.join(data_dir, "Test-28x28_cntk_text.txt"))
check_path(path)
reader_test = create_reader(path, False, input_dim, num_output_classes)
input_map = {
input : reader_test.streams.features,
label : reader_test.streams.labels
}
# Test data for trained model
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size, input_map=input_map)
eval_error = trainer.test_minibatch(mb)
test_result = test_result + eval_error
# Average of evaluation errors of all test minibatches
return test_result / num_minibatches_to_test
def convnet_cifar10(debug_output=False):
set_computation_network_trace_level(0)
image_height = 32
image_width = 32
num_channels = 3
input_dim = image_height * image_width * num_channels
num_output_classes = 10
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width), np.float32)
label_var = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
input_removemean = minus(input_var, constant(128))
scaled_input = element_times(constant(0.00390625), input_removemean)
with default_options (activation=relu, pad=True):
z = Sequential([
LayerStack(2, lambda : [
Convolution((3,3), 64),
Convolution((3,3), 64),
MaxPooling((3,3), (2,2))
]),
LayerStack(2, lambda i: [
Dense([256,128][i]),
Dropout(0.5)
]),
Dense(num_output_classes, activation=None)
])(scaled_input)
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
reader_train = create_reader(os.path.join(data_path, 'Train_cntk_text.txt'), True, input_dim, num_output_classes)
# training config
epoch_size = 50000 # for now we manually specify epoch size
minibatch_size = 64
# Set learning parameters
lr_per_sample = [0.0015625]*10+[0.00046875]*10+[0.00015625]
lr_schedule = learning_rate_schedule(lr_per_sample, epoch_size=epoch_size)
momentum_time_constant = [0]*20+[-minibatch_size/np.log(0.9)]
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant, epoch_size=epoch_size)
l2_reg_weight = 0.002
# Instantiate the trainer object to drive the model training
learner = momentum_sgd(z.parameters, lr_schedule, mm_schedule, l2_regularization_weight = l2_reg_weight)
trainer = Trainer(z, ce, pe, learner)
# define mapping from reader streams to network inputs
input_map = {
input_var : reader_train.streams.features,
label_var : reader_train.streams.labels
}
log_number_of_parameters(z) ; print()
progress_printer = ProgressPrinter(tag='Training')
# Get minibatches of images to train with and perform model training
max_epochs = 30
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(min(minibatch_size, epoch_size - sample_count), input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += data[label_var].num_samples # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
persist.save_model(z, os.path.join(model_path, "ConvNet_CIFAR10_{}.dnn".format(epoch)))
# Load test data
reader_test = create_reader(os.path.join(data_path, 'Test_cntk_text.txt'), False, input_dim, num_output_classes)
input_map = {
input_var : reader_test.streams.features,
label_var : reader_test.streams.labels
}
# Test data for trained model
epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch, input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom))
print("")
return metric_numer/metric_denom
def train_and_evaluate(create_train_reader, test_reader, network_name, max_epochs, create_dist_learner, scale_up=False):
set_computation_network_trace_level(0)
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# create model, and configure learning parameters
if network_name == 'resnet20':
z = create_cifar10_model(input_var, 3, num_classes)
lr_per_mb = [1.0]*80+[0.1]*40+[0.01]
elif network_name == 'resnet110':
z = create_cifar10_model(input_var, 18, num_classes)
lr_per_mb = [0.1]*1+[1.0]*80+[0.1]*40+[0.01]
else:
return RuntimeError("Unknown model name!")
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# shared training parameters
epoch_size = 50000 # for now we manually specify epoch size
# NOTE: scaling up minibatch_size increases sample throughput. In 8-GPU machine,
# ResNet110 samples-per-second is ~7x of single GPU, comparing to ~3x without scaling
# up. However, bigger minimatch size on the same number of samples means less updates,
# thus leads to higher training error. This is a trade-off of speed and accuracy
minibatch_size = 128 * (distributed.Communicator.num_workers() if scale_up else 1)
momentum_time_constant = -minibatch_size/np.log(0.9)
l2_reg_weight = 0.0001
# Set learning parameters
lr_per_sample = [lr/minibatch_size for lr in lr_per_mb]
lr_schedule = learning_rate_schedule(lr_per_sample, epoch_size=epoch_size, unit=UnitType.sample)
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant)
# trainer object
learner = create_dist_learner(momentum_sgd(z.parameters, lr_schedule, mm_schedule,
l2_regularization_weight = l2_reg_weight))
trainer = Trainer(z, ce, pe, learner)
total_number_of_samples = max_epochs * epoch_size
train_reader=create_train_reader(total_number_of_samples)
# define mapping from reader streams to network inputs
input_map = {
input_var: train_reader.streams.features,
label_var: train_reader.streams.labels
}
log_number_of_parameters(z) ; print()
progress_printer = ProgressPrinter(tag='Training')
# perform model training
current_epoch=0
updated=True
while updated:
data=train_reader.next_minibatch(minibatch_size, input_map=input_map) # fetch minibatch.
updated=trainer.train_minibatch(data) # update model with it
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
epoch_index = int(trainer.total_number_of_samples_seen/epoch_size)
if current_epoch != epoch_index: # new epoch reached
progress_printer.epoch_summary(with_metric=True)
current_epoch=epoch_index
trainer.save_checkpoint(os.path.join(model_path, network_name + "_{}.dnn".format(current_epoch)))
# Evaluation parameters
epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while True:
data = test_reader.next_minibatch(minibatch_size, input_map=input_map)
if not data: break;
local_mb_samples=data[label_var].num_samples
metric_numer += trainer.test_minibatch(data) * local_mb_samples
metric_denom += local_mb_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom))
print("")
return metric_numer/metric_denom
def train_and_evaluate(reader_train,
reader_test,
network_name,
epoch_size,
max_epochs,
minibatch_size,
model_dir=None,
log_dir=None,
tensorboard_logdir=None,
gen_heartbeat=False,
fp16=False):
"""
:param reader_train:
:param reader_test:
:param network_name:
:param epoch_size: 一个epoch有多少样本
:param max_epochs: 训练多少个epoch
:param model_dir:
:param log_dir:
:param tensorboard_logdir:
:param gen_heartbeat:
:param fp16:
:return:准确率,用时
"""
set_computation_network_trace_level(0)
# Input variables denoting the features and label data
input_var = C.input_variable((num_channels, image_height, image_width),
name='features')
label_var = C.input_variable((num_classes))
with C.default_options(dtype=np.float32):
# create model, and configure learning parameters
model = create_cifar10_model(input_var, 3, num_classes)
# loss and metric
loss = cross_entropy_with_softmax(model, label_var)
error_rate = classification_error(model, label_var)
# shared training parameters
# Set learning parameters
lr_per_sample = []
check_point = [80, 120, 160, 180]
lrs = [3e-2, 3e-3, 3e-4, 3e-4, 5e-5]
for i in range(max_epochs + 1):
if i in range(0, check_point[0]):
lr_per_sample.append(lrs[0])
if i in range(check_point[0], check_point[1]):
lr_per_sample.append(lrs[1])
if i in range(check_point[1], check_point[2]):
lr_per_sample.append(lrs[2])
if i in range(check_point[2], check_point[3]):
lr_per_sample.append(lrs[3])
if i > check_point[3]:
lr_per_sample.append(lrs[4])
lr_schedule = learning_parameter_schedule(lr_per_sample,
minibatch_size=minibatch_size,
epoch_size=epoch_size)
mm_schedule = momentum_schedule(0.9, minibatch_size) #动量
# progress writers
progress_writers = [
ProgressPrinter(tag='Training',
num_epochs=max_epochs,
gen_heartbeat=gen_heartbeat)
]
tensorboard_writer = None
if tensorboard_logdir is not None:
tensorboard_writer = TensorBoardProgressWriter(
freq=10, log_dir=tensorboard_logdir, model=model)
progress_writers.append(tensorboard_writer)
# trainer object
l2_reg_weight = 0.0001
learner = adam(model.parameters,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(model, (loss, error_rate), learner, progress_writers)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(model)
print("*********Training Start*********")
start = time.clock()
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(
min(minibatch_size, epoch_size - sample_count),
input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
trainer.summarize_training_progress()
# Log mean of each parameter tensor, so that we can confirm that the parameters change indeed.
if tensorboard_writer:
for parameter in model.parameters:
tensorboard_writer.write_value(parameter.uid + "/mean",
reduce_mean(parameter).eval(),
epoch)
if model_dir:
model.save(
os.path.join(model_dir,
network_name + "_{}.dnn".format(epoch)))
enable_profiler() # begin to collect profiler data after first epoch
# Evaluation parameters
test_epoch_size = 10000
minibatch_size = 32
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
while sample_count < test_epoch_size:
current_minibatch = min(minibatch_size, test_epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch,
input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
print("")
trainer.summarize_test_progress()
print("")
elapsed = (time.clock() - start)
return 1 - metric_numer / metric_denom, elapsed
def sequence_to_sequence_translator(debug_output=False, run_test=False):
input_vocab_dim = 69
label_vocab_dim = 69
# network complexity; initially low for faster testing
hidden_dim = 256
num_layers = 1
# Source and target inputs to the model
batch_axis = Axis.default_batch_axis()
input_seq_axis = Axis('inputAxis')
label_seq_axis = Axis('labelAxis')
input_dynamic_axes = [batch_axis, input_seq_axis]
raw_input = input_variable(shape=(input_vocab_dim),
dynamic_axes=input_dynamic_axes,
name='raw_input')
label_dynamic_axes = [batch_axis, label_seq_axis]
raw_labels = input_variable(shape=(label_vocab_dim),
dynamic_axes=label_dynamic_axes,
name='raw_labels')
# Instantiate the sequence to sequence translation model
input_sequence = raw_input
# Drop the sentence start token from the label, for decoder training
label_sequence = slice(raw_labels, label_seq_axis, 1,
0) # <s> A B C </s> --> A B C </s>
label_sentence_start = sequence.first(raw_labels) # <s>
is_first_label = sequence.is_first(label_sequence) # <s> 0 0 0 ...
label_sentence_start_scattered = sequence.scatter(label_sentence_start,
is_first_label)
# Encoder
encoder_outputH = stabilize(input_sequence)
for i in range(0, num_layers):
(encoder_outputH,
encoder_outputC) = LSTMP_component_with_self_stabilization(
encoder_outputH.output, hidden_dim, hidden_dim, future_value,
future_value)
thought_vectorH = sequence.first(encoder_outputH)
thought_vectorC = sequence.first(encoder_outputC)
thought_vector_broadcastH = sequence.broadcast_as(thought_vectorH,
label_sequence)
thought_vector_broadcastC = sequence.broadcast_as(thought_vectorC,
label_sequence)
# Decoder
decoder_history_hook = alias(
label_sequence, name='decoder_history_hook') # copy label_sequence
decoder_input = element_select(is_first_label,
label_sentence_start_scattered,
past_value(decoder_history_hook))
decoder_outputH = stabilize(decoder_input)
for i in range(0, num_layers):
if (i > 0):
recurrence_hookH = past_value
recurrence_hookC = past_value
else:
isFirst = sequence.is_first(label_sequence)
recurrence_hookH = lambda operand: element_select(
isFirst, thought_vector_broadcastH, past_value(operand))
recurrence_hookC = lambda operand: element_select(
isFirst, thought_vector_broadcastC, past_value(operand))
(decoder_outputH,
encoder_outputC) = LSTMP_component_with_self_stabilization(
decoder_outputH.output, hidden_dim, hidden_dim, recurrence_hookH,
recurrence_hookC)
decoder_output = decoder_outputH
# Softmax output layer
z = linear_layer(stabilize(decoder_output), label_vocab_dim)
# Criterion nodes
ce = cross_entropy_with_softmax(z, label_sequence)
errs = classification_error(z, label_sequence)
# network output for decoder history
net_output = hardmax(z)
# make a clone of the graph where the ground truth is replaced by the network output
ng = z.clone(CloneMethod.share,
{decoder_history_hook.output: net_output.output})
# Instantiate the trainer object to drive the model training
lr = 0.007
minibatch_size = 72
momentum_time_constant = 1100
m_schedule = momentum_schedule(momentum_time_constant)
clipping_threshold_per_sample = 2.3
gradient_clipping_with_truncation = True
learner = momentum_sgd(z.parameters, lr, m_schedule,
clipping_threshold_per_sample,
gradient_clipping_with_truncation)
trainer = Trainer(z, ce, errs, learner)
# setup data
rel_path = r"../../../../Examples/SequenceToSequence/CMUDict/Data/cmudict-0.7b.train-dev-20-21.ctf"
train_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
rel_path)
valid_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"tiny.ctf")
feature_stream_name = 'features'
labels_stream_name = 'labels'
# readers
randomize_data = True
if run_test:
randomize_data = False # because we want to get an exact error
train_reader = text_format_minibatch_source(train_path, [
StreamConfiguration(feature_stream_name, input_vocab_dim, True, 'S0'),
StreamConfiguration(labels_stream_name, label_vocab_dim, True, 'S1')
],
randomize=randomize_data)
features_si_tr = train_reader.stream_info(feature_stream_name)
labels_si_tr = train_reader.stream_info(labels_stream_name)
valid_reader = text_format_minibatch_source(valid_path, [
StreamConfiguration(feature_stream_name, input_vocab_dim, True, 'S0'),
StreamConfiguration(labels_stream_name, label_vocab_dim, True, 'S1')
],
randomize=False)
features_si_va = valid_reader.stream_info(feature_stream_name)
labels_si_va = valid_reader.stream_info(labels_stream_name)
# get the vocab for printing output sequences in plaintext
rel_path = r"../../../../Examples/SequenceToSequence/CMUDict/Data/cmudict-0.7b.mapping"
vocab_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
rel_path)
vocab = [w.strip() for w in open(vocab_path).readlines()]
i2w = {i: ch for i, ch in enumerate(vocab)}
# Get minibatches of sequences to train with and perform model training
i = 0
mbs = 0
epoch_size = 908241
max_epochs = 10
training_progress_output_freq = 500
# make things more basic for running a quicker test
if run_test:
epoch_size = 5000
max_epochs = 1
training_progress_output_freq = 30
for epoch in range(max_epochs):
loss_numer = 0
metric_numer = 0
denom = 0
while i < (epoch + 1) * epoch_size:
# get next minibatch of training data
mb_train = train_reader.next_minibatch(minibatch_size)
train_args = {
'raw_input': mb_train[features_si_tr],
'raw_labels': mb_train[labels_si_tr]
}
trainer.train_minibatch(train_args)
# collect epoch-wide stats
samples = trainer.previous_minibatch_sample_count
loss_numer += trainer.previous_minibatch_loss_average * samples
metric_numer += trainer.previous_minibatch_evaluation_average * samples
denom += samples
# every N MBs evaluate on a test sequence to visually show how we're doing
if mbs % training_progress_output_freq == 0:
mb_valid = valid_reader.next_minibatch(minibatch_size)
valid_args = {
'raw_input': mb_valid[features_si_va],
'raw_labels': mb_valid[labels_si_va]
}
e = ng.eval(valid_args)
print_sequences(e, i2w)
print_training_progress(trainer, mbs,
training_progress_output_freq)
i += mb_train[labels_si_tr].num_samples
mbs += 1
print("--- EPOCH %d DONE: loss = %f, errs = %f ---" %
(epoch, loss_numer / denom, 100.0 * (metric_numer / denom)))
# now setup a test run
rel_path = r"../../../../Examples/SequenceToSequence/CMUDict/Data/cmudict-0.7b.test.ctf"
test_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
rel_path)
test_reader = text_format_minibatch_source(test_path, [
StreamConfiguration(feature_stream_name, input_vocab_dim, True, 'S0'),
StreamConfiguration(labels_stream_name, label_vocab_dim, True, 'S1')
],
10000,
randomize=False)
features_si_te = test_reader.stream_info(feature_stream_name)
labels_si_te = test_reader.stream_info(labels_stream_name)
test_minibatch_size = 1024
# Get minibatches of sequences to test and perform testing
i = 0
total_error = 0.0
while True:
mb = test_reader.next_minibatch(test_minibatch_size)
if len(mb) == 0:
break
# Specify the mapping of input variables in the model to actual
# minibatch data to be tested with
arguments = {
raw_input: mb[features_si_te],
raw_labels: mb[labels_si_te]
}
mb_error = trainer.test_minibatch(arguments)
total_error += mb_error
if debug_output:
print("Minibatch {}, Error {} ".format(i, mb_error))
i += 1
# Average of evaluation errors of all test minibatches
return total_error / i
def cifar_resnet(base_path, debug_output=False):
image_height = 32
image_width = 32
num_channels = 3
num_classes = 10
feats_stream_name = 'features'
labels_stream_name = 'labels'
minibatch_source = create_mb_source(feats_stream_name, labels_stream_name,
image_height, image_width,
num_channels, num_classes, base_path)
features_si = minibatch_source[feats_stream_name]
labels_si = minibatch_source[labels_stream_name]
# Input variables denoting the features and label data
image_input = input_variable((num_channels, image_height, image_width),
features_si.m_element_type)
label_var = input_variable((num_classes), features_si.m_element_type)
# Instantiate the resnet classification model
classifier_output = resnet_classifer(image_input, num_classes)
ce = cross_entropy_with_softmax(classifier_output, label_var)
pe = classification_error(classifier_output, label_var)
# Instantiate the trainer object to drive the model training
trainer = Trainer(classifier_output, ce, pe,
[sgd(classifier_output.parameters(), lr=0.0078125)])
# Get minibatches of images to train with and perform model training
mb_size = 32
training_progress_output_freq = 60
num_mbs = 1000
if debug_output:
training_progress_output_freq = training_progress_output_freq / 3
for i in range(0, num_mbs):
mb = minibatch_source.get_next_minibatch(mb_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {image_input: mb[features_si], label_var: mb[labels_si]}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
test_minibatch_source = create_test_mb_source(feats_stream_name,
labels_stream_name,
image_height, image_width,
num_channels, num_classes,
base_path)
features_si = test_minibatch_source[feats_stream_name]
labels_si = test_minibatch_source[labels_stream_name]
mb_size = 64
num_mbs = 300
total_error = 0.0
for i in range(0, num_mbs):
mb = test_minibatch_source.get_next_minibatch(mb_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {image_input: mb[features_si], label_var: mb[labels_si]}
error = trainer.test_minibatch(arguments)
total_error += error
return total_error / num_mbs
def simple_mnist():
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
features = input_variable(input_dim, np.float32)
label = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), features)
netout = fully_connected_classifier_net(scaled_input, num_output_classes,
hidden_layers_dim,
num_hidden_layers, relu)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
try:
rel_path = os.path.join(
os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'],
*"Image/MNIST/v0/Train-28x28_cntk_text.txt".split("/"))
except KeyError:
rel_path = os.path.join(
*"../Image/DataSets/MNIST/Train-28x28_cntk_text.txt".split("/"))
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
features: reader_train.streams.features,
label: reader_train.streams.labels
}
# Instantiate progress writers.
logdir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"mnist_log")
tensorboard_writer = TensorBoardProgressWriter(freq=1,
log_dir=logdir,
model=netout)
progress_printer = ProgressPrinter(freq=10, tag='Training')
# Instantiate the trainer object to drive the model training
lr_per_minibatch = learning_rate_schedule(0.2, UnitType.minibatch)
learner = sgd(netout.parameters, lr=lr_per_minibatch)
trainer = Trainer(netout, (ce, pe), learner,
[tensorboard_writer, progress_printer])
# Get minibatches of images to train with and perform model training
minibatch_size = 64
num_samples_per_sweep = 6000
num_sweeps_to_train_with = 2
num_minibatches_to_train = (num_samples_per_sweep *
num_sweeps_to_train_with) / minibatch_size
for minibatch_idx in range(0, int(num_minibatches_to_train)):
trainer.train_minibatch(
reader_train.next_minibatch(minibatch_size, input_map=input_map))
# Log max/min/mean of each parameter tensor, so that we can confirm that the parameters change indeed.
# Don't want to do that very often though, otherwise will spend too much time computing min/max/mean.
if minibatch_idx % 10 == 9:
for p in netout.parameters:
tensorboard_writer.write_value(p.uid + "/max",
reduce_max(p).eval(),
minibatch_idx)
tensorboard_writer.write_value(p.uid + "/min",
reduce_min(p).eval(),
minibatch_idx)
tensorboard_writer.write_value(p.uid + "/mean",
reduce_mean(p).eval(),
minibatch_idx)
trainer.summarize_training_progress()
# Load test data
try:
rel_path = os.path.join(
os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'],
*"Image/MNIST/v0/Test-28x28_cntk_text.txt".split("/"))
except KeyError:
rel_path = os.path.join(
*"../Image/DataSets/MNIST/Test-28x28_cntk_text.txt".split("/"))
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
reader_test = create_reader(path, False, input_dim, num_output_classes)
input_map = {
features: reader_test.streams.features,
label: reader_test.streams.labels
}
# Test data for trained model
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size,
input_map=input_map)
test_result += trainer.test_minibatch(mb)
# Average of evaluation errors of all test minibatches
trainer.summarize_test_progress()
return test_result / num_minibatches_to_test
def train_and_evaluate(reader,
reader_test,
model,
epoch_size=50000,
max_epochs=5):
# declare the model's input dimension
# Training does not require this, but it is needed for deployment.
model.update_signature((num_channels, image_height, image_width))
# criterion function. This is what is being trained trained.
# Model gets "sandwiched" between normalization (not part of model proper) and criterion.
criterion = create_criterion_function(model, normalize=lambda x: x / 256)
#debughelpers.dump_function(criterion, 'criterion')
#from cntk.logging.graph import plot
#plot(criterion, filename=os.path.join(model_path, "ConvNet_CIFAR10_DataAug.pdf"))
# iteration parameters
minibatch_size = 64
#epoch_size = 1000 ; max_epochs = 1 # for faster testing
# learning parameters
learner = momentum_sgd(
model.parameters,
lr=learning_rate_schedule([0.0015625] * 20 + [0.00046875] * 20 +
[0.00015625] * 20 + [0.000046875] * 10 +
[0.000015625],
unit=UnitType.sample,
epoch_size=epoch_size),
momentum=momentum_as_time_constant_schedule([0] * 20 + [600] * 20 +
[1200],
epoch_size=epoch_size),
l2_regularization_weight=0.002)
# trainer object
trainer = Trainer(None, criterion, learner)
# perform model training
log_number_of_parameters(model)
print()
progress_printer = ProgressPrinter(tag='Training')
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
mb = reader.next_minibatch(
min(minibatch_size,
epoch_size - sample_count)) # fetch minibatch.
#trainer.train_minibatch(mb[reader.streams.features], mb[reader.streams.labels])
trainer.train_minibatch({
criterion.arguments[0]:
mb[reader.streams.features],
criterion.arguments[1]:
mb[reader.streams.labels]
})
sample_count += mb[
reader.streams.
labels].num_samples # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
loss, metric, actual_samples = progress_printer.epoch_summary(
with_metric=True)
model.save(
os.path.join(model_path,
"ConvNet_CIFAR10_DataAug_{}.dnn".format(epoch)))
progress_printer.epoch_summary(with_metric=True)
# TODO: we should be done here
#return metric_numer/metric_denom
### Evaluation action
# evaluate with current Trainer instance; just to make sure we save and load the model correctly and BN works now --TODO: delete once confirmed
epoch_size = 10000
minibatch_size = 16
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while sample_count < epoch_size:
mbsize = min(minibatch_size, epoch_size - sample_count)
mb = reader_test.next_minibatch(mbsize)
metric_numer += mbsize * trainer.test_minibatch(
{
criterion.arguments[0]: mb[reader_test.streams.features],
criterion.arguments[1]: mb[reader_test.streams.labels]
})
metric_denom += mbsize
sample_count += mb[reader_test.streams.labels].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom,
metric_denom))
print("")
# return evaluation error.
return loss, metric # return values from last epoch
def conv3d_ucf11(train_reader, test_reader, max_epochs=30):
# Replace 0 with 1 to get detailed log.
set_computation_network_trace_level(0)
# These values must match for both train and test reader.
image_height = train_reader.height
image_width = train_reader.width
num_channels = train_reader.channel_count
sequence_length = train_reader.sequence_length
num_output_classes = train_reader.label_count
# Input variables denoting the features and label data
input_var = input(
(num_channels, sequence_length, image_height, image_width), np.float32)
label_var = input(num_output_classes, np.float32)
# Instantiate simple 3D Convolution network inspired by VGG network
# and http://vlg.cs.dartmouth.edu/c3d/c3d_video.pdf
with default_options(activation=relu):
z = Sequential([
Convolution3D((3, 3, 3), 64, pad=True),
MaxPooling((1, 2, 2), (1, 2, 2)),
For(
range(3), lambda i: [
Convolution3D((3, 3, 3), [96, 128, 128][i], pad=True),
Convolution3D((3, 3, 3), [96, 128, 128][i], pad=True),
MaxPooling((2, 2, 2), (2, 2, 2))
]),
For(range(2), lambda: [Dense(1024), Dropout(0.5)]),
Dense(num_output_classes, activation=None)
])(input_var)
# loss and classification error.
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# training config
epoch_size = 1322 # for now we manually specify epoch size
minibatch_size = 4
# Set learning parameters
lr_per_sample = [0.01] * 10 + [0.001] * 10 + [0.0001]
lr_schedule = learning_rate_schedule(lr_per_sample,
epoch_size=epoch_size,
unit=UnitType.sample)
momentum_time_constant = 4096
mm_schedule = momentum_as_time_constant_schedule([momentum_time_constant],
epoch_size=epoch_size)
# Instantiate the trainer object to drive the model training
learner = momentum_sgd(z.parameters, lr_schedule, mm_schedule, True)
progress_printer = ProgressPrinter(tag='Training', num_epochs=max_epochs)
trainer = Trainer(z, (ce, pe), learner, progress_printer)
log_number_of_parameters(z)
print()
# Get minibatches of images to train with and perform model training
for epoch in range(max_epochs): # loop over epochs
train_reader.reset()
while train_reader.has_more():
videos, labels, current_minibatch = train_reader.next_minibatch(
minibatch_size)
trainer.train_minibatch({input_var: videos, label_var: labels})
trainer.summarize_training_progress()
# Test data for trained model
epoch_size = 332
minibatch_size = 2
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
minibatch_index = 0
test_reader.reset()
while test_reader.has_more():
videos, labels, current_minibatch = test_reader.next_minibatch(
minibatch_size)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch({
input_var: videos,
label_var: labels
}) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom,
metric_denom))
print("")
return metric_numer / metric_denom
def main(base_folder, training_mode='majority', model_name='VGG13', max_epochs = 100):
# create needed folders.
output_model_path = os.path.join(base_folder, R'models')
output_model_folder = os.path.join(output_model_path, model_name + '_' + training_mode)
if not os.path.exists(output_model_folder):
os.makedirs(output_model_folder)
# creating logging file
logging.basicConfig(filename = os.path.join(output_model_folder, "train.log"), filemode = 'w', level = logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler())
logging.info("Starting with training mode {} using {} model and max epochs {}.".format(training_mode, model_name, max_epochs))
# create the model
num_classes = len(emotion_table)
model = build_model(num_classes, model_name)
# set the input variables.
input_var = ct.input_variable((1, model.input_height, model.input_width), np.float32)
label_var = ct.input_variable((num_classes), np.float32)
# read FER+ dataset.
logging.info("Loading data...")
train_params = FERPlusParameters(num_classes, model.input_height, model.input_width, training_mode, False)
test_and_val_params = FERPlusParameters(num_classes, model.input_height, model.input_width, "majority", True)
train_data_reader = FERPlusReader.create(base_folder, train_folders, "label.csv", train_params)
val_data_reader = FERPlusReader.create(base_folder, valid_folders, "label.csv", test_and_val_params)
test_data_reader = FERPlusReader.create(base_folder, test_folders, "label.csv", test_and_val_params)
# print summary of the data.
display_summary(train_data_reader, val_data_reader, test_data_reader)
# get the probalistic output of the model.
z = model.model(input_var)
pred = ct.softmax(z)
epoch_size = train_data_reader.size()
minibatch_size = 32
# Training config
lr_per_minibatch = [model.learning_rate]*20 + [model.learning_rate / 2.0]*20 + [model.learning_rate / 10.0]
mm_time_constant = -minibatch_size/np.log(0.9)
lr_schedule = learning_rate_schedule(lr_per_minibatch, unit=UnitType.minibatch, epoch_size=epoch_size)
mm_schedule = momentum_as_time_constant_schedule(mm_time_constant)
# loss and error cost
train_loss = cost_func(training_mode, pred, label_var)
pe = classification_error(z, label_var)
# construct the trainer
learner = momentum_sgd(z.parameters, lr_schedule, mm_schedule)
trainer = Trainer(z, (train_loss, pe), learner)
# Get minibatches of images to train with and perform model training
max_val_accuracy = 0.0
final_test_accuracy = 0.0
best_test_accuracy = 0.0
logging.info("Start training...")
epoch = 0
best_epoch = 0
while epoch < max_epochs:
train_data_reader.reset()
val_data_reader.reset()
test_data_reader.reset()
# Training
start_time = time.time()
training_loss = 0
training_accuracy = 0
while train_data_reader.has_more():
images, labels, current_batch_size = train_data_reader.next_minibatch(minibatch_size)
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
trainer.train_minibatch({input_var : images, label_var : labels})
# keep track of statistics.
training_loss += trainer.previous_minibatch_loss_average * current_batch_size
training_accuracy += trainer.previous_minibatch_evaluation_average * current_batch_size
training_accuracy /= train_data_reader.size()
training_accuracy = 1.0 - training_accuracy
# Validation
val_accuracy = 0
while val_data_reader.has_more():
images, labels, current_batch_size = val_data_reader.next_minibatch(minibatch_size)
val_accuracy += trainer.test_minibatch({input_var : images, label_var : labels}) * current_batch_size
val_accuracy /= val_data_reader.size()
val_accuracy = 1.0 - val_accuracy
# if validation accuracy goes higher, we compute test accuracy
test_run = False
if val_accuracy > max_val_accuracy:
best_epoch = epoch
max_val_accuracy = val_accuracy
trainer.save_checkpoint(os.path.join(output_model_folder, "model_{}".format(best_epoch)))
test_run = True
test_accuracy = 0
while test_data_reader.has_more():
images, labels, current_batch_size = test_data_reader.next_minibatch(minibatch_size)
test_accuracy += trainer.test_minibatch({input_var : images, label_var : labels}) * current_batch_size
test_accuracy /= test_data_reader.size()
test_accuracy = 1.0 - test_accuracy
final_test_accuracy = test_accuracy
if final_test_accuracy > best_test_accuracy:
best_test_accuracy = final_test_accuracy
logging.info("Epoch {}: took {:.3f}s".format(epoch, time.time() - start_time))
logging.info(" training loss:\t{:e}".format(training_loss))
logging.info(" training accuracy:\t\t{:.2f} %".format(training_accuracy * 100))
logging.info(" validation accuracy:\t\t{:.2f} %".format(val_accuracy * 100))
if test_run:
logging.info(" test accuracy:\t\t{:.2f} %".format(test_accuracy * 100))
epoch += 1
logging.info("")
logging.info("Best validation accuracy:\t\t{:.2f} %, epoch {}".format(max_val_accuracy * 100, best_epoch))
logging.info("Test accuracy corresponding to best validation:\t\t{:.2f} %".format(final_test_accuracy * 100))
logging.info("Best test accuracy:\t\t{:.2f} %".format(best_test_accuracy * 100))
def convnet_cifar10_dataaug(reader_train, reader_test, max_epochs = 80):
set_computation_network_trace_level(0)
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# apply model to input
scaled_input = element_times(constant(0.00390625), input_var)
with default_options (activation=relu, pad=True):
z = Sequential([
LayerStack(2, lambda : [
Convolution((3,3), 64),
Convolution((3,3), 64),
MaxPooling((3,3), (2,2))
]),
LayerStack(2, lambda i: [
Dense([256,128][i]),
Dropout(0.5)
]),
Dense(num_classes, activation=None)
])(scaled_input)
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# training config
epoch_size = 50000 # for now we manually specify epoch size
minibatch_size = 64
# Set learning parameters
lr_per_sample = [0.0015625]*20+[0.00046875]*20+[0.00015625]*20+[0.000046875]*10+[0.000015625]
lr_schedule = learning_rate_schedule(lr_per_sample, unit=UnitType.sample, epoch_size=epoch_size)
mm_time_constant = [0]*20+[600]*20+[1200]
mm_schedule = momentum_as_time_constant_schedule(mm_time_constant, epoch_size=epoch_size)
l2_reg_weight = 0.002
# trainer object
learner = momentum_sgd(z.parameters, lr_schedule, mm_schedule,
l2_regularization_weight = l2_reg_weight)
trainer = Trainer(z, ce, pe, learner)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z) ; print()
progress_printer = ProgressPrinter(tag='Training')
# perform model training
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(min(minibatch_size, epoch_size-sample_count), input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
persist.save_model(z, os.path.join(model_path, "ConvNet_CIFAR10_DataAug_{}.dnn".format(epoch)))
### Evaluation action
epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch, input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom))
print("")
return metric_numer/metric_denom
def train_and_evaluate(reader_train,
reader_test,
network_name,
max_epochs,
distributed_trainer,
scale_up=False):
set_computation_network_trace_level(0)
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# create model, and configure learning parameters
if network_name == 'resnet20':
z = create_cifar10_model(input_var, 3, num_classes)
lr_per_mb = [1.0] * 80 + [0.1] * 40 + [0.01]
elif network_name == 'resnet110':
z = create_cifar10_model(input_var, 18, num_classes)
lr_per_mb = [0.1] * 1 + [1.0] * 80 + [0.1] * 40 + [0.01]
else:
return RuntimeError("Unknown model name!")
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# shared training parameters
epoch_size = 50000 # for now we manually specify epoch size
# NOTE: scaling up minibatch_size increases sample throughput. In 8-GPU machine,
# ResNet110 samples-per-second is ~7x of single GPU, comparing to ~3x without scaling
# up. However, bigger minimatch size on the same number of samples means less updates,
# thus leads to higher training error. This is a trade-off of speed and accuracy
minibatch_size = 128 * (len(distributed_trainer.communicator().workers())
if scale_up else 1)
momentum_time_constant = -minibatch_size / np.log(0.9)
l2_reg_weight = 0.0001
# Set learning parameters
lr_per_sample = [lr / minibatch_size for lr in lr_per_mb]
lr_schedule = learning_rate_schedule(lr_per_sample,
epoch_size=epoch_size,
unit=UnitType.sample)
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant)
# trainer object
learner = momentum_sgd(z.parameters,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(z, ce, pe, learner, distributed_trainer)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z)
print()
progress_printer = ProgressPrinter(tag='Training')
# perform model training
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(
min(minibatch_size, epoch_size - sample_count),
input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
# save model only in worker0, otherwise there will be file write conflicts for multi GPU on the same machine
if distributed_trainer.communicator().current_worker(
).global_rank == 0:
z.save_model(
os.path.join(model_path,
network_name + "_{}.dnn".format(epoch)))
# Evaluation parameters
epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch,
input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom,
metric_denom))
print("")
return metric_numer / metric_denom
def train_and_evaluate(reader_train, reader_test, max_epochs):
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# Normalize the input
feature_scale = 1.0 / 256.0
input_var_norm = element_times(feature_scale, input_var)
# apply model to input
z = create_vgg9_model(input_var_norm, 10)
#
# Training action
#
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# training config
epoch_size = 50000
minibatch_size = 64
# Set learning parameters
lr_per_minibatch = learning_rate_schedule(
[0.01] * 10 + [0.003] * 10 + [0.001], epoch_size, UnitType.minibatch)
momentum_time_constant = momentum_as_time_constant_schedule(
-minibatch_size / np.log(0.9))
l2_reg_weight = 0.0001
# trainer object
learner = momentum_sgd(z.parameters,
lr=lr_per_minibatch,
momentum=momentum_time_constant,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(z, ce, pe, learner)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z)
print()
progress_printer = ProgressPrinter(tag='Training')
# perform model training
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(
min(minibatch_size, epoch_size - sample_count),
input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += data[
label_var].num_samples # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
#
# Evaluation action
#
epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
#progress_printer = ProgressPrinter(freq=100, first=10, tag='Eval')
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch,
input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.1f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom,
metric_denom))
print("")
# return evaluation error.
return metric_numer / metric_denom
def cifar_resnet_distributed(data_path,
run_test,
num_epochs,
communicator=None,
save_model_filename=None,
load_model_filename=None,
debug_output=False):
image_height = 32
image_width = 32
num_channels = 3
num_classes = 10
feats_stream_name = 'features'
labels_stream_name = 'labels'
minibatch_source = create_reader(os.path.join(data_path, 'train_map.txt'),
os.path.join(data_path,
'CIFAR-10_mean.xml'),
True,
distributed_communicator=communicator)
features_si = minibatch_source[feats_stream_name]
labels_si = minibatch_source[labels_stream_name]
# Instantiate the resnet classification model, or load from file
if load_model_filename:
print("Loading model:", load_model_filename)
classifier_output = persist.load_model(load_model_filename)
image_input = classifier_output.arguments[0]
else:
image_input = input_variable((num_channels, image_height, image_width),
features_si.m_element_type)
classifier_output = create_resnet_model(image_input, num_classes)
# Input variables denoting the features and label data
label_var = input_variable((num_classes), features_si.m_element_type)
ce = cross_entropy_with_softmax(classifier_output, label_var)
pe = classification_error(classifier_output, label_var)
# Instantiate the trainer object to drive the model training
mb_size = 128
num_mb_per_epoch = 100
num_mbs = num_mb_per_epoch * num_epochs
lr_schedule = [1.0 / mb_size] * 80 + [0.1 / mb_size] * 40 + [
0.01 / mb_size
]
lr_per_minibatch = learning_rate_schedule(lr_schedule, UnitType.minibatch,
mb_size * num_mb_per_epoch)
momentum_time_constant = momentum_as_time_constant_schedule(-mb_size /
np.log(0.9))
# create data parallel distributed trainer if needed
dist_trainer = distributed.data_parallel_distributed_trainer(
communicator, False) if communicator else None
# Instantiate the trainer object to drive the model training
trainer = Trainer(classifier_output,
ce,
pe, [
momentum_sgd(classifier_output.parameters,
lr=lr_per_minibatch,
momentum=momentum_time_constant,
l2_regularization_weight=0.0001)
],
distributed_trainer=dist_trainer)
# Get minibatches of images to train with and perform model training
training_progress_output_freq = 100 if communicator else 20
if debug_output:
training_progress_output_freq = training_progress_output_freq / 4
for i in range(0, num_mbs):
# NOTE: depends on network, the mb_size can be changed dynamically here
mb = minibatch_source.next_minibatch(mb_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {image_input: mb[features_si], label_var: mb[labels_si]}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
if save_model_filename:
print("Saving model:", save_model_filename)
persist.save_model(classifier_output, save_model_filename)
if run_test:
test_minibatch_source = create_reader(
os.path.join(data_path, 'test_map.txt'),
os.path.join(data_path, 'CIFAR-10_mean.xml'), False)
features_si = test_minibatch_source[feats_stream_name]
labels_si = test_minibatch_source[labels_stream_name]
mb_size = 128
num_mbs = 100
total_error = 0.0
for i in range(0, num_mbs):
mb = test_minibatch_source.next_minibatch(mb_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {
image_input: mb[features_si],
label_var: mb[labels_si]
}
error = trainer.test_minibatch(arguments)
total_error += error
return total_error / num_mbs
else:
return 0
def train_and_evaluate(reader_train, reader_test, max_epochs, model_func):
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# Normalize the input
feature_scale = 1.0 / 256.0
input_var_norm = element_times(feature_scale, input_var)
# apply model to input
z = model_func(input_var_norm, out_dims=num_classes)
#
# Training action
#
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# training config
epoch_size = 20000
minibatch_size = 64
# Set training parameters
lr_per_minibatch = learning_rate_schedule([0.01]*10 + [0.003]*10 + [0.001], UnitType.minibatch, epoch_size)
momentum_time_constant = momentum_as_time_constant_schedule(-minibatch_size/np.log(0.9))
l2_reg_weight = 0.001
# trainer object
progress_printer = ProgressPrinter(0)
learner = momentum_sgd(z.parameters,
lr = lr_per_minibatch, momentum = momentum_time_constant,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(z, (ce, pe), [learner], [progress_printer])
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z) ; print()
#progress_printer = ProgressPrinter(tag='Training')
# perform model training
stop_run=False
batch_index = 0
plot_data = {'batchindex':[], 'loss':[], 'error':[]}
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(min(minibatch_size, epoch_size - sample_count), input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += data[label_var].num_samples # count samples processed so far
# For visualization...
plot_data['batchindex'].append(batch_index)
plot_data['loss'].append(trainer.previous_minibatch_loss_average)
plot_data['error'].append(trainer.previous_minibatch_evaluation_average)
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
batch_index += 1
if trainer.previous_minibatch_evaluation_average < 0.025:
stop_run=True
break
if stop_run:
break
progress_printer.epoch_summary(with_metric=True)
#trainer.save_checkpoint(model_temp_file)
#
# Evaluation action
#
epoch_size = 6600
minibatch_size = 32
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
input_map = {
input_var: reader_test.streams.features,
label_var: reader_test.streams.labels
}
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch, input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.1f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom))
print("")
# Visualize training result:
window_width = 32
loss_cumsum = np.cumsum(np.insert(plot_data['loss'], 0, 0))
error_cumsum = np.cumsum(np.insert(plot_data['error'], 0, 0))
# Moving average.
plot_data['batchindex'] = np.insert(plot_data['batchindex'], 0, 0)[window_width:]
plot_data['avg_loss'] = (loss_cumsum[window_width:] - loss_cumsum[:-window_width]) / window_width
plot_data['avg_error'] = (error_cumsum[window_width:] - error_cumsum[:-window_width]) / window_width
plt.figure(1)
plt.subplot(211)
plt.plot(plot_data["batchindex"], plot_data["avg_loss"], 'b--')
plt.xlabel('Minibatch number')
plt.ylabel('Loss')
plt.title('Minibatch run vs. Training loss ')
plt.show()
plt.subplot(212)
plt.plot(plot_data["batchindex"], plot_data["avg_error"], 'r--')
plt.xlabel('Minibatch number')
plt.ylabel('Label Prediction Error')
plt.title('Minibatch run vs. Label Prediction Error ')
plt.show()
return softmax(z)
def simple_mnist(debug_output=False):
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
input = input_variable(input_dim, np.float32)
label = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), input)
netout = fully_connected_classifier_net(scaled_input, num_output_classes,
hidden_layers_dim,
num_hidden_layers, relu)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
try:
rel_path = os.path.join(
os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'],
*"Image/MNIST/v0/Train-28x28_cntk_text.txt".split("/"))
except KeyError:
rel_path = os.path.join(
*
"../../../../Examples/Image/DataSets/MNIST/Train-28x28_cntk_text.txt"
.split("/"))
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
input: reader_train.streams.features,
label: reader_train.streams.labels
}
# Instantiate the trainer object to drive the model training
trainer = Trainer(netout, ce, pe, sgd(netout.parameters, lr=0.003125))
# Get minibatches of images to train with and perform model training
minibatch_size = 64
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 10
num_minibatches_to_train = (num_samples_per_sweep *
num_sweeps_to_train_with) / minibatch_size
training_progress_output_freq = 500
if debug_output:
training_progress_output_freq = training_progress_output_freq / 4
for i in range(0, int(num_minibatches_to_train)):
mb = reader_train.next_minibatch(minibatch_size, input_map=input_map)
trainer.train_minibatch(mb)
print_training_progress(trainer, i, training_progress_output_freq)
# Load test data
try:
rel_path = os.path.join(
os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'],
*"Image/MNIST/v0/Test-28x28_cntk_text.txt".split("/"))
except KeyError:
rel_path = os.path.join(
*
"../../../../Examples/Image/DataSets/MNIST/Test-28x28_cntk_text.txt"
.split("/"))
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
reader_test = create_reader(path, False, input_dim, num_output_classes)
input_map = {
input: reader_test.streams.features,
label: reader_test.streams.labels
}
# Test data for trained model
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size,
input_map=input_map)
eval_error = trainer.test_minibatch(mb)
test_result = test_result + eval_error
# Average of evaluation errors of all test minibatches
return test_result / num_minibatches_to_test
def convnet_cifar10(debug_output=False):
set_computation_network_trace_level(0)
image_height = 32
image_width = 32
num_channels = 3
input_dim = image_height * image_width * num_channels
num_output_classes = 10
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width),
np.float32)
label_var = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
input_removemean = minus(input_var, constant(128))
scaled_input = element_times(constant(0.00390625), input_removemean)
with default_options(activation=relu, pad=True):
z = Sequential([
LayerStack(
2, lambda: [
Convolution((3, 3), 64),
Convolution((3, 3), 64),
MaxPooling((3, 3), (2, 2))
]),
LayerStack(2, lambda i: [Dense([256, 128][i]),
Dropout(0.5)]),
Dense(num_output_classes, activation=None)
])(scaled_input)
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
reader_train = create_reader(
os.path.join(data_path, 'Train_cntk_text.txt'), True, input_dim,
num_output_classes)
# training config
epoch_size = 50000 # for now we manually specify epoch size
minibatch_size = 64
# Set learning parameters
lr_per_sample = [0.0015625] * 10 + [0.00046875] * 10 + [0.00015625]
lr_schedule = learning_rate_schedule(lr_per_sample,
epoch_size=epoch_size,
unit=UnitType.sample)
momentum_time_constant = [0] * 20 + [-minibatch_size / np.log(0.9)]
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant,
epoch_size=epoch_size)
l2_reg_weight = 0.002
# Instantiate the trainer object to drive the model training
learner = momentum_sgd(z.parameters,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(z, ce, pe, learner)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z)
print()
progress_printer = ProgressPrinter(tag='Training')
# Get minibatches of images to train with and perform model training
max_epochs = 30
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(
min(minibatch_size, epoch_size - sample_count),
input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
persist.save_model(
z, os.path.join(model_path,
"ConvNet_CIFAR10_{}.dnn".format(epoch)))
# Load test data
reader_test = create_reader(os.path.join(data_path, 'Test_cntk_text.txt'),
False, input_dim, num_output_classes)
input_map = {
input_var: reader_test.streams.features,
label_var: reader_test.streams.labels
}
# Test data for trained model
epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch,
input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom,
metric_denom))
print("")
return metric_numer / metric_denom
def simple_mnist(debug_output=False):
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
input = input_variable(input_dim, np.float32)
label = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), input)
z = fully_connected_classifier_net(
scaled_input, num_output_classes, hidden_layers_dim, num_hidden_layers, relu)
ce = cross_entropy_with_softmax(z, label)
pe = classification_error(z, label)
try:
rel_path = os.path.join(os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'],
*"Image/MNIST/v0/Train-28x28_cntk_text.txt".split("/"))
except KeyError:
rel_path = os.path.join(abs_path, "..", "..", "..", "..", "..", "Examples", "Image", "DataSets", "MNIST", "Train-28x28_cntk_text.txt")
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
input : reader_train.streams.features,
label : reader_train.streams.labels
}
lr_per_minibatch=learning_rate_schedule(0.2, UnitType.minibatch)
# Instantiate the trainer object to drive the model training
trainer = Trainer(z, ce, pe, sgd(z.parameters, lr=lr_per_minibatch))
# Get minibatches of images to train with and perform model training
minibatch_size = 64
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 10
num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size
training_progress_output_freq = 500
if debug_output:
training_progress_output_freq = training_progress_output_freq/4
for i in range(0, int(num_minibatches_to_train)):
mb = reader_train.next_minibatch(minibatch_size, input_map=input_map)
trainer.train_minibatch(mb)
print_training_progress(trainer, i, training_progress_output_freq)
# Load test data
try:
rel_path = os.path.join(os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'],
*"Image/MNIST/v0/Test-28x28_cntk_text.txt".split("/"))
except KeyError:
rel_path = os.path.join(abs_path, "..", "..", "..", "..", "..", "Examples", "Image", "DataSets", "MNIST", "Test-28x28_cntk_text.txt")
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
reader_test = create_reader(path, False, input_dim, num_output_classes)
input_map = {
input : reader_test.streams.features,
label : reader_test.streams.labels
}
# Test data for trained model
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size, input_map=input_map)
eval_error = trainer.test_minibatch(mb)
test_result = test_result + eval_error
# Average of evaluation errors of all test minibatches
return test_result / num_minibatches_to_test
def train_and_evaluate(reader_train, reader_test, max_epochs):
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# Normalize the input
feature_scale = 1.0 / 256.0
input_var_norm = element_times(feature_scale, input_var)
# apply model to input
z = create_vgg9_model(input_var_norm, 10)
#
# Training action
#
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# training config
epoch_size = 50000
minibatch_size = 64
# Set learning parameters
lr_per_minibatch = learning_rate_schedule([0.01]*10 + [0.003]*10 + [0.001], epoch_size, UnitType.minibatch)
momentum_time_constant = momentum_as_time_constant_schedule(-minibatch_size/np.log(0.9))
l2_reg_weight = 0.0001
# trainer object
learner = momentum_sgd(z.parameters,
lr = lr_per_minibatch, momentum = momentum_time_constant,
l2_regularization_weight = l2_reg_weight)
trainer = Trainer(z, ce, pe, learner)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z) ; print()
progress_printer = ProgressPrinter(tag='Training')
# perform model training
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(min(minibatch_size, epoch_size - sample_count), input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += data[label_var].num_samples # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
#
# Evaluation action
#
epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
#progress_printer = ProgressPrinter(freq=100, first=10, tag='Eval')
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch, input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.1f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom))
print("")
# return evaluation error.
return metric_numer/metric_denom
def sequence_to_sequence_translator(debug_output=False):
input_vocab_dim = 69
label_vocab_dim = 69
hidden_dim = 512
num_layers = 2
# Source and target inputs to the model
batch_axis = Axis.default_batch_axis()
input_seq_axis = Axis("inputAxis")
label_seq_axis = Axis("labelAxis")
input_dynamic_axes = [batch_axis, input_seq_axis]
raw_input = input_variable(shape=(input_vocab_dim), dynamic_axes=input_dynamic_axes)
label_dynamic_axes = [batch_axis, label_seq_axis]
raw_labels = input_variable(shape=(label_vocab_dim), dynamic_axes=label_dynamic_axes)
# Instantiate the sequence to sequence translation model
input_sequence = raw_input
# Drop the sentence start token from the label, for decoder training
label_sequence = slice(raw_labels, label_seq_axis, 1, 0)
label_sentence_start = sequence.first(raw_labels)
is_first_label = sequence.is_first(label_sequence)
label_sentence_start_scattered = sequence.scatter(label_sentence_start, is_first_label)
# Encoder
encoder_outputH = stabilize(input_sequence)
for i in range(0, num_layers):
(encoder_outputH, encoder_outputC) = LSTMP_component_with_self_stabilization(
encoder_outputH.output(), hidden_dim, hidden_dim, future_value, future_value
)
thought_vectorH = sequence.first(encoder_outputH)
thought_vectorC = sequence.first(encoder_outputC)
thought_vector_broadcastH = sequence.broadcast_as(thought_vectorH, label_sequence)
thought_vector_broadcastC = sequence.broadcast_as(thought_vectorC, label_sequence)
# Decoder
decoder_history_from_ground_truth = label_sequence
decoder_input = element_select(
is_first_label, label_sentence_start_scattered, past_value(decoder_history_from_ground_truth)
)
decoder_outputH = stabilize(decoder_input)
for i in range(0, num_layers):
if i > 0:
recurrence_hookH = past_value
recurrence_hookC = past_value
else:
isFirst = sequence.is_first(label_sequence)
recurrence_hookH = lambda operand: element_select(isFirst, thought_vector_broadcastH, past_value(operand))
recurrence_hookC = lambda operand: element_select(isFirst, thought_vector_broadcastC, past_value(operand))
(decoder_outputH, encoder_outputC) = LSTMP_component_with_self_stabilization(
decoder_outputH.output(), hidden_dim, hidden_dim, recurrence_hookH, recurrence_hookC
)
decoder_output = decoder_outputH
decoder_dim = hidden_dim
# Softmax output layer
z = linear_layer(stabilize(decoder_output), label_vocab_dim)
ce = cross_entropy_with_softmax(z, label_sequence)
errs = classification_error(z, label_sequence)
# Instantiate the trainer object to drive the model training
lr = 0.007
momentum_time_constant = 1100
momentum_per_sample = momentums_per_sample(math.exp(-1.0 / momentum_time_constant))
clipping_threshold_per_sample = 2.3
gradient_clipping_with_truncation = True
trainer = Trainer(
z,
ce,
errs,
[
momentum_sgd(
z.parameters(),
lr,
momentum_per_sample,
clipping_threshold_per_sample,
gradient_clipping_with_truncation,
)
],
)
rel_path = r"../../../../Examples/SequenceToSequence/CMUDict/Data/cmudict-0.7b.train-dev-20-21.ctf"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
feature_stream_name = "features"
labels_stream_name = "labels"
mb_source = text_format_minibatch_source(
path,
[
StreamConfiguration(feature_stream_name, input_vocab_dim, True, "S0"),
StreamConfiguration(labels_stream_name, label_vocab_dim, True, "S1"),
],
10000,
)
features_si = mb_source[feature_stream_name]
labels_si = mb_source[labels_stream_name]
# Get minibatches of sequences to train with and perform model training
minibatch_size = 72
training_progress_output_freq = 30
if debug_output:
training_progress_output_freq = training_progress_output_freq / 3
while True:
mb = mb_source.get_next_minibatch(minibatch_size)
if len(mb) == 0:
break
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {raw_input: mb[features_si], raw_labels: mb[labels_si]}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
i += 1
rel_path = r"../../../../Examples/SequenceToSequence/CMUDict/Data/cmudict-0.7b.test.ctf"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
test_mb_source = text_format_minibatch_source(
path,
[
StreamConfiguration(feature_stream_name, input_vocab_dim, True, "S0"),
StreamConfiguration(labels_stream_name, label_vocab_dim, True, "S1"),
],
10000,
False,
)
features_si = test_mb_source[feature_stream_name]
labels_si = test_mb_source[labels_stream_name]
# choose this to be big enough for the longest sentence
train_minibatch_size = 1024
# Get minibatches of sequences to test and perform testing
i = 0
total_error = 0.0
while True:
mb = test_mb_source.get_next_minibatch(train_minibatch_size)
if len(mb) == 0:
break
# Specify the mapping of input variables in the model to actual
# minibatch data to be tested with
arguments = {raw_input: mb[features_si], raw_labels: mb[labels_si]}
mb_error = trainer.test_minibatch(arguments)
total_error += mb_error
if debug_output:
print("Minibatch {}, Error {} ".format(i, mb_error))
i += 1
# Average of evaluation errors of all test minibatches
return total_error / i
data = training_reader.next_minibatch(minibatch_size, input_map = input_map)
"""
# This is how to get the Numpy typed data from the reader
ldata = data[label].asarray()
fdata = data[feature].asarray()
"""
trainer.train_minibatch(data)
loss = trainer.previous_minibatch_loss_average
if not (loss == "NA"):
plotdata["loss"].append(loss)
if np.abs(trainer.previous_minibatch_loss_average) < 0.0015: #stop once the model is good.
break
#%%
trainer.summarize_training_progress()
test_data = training_reader.next_minibatch(minibatch_size, input_map = input_map)
avg_error = trainer.test_minibatch(test_data)
print("Error rate on an unseen minibatch %f" % avg_error)
#%%
ntldata = data[label].asarray()
ntfdata = data[feature].asarray()
for i in range(10):
print("data {},\tevaluation {},\texpected {}".format(
", ".join(str(v) for v in ntfdata[i][0]),
netout.eval({feature: ntfdata[i]})[0],
ntldata[i][0]))
#%%
import matplotlib.pyplot as plt
plotdata["avgloss"] = moving_average(plotdata["loss"], 100)
plt.figure(1)
plt.subplot(211)
def train_and_evaluate(create_train_reader,
test_reader,
network_name,
max_epochs,
create_dist_learner,
scale_up=False):
set_computation_network_trace_level(0)
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# create model, and configure learning parameters
if network_name == 'resnet20':
z = create_cifar10_model(input_var, 3, num_classes)
lr_per_mb = [1.0] * 80 + [0.1] * 40 + [0.01]
elif network_name == 'resnet110':
z = create_cifar10_model(input_var, 18, num_classes)
lr_per_mb = [0.1] * 1 + [1.0] * 80 + [0.1] * 40 + [0.01]
else:
return RuntimeError("Unknown model name!")
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# shared training parameters
epoch_size = 50000 # for now we manually specify epoch size
# NOTE: scaling up minibatch_size increases sample throughput. In 8-GPU machine,
# ResNet110 samples-per-second is ~7x of single GPU, comparing to ~3x without scaling
# up. However, bigger minimatch size on the same number of samples means less updates,
# thus leads to higher training error. This is a trade-off of speed and accuracy
minibatch_size = 128 * (distributed.Communicator.num_workers()
if scale_up else 1)
momentum_time_constant = -minibatch_size / np.log(0.9)
l2_reg_weight = 0.0001
# Set learning parameters
lr_per_sample = [lr / minibatch_size for lr in lr_per_mb]
lr_schedule = learning_rate_schedule(lr_per_sample,
epoch_size=epoch_size,
unit=UnitType.sample)
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant)
# trainer object
learner = create_dist_learner(
momentum_sgd(z.parameters,
lr_schedule,
mm_schedule,
unit_gain=True,
l2_regularization_weight=l2_reg_weight))
trainer = Trainer(z, ce, pe, learner)
total_number_of_samples = max_epochs * epoch_size
train_reader = create_train_reader(total_number_of_samples)
# define mapping from reader streams to network inputs
input_map = {
input_var: train_reader.streams.features,
label_var: train_reader.streams.labels
}
log_number_of_parameters(z)
print()
progress_printer = ProgressPrinter(tag='Training')
# perform model training
current_epoch = 0
updated = True
while updated:
data = train_reader.next_minibatch(
minibatch_size, input_map=input_map) # fetch minibatch.
updated = trainer.train_minibatch(data) # update model with it
progress_printer.update_with_trainer(trainer,
with_metric=True) # log progress
epoch_index = int(trainer.total_number_of_samples_seen / epoch_size)
if current_epoch != epoch_index: # new epoch reached
progress_printer.epoch_summary(with_metric=True)
current_epoch = epoch_index
trainer.save_checkpoint(
os.path.join(model_path,
network_name + "_{}.dnn".format(current_epoch)))
# Evaluation parameters
epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while True:
data = test_reader.next_minibatch(minibatch_size, input_map=input_map)
if not data: break
local_mb_samples = data[label_var].num_samples
metric_numer += trainer.test_minibatch(data) * local_mb_samples
metric_denom += local_mb_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom,
metric_denom))
print("")
return metric_numer / metric_denom
def convnet_cifar10_dataaug(reader_train,
reader_test,
distributed_trainer,
max_epochs=80):
set_computation_network_trace_level(0)
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# apply model to input
scaled_input = element_times(constant(0.00390625), input_var)
with default_options(activation=relu, pad=True):
z = Sequential([
LayerStack(
2, lambda: [
Convolution((3, 3), 64),
Convolution((3, 3), 64),
MaxPooling((3, 3), (2, 2))
]),
LayerStack(2, lambda i: [Dense([256, 128][i]),
Dropout(0.5)]),
Dense(num_classes, activation=None)
])(scaled_input)
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# training config
epoch_size = 50000 # for now we manually specify epoch size
minibatch_size = 64
# Set learning parameters
lr_per_sample = [0.0015625] * 20 + [0.00046875] * 20 + [
0.00015625
] * 20 + [0.000046875] * 10 + [0.000015625]
lr_schedule = learning_rate_schedule(lr_per_sample,
unit=UnitType.sample,
epoch_size=epoch_size)
momentum_time_constant = [0] * 20 + [600] * 20 + [1200]
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant,
epoch_size=epoch_size)
l2_reg_weight = 0.002
# trainer object
learner = momentum_sgd(z.parameters,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(z, ce, pe, learner, distributed_trainer)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z)
print()
progress_printer = ProgressPrinter(tag='Training')
# perform model training
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(
min(minibatch_size, epoch_size - sample_count),
input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
if distributed_trainer.communicator().current_worker(
).global_rank == 0:
persist.save_model(
z,
os.path.join(model_path,
"ConvNet_CIFAR10_DataAug_{}.dnn".format(epoch)))
### Evaluation action
epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch,
input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += trainer.previous_minibatch_sample_count
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom,
metric_denom))
print("")
return metric_numer / metric_denom
def train_and_evaluate(reader_train, reader_test, network_name):
set_computation_network_trace_level(0)
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# create model, and configure learning parameters
if network_name == 'resnet20':
z = create_cifar10_model(input_var, 3, num_classes)
lr_per_mb = [1.0]*80+[0.1]*40+[0.01]
elif network_name == 'resnet110':
z = create_cifar10_model(input_var, 18, num_classes)
lr_per_mb = [0.1]*1+[1.0]*80+[0.1]*40+[0.01]
else:
return RuntimeError("Unknown model name!")
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# shared training parameters
epoch_size = 50000 # for now we manually specify epoch size
minibatch_size = 128
max_epochs = 160
momentum_time_constant = -minibatch_size/np.log(0.9)
l2_reg_weight = 0.0001
# Set learning parameters
lr_per_sample = [lr/minibatch_size for lr in lr_per_mb]
lr_schedule = learning_rate_schedule(lr_per_sample, epoch_size=epoch_size)
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant)
# trainer object
learner = momentum_sgd(z.parameters, lr_schedule, mm_schedule,
l2_regularization_weight = l2_reg_weight)
trainer = Trainer(z, ce, pe, learner)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z) ; print()
progress_printer = ProgressPrinter(tag='Training')
# perform model training
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(min(minibatch_size, epoch_size-sample_count), input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += data[label_var].num_samples # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
persist.save_model(z, os.path.join(model_path, network_name + "_{}.dnn".format(epoch)))
# Evaluation parameters
epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch, input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom))
print("")
return metric_numer/metric_denom
def train_and_evaluate(reader_train,
reader_test,
network_name,
epoch_size,
max_epochs,
profiler_dir=None,
model_dir=None,
log_dir=None,
tensorboard_logdir=None,
gen_heartbeat=False):
set_computation_network_trace_level(0)
# Input variables denoting the features and label data
input_var = C.input_variable((num_channels, image_height, image_width),
name='features')
label_var = C.input_variable((num_classes))
# create model, and configure learning parameters
if network_name == 'resnet20':
z = create_cifar10_model(input_var, 3, num_classes)
lr_per_mb = [1.0] * 80 + [0.1] * 40 + [0.01]
elif network_name == 'resnet110':
z = create_cifar10_model(input_var, 18, num_classes)
lr_per_mb = [0.1] * 1 + [1.0] * 80 + [0.1] * 40 + [0.01]
else:
raise RuntimeError("Unknown model name!")
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# shared training parameters
minibatch_size = 128
momentum_time_constant = -minibatch_size / np.log(0.9)
l2_reg_weight = 0.0001
# Set learning parameters
lr_per_sample = [lr / minibatch_size for lr in lr_per_mb]
lr_schedule = learning_rate_schedule(lr_per_sample,
epoch_size=epoch_size,
unit=UnitType.sample)
mm_schedule = momentum_as_time_constant_schedule(momentum_time_constant)
# progress writers
progress_writers = [
ProgressPrinter(tag='Training',
log_to_file=log_dir,
num_epochs=max_epochs,
gen_heartbeat=gen_heartbeat)
]
tensorboard_writer = None
if tensorboard_logdir is not None:
tensorboard_writer = TensorBoardProgressWriter(
freq=10, log_dir=tensorboard_logdir, model=z)
progress_writers.append(tensorboard_writer)
# trainer object
learner = momentum_sgd(z.parameters,
lr_schedule,
mm_schedule,
l2_regularization_weight=l2_reg_weight)
trainer = Trainer(z, (ce, pe), learner, progress_writers)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z)
print()
# perform model training
if profiler_dir:
start_profiler(profiler_dir, True)
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(
min(minibatch_size, epoch_size - sample_count),
input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
trainer.summarize_training_progress()
# Log mean of each parameter tensor, so that we can confirm that the parameters change indeed.
if tensorboard_writer:
for parameter in z.parameters:
tensorboard_writer.write_value(parameter.uid + "/mean",
reduce_mean(parameter).eval(),
epoch)
if model_dir:
z.save(
os.path.join(model_dir,
network_name + "_{}.dnn".format(epoch)))
enable_profiler() # begin to collect profiler data after first epoch
if profiler_dir:
stop_profiler()
# Evaluation parameters
test_epoch_size = 10000
minibatch_size = 16
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
while sample_count < test_epoch_size:
current_minibatch = min(minibatch_size, test_epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch,
input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += data[label_var].num_samples
print("")
trainer.summarize_test_progress()
print("")
return metric_numer / metric_denom
def conv3d_ucf11(train_reader, test_reader, max_epochs=30):
# Replace 0 with 1 to get detailed log.
set_computation_network_trace_level(0)
# These values must match for both train and test reader.
image_height = train_reader.height
image_width = train_reader.width
num_channels = train_reader.channel_count
sequence_length = train_reader.sequence_length
num_output_classes = train_reader.label_count
# Input variables denoting the features and label data
input_var = input_variable((num_channels, sequence_length, image_height, image_width), np.float32)
label_var = input_variable(num_output_classes, np.float32)
# Instantiate simple 3D Convolution network inspired by VGG network
# and http://vlg.cs.dartmouth.edu/c3d/c3d_video.pdf
with default_options (activation=relu):
z = Sequential([
Convolution3D((3,3,3), 64, pad=True),
MaxPooling((1,2,2), (1,2,2)),
For(range(3), lambda i: [
Convolution3D((3,3,3), [96, 128, 128][i], pad=True),
Convolution3D((3,3,3), [96, 128, 128][i], pad=True),
MaxPooling((2,2,2), (2,2,2))
]),
For(range(2), lambda : [
Dense(1024),
Dropout(0.5)
]),
Dense(num_output_classes, activation=None)
])(input_var)
# loss and classification error.
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
# training config
epoch_size = 1322 # for now we manually specify epoch size
minibatch_size = 4
# Set learning parameters
lr_per_sample = [0.01]*10+[0.001]*10+[0.0001]
lr_schedule = learning_rate_schedule(lr_per_sample, epoch_size=epoch_size, unit=UnitType.sample)
momentum_time_constant = 4096
mm_schedule = momentum_as_time_constant_schedule([momentum_time_constant], epoch_size=epoch_size)
# Instantiate the trainer object to drive the model training
learner = momentum_sgd(z.parameters, lr_schedule, mm_schedule, True)
progress_printer = ProgressPrinter(tag='Training', num_epochs=max_epochs)
trainer = Trainer(z, (ce, pe), learner, progress_printer)
log_number_of_parameters(z) ; print()
# Get minibatches of images to train with and perform model training
for epoch in range(max_epochs): # loop over epochs
train_reader.reset()
while train_reader.has_more():
videos, labels, current_minibatch = train_reader.next_minibatch(minibatch_size)
trainer.train_minibatch({input_var : videos, label_var : labels})
trainer.summarize_training_progress()
# Test data for trained model
epoch_size = 332
minibatch_size = 2
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
minibatch_index = 0
test_reader.reset()
while test_reader.has_more():
videos, labels, current_minibatch = test_reader.next_minibatch(minibatch_size)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch({input_var : videos, label_var : labels}) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom))
print("")
return metric_numer/metric_denom
def convnet_mnist(debug_output=False):
image_height = 28
image_width = 28
num_channels = 1
input_dim = image_height * image_width * num_channels
num_output_classes = 10
# Input variables denoting the features and label data
input_var = input_variable((num_channels, image_height, image_width),
np.float32)
label_var = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), input_var)
with default_options(activation=relu, pad=False):
conv1 = Convolution((5, 5), 32, pad=True)(scaled_input)
pool1 = MaxPooling((3, 3), (2, 2))(conv1)
conv2 = Convolution((3, 3), 48)(pool1)
pool2 = MaxPooling((3, 3), (2, 2))(conv2)
conv3 = Convolution((3, 3), 64)(pool2)
f4 = Dense(96)(conv3)
drop4 = Dropout(0.5)(f4)
z = Dense(num_output_classes, activation=None)(drop4)
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
reader_train = create_reader(
os.path.join(data_path, 'Train-28x28_cntk_text.txt'), True, input_dim,
num_output_classes)
# training config
epoch_size = 60000 # for now we manually specify epoch size
minibatch_size = 128
# Set learning parameters
lr_per_sample = [0.001] * 10 + [0.0005] * 10 + [0.0001]
lr_schedule = learning_rate_schedule(lr_per_sample, UnitType.sample,
epoch_size)
mm_time_constant = [0] * 5 + [1024]
mm_schedule = momentum_as_time_constant_schedule(mm_time_constant,
epoch_size)
# Instantiate the trainer object to drive the model training
learner = momentum_sgd(z.parameters, lr_schedule, mm_schedule)
trainer = Trainer(z, ce, pe, learner)
# define mapping from reader streams to network inputs
input_map = {
input_var: reader_train.streams.features,
label_var: reader_train.streams.labels
}
log_number_of_parameters(z)
print()
progress_printer = ProgressPrinter(tag='Training')
# Get minibatches of images to train with and perform model training
max_epochs = 40
for epoch in range(max_epochs): # loop over epochs
sample_count = 0
while sample_count < epoch_size: # loop over minibatches in the epoch
data = reader_train.next_minibatch(
min(minibatch_size, epoch_size - sample_count),
input_map=input_map) # fetch minibatch.
trainer.train_minibatch(data) # update model with it
sample_count += data[
label_var].num_samples # count samples processed so far
progress_printer.update_with_trainer(
trainer, with_metric=True) # log progress
progress_printer.epoch_summary(with_metric=True)
z.save_model(
os.path.join(model_path, "ConvNet_MNIST_{}.dnn".format(epoch)))
# Load test data
reader_test = create_reader(
os.path.join(data_path, 'Test-28x28_cntk_text.txt'), False, input_dim,
num_output_classes)
input_map = {
input_var: reader_test.streams.features,
label_var: reader_test.streams.labels
}
# Test data for trained model
epoch_size = 10000
minibatch_size = 1024
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
sample_count = 0
minibatch_index = 0
while sample_count < epoch_size:
current_minibatch = min(minibatch_size, epoch_size - sample_count)
# Fetch next test min batch.
data = reader_test.next_minibatch(current_minibatch,
input_map=input_map)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch(data) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
sample_count += trainer.previous_minibatch_sample_count
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom,
metric_denom))
print("")
return metric_numer / metric_denom
def simple_mnist(debug_output=False):
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
input = input_variable(input_dim, np.float32)
label = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant((), 0.00390625), input)
netout = fully_connected_classifier_net(scaled_input, num_output_classes,
hidden_layers_dim,
num_hidden_layers, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
try:
rel_path = os.path.join(
os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'],
*"Image/MNIST/v0/Train-28x28_cntk_text.txt".split("/"))
except KeyError:
rel_path = os.path.join(
*"../../../../Examples/Image/MNIST/Data/Train-28x28_cntk_text.txt".
split("/"))
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
feature_stream_name = 'features'
labels_stream_name = 'labels'
mb_source = text_format_minibatch_source(path, [
StreamConfiguration(feature_stream_name, input_dim),
StreamConfiguration(labels_stream_name, num_output_classes)
])
features_si = mb_source.stream_info(feature_stream_name)
labels_si = mb_source.stream_info(labels_stream_name)
# Instantiate the trainer object to drive the model training
trainer = Trainer(netout, ce, pe, [sgd(netout.parameters(), lr=0.003125)])
# Get minibatches of images to train with and perform model training
minibatch_size = 32
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 1
num_minibatches_to_train = (num_samples_per_sweep *
num_sweeps_to_train_with) / minibatch_size
training_progress_output_freq = 20
for i in range(0, int(num_minibatches_to_train)):
mb = mb_source.get_next_minibatch(minibatch_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
arguments = {
input: mb[features_si].m_data,
label: mb[labels_si].m_data
}
trainer.train_minibatch(arguments)
if debug_output:
print_training_progress(trainer, i, training_progress_output_freq)
# Load test data
try:
rel_path = os.path.join(
os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'],
*"Image/MNIST/v0/Test-28x28_cntk_text.txt".split("/"))
except KeyError:
rel_path = os.path.join(
*"../../../../Examples/Image/MNIST/Data/Test-28x28_cntk_text.txt".
split("/"))
path = os.path.normpath(os.path.join(abs_path, rel_path))
check_path(path)
test_mb_source = text_format_minibatch_source(path, [
StreamConfiguration(feature_stream_name, input_dim),
StreamConfiguration(labels_stream_name, num_output_classes)
])
features_si = test_mb_source.stream_info(feature_stream_name)
labels_si = test_mb_source.stream_info(labels_stream_name)
# Test data for trained model
test_minibatch_size = 512
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = test_mb_source.get_next_minibatch(test_minibatch_size)
# Specify the mapping of input variables in the model to actual
# minibatch data to be tested with
arguments = {
input: mb[features_si].m_data,
label: mb[labels_si].m_data
}
eval_error = trainer.test_minibatch(arguments)
test_result = test_result + eval_error
# Average of evaluation errors of all test minibatches
return test_result / num_minibatches_to_test