def train_and_test(network, trainer, train_source, test_source, minibatch_size, epoch_size, restore, profiling=False):
# define mapping from intput streams to network inputs
input_map = {
network['feature']: train_source.streams.features,
network['label']: train_source.streams.labels
}
# Train all minibatches
if profiling:
start_profiler(sync_gpu=True)
training_session(
trainer=trainer, mb_source = train_source,
model_inputs_to_streams = input_map,
mb_size = minibatch_size,
progress_frequency=epoch_size,
checkpoint_config = CheckpointConfig(frequency = epoch_size,
filename = os.path.join(model_path, "ConvNet_CIFAR10_DataAug"),
restore = restore),
test_config = TestConfig(source = test_source, mb_size=minibatch_size)
).train()
if profiling:
stop_profiler()
def train_and_test(network, trainer, train_source, minibatch_size, restore,
model_path, model_name, epoch_size, cv_source):
input_map = {
network['feature']: train_source.streams.features,
network['label']: train_source.streams.labels
}
cv_input_map = None if cv_source is None else {
network['feature']: cv_source.streams.features,
network['label']: cv_source.streams.labels
}
cv_checkpoint_interval = 5 # evaluate dev set after every N epochs
checkpoint_config = CheckpointConfig(frequency=epoch_size,
filename=os.path.join(
model_path, model_name),
restore=restore,
preserve_all=True)
cv_checkpoint_config = CrossValidationConfig(
cv_source,
model_inputs_to_streams=cv_input_map,
frequency=epoch_size * cv_checkpoint_interval)
# Train all minibatches
training_session(trainer=trainer,
mb_source=train_source,
model_inputs_to_streams=input_map,
mb_size=minibatch_size_schedule(minibatch_size,
epoch_size),
progress_frequency=epoch_size,
checkpoint_config=checkpoint_config,
cv_config=cv_checkpoint_config).train()
def train_and_test(network, trainer, train_source, test_source, minibatch_size, epoch_size, restore, profiling=False):
# define mapping from intput streams to network inputs
input_map = {
network['feature']: train_source.streams.features,
network['label']: train_source.streams.labels
}
# Train all minibatches
if profiling:
cntk.start_profiler(sync_gpu=True)
training_session(
trainer=trainer, mb_source = train_source,
var_to_stream = input_map,
mb_size = minibatch_size,
progress_frequency=epoch_size,
checkpoint_config = CheckpointConfig(frequency = epoch_size,
filename = os.path.join(model_path, "ConvNet_CIFAR10_DataAug"),
restore = restore),
cv_config = CrossValidationConfig(source = test_source, mb_size=minibatch_size)
).train()
if profiling:
cntk.stop_profiler()
def train_and_test(network,
trainer,
train_source,
test_source,
minibatch_size,
epoch_size,
restore=False,
model_path=model_path,
cv_freq=None):
input_map = {
network['feature']: train_source.streams.amazing_features,
network['label']: train_source.streams.awesome_labels
}
cv_input_map = {
network['feature']: test_source.streams.amazing_features,
network['label']: test_source.streams.awesome_labels
}
training_session(
trainer=trainer,
mb_source=train_source,
model_inputs_to_streams=input_map,
mb_size=minibatch_size,
progress_frequency=epoch_size,
checkpoint_config=CheckpointConfig(frequency=epoch_size,
filename=os.path.join(
model_path,
"HTK_LSTM_Truncated"),
restore=restore),
cv_config=CrossValidationConfig(test_source,
minibatch_size=minibatch_size,
model_inputs_to_streams=cv_input_map,
frequency=cv_freq)).train()
def train_and_test(network, trainer, train_source, minibatch_size, restore, model_path, model_name, epoch_size, cv_source):
input_map = {
network['feature']: train_source.streams.features,
network['label']: train_source.streams.labels
}
cv_input_map = None if cv_source is None else {
network['feature']: cv_source.streams.features,
network['label']: cv_source.streams.labels
}
cv_checkpoint_interval = 5 # evaluate dev set after every N epochs
##A checkpoint configuration for the training session##
checkpoint_config = CheckpointConfig(frequency=epoch_size, #checkpointing period #after every epoch
filename=os.path.join(model_path, model_name), #checkpoint file name
restore=restore, # indicating whether to restore from available checkpoint before the start of the training
preserve_all=True)
## #A cross validation configuration for the training session
cv_checkpoint_config = CrossValidationConfig(cv_source, #minibatch source used for cross validation
model_inputs_to_streams=cv_input_map,
frequency=epoch_size*cv_checkpoint_interval)
####Train all minibatches
training_session( #A training session trains a model using the specified trainer and configs.,.....
trainer=trainer,
mb_source=train_source,
model_inputs_to_streams=input_map,
mb_size=minibatch_size_schedule(minibatch_size, epoch_size),
progress_frequency=epoch_size,
checkpoint_config=checkpoint_config,
cv_config=cv_checkpoint_config
).train() #train
def train_and_test(network,
trainer,
train_source,
test_source,
minibatch_size,
epoch_size,
restore,
profiling=False):
# define mapping from intput streams to network inputs
input_map = {
network['feature']: train_source.streams.features,
network['label']: train_source.streams.labels
}
if profiling:
start_profiler(sync_gpu=True)
training_session(trainer=trainer,
mb_source=train_source,
mb_size=minibatch_size,
var_to_stream=input_map,
checkpoint_config=CheckpointConfig(filename=os.path.join(
model_path, model_name),
restore=restore),
progress_frequency=epoch_size,
test_config=TestConfig(source=test_source,
mb_size=16)).train()
if profiling:
stop_profiler()
def train_and_test(network, trainer, train_source, test_source, max_epochs,
minibatch_size, epoch_size, restore, profiling):
# define mapping from intput streams to network inputs
input_map = {
network['feature']: train_source.streams.features,
network['label']: train_source.streams.labels
}
if profiling:
start_profiler(sync_gpu=True)
training_session(
trainer=trainer,
mb_source=train_source,
model_inputs_to_streams=input_map,
mb_size=minibatch_size,
progress_frequency=epoch_size,
checkpoint_config=CheckpointConfig(frequency=epoch_size,
filename=os.path.join(
model_path,
"BN-Inception_CIFAR10"),
restore=restore),
test_config=TestConfig(test_source,
minibatch_size=minibatch_size)).train()
if profiling:
stop_profiler()
def train_model(network,
trainer,
train_source,
test_source,
validation_source,
minibatch_size,
epoch_size,
restore,
profiling=False):
print('Training the model.')
# define mapping from intput streams to network inputs
input_map = {
network['feature']: train_source.streams.features,
network['label']: train_source.streams.labels
}
# Train all minibatches
if profiling:
start_profiler(sync_gpu=True)
def callback(index, average_error, cv_num_samples, cv_num_minibatches):
print('Epoch:{}, Validation Error: {}'.format(index, average_error))
return True
checkpoint_config = CheckpointConfig(frequency=epoch_size,
filename=os.path.join(
model_path, "resnet34_cp"),
restore=restore)
# test_config = TestConfig(minibatch_source=test_source, minibatch_size=minibatch_size)
validation_config = CrossValidationConfig(
minibatch_source=validation_source,
frequency=epoch_size,
minibatch_size=minibatch_size,
callback=callback)
start = time.time()
training_session(
trainer=trainer,
mb_source=train_source,
model_inputs_to_streams=input_map,
mb_size=minibatch_size,
progress_frequency=epoch_size,
checkpoint_config=checkpoint_config,
# test_config=test_config,
cv_config=validation_config).train()
end = time.time()
print('The Network took {} secs to train.'.format(end - start))
print('Saving the model here {}.'.format(
os.path.join(model_path, model_name)))
# save the model
trainer.model.save(os.path.join(model_path, model_name))
if profiling:
stop_profiler()
def train_and_test(s2smodel, train_reader, test_reader, block_size,
num_quantization_bits, max_epochs, epoch_size,
minibatch_size, progress_printer, warm_up):
from Sequence2Sequence import create_criterion_function, create_model_train
model_train = create_model_train(s2smodel)
criterion = create_criterion_function(model_train)
# Create learner
if block_size is not None and num_quantization_bits != default_quantization_bits:
raise RuntimeError(
"Block momentum cannot be used with quantization, please remove quantized_bits option."
)
lr = 0.001 if use_attention else 0.005 # TODO: can we use the same value for both?
local_learner = fsadagrad(
model_train.parameters,
lr=learning_rate_schedule([lr] * 2 + [lr / 2] * 3 + [lr / 4],
UnitType.sample, epoch_size),
momentum=momentum_as_time_constant_schedule(1100),
gradient_clipping_threshold_per_sample=2.3,
gradient_clipping_with_truncation=True)
if block_size != None:
learner = block_momentum_distributed_learner(local_learner,
block_size=block_size)
else:
learner = data_parallel_distributed_learner(
local_learner,
num_quantization_bits=num_quantization_bits,
distributed_after=warm_up)
trainer = Trainer(None, criterion, learner, progress_printer)
train_bind = {
criterion.arguments[0]: train_reader.streams.features,
criterion.arguments[1]: train_reader.streams.labels
}
training_session(
mb_source=train_reader,
trainer=trainer,
model_inputs_to_streams=train_bind,
mb_size=minibatch_size,
progress_frequency=epoch_size,
checkpoint_config=CheckpointConfig(frequency=epoch_size,
filename=os.path.join(
model_path,
"SequenceToSequence"),
restore=False),
cv_config=CrossValidationConfig(source=test_reader,
mb_size=minibatch_size)).train()
def train_model(network, trainer, train_source, test_source, validation_source, minibatch_size,
epoch_size, restore, profiling=False):
''' Train the model '''
print('Training the model.')
# define mapping from intput streams to network inputs
input_map = {
network['feature']: train_source.streams.features,
network['label']: train_source.streams.labels
}
# Enable profiling
if profiling:
start_profiler(sync_gpu=True)
# Callback functionn called after each epoch
def callback(index, average_error, cv_num_samples, cv_num_minibatches):
print('Epoch:{}, Validation Error: {}'.format(index, average_error))
return True
# Create multiple checkpoints for your model, trainer and minibatches so that you can recover in case of failure.
checkpoint_config = CheckpointConfig(frequency=epoch_size * 5, filename=os.path.join(OUTPUTDIR, "resnet152_cp"), restore=restore)
# test_config = TestConfig(minibatch_source=test_source, minibatch_size=minibatch_size)
validation_config = CrossValidationConfig(
minibatch_source=validation_source,
frequency=epoch_size,
minibatch_size=minibatch_size,
callback=callback)
start = time.time()
# Train the model
training_session(
trainer=trainer,
mb_source=train_source,
model_inputs_to_streams=input_map,
mb_size=minibatch_size,
progress_frequency=epoch_size,
checkpoint_config=checkpoint_config,
# test_config=test_config,
cv_config=validation_config
).train()
end = time.time()
print('The Network took {} secs to train.'.format(end - start))
print('Saving the model here {}.'.format(os.path.join(OUTPUTDIR, model_name)))
# save the model
trainer.model.save(os.path.join(OUTPUTDIR, model_name))
if profiling:
stop_profiler()
def main(params):
# Create output and log directories if they don't exist
if not os.path.isdir(params['output_folder']):
os.makedirs(params['output_folder'])
if not os.path.isdir(params['log_folder']):
os.makedirs(params['log_folder'])
# Create the network
network = create_network()
# Create readers
train_reader = cbf_reader(os.path.join(params['input_folder'], 'train{}.cbf'.format(params['prefix'])), is_training=True,
max_samples=cntk.io.INFINITELY_REPEAT)
cv_reader = cbf_reader(os.path.join(params['input_folder'], 'test{}.cbf'.format(params['prefix'])), is_training=False,
max_samples=cntk.io.FULL_DATA_SWEEP)
test_reader = cbf_reader(os.path.join(params['input_folder'], 'test{}.cbf'.format(params['prefix'])), is_training=False,
max_samples=cntk.io.FULL_DATA_SWEEP)
input_map = {
network['input']: train_reader.streams.front,
network['target']: train_reader.streams.label
}
# Create learner
mm_schedule = momentum_schedule(0.90)
lr_schedule = learning_parameter_schedule([(40, 0.1), (40, 0.01)], minibatch_size=params['minibatch_size'])
learner = cntk.adam(network['model'].parameters, lr_schedule, mm_schedule, l2_regularization_weight=0.0005,
epoch_size=params['epoch_size'], minibatch_size=params['minibatch_size'])
# Use TensorBoard for visual logging
log_file = os.path.join(params['log_folder'], 'log.txt')
pp_writer = cntk.logging.ProgressPrinter(freq=10, tag='Training', num_epochs=params['max_epochs'], log_to_file=log_file)
tb_writer = cntk.logging.TensorBoardProgressWriter(freq=10, log_dir=params['log_folder'], model=network['model'])
# Create trainer and training session
trainer = Trainer(network['model'], (network['loss'], network['metric']), [learner], [pp_writer, tb_writer])
test_config = TestConfig(minibatch_source=test_reader, minibatch_size=params['minibatch_size'], model_inputs_to_streams=input_map)
cv_config = CrossValidationConfig(minibatch_source=cv_reader, frequency=(1, DataUnit.sweep),
minibatch_size=params['minibatch_size'], model_inputs_to_streams=input_map)
checkpoint_config = CheckpointConfig(os.path.join(params['output_folder'], model_name), frequency=(10, DataUnit.sweep), restore=params['restore'])
session = training_session(trainer=trainer,
mb_source=train_reader,
mb_size=params['minibatch_size'],
model_inputs_to_streams=input_map,
max_samples=params['epoch_size'] * params['max_epochs'],
progress_frequency=(1, DataUnit.sweep),
checkpoint_config=checkpoint_config,
cv_config=cv_config,
test_config=test_config)
cntk.logging.log_number_of_parameters(network['model'])
session.train()
# Save the trained model
path = os.path.join(params['output_folder'], 'final_model.dnn')
network['model'].save(path)
print('Saved final model to', path)
def train_and_test(network, trainer, train_source, test_source, minibatch_size, epoch_size, restore):
# define mapping from intput streams to network inputs
input_map = {
network['feature']: train_source.streams.features,
network['label']: train_source.streams.labels
}
# Train all minibatches
training_session(
trainer=trainer, mb_source = train_source,
model_inputs_to_streams = input_map,
mb_size = minibatch_size,
progress_frequency=epoch_size,
checkpoint_config = CheckpointConfig(filename = os.path.join(model_path, model_name), restore=restore),
test_config = TestConfig(source=test_source, mb_size=minibatch_size)
).train()
def train_and_test(network, trainer, train_source, test_source, minibatch_size, epoch_size, restore):
# define mapping from intput streams to network inputs
input_map = {
network['feature']: train_source.streams.features,
network['label']: train_source.streams.labels
}
# Train all minibatches
training_session(
trainer=trainer, mb_source = train_source,
model_inputs_to_streams = input_map,
mb_size = minibatch_size,
progress_frequency=epoch_size,
checkpoint_config = CheckpointConfig(filename=os.path.join(model_path, model_name), restore=restore),
test_config= TestConfig(source=test_source, mb_size=minibatch_size)
).train()
def train_and_test(network, trainer, train_source, test_source, minibatch_size, epoch_size):
input_map = {
network['feature']: train_source.streams.amazing_features,
network['label']: train_source.streams.awesome_labels
}
training_session(
trainer=trainer,
mb_source = train_source,
model_inputs_to_streams = input_map,
mb_size = minibatch_size,
progress_frequency=epoch_size,
checkpoint_config = CheckpointConfig(frequency = epoch_size,
filename = os.path.join(model_path, "HKT_LSTM_Truncated"),
restore = False),
cv_config = CrossValidationConfig(source=test_source, mb_size=minibatch_size)
).train()
def train_and_test(network, trainer, train_reader, test_reader, epoch_size, minibatch_size):
train_bind = {
network['raw_input'] : train_reader.streams.features,
network['raw_labels'] : train_reader.streams.labels
}
training_session(
mb_source = train_reader,
trainer=trainer,
model_inputs_to_streams=train_bind,
mb_size=minibatch_size,
progress_frequency=epoch_size,
checkpoint_config=CheckpointConfig(frequency = epoch_size,
filename = os.path.join(model_path, "SequenceToSequence"),
restore = False),
cv_config=CrossValidationConfig(source=test_reader, mb_size=minibatch_size)
).train()
def train_and_test(s2smodel, train_reader, test_reader, block_size, num_quantization_bits, max_epochs, epoch_size, minibatch_size, progress_printer, warm_up):
from Sequence2Sequence import create_criterion_function, create_model_train
model_train = create_model_train(s2smodel)
criterion = create_criterion_function(model_train)
# Create learner
if block_size is not None and num_quantization_bits != default_quantization_bits:
raise RuntimeError("Block momentum cannot be used with quantization, please remove quantized_bits option.")
lr = 0.001 if use_attention else 0.005 # TODO: can we use the same value for both?
local_learner = fsadagrad(model_train.parameters,
lr = learning_rate_schedule([lr]*2+[lr/2]*3+[lr/4], UnitType.sample, epoch_size),
momentum = momentum_as_time_constant_schedule(1100),
gradient_clipping_threshold_per_sample=2.3,
gradient_clipping_with_truncation=True)
if block_size != None:
learner = block_momentum_distributed_learner(local_learner, block_size=block_size)
else:
learner = data_parallel_distributed_learner(local_learner, num_quantization_bits=num_quantization_bits, distributed_after=warm_up)
trainer = Trainer(None, criterion, learner, progress_printer)
train_bind = {criterion.arguments[0]: train_reader.streams.features,
criterion.arguments[1]: train_reader.streams.labels}
training_session(
mb_source = train_reader,
trainer=trainer,
model_inputs_to_streams=train_bind,
mb_size=minibatch_size,
progress_frequency=epoch_size,
checkpoint_config=CheckpointConfig(frequency = epoch_size,
filename = os.path.join(model_path, "SequenceToSequence"),
restore = False),
cv_config=CrossValidationConfig(source=test_reader, mb_size=minibatch_size)
).train()
def simple_mnist(tensorboard_logdir=None):
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
feature = input(input_dim, np.float32)
label = input(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), feature)
z = fully_connected_classifier_net(scaled_input, num_output_classes,
hidden_layers_dim, num_hidden_layers,
relu)
ce = cross_entropy_with_softmax(z, label)
pe = classification_error(z, label)
data_dir = os.path.join(abs_path, "..", "..", "..", "DataSets", "MNIST")
path = os.path.normpath(os.path.join(data_dir,
"Train-28x28_cntk_text.txt"))
check_path(path)
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
feature: reader_train.streams.features,
label: reader_train.streams.labels
}
# Training config
minibatch_size = 64
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 10
# Instantiate progress writers.
#training_progress_output_freq = 100
progress_writers = [
ProgressPrinter(
#freq=training_progress_output_freq,
tag='Training',
num_epochs=num_sweeps_to_train_with)
]
if tensorboard_logdir is not None:
progress_writers.append(
TensorBoardProgressWriter(freq=10,
log_dir=tensorboard_logdir,
model=z))
# Instantiate the trainer object to drive the model training
trainer = Trainer(z, (ce, pe), adadelta(z.parameters), progress_writers)
training_session(trainer=trainer,
mb_source=reader_train,
mb_size=minibatch_size,
var_to_stream=input_map,
max_samples=num_samples_per_sweep *
num_sweeps_to_train_with,
progress_frequency=num_samples_per_sweep).train()
# Load test data
path = os.path.normpath(os.path.join(data_dir, "Test-28x28_cntk_text.txt"))
check_path(path)
reader_test = create_reader(path, False, input_dim, num_output_classes)
input_map = {
feature: reader_test.streams.features,
label: reader_test.streams.labels
}
# Test data for trained model
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size,
input_map=input_map)
eval_error = trainer.test_minibatch(mb)
test_result = test_result + eval_error
# Average of evaluation errors of all test minibatches
return test_result / num_minibatches_to_test
def simple_mnist(tensorboard_logdir=None):
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
feature = C.input_variable(input_dim, np.float32)
label = C.input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), feature)
z = Sequential([For(range(num_hidden_layers), lambda i: Dense(hidden_layers_dim, activation=relu)),
Dense(num_output_classes)])(scaled_input)
ce = cross_entropy_with_softmax(z, label)
pe = classification_error(z, label)
data_dir = os.path.join(abs_path, "..", "..", "..", "DataSets", "MNIST")
path = os.path.normpath(os.path.join(data_dir, "Train-28x28_cntk_text.txt"))
check_path(path)
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
feature : reader_train.streams.features,
label : reader_train.streams.labels
}
# Training config
minibatch_size = 64
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 10
# Instantiate progress writers.
#training_progress_output_freq = 100
progress_writers = [ProgressPrinter(
#freq=training_progress_output_freq,
tag='Training',
num_epochs=num_sweeps_to_train_with)]
if tensorboard_logdir is not None:
progress_writers.append(TensorBoardProgressWriter(freq=10, log_dir=tensorboard_logdir, model=z))
# Instantiate the trainer object to drive the model training
lr = learning_parameter_schedule_per_sample(1)
trainer = Trainer(z, (ce, pe), adadelta(z.parameters, lr), progress_writers)
training_session(
trainer=trainer,
mb_source = reader_train,
mb_size = minibatch_size,
model_inputs_to_streams = input_map,
max_samples = num_samples_per_sweep * num_sweeps_to_train_with,
progress_frequency=num_samples_per_sweep
).train()
# Load test data
path = os.path.normpath(os.path.join(data_dir, "Test-28x28_cntk_text.txt"))
check_path(path)
reader_test = create_reader(path, False, input_dim, num_output_classes)
input_map = {
feature : reader_test.streams.features,
label : reader_test.streams.labels
}
# Test data for trained model
C.debugging.start_profiler()
C.debugging.enable_profiler()
C.debugging.set_node_timing(True)
#C.cntk_py.disable_cpueval_optimization() # uncomment this to check CPU eval perf without optimization
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size, input_map=input_map)
eval_error = trainer.test_minibatch(mb)
test_result = test_result + eval_error
C.debugging.stop_profiler()
trainer.print_node_timing()
# Average of evaluation errors of all test minibatches
return test_result / num_minibatches_to_test
def train_test(train_reader, test_reader, model_func, epoch_size):
# Instantiate the model function; x is the input (feature) variable
# We will scale the input image pixels within 0-1 range by dividing all input value by 255.
model = model_func(x / 255)
# Instantiate the loss and error function
loss, label_error = create_criterion_function(model, y)
# Instantiate the trainer object to drive the model training
learning_rate = 0.2
lr_schedule = C.learning_rate_schedule(learning_rate, C.UnitType.minibatch)
learner = C.sgd(z.parameters, lr_schedule)
# use a distributed learner for multi-GPU training
# either a data_parallel learner
distributed_learner = C.distributed.data_parallel_distributed_learner(
learner=learner, num_quantization_bits=1)
# or a block momemtum learner
# distributed_learner = C.train.distributed.block_momentum_distributed_learner(learner, block_size=block_size)
distributed_sync_report_freq = None
#if block_size is not None:
# distributed_sync_report_freq = 1
progress_writers = [
C.logging.ProgressPrinter(
freq=None,
tag='Training',
log_to_file=None,
rank=C.train.distributed.Communicator.rank(),
gen_heartbeat=False,
num_epochs=5,
distributed_freq=distributed_sync_report_freq)
]
trainer = C.Trainer(z, (loss, label_error), [distributed_learner],
progress_writers)
# Initialize the parameters for the trainer
minibatch_size = 20000
# Map the data streams to the input and labels.
input_map = {
y: train_reader.streams.labels,
x: train_reader.streams.features
}
# Uncomment below for more detailed logging
training_progress_output_freq = 500
# Start a timer
start = time.time()
training_session(trainer=trainer,
mb_source=train_reader,
model_inputs_to_streams=input_map,
mb_size=minibatch_size,
progress_frequency=epoch_size,
test_config=TestConfig(source=test_reader,
mb_size=minibatch_size)).train()
# Print training time
print("Training took {:.1f} sec".format(time.time() - start))
def simple_mnist():
input_dim = 784
num_output_classes = 10
num_hidden_layers = 2
hidden_layers_dim = 200
# Input variables denoting the features and label data
feature = C.input_variable(input_dim)
label = C.input_variable(num_output_classes)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), feature)
# z = Sequential([
# Dense(hidden_layers_dim, activation=relu),
# Dense(hidden_layers_dim, activation=relu),
# Dense(num_output_classes)])(scaled_input)
with default_options(activation=relu, init=C.glorot_uniform()):
z = Sequential([For(range(num_hidden_layers),
lambda i: Dense(hidden_layers_dim)),
Dense(num_output_classes, activation=None)])(scaled_input)
ce = cross_entropy_with_softmax(z, label)
pe = classification_error(z, label)
# setup the data
path = abs_path + "\Train-28x28_cntk_text.txt"
reader_train = MinibatchSource(CTFDeserializer(path, StreamDefs(
features=StreamDef(field='features', shape=input_dim),
labels=StreamDef(field='labels', shape=num_output_classes))))
input_map = {
feature: reader_train.streams.features,
label: reader_train.streams.labels
}
# Training config
minibatch_size = 64
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 10
# Instantiate progress writers.
progress_writers = [ProgressPrinter(
tag='Training',
num_epochs=num_sweeps_to_train_with)]
# Instantiate the trainer object to drive the model training
lr = learning_rate_schedule(1, UnitType.sample)
trainer = Trainer(z, (ce, pe), [adadelta(z.parameters, lr)], progress_writers)
training_session(
trainer=trainer,
mb_source=reader_train,
mb_size=minibatch_size,
model_inputs_to_streams=input_map,
max_samples=num_samples_per_sweep * num_sweeps_to_train_with,
progress_frequency=num_samples_per_sweep
).train()
# Load test data
path = abs_path + "\Test-28x28_cntk_text.txt"
reader_test = MinibatchSource(CTFDeserializer(path, StreamDefs(
features=StreamDef(field='features', shape=input_dim),
labels=StreamDef(field='labels', shape=num_output_classes))))
input_map = {
feature: reader_test.streams.features,
label: reader_test.streams.labels
}
# Test data for trained model
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size, input_map=input_map)
eval_error = trainer.test_minibatch(mb)
test_result = test_result + eval_error
# Average of evaluation errors of all test minibatches
return test_result / num_minibatches_to_test
def simple_mnist(tensorboard_logdir=None):
input_dim = 784
num_output_classes = 10
num_hidden_layers = 2
hidden_layers_dim = 200
# Input variables denoting the features and label data
feature = C.input_variable(input_dim, np.float32)
label = C.input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), feature)
z = Sequential([
For(range(num_hidden_layers),
lambda i: Dense(hidden_layers_dim, activation=relu)),
Dense(num_output_classes)
])(scaled_input)
ce = cross_entropy_with_softmax(z, label)
pe = classification_error(z, label)
data_dir = os.path.dirname(os.path.abspath(__file__))
path = os.path.join(data_dir, 'Train-28x28_cntk_text.txt')
reader_train = create_reader(path, True, input_dim, num_output_classes)
input_map = {
feature: reader_train.streams.features,
label: reader_train.streams.labels
}
# Training config
minibatch_size = 64
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 10
# Instantiate progress writers.
# training_progress_output_freq = 100
progress_writers = [
ProgressPrinter(
# freq=training_progress_output_freq,
tag='Training',
num_epochs=num_sweeps_to_train_with)
]
if tensorboard_logdir is not None:
progress_writers.append(
TensorBoardProgressWriter(freq=10,
log_dir=tensorboard_logdir,
model=z))
# Instantiate the trainer object to drive the model training
lr = 0.001
trainer = Trainer(z, (ce, pe), sgd(z.parameters, lr), progress_writers)
training_session(trainer=trainer,
mb_source=reader_train,
mb_size=minibatch_size,
model_inputs_to_streams=input_map,
max_samples=num_samples_per_sweep *
num_sweeps_to_train_with,
progress_frequency=num_samples_per_sweep).train()
# Load test data
path = os.path.normpath(os.path.join(data_dir, "Test-28x28_cntk_text.txt"))
check_path(path)
reader_test = create_reader(path, False, input_dim, num_output_classes)
input_map = {
feature: reader_test.streams.features,
label: reader_test.streams.labels
}
# Test data for trained model
C.debugging.start_profiler()
C.debugging.enable_profiler()
C.debugging.set_node_timing(True)
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size,
input_map=input_map)
eval_error = trainer.test_minibatch(mb)
test_result = test_result + eval_error
C.debugging.stop_profiler()
trainer.print_node_timing()
# Average of evaluation errors of all test minibatches
return test_result * 100 / num_minibatches_to_test
def simple_mnist():
input_dim = 784
num_output_classes = 10
num_hidden_layers = 2
hidden_layers_dim = 200
# Input variables denoting the features and label data
feature = C.input_variable(input_dim)
label = C.input_variable(num_output_classes)
# Instantiate the feedforward classification model
scaled_input = element_times(constant(0.00390625), feature)
# z = Sequential([
# Dense(hidden_layers_dim, activation=relu),
# Dense(hidden_layers_dim, activation=relu),
# Dense(num_output_classes)])(scaled_input)
with default_options(activation=relu, init=C.glorot_uniform()):
z = Sequential([
For(range(num_hidden_layers), lambda i: Dense(hidden_layers_dim)),
Dense(num_output_classes, activation=None)
])(scaled_input)
ce = cross_entropy_with_softmax(z, label)
pe = classification_error(z, label)
# setup the data
path = abs_path + "\Train-28x28_cntk_text.txt"
reader_train = MinibatchSource(
CTFDeserializer(
path,
StreamDefs(features=StreamDef(field='features', shape=input_dim),
labels=StreamDef(field='labels',
shape=num_output_classes))))
input_map = {
feature: reader_train.streams.features,
label: reader_train.streams.labels
}
# Training config
minibatch_size = 64
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 10
# Instantiate progress writers.
progress_writers = [
ProgressPrinter(tag='Training', num_epochs=num_sweeps_to_train_with)
]
# Instantiate the trainer object to drive the model training
lr = learning_rate_schedule(1, UnitType.sample)
trainer = Trainer(z, (ce, pe), [adadelta(z.parameters, lr)],
progress_writers)
training_session(trainer=trainer,
mb_source=reader_train,
mb_size=minibatch_size,
model_inputs_to_streams=input_map,
max_samples=num_samples_per_sweep *
num_sweeps_to_train_with,
progress_frequency=num_samples_per_sweep).train()
# Load test data
path = abs_path + "\Test-28x28_cntk_text.txt"
reader_test = MinibatchSource(
CTFDeserializer(
path,
StreamDefs(features=StreamDef(field='features', shape=input_dim),
labels=StreamDef(field='labels',
shape=num_output_classes))))
input_map = {
feature: reader_test.streams.features,
label: reader_test.streams.labels
}
# Test data for trained model
test_minibatch_size = 1024
num_samples = 10000
num_minibatches_to_test = num_samples / test_minibatch_size
test_result = 0.0
for i in range(0, int(num_minibatches_to_test)):
mb = reader_test.next_minibatch(test_minibatch_size,
input_map=input_map)
eval_error = trainer.test_minibatch(mb)
test_result = test_result + eval_error
# Average of evaluation errors of all test minibatches
return test_result / num_minibatches_to_test
