def train_sequence_classifier():
input_dim = 2000
cell_dim = 25
hidden_dim = 25
embedding_dim = 50
num_output_classes = 5
# Input variables denoting the features and label data
features = input_variable(shape=input_dim, is_sparse=True)
label = input_variable(num_output_classes,
dynamic_axes=[Axis.default_batch_axis()])
# Instantiate the sequence classification model
classifier_output = LSTM_sequence_classifer_net(features,
num_output_classes,
embedding_dim, hidden_dim,
cell_dim)
ce = cross_entropy_with_softmax(classifier_output, label)
pe = classification_error(classifier_output, label)
rel_path = r"../../../../Tests/EndToEndTests/Text/SequenceClassification/Data/Train.ctf"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
feature_stream_name = 'features'
labels_stream_name = 'labels'
mb_source = text_format_minibatch_source(path, [
StreamConfiguration(feature_stream_name, input_dim, True, 'x'),
StreamConfiguration(labels_stream_name, num_output_classes, False, 'y')
], 0)
features_si = mb_source.stream_info(features)
labels_si = mb_source.stream_info(label)
# Instantiate the trainer object to drive the model training
lr = lr = learning_rates_per_sample(0.0005)
trainer = Trainer(classifier_output, ce, pe,
[sgd_learner(classifier_output.owner.parameters(), lr)])
# Get minibatches of sequences to train with and perform model training
minibatch_size = 200
training_progress_output_freq = 10
i = 0
while True:
mb = mb_source.get_next_minibatch(minibatch_size)
if len(mb) == 0:
break
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
arguments = {
features: mb[features_si].m_data,
label: mb[labels_si].m_data
}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
i += 1
def cifar_resnet():
dev = 0
cntk_dev = cntk_device(dev)
epoch_size = sys.maxsize
mbs = create_mb_source(epoch_size)
stream_infos = mbs.stream_infos()
for si in stream_infos:
if si.m_name == 'features':
features_si = si
elif si.m_name == 'labels':
labels_si = si
image_shape = features_si.m_sample_layout.dimensions()
image_shape = (image_shape[2], image_shape[0], image_shape[1])
num_classes = labels_si.m_sample_layout.dimensions()[0]
image_input = variable(image_shape,
features_si.m_element_type,
needs_gradient=False,
name="Images")
classifier_output = resnet_classifer(image_input, num_classes, dev,
"classifierOutput")
label_var = variable((num_classes),
features_si.m_element_type,
needs_gradient=False,
name="Labels")
ce = cross_entropy_with_softmax(classifier_output, label_var)
pe = classification_error(classifier_output, label_var)
#TODO: add save and load module code
image_classifier = combine([ce, pe, classifier_output], "ImageClassifier")
lr = learning_rates_per_sample(0.0078125)
mb_size = 32
num_mbs = 1000
trainer = Trainer(classifier_output, ce, pe,
[sgdlearner(classifier_output.owner.parameters(), lr)])
for i in range(0, num_mbs):
mb = mbs.get_next_minibatch(mb_size, cntk_dev)
arguments = dict()
arguments[image_input] = mb[features_si].m_data
arguments[label_var] = mb[labels_si].m_data
trainer.train_minibatch(arguments, cntk_dev)
freq = 20
if i % freq == 0:
training_loss = get_train_loss(trainer)
eval_crit = get_train_eval_criterion(trainer)
print(
"Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}"
.format(i, training_loss, eval_crit))
def simple_mnist():
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
input = input_variable(input_dim, np.float32)
label = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant((), 0.00390625), input)
netout = fully_connected_classifier_net(scaled_input, num_output_classes,
hidden_layers_dim,
num_hidden_layers, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
rel_path = r"../../../../Examples/Image/MNIST/Data/Train-28x28_cntk_text.txt"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
feature_stream_name = 'features'
labels_stream_name = 'labels'
mb_source = text_format_minibatch_source(path, [
StreamConfiguration(feature_stream_name, input_dim),
StreamConfiguration(labels_stream_name, num_output_classes)
])
features_si = mb_source.stream_info(feature_stream_name)
labels_si = mb_source.stream_info(labels_stream_name)
# Instantiate the trainer object to drive the model training
lr = learning_rates_per_sample(0.003125)
trainer = Trainer(netout, ce, pe,
[sgd_learner(netout.owner.parameters(), lr)])
# Get minibatches of images to train with and perform model training
minibatch_size = 32
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 1
num_minibatches_to_train = (num_samples_per_sweep *
num_sweeps_to_train_with) / minibatch_size
training_progress_output_freq = 20
for i in range(0, int(num_minibatches_to_train)):
mb = mb_source.get_next_minibatch(minibatch_size)
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
arguments = {
input: mb[features_si].m_data,
label: mb[labels_si].m_data
}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
def ffnet():
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
epoch_size = sys.maxsize
minibatch_size = 25
num_samples_per_sweep = 10000
num_sweeps_to_train_with = 2
num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size
lr = learning_rates_per_sample(0.02)
input = variable((input_dim,), np.float32, needs_gradient=False, name="features")
label = variable((num_output_classes,), np.float32, needs_gradient=False, name="labels")
dev = -1
cntk_dev = cntk_device(dev)
netout = fully_connected_classifier_net(input, num_output_classes, hidden_layers_dim, num_hidden_layers, dev, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
#TODO: add save and load module code
ffnet = combine([ce, pe, netout], "classifier_model")
rel_path = r"../../../../Examples/Other/Simple2d/Data/SimpleDataTrain_cntk_text.txt"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
cm = create_text_mb_source(path, input_dim, num_output_classes, epoch_size)
stream_infos = cm.stream_infos()
for si in stream_infos:
if si.m_name == 'features':
features_si = si
elif si.m_name == 'labels':
labels_si = si
trainer = Trainer(netout, ce, pe, [sgdlearner(netout.owner.parameters(), lr)])
for i in range(0,int(num_minibatches_to_train)):
mb=cm.get_next_minibatch(minibatch_size, cntk_dev)
arguments = dict()
arguments[input] = mb[features_si].m_data
arguments[label] = mb[labels_si].m_data
trainer.train_minibatch(arguments, cntk_dev)
freq = 20
if i % freq == 0:
training_loss = get_train_loss(trainer)
eval_crit = get_train_eval_criterion(trainer)
print ("Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}".format(i, training_loss, eval_crit))
def simple_mnist():
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
# Input variables denoting the features and label data
input = input_variable(input_dim, np.float32)
label = input_variable(num_output_classes, np.float32)
# Instantiate the feedforward classification model
scaled_input = element_times(constant((), 0.00390625), input)
netout = fully_connected_classifier_net(scaled_input, num_output_classes, hidden_layers_dim, num_hidden_layers, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
rel_path = os.path.join(*"../../../../Examples/Image/MNIST/Data/Train-28x28_cntk_text.txt".split("/"))
path = os.path.normpath(os.path.join(abs_path, rel_path))
if not os.path.exists(path):
readme_file = os.path.normpath(os.path.join(os.path.dirname(path), "..", "README.md"))
raise RuntimeError("File '%s' does not exist. Please follow the instructions at %s to download and prepare it."%(path, readme_file))
feature_stream_name = 'features'
labels_stream_name = 'labels'
mb_source = text_format_minibatch_source(path, [
StreamConfiguration( feature_stream_name, input_dim ),
StreamConfiguration( labels_stream_name, num_output_classes) ])
features_si = mb_source.stream_info(feature_stream_name)
labels_si = mb_source.stream_info(labels_stream_name)
# Instantiate the trainer object to drive the model training
lr = learning_rates_per_sample(0.003125)
trainer = Trainer(netout, ce, pe, [sgd_learner(netout.owner.parameters(), lr)])
# Get minibatches of images to train with and perform model training
minibatch_size = 32
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 1
num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size
training_progress_output_freq = 20
for i in range(0, int(num_minibatches_to_train)):
mb = mb_source.get_next_minibatch(minibatch_size)
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
arguments = {input : mb[features_si].m_data, label : mb[labels_si].m_data}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
def train_sequence_classifier():
input_dim = 2000;
cell_dim = 25;
hidden_dim = 25;
embedding_dim = 50;
num_output_classes = 5;
# Input variables denoting the features and label data
features = input_variable(shape=input_dim, is_sparse=True)
label = input_variable(num_output_classes, dynamic_axes = [Axis.default_batch_axis()])
# Instantiate the sequence classification model
classifier_output = LSTM_sequence_classifer_net(features, num_output_classes, embedding_dim, hidden_dim, cell_dim)
ce = cross_entropy_with_softmax(classifier_output, label)
pe = classification_error(classifier_output, label)
rel_path = r"../../../../Tests/EndToEndTests/Text/SequenceClassification/Data/Train.ctf"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
feature_stream_name = 'features'
labels_stream_name = 'labels'
mb_source = text_format_minibatch_source(path, [
StreamConfiguration( feature_stream_name, input_dim, True, 'x' ),
StreamConfiguration( labels_stream_name, num_output_classes, False, 'y')], 0)
features_si = mb_source.stream_info(features)
labels_si = mb_source.stream_info(label)
# Instantiate the trainer object to drive the model training
lr = lr = learning_rates_per_sample(0.0005)
trainer = Trainer(classifier_output, ce, pe, [sgd_learner(classifier_output.owner.parameters(), lr)])
# Get minibatches of sequences to train with and perform model training
minibatch_size = 200
training_progress_output_freq = 10
i = 0;
while True:
mb = mb_source.get_next_minibatch(minibatch_size)
if len(mb) == 0:
break
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
arguments = {features : mb[features_si].m_data, label : mb[labels_si].m_data}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
i += 1
def cifar_resnet():
image_height = 32
image_width = 32
num_channels = 3
num_classes = 10
feats_stream_name = 'features'
labels_stream_name = 'labels'
minibatch_source = create_mb_source(feats_stream_name, labels_stream_name,
image_height, image_width,
num_channels, num_classes)
features_si = minibatch_source.stream_info(feats_stream_name)
labels_si = minibatch_source.stream_info(labels_stream_name)
# Input variables denoting the features and label data
image_input = input_variable((num_channels, image_height, image_width),
features_si.m_element_type)
label_var = input_variable((num_classes), features_si.m_element_type)
# Instantiate the resnet classification model
classifier_output = resnet_classifer(image_input, num_classes)
ce = cross_entropy_with_softmax(classifier_output, label_var)
pe = classification_error(classifier_output, label_var)
# Instantiate the trainer object to drive the model training
lr = learning_rates_per_sample(0.0078125)
trainer = Trainer(classifier_output, ce, pe,
[sgd_learner(classifier_output.owner.parameters(), lr)])
# Get minibatches of images to train with and perform model training
mb_size = 32
training_progress_output_freq = 20
num_mbs = 1000
for i in range(0, num_mbs):
mb = minibatch_source.get_next_minibatch(mb_size)
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
arguments = {
image_input: mb[features_si].m_data,
label_var: mb[labels_si].m_data
}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
def cifar_resnet():
image_height = 32
image_width = 32
num_channels = 3
num_classes = 10
feats_stream_name = 'features'
labels_stream_name = 'labels'
minibatch_source = create_mb_source(feats_stream_name, labels_stream_name,
image_height, image_width, num_channels, num_classes)
features_si = minibatch_source.stream_info(feats_stream_name)
labels_si = minibatch_source.stream_info(labels_stream_name)
# Input variables denoting the features and label data
image_input = input_variable((num_channels, image_height, image_width), features_si.m_element_type)
label_var = input_variable((num_classes), features_si.m_element_type)
# Instantiate the resnet classification model
classifier_output = resnet_classifer(image_input, num_classes)
ce = cross_entropy_with_softmax(classifier_output, label_var)
pe = classification_error(classifier_output, label_var)
# Instantiate the trainer object to drive the model training
lr = learning_rates_per_sample(0.0078125)
trainer = Trainer(classifier_output, ce, pe, [sgd_learner(classifier_output.owner.parameters(), lr)])
# Get minibatches of images to train with and perform model training
mb_size = 32
training_progress_output_freq = 20
num_mbs = 1000
for i in range(0, num_mbs):
mb=minibatch_source.get_next_minibatch(mb_size)
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
arguments = {image_input : mb[features_si].m_data, label_var : mb[labels_si].m_data}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
def ffnet():
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
input = input_variable((input_dim), np.float32)
label = input_variable((num_output_classes), np.float32)
# Instantiate the feedforward classification model
netout = fully_connected_classifier_net(input, num_output_classes,
hidden_layers_dim,
num_hidden_layers, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
# Instantiate the trainer object to drive the model training
lr = learning_rates_per_sample(0.02)
trainer = Trainer(netout, ce, pe,
[sgd_learner(netout.owner.parameters(), lr)])
# Get minibatches of training data and perform model training
minibatch_size = 25
num_samples_per_sweep = 10000
num_sweeps_to_train_with = 2
num_minibatches_to_train = (num_samples_per_sweep *
num_sweeps_to_train_with) / minibatch_size
training_progress_output_freq = 20
for i in range(0, int(num_minibatches_to_train)):
features, labels = generate_random_data(minibatch_size, input_dim,
num_output_classes)
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
trainer.train_minibatch({input: features, label: labels})
print_training_progress(trainer, i, training_progress_output_freq)
def train_sequence_to_sequence_translator():
input_vocab_dim = 69
label_vocab_dim = 69
hidden_dim = 512
num_layers = 2
# Source and target inputs to the model
input_dynamic_axes = [ Axis('inputAxis'), Axis.default_batch_axis() ]
raw_input = input_variable(shape=(input_vocab_dim), dynamic_axes = input_dynamic_axes)
label_dynamic_axes = [ Axis('labelAxis'), Axis.default_batch_axis() ]
raw_labels = input_variable(shape=(label_vocab_dim), dynamic_axes = label_dynamic_axes)
# Instantiate the sequence to sequence translation model
input_sequence = raw_input
# Drop the sentence start token from the label, for decoder training
label_sequence = slice(raw_labels, label_dynamic_axes[0], 1, 0)
label_sentence_start = sequence.first(raw_labels)
is_first_label = sequence.is_first(label_sequence)
label_sentence_start_scattered = sequence.scatter(label_sentence_start, is_first_label)
# Encoder
encoder_outputH = stabilize(input_sequence)
for i in range(0, num_layers):
(encoder_outputH, encoder_outputC) = LSTMP_component_with_self_stabilization(encoder_outputH, hidden_dim, hidden_dim, future_value, future_value)
thought_vectorH = sequence.first(encoder_outputH)
thought_vectorC = sequence.first(encoder_outputC)
thought_vector_broadcastH = sequence.broadcast_as(thought_vectorH, label_sequence)
thought_vector_broadcastC = sequence.broadcast_as(thought_vectorC, label_sequence)
# Decoder
decoder_history_from_ground_truth = label_sequence
decoder_input = element_select(is_first_label, label_sentence_start_scattered, past_value(decoder_history_from_ground_truth))
decoder_outputH = stabilize(decoder_input)
for i in range(0, num_layers):
if (i == 0):
recurrence_hookH = past_value
recurrence_hookC = past_value
else:
isFirst = sequence.is_first(label_sequence)
recurrence_hookH = lambda operand: element_select(isFirst, thought_vector_broadcastH, past_value(operand))
recurrence_hookC = lambda operand: element_select(isFirst, thought_vector_broadcastC, past_value(operand))
(decoder_outputH, encoder_outputC) = LSTMP_component_with_self_stabilization(decoder_outputH, hidden_dim, hidden_dim, recurrence_hookH, recurrence_hookC)
decoder_output = decoder_outputH
decoder_dim = hidden_dim
# Softmax output layer
z = linear_layer(stabilize(decoder_output), label_vocab_dim)
ce = cross_entropy_with_softmax(z, label_sequence)
errs = classification_error(z, label_sequence)
rel_path = r"../../../../Examples/SequenceToSequence/CMUDict/Data/cmudict-0.7b.train-dev-20-21.ctf"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
feature_stream_name = 'features'
labels_stream_name = 'labels'
mb_source = text_format_minibatch_source(path, [
StreamConfiguration( feature_stream_name, input_vocab_dim, True, 'S0' ),
StreamConfiguration( labels_stream_name, label_vocab_dim, True, 'S1') ], 10000)
features_si = mb_source.stream_info(feature_stream_name)
labels_si = mb_source.stream_info(labels_stream_name)
# Instantiate the trainer object to drive the model training
lr = learning_rates_per_sample(0.007)
momentum_time_constant = 1100
momentum_per_sample = momentums_per_sample(math.exp(-1.0 / momentum_time_constant))
clipping_threshold_per_sample = 2.3
gradient_clipping_with_truncation = True
trainer = Trainer(z, ce, errs, [momentum_sgd_learner(z.owner.parameters(), lr, momentum_per_sample, clipping_threshold_per_sample, gradient_clipping_with_truncation)])
# Get minibatches of sequences to train with and perform model training
minibatch_size = 72
training_progress_output_freq = 10
while True:
mb = mb_source.get_next_minibatch(minibatch_size)
if len(mb) == 0:
break
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
arguments = {raw_input : mb[features_si].m_data, raw_labels : mb[labels_si].m_data}
trainer.train_minibatch(arguments)
print_training_progress(trainer, i, training_progress_output_freq)
i += 1
def train_sequence_classifier(device):
input_dim = 2000
cell_dim = 25
hidden_dim = 25
embedding_dim = 50
num_output_classes = 5
features = variable(shape=input_dim, is_sparse=True, name="features")
classifier_output = LSTM_sequence_classifer_net(features,
num_output_classes,
embedding_dim, hidden_dim,
cell_dim, device)
label = variable(num_output_classes,
dynamic_axes=[Axis.default_batch_axis()],
name="labels")
ce = cross_entropy_with_softmax(classifier_output, label)
pe = classification_error(classifier_output, label)
#TODO: add save and load module code
lstm_net = combine([ce, pe, classifier_output], "classifier_model")
rel_path = r"../../../../Tests/EndToEndTests/Text/SequenceClassification/Data/Train.ctf"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
cm = create_text_mb_source(path, input_dim, num_output_classes, 0, True,
False, "x", "y")
stream_infos = cm.stream_infos()
for si in stream_infos:
if si.m_name == 'features':
features_si = si
elif si.m_name == 'labels':
labels_si = si
minibatch_size = 200
lr = lr = learning_rates_per_sample(0.0005)
trainer = Trainer(classifier_output, ce, pe,
[sgdlearner(classifier_output.owner.parameters(), lr)])
freq = 1
i = 0
cntk_dev = cntk_device(device)
while True:
mb = cm.get_next_minibatch(minibatch_size, cntk_dev)
if len(mb) == 0:
break
arguments = dict()
arguments[features] = mb[features_si].m_data
arguments[label] = mb[labels_si].m_data
trainer.train_minibatch(arguments, cntk_dev)
if i % freq == 0:
training_loss = get_train_loss(trainer)
eval_crit = get_train_eval_criterion(trainer)
print(
"Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}"
.format(i, training_loss, eval_crit))
i += 1
