def build_graph(noise_shape, image_shape, G_progress_printer, D_progress_printer):
input_dynamic_axes = [C.Axis.default_batch_axis()]
Z = C.input_variable(noise_shape, dynamic_axes=input_dynamic_axes)
X_real = C.input_variable(image_shape, dynamic_axes=input_dynamic_axes)
X_real_scaled = 2*(X_real / 255.0) - 1.0
# Create the model function for the generator and discriminator models
X_fake = generator(Z)
D_real = discriminator(X_real_scaled)
D_fake = D_real.clone(
method = 'share',
substitutions = {X_real_scaled.output: X_fake.output}
)
# Create loss functions and configure optimazation algorithms
G_loss = 1.0 - C.log(D_fake)
D_loss = -(C.log(D_real) + C.log(1.0 - D_fake))
G_learner = C.fsadagrad(
parameters = X_fake.parameters,
lr = C.learning_parameter_schedule_per_sample(lr),
momentum = C.momentum_schedule_per_sample(0.9985724484938566)
)
D_learner = C.fsadagrad(
parameters = D_real.parameters,
lr = C.learning_parameter_schedule_per_sample(lr),
momentum = C.momentum_schedule_per_sample(0.9985724484938566)
)
DistG_learner = C.train.distributed.data_parallel_distributed_learner(G_learner)
# The following API marks a learner as the matric aggregator, which is used by
# the trainer to determine the training progress.
# It is required, only when more than one learner is provided to a *single* trainer.
# In this example, we use two trainers each with a single learner, so it
# is not required and automatically set by CNTK for each single learner. However, if you
# plan to use both learners with a single trainer, then it needs to be call before
# creating the trainer.
#DistG_learner.set_as_metric_aggregator()
DistD_learner = C.train.distributed.data_parallel_distributed_learner(D_learner)
# Instantiate the trainers
G_trainer = C.Trainer(
X_fake,
(G_loss, None),
DistG_learner,
G_progress_printer
)
D_trainer = C.Trainer(
D_real,
(D_loss, None),
DistD_learner,
D_progress_printer
)
return X_real, X_fake, Z, G_trainer, D_trainer
def train_lm(testing=False):
data = DataReader(token_to_id_path, segment_sepparator)
# Create model nodes for the source and target inputs
input_sequence, label_sequence = create_inputs(data.vocab_dim)
# Create the model. It has three output nodes
# z: the input to softmax that provides the latent representation of the next token
# cross_entropy: this is used training criterion
# error: this a binary indicator if the model predicts the correct token
z, cross_entropy, error = create_model(input_sequence, label_sequence, data.vocab_dim, hidden_dim)
# For measurement we use the (build in) full softmax.
full_ce = C.cross_entropy_with_softmax(z, label_sequence)
# print out some useful training information
log_number_of_parameters(z) ; print()
# Run the training loop
num_trained_samples = 0
num_trained_samples_since_last_report = 0
# Instantiate the trainer object to drive the model training
lr_schedule = C.learning_parameter_schedule_per_sample(learning_rate)
momentum_schedule = C.momentum_schedule_per_sample(momentum_per_sample)
gradient_clipping_with_truncation = True
learner = momentum_sgd(z.parameters, lr_schedule, momentum_schedule,
gradient_clipping_threshold_per_sample=clipping_threshold_per_sample,
gradient_clipping_with_truncation=gradient_clipping_with_truncation)
trainer = Trainer(z, (cross_entropy, error), learner)
last_avg_ce = 0
for epoch_count in range(num_epochs):
for features, labels, token_count in data.minibatch_generator(train_file_path, sequence_length, sequences_per_batch):
arguments = ({input_sequence : features, label_sequence : labels})
t_start = timeit.default_timer()
trainer.train_minibatch(arguments)
t_end = timeit.default_timer()
samples_per_second = token_count / (t_end - t_start)
# Print progress report every num_samples_between_progress_report samples
if num_trained_samples_since_last_report >= num_samples_between_progress_report or num_trained_samples == 0:
av_ce = average_cross_entropy(full_ce, input_sequence, label_sequence, data)
print_progress(samples_per_second, av_ce, num_trained_samples, t_start)
num_trained_samples_since_last_report = 0
last_avg_ce = av_ce
num_trained_samples += token_count
num_trained_samples_since_last_report += token_count
if not testing:
# after each epoch save the model
model_filename = "models/lm_epoch%d.dnn" % epoch_count
z.save(model_filename)
print("Saved model to '%s'" % model_filename)
return last_avg_ce
def create_trainer(self):
try:
p = self.output.parameters
# Three of four parameters are learned by block_momentum_distributed_learner.
bmd_learner = cntk.block_momentum_distributed_learner(
cntk.momentum_sgd(
[p[0], p[1], p[2]],
cntk.learning_parameter_schedule(0.0001),
cntk.momentum_as_time_constant_schedule(1000)),
block_size=1000,
block_learning_rate=0.01,
block_momentum_as_time_constant=1000)
# New API to mark which learner is to use for metric aggregaion.
bmd_learner.set_as_metric_aggregator()
# The last parameter is learned by the data_parallel_distributed_learner.
momentum_schedule = cntk.momentum_schedule_per_sample(
0.9990913221888589)
lr_per_sample = cntk.learning_parameter_schedule_per_sample(0.007)
dpd_learner = cntk.data_parallel_distributed_learner(
cntk.momentum_sgd([p[3]], lr_per_sample, momentum_schedule,
True))
comm_rank = cntk.distributed.Communicator.rank()
self.trainer = cntk.Trainer(
self.output, (self.ce, self.err), [bmd_learner, dpd_learner], [
cntk.logging.ProgressPrinter(
freq=progress_freq, tag="Training", rank=comm_rank)
])
except RuntimeError:
self.trainer = None
return
def test_htk_deserializers():
mbsize = 640
epoch_size = 1000 * mbsize
lr = [0.001]
feature_dim = 33
num_classes = 132
context = 2
os.chdir(data_path)
features_file = "glob_0000.scp"
labels_file = "glob_0000.mlf"
label_mapping_file = "state.list"
fd = HTKFeatureDeserializer(
StreamDefs(amazing_features=StreamDef(
shape=feature_dim, context=(context, context), scp=features_file)))
ld = HTKMLFDeserializer(
label_mapping_file,
StreamDefs(
awesome_labels=StreamDef(shape=num_classes, mlf=labels_file)))
reader = MinibatchSource([fd, ld])
features = C.sequence.input_variable(((2 * context + 1) * feature_dim))
labels = C.sequence.input_variable((num_classes))
model = Sequential(
[For(range(3), lambda: Recurrence(LSTM(256))),
Dense(num_classes)])
z = model(features)
ce = C.cross_entropy_with_softmax(z, labels)
errs = C.classification_error(z, labels)
learner = C.fsadagrad(
z.parameters,
lr=C.learning_parameter_schedule_per_sample(lr, epoch_size=epoch_size),
momentum=C.momentum_schedule_per_sample(0.9990913221888589),
gradient_clipping_threshold_per_sample=15,
gradient_clipping_with_truncation=True)
progress_printer = C.logging.ProgressPrinter(freq=0)
trainer = C.Trainer(z, (ce, errs), learner, progress_printer)
input_map = {
features: reader.streams.amazing_features,
labels: reader.streams.awesome_labels
}
# just run and verify it doesn't crash
for i in range(3):
mb_data = reader.next_minibatch(mbsize, input_map=input_map)
trainer.train_minibatch(mb_data)
assert True
os.chdir(abs_path)
def create_learner(model):
'''Create the optimized method'''
lr_per_minibatch = C.learning_parameter_schedule(opt.lr)
momentum_schedule = C.momentum_schedule_per_sample(0.9990913221888589)
if opt.optim == 'sgd':
return C.sgd(model.parameters, lr=lr_per_minibatch)
elif opt.optim == 'adam':
return C.adam(model.parameters, lr=lr_per_minibatch, momentum=momentum_schedule)
elif opt.optim == 'adagrad':
return C.adagrad(model.parameters, lr=lr_per_minibatch)
else:
raise RuntimeError("Invalid optim method: " + opt.optim)
def create_learner(model):
'''Create the optimized method'''
optim = "momentum_sgd"
lr = 0.001
lr_per_sample = C.learning_parameter_schedule_per_sample(lr)
momentum_schedule = C.momentum_schedule_per_sample(0.9990913221888589)
if optim == 'momentum_sgd':
clipping_threshold_per_sample = 5.0
gradient_clipping_with_truncation = True
return C.momentum_sgd(model.parameters, lr_per_sample, momentum_schedule,
gradient_clipping_threshold_per_sample=clipping_threshold_per_sample,
gradient_clipping_with_truncation=gradient_clipping_with_truncation)
def Loss(self):
# Evaluating old actions and values :
logprobs, state_value, dist_entropy = self.policy.evaluate()
# Finding the ratio (pi_theta / pi_theta__old): # (importance sampling)
c_old_logprobs = C.input_variable(logprobs.shape, name='old_log_probs')
ratios = C.exp(logprobs - C.stop_gradient(c_old_logprobs))
c_rewards = C.input_variable(1, name='rewards')
advantages = c_rewards - C.stop_gradient(state_value)
# Finding Surrogate Loss:
surr1 = ratios * advantages
surr2 = C.clip(ratios, 1 - self.eps_clip,
1 + self.eps_clip) * advantages
neglog_loss = -C.element_min(surr1, surr2)
entropy_loss = -0.01 * dist_entropy
actor_loss = C.reduce_mean(neglog_loss + entropy_loss)
critic_loss = 0.5 * C.reduce_mean(C.square(state_value - c_rewards))
loss = actor_loss + critic_loss
chunk = {
'neglog_loss': neglog_loss,
'entropy_loss': entropy_loss,
'actor_loss': actor_loss,
'critic_loss': critic_loss
}
trainer = C.Trainer(
loss, (loss, None),
C.adam(loss.parameters,
C.learning_parameter_schedule_per_sample(self.lr),
C.momentum_schedule_per_sample(self.betas[0]),
variance_momentum=C.momentum_schedule_per_sample(
self.betas[1])))
# trainer = C.Trainer(loss, (loss, None), C.adam(loss.parameters, C.learning_parameter_schedule(10), C.momentum_schedule(0.9), variance_momentum=C.momentum_schedule(0.999))) # higher learning rate
return loss, chunk, trainer
def create_learner(model):
'''Create the optimized method'''
lr_per_minibatch = C.learning_parameter_schedule(opt.lr)
momentum_schedule = C.momentum_schedule_per_sample(0.9990913221888589)
if opt.optim == 'sgd':
return C.sgd(model.parameters, lr=lr_per_minibatch)
elif opt.optim == 'adam':
return C.adam(model.parameters,
lr=lr_per_minibatch,
momentum=momentum_schedule)
elif opt.optim == 'adagrad':
return C.adagrad(model.parameters, lr=lr_per_minibatch)
else:
raise RuntimeError("Invalid optim method: " + opt.optim)
def test_htk_deserializers():
mbsize = 640
epoch_size = 1000 * mbsize
lr = [0.001]
feature_dim = 33
num_classes = 132
context = 2
os.chdir(data_path)
features_file = "glob_0000.scp"
labels_file = "glob_0000.mlf"
label_mapping_file = "state.list"
fd = HTKFeatureDeserializer(StreamDefs(
amazing_features = StreamDef(shape=feature_dim, context=(context,context), scp=features_file)))
ld = HTKMLFDeserializer(label_mapping_file, StreamDefs(
awesome_labels = StreamDef(shape=num_classes, mlf=labels_file)))
reader = MinibatchSource([fd,ld])
features = C.sequence.input_variable(((2*context+1)*feature_dim))
labels = C.sequence.input_variable((num_classes))
model = Sequential([For(range(3), lambda : Recurrence(LSTM(256))),
Dense(num_classes)])
z = model(features)
ce = C.cross_entropy_with_softmax(z, labels)
errs = C.classification_error (z, labels)
learner = C.fsadagrad(z.parameters,
lr=C.learning_parameter_schedule_per_sample(lr, epoch_size=epoch_size),
momentum=C.momentum_schedule_per_sample(0.9990913221888589),
gradient_clipping_threshold_per_sample=15, gradient_clipping_with_truncation=True)
progress_printer = C.logging.ProgressPrinter(freq=0)
trainer = C.Trainer(z, (ce, errs), learner, progress_printer)
input_map={ features: reader.streams.amazing_features, labels: reader.streams.awesome_labels }
# just run and verify it doesn't crash
for i in range(3):
mb_data = reader.next_minibatch(mbsize, input_map=input_map)
trainer.train_minibatch(mb_data)
assert True
os.chdir(abs_path)
def train_lm(testing=False):
data = DataReader(token_to_id_path, segment_sepparator)
# Create model nodes for the source and target inputs
input_sequence, label_sequence = create_inputs(data.vocab_dim)
# Create the model. It has three output nodes
# z: the input to softmax that provides the latent representation of the next token
# cross_entropy: this is used training criterion
# error: this a binary indicator if the model predicts the correct token
z, cross_entropy, error = create_model(input_sequence, label_sequence,
data.vocab_dim, hidden_dim)
# For measurement we use the (build in) full softmax.
full_ce = C.cross_entropy_with_softmax(z, label_sequence)
# print out some useful training information
log_number_of_parameters(z)
print()
# Run the training loop
num_trained_samples = 0
num_trained_samples_since_last_report = 0
# Instantiate the trainer object to drive the model training
lr_schedule = C.learning_parameter_schedule_per_sample(learning_rate)
momentum_schedule = C.momentum_schedule_per_sample(momentum_per_sample)
gradient_clipping_with_truncation = True
learner = momentum_sgd(
z.parameters,
lr_schedule,
momentum_schedule,
gradient_clipping_threshold_per_sample=clipping_threshold_per_sample,
gradient_clipping_with_truncation=gradient_clipping_with_truncation)
trainer = Trainer(z, (cross_entropy, error), learner)
last_avg_ce = 0
for epoch_count in range(num_epochs):
for features, labels, token_count in data.minibatch_generator(
train_file_path, sequence_length, sequences_per_batch):
arguments = ({input_sequence: features, label_sequence: labels})
t_start = timeit.default_timer()
trainer.train_minibatch(arguments)
t_end = timeit.default_timer()
samples_per_second = token_count / (t_end - t_start)
# Print progress report every num_samples_between_progress_report samples
if num_trained_samples_since_last_report >= num_samples_between_progress_report or num_trained_samples == 0:
av_ce = average_cross_entropy(full_ce, input_sequence,
label_sequence, data)
print_progress(samples_per_second, av_ce, num_trained_samples,
t_start)
num_trained_samples_since_last_report = 0
last_avg_ce = av_ce
num_trained_samples += token_count
num_trained_samples_since_last_report += token_count
if not testing:
# after each epoch save the model
model_filename = "models/lm_epoch%d.dnn" % epoch_count
z.save(model_filename)
print("Saved model to '%s'" % model_filename)
return last_avg_ce
results = re.findall("Completed successfully.", str_out)
if len(results) != 2:
print(str_out)
assert False
if __name__=='__main__':
in1 = C.input_variable(shape=1)
labels = C.input_variable(shape=1)
p1 = parameter(shape=1)
p2 = parameter(shape=1)
n = plus(in1, p1, name='n')
z = plus(n, p2, name='z')
ce = squared_error(z, labels)
momentum_schedule = C.momentum_schedule_per_sample(0.9990913221888589)
lr_per_sample = C.learning_parameter_schedule_per_sample(0.007)
dist_learners = [
C.distributed.data_parallel_distributed_learner(C.momentum_sgd([p1], lr_per_sample, momentum_schedule, True)),
C.distributed.data_parallel_distributed_learner(C.momentum_sgd([p2], lr_per_sample, momentum_schedule, True))
]
trainer = C.Trainer(z, ce, dist_learners)
in1_value = [[1]]
label_value = [[0]]
arguments = {in1: in1_value, labels: label_value}
z_output = z.output
def check_samples(learners, expected_number_of_samples):
for learner in learners:
if learner.total_number_of_samples_seen != expected_number_of_samples:
def build_graph(noise_shape, image_shape,
G_progress_printer, D_progress_printer):
'''
The rest of the computational graph is mostly responsible for
coordinating the training algorithms and parameter updates,
which is particularly tricky with GANs for couple reasons.
First, the discriminator must be used on both the real MNIST
images and fake images generated by the generator function.
One way to represent this in the computational graph is to
create a clone of the output of the discriminator function,
but with substituted inputs. Setting method=share in the
clone function ensures that both paths through the
discriminator model use the same set of parameters.
Second, we need to update the parameters for the generator
and discriminator model separately using the gradients from
different loss functions. We can get the parameters for a
Function in the graph with the parameters attribute. However,
when updating the model parameters, update only the parameters
of the respective models while keeping the other parameters
unchanged. In other words, when updating the generator we will
update only the parameters of the GG function while keeping
the parameters of the DD function fixed and vice versa.
'''
input_dynamic_axes = [C.Axis.default_batch_axis()]
Z = C.input_variable(noise_shape, dynamic_axes=input_dynamic_axes)
X_real = C.input_variable(image_shape, dynamic_axes=input_dynamic_axes)
X_real_scaled = 2*(X_real / 255.0) - 1.0
# Create the model function for the generator and discriminator models
X_fake = generator(Z)
D_real = discriminator(X_real_scaled)
D_fake = D_real.clone(
method = 'share',
substitutions = {X_real_scaled.output: X_fake.output}
)
# Create loss functions and configure optimazation algorithms
G_loss = 1.0 - C.log(D_fake)
D_loss = -(C.log(D_real) + C.log(1.0 - D_fake))
G_learner = C.fsadagrad(
parameters = X_fake.parameters,
lr = C.learning_parameter_schedule_per_sample(lr),
momentum = C.momentum_schedule_per_sample(0.9985724484938566)
)
D_learner = C.fsadagrad(
parameters = D_real.parameters,
lr = C.learning_parameter_schedule_per_sample(lr),
momentum = C.momentum_schedule_per_sample(0.9985724484938566)
)
# Instantiate the trainers
G_trainer = C.Trainer(
X_fake,
(G_loss, None),
G_learner,
G_progress_printer
)
D_trainer = C.Trainer(
D_real,
(D_loss, None),
D_learner,
D_progress_printer
)
return X_real, X_fake, Z, G_trainer, D_trainer
def conv3d_ucf11(train_reader, test_reader, max_epochs=30):
# Replace 0 with 1 to get detailed log.
set_computation_network_trace_level(0)
# These values must match for both train and test reader.
image_height = train_reader.height
image_width = train_reader.width
num_channels = train_reader.channel_count
sequence_length = train_reader.sequence_length
num_output_classes = train_reader.label_count
# Input variables denoting the features and label data
input_var = C.input_variable(
(num_channels, sequence_length, image_height, image_width), np.float32)
label_var = C.input_variable(num_output_classes, np.float32)
# Instantiate simple 3D Convolution network inspired by VGG network
# and http://vlg.cs.dartmouth.edu/c3d/c3d_video.pdf
with C.default_options(activation=C.relu):
z = C.layers.Sequential([
C.layers.Convolution3D((3, 3, 3), 64, pad=True),
C.layers.MaxPooling((1, 2, 2), (1, 2, 2)),
C.layers.For(
range(3), lambda i: [
C.layers.Convolution3D(
(3, 3, 3), [96, 128, 128][i], pad=True),
C.layers.Convolution3D(
(3, 3, 3), [96, 128, 128][i], pad=True),
C.layers.MaxPooling((2, 2, 2), (2, 2, 2))
]),
C.layers.For(range(2),
lambda: [C.layers.Dense(1024),
C.layers.Dropout(0.5)]),
C.layers.Dense(num_output_classes, activation=None)
])(input_var)
# loss and classification error.
ce = C.cross_entropy_with_softmax(z, label_var)
pe = C.classification_error(z, label_var)
# training config
train_epoch_size = train_reader.size()
train_minibatch_size = 2
# Set learning parameters
lr_per_sample = [0.01] * 10 + [0.001] * 10 + [0.0001]
lr_schedule = C.learning_parameter_schedule_per_sample(
lr_per_sample, epoch_size=train_epoch_size)
momentum_per_sample = 0.9997558891748972
mm_schedule = C.momentum_schedule_per_sample([momentum_per_sample])
# Instantiate the trainer object to drive the model training
learner = C.momentum_sgd(z.parameters, lr_schedule, mm_schedule, True)
progress_printer = ProgressPrinter(tag='Training', num_epochs=max_epochs)
trainer = C.Trainer(z, (ce, pe), learner, progress_printer)
log_number_of_parameters(z)
print()
# Get minibatches of images to train with and perform model training
for epoch in range(max_epochs): # loop over epochs
train_reader.reset()
while train_reader.has_more():
videos, labels, current_minibatch = train_reader.next_minibatch(
train_minibatch_size)
trainer.train_minibatch({input_var: videos, label_var: labels})
trainer.summarize_training_progress()
# Test data for trained model
epoch_size = test_reader.size()
test_minibatch_size = 2
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
minibatch_index = 0
test_reader.reset()
while test_reader.has_more():
videos, labels, current_minibatch = test_reader.next_minibatch(
test_minibatch_size)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch({
input_var: videos,
label_var: labels
}) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(
minibatch_index + 1, (metric_numer * 100.0) / metric_denom,
metric_denom))
print("")
return metric_numer / metric_denom
def conv3d_ucf11(train_reader, test_reader, max_epochs=30):
# Replace 0 with 1 to get detailed log.
set_computation_network_trace_level(0)
# These values must match for both train and test reader.
image_height = train_reader.height
image_width = train_reader.width
num_channels = train_reader.channel_count
sequence_length = train_reader.sequence_length
num_output_classes = train_reader.label_count
# Input variables denoting the features and label data
input_var = C.input_variable((num_channels, sequence_length, image_height, image_width), np.float32)
label_var = C.input_variable(num_output_classes, np.float32)
# Instantiate simple 3D Convolution network inspired by VGG network
# and http://vlg.cs.dartmouth.edu/c3d/c3d_video.pdf
with C.default_options (activation=C.relu):
z = C.layers.Sequential([
C.layers.Convolution3D((3,3,3), 64, pad=True),
C.layers.MaxPooling((1,2,2), (1,2,2)),
C.layers.For(range(3), lambda i: [
C.layers.Convolution3D((3,3,3), [96, 128, 128][i], pad=True),
C.layers.Convolution3D((3,3,3), [96, 128, 128][i], pad=True),
C.layers.MaxPooling((2,2,2), (2,2,2))
]),
C.layers.For(range(2), lambda : [
C.layers.Dense(1024),
C.layers.Dropout(0.5)
]),
C.layers.Dense(num_output_classes, activation=None)
])(input_var)
# loss and classification error.
ce = C.cross_entropy_with_softmax(z, label_var)
pe = C.classification_error(z, label_var)
# training config
train_epoch_size = train_reader.size()
train_minibatch_size = 2
# Set learning parameters
lr_per_sample = [0.01]*10+[0.001]*10+[0.0001]
lr_schedule = C.learning_parameter_schedule_per_sample(lr_per_sample, epoch_size=train_epoch_size)
momentum_per_sample = 0.9997558891748972
mm_schedule = C.momentum_schedule_per_sample([momentum_per_sample])
# Instantiate the trainer object to drive the model training
learner = C.momentum_sgd(z.parameters, lr_schedule, mm_schedule, True)
progress_printer = ProgressPrinter(tag='Training', num_epochs=max_epochs)
trainer = C.Trainer(z, (ce, pe), learner, progress_printer)
log_number_of_parameters(z) ; print()
# Get minibatches of images to train with and perform model training
for epoch in range(max_epochs): # loop over epochs
train_reader.reset()
while train_reader.has_more():
videos, labels, current_minibatch = train_reader.next_minibatch(train_minibatch_size)
trainer.train_minibatch({input_var : videos, label_var : labels})
trainer.summarize_training_progress()
# Test data for trained model
epoch_size = test_reader.size()
test_minibatch_size = 2
# process minibatches and evaluate the model
metric_numer = 0
metric_denom = 0
minibatch_index = 0
test_reader.reset()
while test_reader.has_more():
videos, labels, current_minibatch = test_reader.next_minibatch(test_minibatch_size)
# minibatch data to be trained with
metric_numer += trainer.test_minibatch({input_var : videos, label_var : labels}) * current_minibatch
metric_denom += current_minibatch
# Keep track of the number of samples processed so far.
minibatch_index += 1
print("")
print("Final Results: Minibatch[1-{}]: errs = {:0.2f}% * {}".format(minibatch_index+1, (metric_numer*100.0)/metric_denom, metric_denom))
print("")
return metric_numer/metric_denom
results = re.findall("Completed successfully.", str_out)
if len(results) != 2:
print(str_out)
assert False
if __name__ == '__main__':
in1 = C.input_variable(shape=1)
labels = C.input_variable(shape=1)
p1 = parameter(shape=1)
p2 = parameter(shape=1)
n = plus(in1, p1, name='n')
z = plus(n, p2, name='z')
ce = squared_error(z, labels)
momentum_schedule = C.momentum_schedule_per_sample(0.9990913221888589)
lr_per_sample = C.learning_parameter_schedule_per_sample(0.007)
learner1 = C.distributed.data_parallel_distributed_learner(
C.momentum_sgd([p1], lr_per_sample, momentum_schedule, True))
learner1.set_as_metric_aggregator()
dist_learners = [
learner1,
C.distributed.data_parallel_distributed_learner(
C.momentum_sgd([p2], lr_per_sample, momentum_schedule, True))
]
trainer = C.Trainer(z, ce, dist_learners)
in1_value = [[1]]
label_value = [[0]]
arguments = {in1: in1_value, labels: label_value}
z_output = z.output
def trainNetwork():
mapper, gens = loadData(dir + fileName,
'./data/Shakespeare',
batchSize,
timeSteps,
timeShift,
load=False,
lineShape=(0, 40000))
# Input with dynamic sequence axis
# consisting of a matrix of [steps-in-time X number-of-possible-characters]
inputSeqAxis = cntk.Axis('inputAxis')
input = cntk.sequence.input_variable((timeSteps, mapper.numClasses),
sequence_axis=inputSeqAxis,
name='input')
model = createNetwork(input, layers, mapper.numClasses)
label = cntk.sequence.input_variable(mapper.numClasses,
sequence_axis=inputSeqAxis,
name='label')
z = model(input)
loss = cntk.cross_entropy_with_softmax(z, label)
error = cntk.classification_error(z, label)
printer = cntk.logging.ProgressPrinter(tag='Training',
freq=100,
num_epochs=maxEpochs)
lr_per_sample = cntk.learning_parameter_schedule_per_sample(0.001)
momentum_schedule = cntk.momentum_schedule_per_sample(0.9990913221888589)
learner = cntk.momentum_sgd(z.parameters,
lr_per_sample,
momentum_schedule,
gradient_clipping_threshold_per_sample=5.0,
gradient_clipping_with_truncation=True)
#learner = cntk.momentum_sgd(z.parameters, lr, 0.9, minibatch_size=batchSize)
#learner = cntk.fsadagrad(model.parameters, lr=lr, minibatch_size=batchSize, momentum=0.9, unit_gain=True)
trainer = cntk.Trainer(z, (loss, error), learner, [printer])
numMinibatch = mapper.samples // batchSize
print("Input sequence length: {}; unique characters {};".format(
timeSteps, mapper.numClasses))
cntk.logging.log_number_of_parameters(z)
print("Datset size {}; {} Epochs; {} minibatches per epoch".format(
mapper.samples, maxEpochs, numMinibatch))
for epoch in range(maxEpochs):
mask = [True]
for mb in range(numMinibatch):
X, Y = next(gens['train'])
#X, Y = get_data(mb, batchSize, data, mapper)
arguments = ({input: X, label: Y}, mask)
mask = [False]
trainer.train_minibatch(arguments)
if mb % 100 == 0:
print(generateText(z, mapper, 200) + '\n')
trainer.summarize_training_progress()
print(generateText(z, mapper, 100))
bn_update = []
for f in flows:
q, log_det_J = f.forward(q, log_det_J)
base_dist = MultivariateNormalDiag(loc=[0.]*c_dim, scale_diag=[1.]*c_dim)
prior_logprob = base_dist.log_prob(q) # or C.log(base_dist.pdf(q))
loss = -C.reduce_mean(prior_logprob + log_det_J)
v = np.r_[np.random.randn(512 // 2, 2) + np.array([5, 3]),
np.random.randn(512 // 2, 2) + np.array([-5, 3])]
v = (v - v.mean(axis=0)) / v.std(axis=0)
lr_rate = 5e-3
learner = C.adam(loss.parameters, C.learning_parameter_schedule_per_sample(lr_rate), C.momentum_schedule_per_sample(0.99))
# lr_rate = 1e-2
# learner = C.adam(loss.parameters, C.learning_parameter_schedule(lr_rate), C.momentum_schedule(0.99))
trainer = C.Trainer(loss, (loss, None), [learner])
for i in tqdm(range(500)):
# v = np.random.uniform(size=(1000,c_dim))
# v = datasets.make_moons(n_samples=1000, noise=.05)[0].astype(np.float32)
# v = np.r_[np.random.randn(512 // 2, 2) + np.array([5, 3]),
# np.random.randn(512 // 2, 2) + np.array([-5, 3])]
out = trainer.train_minibatch({loss.arguments[0]:v}, outputs=[prior_logprob, log_det_J])
if i%100 == 0:
logprob = out[1][prior_logprob].mean() + out[1][log_det_J].mean()