def create_main_loop(save_path):
model, bn_model, bn_updates = create_models()
gan, = bn_model.top_bricks
discriminator_loss, generator_loss = bn_model.outputs
step_rule = Adam(learning_rate=LEARNING_RATE, beta1=BETA1)
algorithm = ali_algorithm(discriminator_loss, gan.discriminator_parameters,
step_rule, generator_loss,
gan.generator_parameters, step_rule)
algorithm.add_updates(bn_updates)
streams = create_gaussian_mixture_data_streams(
batch_size=BATCH_SIZE,
monitoring_batch_size=MONITORING_BATCH_SIZE,
means=MEANS,
variances=VARIANCES,
priors=PRIORS)
main_loop_stream, train_monitor_stream, valid_monitor_stream = streams
bn_monitored_variables = (
[v for v in bn_model.auxiliary_variables if 'norm' not in v.name] +
bn_model.outputs)
monitored_variables = (
[v for v in model.auxiliary_variables if 'norm' not in v.name] +
model.outputs)
extensions = [
Timing(),
FinishAfter(after_n_epochs=NUM_EPOCHS),
DataStreamMonitoring(bn_monitored_variables,
train_monitor_stream,
prefix="train",
updates=bn_updates),
DataStreamMonitoring(monitored_variables,
valid_monitor_stream,
prefix="valid"),
Checkpoint(save_path,
after_epoch=True,
after_training=True,
use_cpickle=True),
ProgressBar(),
Printing(),
]
main_loop = MainLoop(model=bn_model,
data_stream=main_loop_stream,
algorithm=algorithm,
extensions=extensions)
return main_loop
def train_model(cost,
error_rate,
train_stream,
load_location=None,
save_location=None):
cost.name = "Cross_entropy"
error_rate.name = 'Error_rate'
# Define the model
model = Model(cost)
# Load the parameters from a dumped model
if load_location is not None:
logger.info('Loading parameters...')
model.set_param_values(load_parameter_values(load_location))
cg = ComputationGraph(cost)
step_rule = Momentum(learning_rate=0.1, momentum=0.9)
algorithm = GradientDescent(cost=cost,
step_rule=step_rule,
params=cg.parameters)
main_loop = MainLoop(
model=model,
data_stream=train_stream,
algorithm=algorithm,
extensions=[
# DataStreamMonitoring([cost], test_stream, prefix='test',
# after_epoch=False, every_n_epochs=10),
DataStreamMonitoring([cost],
train_stream,
prefix='train',
after_epoch=True),
Printing(after_epoch=True)
])
main_loop.run()
# Save the main loop
if save_location is not None:
logger.info('Saving the main loop...')
dump_manager = MainLoopDumpManager(save_location)
dump_manager.dump(main_loop)
logger.info('Saved')
def build_extensions_list(self):
epochs = self.config['epochs']
save_freq = self.config['save_freq']
original_save = self.config['original_save'] if self.config.has_key('original_save') \
else False
tracked_train = self.model.linksholder.filter_and_broadcast_request(
'train', 'TrackingLink')
tracked_valid = self.model.linksholder.filter_and_broadcast_request(
'valid', 'TrackingLink')
extensions = [
Timing(),
Experiment(self.filemanager.exp_name, self.filemanager.local_path,
self.filemanager.network_path,
self.filemanager.full_dump),
FinishAfter(after_n_epochs=epochs),
TrainingDataMonitoring(tracked_train,
prefix="train",
after_epoch=True),
DataStreamMonitoring(tracked_valid,
self.streams['valid'],
prefix="valid",
after_epoch=True),
#Ipdb(after_batch=True),
SaveExperiment(self.model.parameters,
original_save=original_save,
every_n_epochs=save_freq),
]
extensions += [
#LoadExperiment(
# parameters,
# full_load=False),
ProgressBar(),
Printing(),
]
self.extensions = extensions
def test_training_interrupt():
def process_batch(batch):
time.sleep(0.1)
algorithm = MockAlgorithm()
algorithm.process_batch = process_batch
main_loop = MockMainLoop(algorithm=algorithm,
data_stream=IterableDataset(
count()).get_example_stream(),
extensions=[Printing()])
p = Process(target=main_loop.run)
p.start()
time.sleep(0.1)
os.kill(p.pid, signal.SIGINT)
time.sleep(0.1)
assert p.is_alive()
os.kill(p.pid, signal.SIGINT)
time.sleep(0.2)
assert not p.is_alive()
p.join()
def train(self, data_file, output_data_file, n_epochs=0):
training_data = dataset.T_H5PYDataset(data_file, which_sets=('train',))
test_data = dataset.T_H5PYDataset(data_file, which_sets=('test',))
session = Session(root_url='http://localhost:5006')
if self.MainLoop is None:
step_rules = [RMSProp(learning_rate=0.2, decay_rate=0.95), StepClipping(1)]
algorithm = GradientDescent(cost=self.Cost,
parameters=self.ComputationGraph.parameters,
step_rule=CompositeRule(step_rules),
on_unused_sources='ignore')
train_stream = DataStream.default_stream(
training_data, iteration_scheme=SequentialScheme(
training_data.num_examples, batch_size=100))
test_stream = DataStream.default_stream(
test_data, iteration_scheme=SequentialScheme(
test_data.num_examples, batch_size=100))
self.MainLoop = MainLoop(
model=Model(self.Cost),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
FinishAfter(after_n_epochs=n_epochs),
Printing(),
Checkpoint(output_data_file, every_n_epochs=50),
TrainingDataMonitoring([self.Cost], after_batch=True, prefix='train'),
DataStreamMonitoring([self.Cost], after_batch=True, data_stream=test_stream, prefix='test'),
Plot(output_data_file, channels=[['train_cost', 'test_cost']])
])
self.MainLoop.run()
def create_main_loop(save_path, backup_path):
model, bn_model, bn_updates = create_models()
ali, = bn_model.top_bricks
discriminator_loss, generator_loss = bn_model.outputs
step_rule = Adam(learning_rate=LEARNING_RATE, beta1=BETA1)
algorithm = ali_algorithm(discriminator_loss, ali.discriminator_parameters,
step_rule, generator_loss,
ali.generator_parameters, step_rule)
algorithm.add_updates(bn_updates)
streams = create_svhn_108_data_streams(BATCH_SIZE, MONITORING_BATCH_SIZE)
main_loop_stream, train_monitor_stream, valid_monitor_stream = streams
bn_monitored_variables = (
[v for v in bn_model.auxiliary_variables if 'norm' not in v.name] +
bn_model.outputs)
monitored_variables = (
[v for v in model.auxiliary_variables if 'norm' not in v.name] +
model.outputs)
extensions = [
Timing(),
FinishAfter(after_n_epochs=NUM_EPOCHS),
DataStreamMonitoring(
bn_monitored_variables, train_monitor_stream, prefix="train",
updates=bn_updates),
DataStreamMonitoring(
monitored_variables, valid_monitor_stream, prefix="valid"),
Checkpoint(save_path, after_epoch=True, after_training=True,
use_cpickle=True),
BackupModel(backup_path, save_path, every_n_epochs=5, after_training=True),
PlotLoss(backup_path, "ALI SVHN 0, 8", every_n_epochs=5, after_training=True),
PlotAccuracy(backup_path, "ALI SVHN 0, 8", every_n_epochs=5, after_training=True),
ProgressBar(),
Printing(),
]
main_loop = MainLoop(model=bn_model, data_stream=main_loop_stream,
algorithm=algorithm, extensions=extensions)
return main_loop
def train(self, training_data):
step_rules = [Adam(), StepClipping(1.0)]
algorithm = GradientDescent(
cost=self.Cost,
parameters=self.ComputationGraph.parameters,
step_rule=CompositeRule(step_rules))
train_stream = DataStream.default_stream(
training_data,
iteration_scheme=SequentialScheme(training_data.num_examples,
batch_size=20))
main = MainLoop(model=Model(self.Cost),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
FinishAfter(),
Printing(),
Checkpoint('trainingdata.tar', every_n_epochs=10)
])
main.run()
valid_stream,
after_epoch=True,
#before_first_epoch = False,
prefix="valid")
extensions = extensions = [
Timing(every_n_batches=n_batches), train_monitor, valid_monitor,
TrackTheBest('valid_nll', after_epoch=True),
Plot(save_dir + experiment_name + ".png", [['train_nll', 'valid_nll']],
every_n_batches=4 * n_batches,
email=True),
Checkpoint(save_dir + experiment_name + ".pkl",
use_cpickle=True,
every_n_batches=n_batches * 8,
after_epoch=True),
Checkpoint(save_dir + "best_" + experiment_name + ".pkl",
after_epoch=True,
use_cpickle=True).add_condition(
['after_epoch'],
predicate=OnLogRecord('valid_nll_best_so_far')),
Printing(every_n_batches=n_batches, after_epoch=True),
FinishAfter(after_n_epochs=2),
SaveComputationGraph(emit)
]
main_loop = MainLoop(model=model,
data_stream=train_stream,
algorithm=algorithm,
extensions=extensions)
main_loop.run()
def main(name, dataset, epochs, batch_size, learning_rate, attention, n_iter,
enc_dim, dec_dim, z_dim, oldmodel):
image_size, data_train, data_valid, data_test = datasets.get_data(dataset)
train_stream = Flatten(
DataStream(data_train,
iteration_scheme=SequentialScheme(data_train.num_examples,
batch_size)))
valid_stream = Flatten(
DataStream(data_valid,
iteration_scheme=SequentialScheme(data_valid.num_examples,
batch_size)))
test_stream = Flatten(
DataStream(data_test,
iteration_scheme=SequentialScheme(data_test.num_examples,
batch_size)))
if name is None:
name = dataset
img_height, img_width = image_size
x_dim = img_height * img_width
rnninits = {
#'weights_init': Orthogonal(),
'weights_init': IsotropicGaussian(0.01),
'biases_init': Constant(0.),
}
inits = {
#'weights_init': Orthogonal(),
'weights_init': IsotropicGaussian(0.01),
'biases_init': Constant(0.),
}
if attention != "":
read_N, write_N = attention.split(',')
read_N = int(read_N)
write_N = int(write_N)
read_dim = 2 * read_N**2
reader = AttentionReader(x_dim=x_dim,
dec_dim=dec_dim,
width=img_width,
height=img_height,
N=read_N,
**inits)
writer = AttentionWriter(input_dim=dec_dim,
output_dim=x_dim,
width=img_width,
height=img_height,
N=write_N,
**inits)
attention_tag = "r%d-w%d" % (read_N, write_N)
else:
read_dim = 2 * x_dim
reader = Reader(x_dim=x_dim, dec_dim=dec_dim, **inits)
writer = Writer(input_dim=dec_dim, output_dim=x_dim, **inits)
attention_tag = "full"
#----------------------------------------------------------------------
# Learning rate
def lr_tag(value):
""" Convert a float into a short tag-usable string representation. E.g.:
0.1 -> 11
0.01 -> 12
0.001 -> 13
0.005 -> 53
"""
exp = np.floor(np.log10(value))
leading = ("%e" % value)[0]
return "%s%d" % (leading, -exp)
lr_str = lr_tag(learning_rate)
subdir = time.strftime("%Y%m%d-%H%M%S") + "-" + name
longname = "%s-%s-t%d-enc%d-dec%d-z%d-lr%s" % (
dataset, attention_tag, n_iter, enc_dim, dec_dim, z_dim, lr_str)
pickle_file = subdir + "/" + longname + ".pkl"
print("\nRunning experiment %s" % longname)
print(" dataset: %s" % dataset)
print(" subdirectory: %s" % subdir)
print(" learning rate: %g" % learning_rate)
print(" attention: %s" % attention)
print(" n_iterations: %d" % n_iter)
print(" encoder dimension: %d" % enc_dim)
print(" z dimension: %d" % z_dim)
print(" decoder dimension: %d" % dec_dim)
print(" batch size: %d" % batch_size)
print(" epochs: %d" % epochs)
print()
#----------------------------------------------------------------------
encoder_rnn = LSTM(dim=enc_dim, name="RNN_enc", **rnninits)
decoder_rnn = LSTM(dim=dec_dim, name="RNN_dec", **rnninits)
encoder_mlp = MLP([Identity()], [(read_dim + dec_dim), 4 * enc_dim],
name="MLP_enc",
**inits)
decoder_mlp = MLP([Identity()], [z_dim, 4 * dec_dim],
name="MLP_dec",
**inits)
q_sampler = Qsampler(input_dim=enc_dim, output_dim=z_dim, **inits)
draw = DrawModel(n_iter,
reader=reader,
encoder_mlp=encoder_mlp,
encoder_rnn=encoder_rnn,
sampler=q_sampler,
decoder_mlp=decoder_mlp,
decoder_rnn=decoder_rnn,
writer=writer)
draw.initialize()
#------------------------------------------------------------------------
x = tensor.matrix('features')
#x_recons = 1. + x
x_recons, kl_terms = draw.reconstruct(x)
#x_recons, _, _, _, _ = draw.silly(x, n_steps=10, batch_size=100)
#x_recons = x_recons[-1,:,:]
#samples = draw.sample(100)
#x_recons = samples[-1, :, :]
#x_recons = samples[-1, :, :]
recons_term = BinaryCrossEntropy().apply(x, x_recons)
recons_term.name = "recons_term"
cost = recons_term + kl_terms.sum(axis=0).mean()
cost.name = "nll_bound"
#------------------------------------------------------------
cg = ComputationGraph([cost])
params = VariableFilter(roles=[PARAMETER])(cg.variables)
algorithm = GradientDescent(
cost=cost,
params=params,
step_rule=CompositeRule([
StepClipping(10.),
Adam(learning_rate),
])
#step_rule=RMSProp(learning_rate),
#step_rule=Momentum(learning_rate=learning_rate, momentum=0.95)
)
#algorithm.add_updates(scan_updates)
#------------------------------------------------------------------------
# Setup monitors
monitors = [cost]
for t in range(n_iter):
kl_term_t = kl_terms[t, :].mean()
kl_term_t.name = "kl_term_%d" % t
#x_recons_t = T.nnet.sigmoid(c[t,:,:])
#recons_term_t = BinaryCrossEntropy().apply(x, x_recons_t)
#recons_term_t = recons_term_t.mean()
#recons_term_t.name = "recons_term_%d" % t
monitors += [kl_term_t]
train_monitors = monitors[:]
train_monitors += [aggregation.mean(algorithm.total_gradient_norm)]
train_monitors += [aggregation.mean(algorithm.total_step_norm)]
# Live plotting...
plot_channels = [
["train_nll_bound", "test_nll_bound"],
["train_kl_term_%d" % t for t in range(n_iter)],
#["train_recons_term_%d" % t for t in range(n_iter)],
["train_total_gradient_norm", "train_total_step_norm"]
]
#------------------------------------------------------------
if not os.path.exists(subdir):
os.makedirs(subdir)
main_loop = MainLoop(
model=Model(cost),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
Timing(),
FinishAfter(after_n_epochs=epochs),
TrainingDataMonitoring(train_monitors,
prefix="train",
after_epoch=True),
# DataStreamMonitoring(
# monitors,
# valid_stream,
## updates=scan_updates,
# prefix="valid"),
DataStreamMonitoring(
monitors,
test_stream,
# updates=scan_updates,
prefix="test"),
Checkpoint(name,
before_training=False,
after_epoch=True,
save_separately=['log', 'model']),
#Checkpoint(image_size=image_size, save_subdir=subdir, path=pickle_file, before_training=False, after_epoch=True, save_separately=['log', 'model']),
Plot(name, channels=plot_channels),
ProgressBar(),
Printing()
])
if oldmodel is not None:
print("Initializing parameters with old model %s" % oldmodel)
with open(oldmodel, "rb") as f:
oldmodel = pickle.load(f)
main_loop.model.set_param_values(oldmodel.get_param_values())
del oldmodel
main_loop.run()
def main(config,
tr_stream,
dev_stream,
source_vocab,
target_vocab,
use_bokeh=False):
# Create Theano variables
logger.info('Creating theano variables')
source_sentence = tensor.lmatrix('source')
source_sentence_mask = tensor.matrix('source_mask')
target_sentence = tensor.lmatrix('target')
target_sentence_mask = tensor.matrix('target_mask')
initial_context = tensor.matrix('initial_context')
# Construct model
logger.info('Building RNN encoder-decoder')
encoder = BidirectionalEncoder(config['src_vocab_size'],
config['enc_embed'], config['enc_nhids'])
# let user specify the target transition class name in config,
# eval it and pass to decoder
target_transition_name = config.get(
'target_transition', 'GRUInitialStateWithInitialStateSumContext')
target_transition = eval(target_transition_name)
logger.info('Using target transition: {}'.format(target_transition_name))
decoder = InitialContextDecoder(config['trg_vocab_size'],
config['dec_embed'], config['dec_nhids'],
config['enc_nhids'] * 2,
config['context_dim'], target_transition)
cost = decoder.cost(encoder.apply(source_sentence, source_sentence_mask),
source_sentence_mask, target_sentence,
target_sentence_mask, initial_context)
cost.name = 'decoder_cost'
# Initialize model
logger.info('Initializing model')
encoder.weights_init = decoder.weights_init = IsotropicGaussian(
config['weight_scale'])
encoder.biases_init = decoder.biases_init = Constant(0)
encoder.push_initialization_config()
decoder.push_initialization_config()
encoder.bidir.prototype.weights_init = Orthogonal()
decoder.transition.weights_init = Orthogonal()
encoder.initialize()
decoder.initialize()
logger.info('Creating computational graph')
cg = ComputationGraph(cost)
# GRAPH TRANSFORMATIONS FOR BETTER TRAINING
# TODO: validate performance with/without regularization
if config.get('l2_regularization', False) is True:
l2_reg_alpha = config['l2_regularization_alpha']
logger.info(
'Applying l2 regularization with alpha={}'.format(l2_reg_alpha))
model_weights = VariableFilter(roles=[WEIGHT])(cg.variables)
for W in model_weights:
cost = cost + (l2_reg_alpha * (W**2).sum())
# why do we need to name the cost variable? Where did the original name come from?
cost.name = 'decoder_cost_cost'
cg = ComputationGraph(cost)
# apply dropout for regularization
if config['dropout'] < 1.0:
# dropout is applied to the output of maxout in ghog
# this is the probability of dropping out, so you probably want to make it <=0.5
logger.info('Applying dropout')
dropout_inputs = [
x for x in cg.intermediary_variables
if x.name == 'maxout_apply_output'
]
cg = apply_dropout(cg, dropout_inputs, config['dropout'])
# Print shapes
shapes = [param.get_value().shape for param in cg.parameters]
logger.info("Parameter shapes: ")
for shape, count in Counter(shapes).most_common():
logger.info(' {:15}: {}'.format(shape, count))
logger.info("Total number of parameters: {}".format(len(shapes)))
# Print parameter names
enc_dec_param_dict = merge(
Selector(encoder).get_parameters(),
Selector(decoder).get_parameters())
logger.info("Parameter names: ")
for name, value in enc_dec_param_dict.items():
logger.info(' {:15}: {}'.format(value.get_value().shape, name))
logger.info("Total number of parameters: {}".format(
len(enc_dec_param_dict)))
# Set up training model
logger.info("Building model")
training_model = Model(cost)
# create the training directory, and copy this config there if directory doesn't exist
if not os.path.isdir(config['saveto']):
os.makedirs(config['saveto'])
shutil.copy(config['config_file'], config['saveto'])
# Set extensions
# TODO: add checking for existing model and loading
logger.info("Initializing extensions")
extensions = [
FinishAfter(after_n_batches=config['finish_after']),
TrainingDataMonitoring([cost], after_batch=True),
Printing(after_batch=True),
CheckpointNMT(config['saveto'], every_n_batches=config['save_freq'])
]
# Create the theano variables that we need for the sampling graph
sampling_input = tensor.lmatrix('input')
sampling_context = tensor.matrix('context_input')
# Set up beam search and sampling computation graphs if necessary
if config['hook_samples'] >= 1 or config.get('bleu_script',
None) is not None:
logger.info("Building sampling model")
sampling_representation = encoder.apply(
sampling_input, tensor.ones(sampling_input.shape))
generated = decoder.generate(sampling_input, sampling_representation,
sampling_context)
search_model = Model(generated)
_, samples = VariableFilter(
bricks=[decoder.sequence_generator], name="outputs")(
ComputationGraph(generated[1])) # generated[1] is next_outputs
# Add sampling
if config['hook_samples'] >= 1:
logger.info("Building sampler")
extensions.append(
Sampler(
model=search_model,
data_stream=tr_stream,
hook_samples=config['hook_samples'],
every_n_batches=config['sampling_freq'],
src_vocab=source_vocab,
trg_vocab=target_vocab,
src_vocab_size=config['src_vocab_size'],
))
# Add early stopping based on bleu
if config.get('bleu_script', None) is not None:
logger.info("Building bleu validator")
extensions.append(
BleuValidator(sampling_input,
sampling_context,
samples=samples,
config=config,
model=search_model,
data_stream=dev_stream,
src_vocab=source_vocab,
trg_vocab=target_vocab,
normalize=config['normalized_bleu'],
every_n_batches=config['bleu_val_freq']))
# Add early stopping based on Meteor
if config.get('meteor_directory', None) is not None:
logger.info("Building meteor validator")
extensions.append(
MeteorValidator(sampling_input,
sampling_context,
samples=samples,
config=config,
model=search_model,
data_stream=dev_stream,
src_vocab=source_vocab,
trg_vocab=target_vocab,
normalize=config['normalized_bleu'],
every_n_batches=config['bleu_val_freq']))
# Reload model if necessary
if config['reload']:
extensions.append(LoadNMT(config['saveto']))
# Plot cost in bokeh if necessary
if use_bokeh and BOKEH_AVAILABLE:
extensions.append(
Plot(config['model_save_directory'],
channels=[[
'decoder_cost', 'validation_set_bleu_score',
'validation_set_meteor_score'
]],
every_n_batches=10))
# Set up training algorithm
logger.info("Initializing training algorithm")
# if there is dropout or random noise, we need to use the output of the modified graph
if config['dropout'] < 1.0 or config['weight_noise_ff'] > 0.0:
algorithm = GradientDescent(cost=cg.outputs[0],
parameters=cg.parameters,
step_rule=CompositeRule([
StepClipping(config['step_clipping']),
eval(config['step_rule'])()
]))
else:
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=CompositeRule([
StepClipping(config['step_clipping']),
eval(config['step_rule'])()
]))
# enrich the logged information
extensions.append(Timing(every_n_batches=100))
# Initialize main loop
logger.info("Initializing main loop")
main_loop = MainLoop(model=training_model,
algorithm=algorithm,
data_stream=tr_stream,
extensions=extensions)
# Train!
main_loop.run()
def main(save_to, model, train, test, num_epochs, input_size = (150,150), learning_rate=0.01,
batch_size=50, num_batches=None, flatten_stream=False):
"""
save_to : where to save trained model
model : model given in input must be already initialised (works with convnet and mlp)
input_size : the shape of the reshaped image in input (before flattening is applied if flatten_stream is True)
"""
if flatten_stream :
x = tensor.matrix('image_features')
else :
x = tensor.tensor4('image_features')
y = tensor.lmatrix('targets')
#Data augmentation
#insert data augmentation here
#Generating stream
train_stream = DataStream.default_stream(
train,
iteration_scheme=ShuffledScheme(train.num_examples, batch_size)
)
test_stream = DataStream.default_stream(
test,
iteration_scheme=ShuffledScheme(test.num_examples, batch_size)
)
#Reshaping procedure
#Add a crop option in scikitresize so that the image is not deformed
#Resize to desired square shape
train_stream = ScikitResize(train_stream, input_size, which_sources=('image_features',))
test_stream = ScikitResize(test_stream, input_size, which_sources=('image_features',))
#Flattening the stream
if flatten_stream is True:
train_stream = Flatten(train_stream, which_sources=('image_features',))
test_stream = Flatten(test_stream, which_sources=('image_features',))
# Apply input to model
probs = model.apply(x)
#Defining cost and various indices to watch
#print(probs)
#cost = SquaredError().apply(y.flatten(),probs)
cost = CategoricalCrossEntropy().apply(y.flatten(), probs).copy(name='cost')
error_rate = MisclassificationRate().apply(y.flatten(), probs).copy(
name='error_rate')
#Building Computation Graph
cg = ComputationGraph([cost, error_rate])
# Train with simple SGD
algorithm = GradientDescent(
cost=cost, parameters=cg.parameters,
step_rule=Scale(learning_rate=learning_rate))
#Defining extensions
extensions = [Timing(),
FinishAfter(after_n_epochs=num_epochs,
after_n_batches=num_batches),
TrainingDataMonitoring([cost, error_rate,aggregation.mean(algorithm.total_gradient_norm)], prefix="train", every_n_batches=5),
DataStreamMonitoring([cost, error_rate],test_stream,prefix="test", every_n_batches=25),
Checkpoint(save_to),
ProgressBar(),
Printing(every_n_batches=5)]
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
model = Model(cost)
main_loop = MainLoop(
algorithm,
train_stream,
model=model,
extensions=extensions)
main_loop.run()
def rf_lstm_experiment(data_name, exp_network, in_dim, out_dim, num_layers,
start_neurons, num_neurons, batch_size, num_epochs):
"""LSTM Experiment."""
# load dataset
train_set = IterableDataset(
ds.transform_sequence(data_name, "train", batch_size))
test_set = IterableDataset(
ds.transform_sequence(data_name, "test", batch_size))
stream_train = DataStream(dataset=train_set)
stream_test = DataStream(dataset=test_set)
methods = ['sgd', 'momentum', 'adagrad', 'rmsprop']
for n_layers in xrange(1, num_layers + 1):
for n_neurons in xrange(start_neurons, num_neurons + 5, 5):
for method in methods:
X = T.tensor3("features")
y = T.matrix("targets")
x_to_h = Linear(in_dim,
n_neurons * 4,
name='x_to_h',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
lstm = LSTM(n_neurons,
name='lstm',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
h_to_o = nc.setup_ff_network(n_neurons, out_dim, n_layers - 1,
n_neurons)
X_trans = x_to_h.apply(X)
h, c = lstm.apply(X_trans)
y_hat = h_to_o.apply(h[-1])
cost, cg = nc.create_cg_and_cost(y, y_hat, "none")
lstm.initialize()
x_to_h.initialize()
h_to_o.initialize()
algorithm = nc.setup_algorithms(cost, cg, method, type="RNN")
test_monitor = DataStreamMonitoring(variables=[cost],
data_stream=stream_test,
prefix="test")
train_monitor = TrainingDataMonitoring(variables=[cost],
prefix="train",
after_epoch=True)
main_loop = MainLoop(
algorithm=algorithm,
data_stream=stream_train,
extensions=[
test_monitor, train_monitor,
FinishAfter(after_n_epochs=num_epochs),
Printing(),
ProgressBar()
])
main_loop.run()
# Saving results
exp_id = ds.create_exp_id(exp_network, n_layers, n_neurons,
batch_size, num_epochs, method,
"none")
# prepare related functions
predict = theano.function([X], y_hat)
# prepare related data
train_features, train_targets = ds.get_iter_data(train_set)
test_features, test_targets = ds.get_iter_data(test_set)
# Prediction of result
train_predicted = gen_prediction(predict, train_features)
test_predicted = gen_prediction(predict, test_features)
# Get cost
cost = ds.get_cost_data(test_monitor, train_set.num_examples,
num_epochs)
# logging
ds.save_experiment(train_targets, train_predicted,
test_targets, test_predicted, cost,
exp_network, n_layers, n_neurons,
batch_size, num_epochs, method, "none",
exp_id, "../results/")
def construct_main_loop(name, task_name, patch_shape, batch_size,
n_spatial_dims, n_patches, max_epochs, patience_epochs,
learning_rate, gradient_limiter, hyperparameters,
**kwargs):
task = tasks.get_task(**hyperparameters)
hyperparameters["n_channels"] = task.n_channels
extensions = []
# let theta noise decay as training progresses
for key in "location_std scale_std".split():
hyperparameters[key] = theano.shared(hyperparameters[key], name=key)
extensions.append(
util.ExponentialDecay(hyperparameters[key],
hyperparameters["%s_decay" % key],
after_batch=True))
print "constructing graphs..."
graphs, outputs, updates = construct_graphs(task=task, **hyperparameters)
print "setting up main loop..."
from blocks.model import Model
model = Model(outputs["train"]["cost"])
from blocks.algorithms import GradientDescent, CompositeRule, StepClipping, Adam, RMSProp
from extensions import Compressor
if gradient_limiter == "clip":
limiter = StepClipping(1.)
elif gradient_limiter == "compress":
limiter = Compressor()
else:
raise ValueError()
algorithm = GradientDescent(
cost=outputs["train"]["cost"],
parameters=graphs["train"].parameters,
step_rule=CompositeRule([limiter,
Adam(learning_rate=learning_rate)]))
algorithm.add_updates(updates["train"])
extensions.extend(
construct_monitors(algorithm=algorithm,
task=task,
model=model,
graphs=graphs,
outputs=outputs,
updates=updates,
**hyperparameters))
from blocks.extensions import FinishAfter, Printing, ProgressBar, Timing
from blocks.extensions.stopping import FinishIfNoImprovementAfter
from blocks.extensions.training import TrackTheBest
from blocks.extensions.saveload import Checkpoint
from dump import DumpBest, LightCheckpoint, PrintingTo, DumpGraph, DumpLog
extensions.extend([
TrackTheBest("valid_error_rate", "best_valid_error_rate"),
FinishIfNoImprovementAfter("best_valid_error_rate",
epochs=patience_epochs),
FinishAfter(after_n_epochs=max_epochs),
DumpBest("best_valid_error_rate", name + "_best.zip"),
Checkpoint(hyperparameters["checkpoint_save_path"],
on_interrupt=False,
every_n_epochs=10,
use_cpickle=True),
DumpLog("log.pkl", after_epoch=True),
ProgressBar(),
Timing(),
Printing(),
PrintingTo(name + "_log"),
DumpGraph(name + "_grad_graph")
])
from blocks.main_loop import MainLoop
main_loop = MainLoop(data_stream=task.get_stream("train"),
algorithm=algorithm,
extensions=extensions,
model=model)
from tabulate import tabulate
print "parameter sizes:"
print tabulate(
(key, "x".join(map(str,
value.get_value().shape)), value.get_value().size)
for key, value in main_loop.model.get_parameter_dict().items())
return main_loop
cost
]
# Construct the main loop and start training!
average_monitoring = TrainingDataMonitoring(observables,
prefix="average",
every_n_batches=10)
checkpoint_after = n_epochs / 5
#checkpoint_after=100;
main_loop = MainLoop(
model=model,
data_stream=data_stream,
algorithm=algo,
extensions=[
Timing(),
TrainingDataMonitoring(observables, after_batch=True),
average_monitoring,
# FinishAfter(after_n_batches=n_batches)
FinishAfter(after_n_epochs=n_epochs)
# This shows a way to handle NaN emerging during training: simply finish it.
.add_condition(["after_batch"], _is_nan),
# Saving the model and the log separately is convenient,
# because loading the whole pickle takes quite some time.
Checkpoint(f_model, every_n_epochs=1, save_separately=["model",
"log"]),
Printing(every_n_batches=100)
])
main_loop.run()
def test_printing():
main_loop = setup_mainloop(Printing())
# We are happy if it does not crash or raise any exceptions
main_loop.run()
def main(save_to, num_epochs,
weight_decay=0.0001, noise_pressure=0, subset=None, num_batches=None,
batch_size=None, histogram=None, resume=False):
output_size = 10
prior_noise_level = -10
noise_step_rule = Scale(1e-6)
noise_rate = theano.shared(numpy.asarray(1e-5, dtype=theano.config.floatX))
convnet = create_res_net(out_noise=True, tied_noise=True, tied_sigma=True,
noise_rate=noise_rate,
prior_noise_level=prior_noise_level)
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
test_probs = convnet.apply(x)
test_cost = (CategoricalCrossEntropy().apply(y.flatten(), test_probs)
.copy(name='cost'))
test_error_rate = (MisclassificationRate().apply(y.flatten(), test_probs)
.copy(name='error_rate'))
test_confusion = (ConfusionMatrix().apply(y.flatten(), test_probs)
.copy(name='confusion'))
test_confusion.tag.aggregation_scheme = Sum(test_confusion)
test_cg = ComputationGraph([test_cost, test_error_rate])
# Apply dropout to all layer outputs except final softmax
# dropout_vars = VariableFilter(
# roles=[OUTPUT], bricks=[Convolutional],
# theano_name_regex="^conv_[25]_apply_output$")(test_cg.variables)
# drop_cg = apply_dropout(test_cg, dropout_vars, 0.5)
# Apply 0.2 dropout to the pre-averaging layer
# dropout_vars_2 = VariableFilter(
# roles=[OUTPUT], bricks=[Convolutional],
# theano_name_regex="^conv_8_apply_output$")(test_cg.variables)
# train_cg = apply_dropout(test_cg, dropout_vars_2, 0.2)
# Apply 0.2 dropout to the input, as in the paper
# train_cg = apply_dropout(test_cg, [x], 0.2)
# train_cg = drop_cg
# train_cg = apply_batch_normalization(test_cg)
# train_cost, train_error_rate, train_components = train_cg.outputs
with batch_normalization(convnet):
with training_noise(convnet):
train_probs = convnet.apply(x)
train_cost = (CategoricalCrossEntropy().apply(y.flatten(), train_probs)
.copy(name='cost'))
train_components = (ComponentwiseCrossEntropy().apply(y.flatten(),
train_probs).copy(name='components'))
train_error_rate = (MisclassificationRate().apply(y.flatten(),
train_probs).copy(name='error_rate'))
train_cg = ComputationGraph([train_cost,
train_error_rate, train_components])
population_updates = get_batch_normalization_updates(train_cg)
bn_alpha = 0.9
extra_updates = [(p, p * bn_alpha + m * (1 - bn_alpha))
for p, m in population_updates]
# for annealing
nit_penalty = theano.shared(numpy.asarray(noise_pressure, dtype=theano.config.floatX))
nit_penalty.name = 'nit_penalty'
# Compute noise rates for training graph
train_logsigma = VariableFilter(roles=[LOG_SIGMA])(train_cg.variables)
train_mean_log_sigma = tensor.concatenate([n.flatten() for n in train_logsigma]).mean()
train_mean_log_sigma.name = 'mean_log_sigma'
train_nits = VariableFilter(roles=[NITS])(train_cg.auxiliary_variables)
train_nit_rate = tensor.concatenate([n.flatten() for n in train_nits]).mean()
train_nit_rate.name = 'nit_rate'
train_nit_regularization = nit_penalty * train_nit_rate
train_nit_regularization.name = 'nit_regularization'
# Apply regularization to the cost
trainable_parameters = VariableFilter(roles=[WEIGHT, BIAS])(
train_cg.parameters)
mask_parameters = [p for p in trainable_parameters
if get_brick(p).name == 'mask']
noise_parameters = VariableFilter(roles=[NOISE])(train_cg.parameters)
biases = VariableFilter(roles=[BIAS])(train_cg.parameters)
weights = VariableFilter(roles=[WEIGHT])(train_cg.variables)
nonmask_weights = [p for p in weights if get_brick(p).name != 'mask']
l2_norm = sum([(W ** 2).sum() for W in nonmask_weights])
l2_norm.name = 'l2_norm'
l2_regularization = weight_decay * l2_norm
l2_regularization.name = 'l2_regularization'
# testversion
test_cost = test_cost + l2_regularization
test_cost.name = 'cost_with_regularization'
# Training version of cost
train_cost_without_regularization = train_cost
train_cost_without_regularization.name = 'cost_without_regularization'
train_cost = train_cost + l2_regularization + train_nit_regularization
train_cost.name = 'cost_with_regularization'
cifar10_train = CIFAR10(("train",))
cifar10_train_stream = RandomPadCropFlip(
NormalizeBatchLevels(DataStream.default_stream(
cifar10_train, iteration_scheme=ShuffledScheme(
cifar10_train.num_examples, batch_size)),
which_sources=('features',)),
(32, 32), pad=4, which_sources=('features',))
test_batch_size = 128
cifar10_test = CIFAR10(("test",))
cifar10_test_stream = NormalizeBatchLevels(DataStream.default_stream(
cifar10_test,
iteration_scheme=ShuffledScheme(
cifar10_test.num_examples, test_batch_size)),
which_sources=('features',))
momentum = Momentum(0.01, 0.9)
# Create a step rule that doubles the learning rate of biases, like Caffe.
# scale_bias = Restrict(Scale(2), biases)
# step_rule = CompositeRule([scale_bias, momentum])
# Create a step rule that reduces the learning rate of noise
scale_mask = Restrict(noise_step_rule, mask_parameters)
step_rule = CompositeRule([scale_mask, momentum])
# from theano.compile.nanguardmode import NanGuardMode
# Train with simple SGD
algorithm = GradientDescent(
cost=train_cost, parameters=trainable_parameters,
step_rule=step_rule)
algorithm.add_updates(extra_updates)
#,
# theano_func_kwargs={
# 'mode': NanGuardMode(
# nan_is_error=True, inf_is_error=True, big_is_error=True)})
exp_name = save_to.replace('.%d', '')
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = [Timing(),
FinishAfter(after_n_epochs=num_epochs,
after_n_batches=num_batches),
EpochSchedule(momentum.learning_rate, [
(0, 0.01), # Warm up with 0.01 learning rate
(50, 0.1), # Then go back to 0.1
(100, 0.01),
(150, 0.001)
# (83, 0.01), # Follow the schedule in the paper
# (125, 0.001)
]),
EpochSchedule(noise_step_rule.learning_rate, [
(0, 1e-2),
(2, 1e-1),
(4, 1)
# (0, 1e-6),
# (2, 1e-5),
# (4, 1e-4)
]),
EpochSchedule(noise_rate, [
(0, 1e-2),
(2, 1e-1),
(4, 1)
# (0, 1e-6),
# (2, 1e-5),
# (4, 1e-4),
# (6, 3e-4),
# (8, 1e-3), # Causes nit rate to jump
# (10, 3e-3),
# (12, 1e-2),
# (15, 3e-2),
# (19, 1e-1),
# (24, 3e-1),
# (30, 1)
]),
NoiseExtension(
noise_parameters=noise_parameters),
NoisyDataStreamMonitoring(
[test_cost, test_error_rate, test_confusion],
cifar10_test_stream,
noise_parameters=noise_parameters,
prefix="test"),
TrainingDataMonitoring(
[train_cost, train_error_rate, train_nit_rate,
train_cost_without_regularization,
l2_regularization,
train_nit_regularization,
momentum.learning_rate,
train_mean_log_sigma,
aggregation.mean(algorithm.total_gradient_norm)],
prefix="train",
every_n_batches=17),
# after_epoch=True),
Plot('Training performance for ' + exp_name,
channels=[
['train_cost_with_regularization',
'train_cost_without_regularization',
'train_nit_regularization',
'train_l2_regularization'],
['train_error_rate'],
['train_total_gradient_norm'],
['train_mean_log_sigma'],
],
every_n_batches=17),
Plot('Test performance for ' + exp_name,
channels=[[
'train_error_rate',
'test_error_rate',
]],
after_epoch=True),
EpochCheckpoint(save_to, use_cpickle=True, after_epoch=True),
ProgressBar(),
Printing()]
if histogram:
attribution = AttributionExtension(
components=train_components,
parameters=cg.parameters,
components_size=output_size,
after_batch=True)
extensions.insert(0, attribution)
if resume:
extensions.append(Load(exp_name, True, True))
model = Model(train_cost)
main_loop = MainLoop(
algorithm,
cifar10_train_stream,
model=model,
extensions=extensions)
main_loop.run()
if histogram:
save_attributions(attribution, filename=histogram)
with open('execution-log.json', 'w') as outfile:
json.dump(main_loop.log, outfile, cls=NumpyEncoder)
def main(args):
"""Run experiment. """
lr_tag = float_tag(args.learning_rate)
x_dim, train_stream, valid_stream, test_stream = datasets.get_streams(
args.data, args.batch_size)
#------------------------------------------------------------
# Setup model
deterministic_act = Tanh
deterministic_size = 1.
if args.method == 'vae':
sizes_tag = args.layer_spec.replace(",", "-")
layer_sizes = [int(i) for i in args.layer_spec.split(",")]
layer_sizes, z_dim = layer_sizes[:-1], layer_sizes[-1]
name = "%s-%s-%s-lr%s-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.n_samples, sizes_tag)
if args.activation == "tanh":
hidden_act = Tanh()
elif args.activation == "logistic":
hidden_act = Logistic()
elif args.activation == "relu":
hidden_act = Rectifier()
else:
raise "Unknown hidden nonlinearity %s" % args.hidden_act
model = VAE(x_dim=x_dim,
hidden_layers=layer_sizes,
hidden_act=hidden_act,
z_dim=z_dim,
batch_norm=args.batch_normalization)
model.initialize()
elif args.method == 'dvae':
sizes_tag = args.layer_spec.replace(",", "-")
layer_sizes = [int(i) for i in args.layer_spec.split(",")]
layer_sizes, z_dim = layer_sizes[:-1], layer_sizes[-1]
name = "%s-%s-%s-lr%s-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.n_samples, sizes_tag)
if args.activation == "tanh":
hidden_act = Tanh()
elif args.activation == "logistic":
hidden_act = Logistic()
elif args.activation == "relu":
hidden_act = Rectifier()
else:
raise "Unknown hidden nonlinearity %s" % args.hidden_act
model = DVAE(x_dim=x_dim,
hidden_layers=layer_sizes,
hidden_act=hidden_act,
z_dim=z_dim,
batch_norm=args.batch_normalization)
model.initialize()
elif args.method == 'rws':
sizes_tag = args.layer_spec.replace(",", "-")
qbase = "" if not args.no_qbaseline else "noqb-"
name = "%s-%s-%s-%slr%s-dl%d-spl%d-%s" % \
(args.data, args.method, args.name, qbase, lr_tag, args.deterministic_layers, args.n_samples, sizes_tag)
p_layers, q_layers = create_layers(args.layer_spec, x_dim,
args.deterministic_layers,
deterministic_act,
deterministic_size)
model = ReweightedWakeSleep(
p_layers,
q_layers,
qbaseline=(not args.no_qbaseline),
)
model.initialize()
elif args.method == 'bihm-rws':
sizes_tag = args.layer_spec.replace(",", "-")
name = "%s-%s-%s-lr%s-dl%d-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.deterministic_layers, args.n_samples, sizes_tag)
p_layers, q_layers = create_layers(args.layer_spec, x_dim,
args.deterministic_layers,
deterministic_act,
deterministic_size)
model = BiHM(
p_layers,
q_layers,
l1reg=args.l1reg,
l2reg=args.l2reg,
)
model.initialize()
elif args.method == 'continue':
import cPickle as pickle
from os.path import basename, splitext
with open(args.model_file, 'rb') as f:
m = pickle.load(f)
if isinstance(m, MainLoop):
m = m.model
model = m.get_top_bricks()[0]
while len(model.parents) > 0:
model = model.parents[0]
assert isinstance(model, (BiHM, ReweightedWakeSleep, VAE))
mname, _, _ = basename(args.model_file).rpartition("_model.pkl")
name = "%s-cont-%s-lr%s-spl%s" % (mname, args.name, lr_tag,
args.n_samples)
else:
raise ValueError("Unknown training method '%s'" % args.method)
#------------------------------------------------------------
x = tensor.matrix('features')
#------------------------------------------------------------
# Testset monitoring
train_monitors = []
valid_monitors = []
test_monitors = []
for s in [1, 10, 100, 1000]:
log_p, log_ph = model.log_likelihood(x, s)
log_p = -log_p.mean()
log_ph = -log_ph.mean()
log_p.name = "log_p_%d" % s
log_ph.name = "log_ph_%d" % s
#train_monitors += [log_p, log_ph]
#valid_monitors += [log_p, log_ph]
test_monitors += [log_p, log_ph]
#------------------------------------------------------------
# Z estimation
#for s in [100000]:
# z2 = tensor.exp(model.estimate_log_z2(s)) / s
# z2.name = "z2_%d" % s
#
# valid_monitors += [z2]
# test_monitors += [z2]
#------------------------------------------------------------
# Gradient and training monitoring
if args.method in ['vae', 'dvae']:
log_p_bound, gradients = model.get_gradients(x, args.n_samples)
log_p_bound = -log_p_bound.mean()
log_p_bound.name = "log_p_bound"
cost = log_p_bound
train_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
valid_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
test_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
else:
log_p, log_ph, gradients = model.get_gradients(x, args.n_samples)
log_p = -log_p.mean()
log_ph = -log_ph.mean()
log_p.name = "log_p"
log_ph.name = "log_ph"
cost = log_ph
train_monitors += [log_p, log_ph]
valid_monitors += [log_p, log_ph]
#------------------------------------------------------------
# Detailed monitoring
"""
n_layers = len(p_layers)
log_px, w, log_p, log_q, samples = model.log_likelihood(x, n_samples)
exp_samples = []
for l in xrange(n_layers):
e = (w.dimshuffle(0, 1, 'x')*samples[l]).sum(axis=1)
e.name = "inference_h%d" % l
e.tag.aggregation_scheme = aggregation.TakeLast(e)
exp_samples.append(e)
s1 = samples[1]
sh1 = s1.shape
s1_ = s1.reshape([sh1[0]*sh1[1], sh1[2]])
s0, _ = model.p_layers[0].sample_expected(s1_)
s0 = s0.reshape([sh1[0], sh1[1], s0.shape[1]])
s0 = (w.dimshuffle(0, 1, 'x')*s0).sum(axis=1)
s0.name = "inference_h0^"
s0.tag.aggregation_scheme = aggregation.TakeLast(s0)
exp_samples.append(s0)
# Draw P-samples
p_samples, _, _ = model.sample_p(100)
#weights = model.importance_weights(samples)
#weights = weights / weights.sum()
for i, s in enumerate(p_samples):
s.name = "psamples_h%d" % i
s.tag.aggregation_scheme = aggregation.TakeLast(s)
#
samples = model.sample(100, oversample=100)
for i, s in enumerate(samples):
s.name = "samples_h%d" % i
s.tag.aggregation_scheme = aggregation.TakeLast(s)
"""
cg = ComputationGraph([cost])
#------------------------------------------------------------
if args.step_rule == "momentum":
step_rule = Momentum(args.learning_rate, 0.95)
elif args.step_rule == "rmsprop":
step_rule = RMSProp(args.learning_rate)
elif args.step_rule == "adam":
step_rule = Adam(args.learning_rate)
else:
raise "Unknown step_rule %s" % args.step_rule
#parameters = cg.parameters[:4] + cg.parameters[5:]
parameters = cg.parameters
algorithm = GradientDescent(
cost=cost,
parameters=parameters,
gradients=gradients,
step_rule=CompositeRule([
#StepClipping(25),
step_rule,
#RemoveNotFinite(1.0),
]))
#------------------------------------------------------------
train_monitors += [
aggregation.mean(algorithm.total_gradient_norm),
aggregation.mean(algorithm.total_step_norm)
]
#------------------------------------------------------------
# Live plotting?
plotting_extensions = []
if args.live_plotting:
plotting_extensions = [
PlotManager(
name,
[
Plotter(channels=[[
"valid_%s" % cost.name, "valid_log_p"
], ["train_total_gradient_norm", "train_total_step_norm"]],
titles=[
"validation cost",
"norm of training gradient and step"
]),
DisplayImage(
[
WeightDisplay(model.p_layers[0].mlp.
linear_transformations[0].W,
n_weights=100,
image_shape=(28, 28))
]
#ImageDataStreamDisplay(test_stream, image_shape=(28,28))]
)
])
]
main_loop = MainLoop(
model=Model(cost),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
Timing(),
ProgressBar(),
TrainingDataMonitoring(
train_monitors, prefix="train", after_epoch=True),
DataStreamMonitoring(
valid_monitors, data_stream=valid_stream, prefix="valid"),
DataStreamMonitoring(test_monitors,
data_stream=test_stream,
prefix="test",
after_epoch=False,
after_training=True,
every_n_epochs=10),
#SharedVariableModifier(
# algorithm.step_rule.components[0].learning_rate,
# half_lr_func,
# before_training=False,
# after_epoch=False,
# after_batch=False,
# every_n_epochs=half_lr),
TrackTheBest('valid_%s' % cost.name),
Checkpoint(name + ".pkl", save_separately=['log', 'model']),
FinishIfNoImprovementAfter('valid_%s_best_so_far' % cost.name,
epochs=args.patience),
FinishAfter(after_n_epochs=args.max_epochs),
Printing()
] + plotting_extensions)
main_loop.run()
def main():
nclasses = 27
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--seed", type=int, default=1)
parser.add_argument("--length", type=int, default=180)
parser.add_argument("--num-epochs", type=int, default=100)
parser.add_argument("--batch-size", type=int, default=64)
parser.add_argument("--learning-rate", type=float, default=1e-3)
parser.add_argument("--epsilon", type=float, default=1e-5)
parser.add_argument("--num-hidden", type=int, default=1000)
parser.add_argument("--baseline", action="store_true")
parser.add_argument("--initialization",
choices="identity glorot orthogonal uniform".split(),
default="identity")
parser.add_argument("--initial-gamma", type=float, default=1e-1)
parser.add_argument("--initial-beta", type=float, default=0)
parser.add_argument("--cluster", action="store_true")
parser.add_argument("--activation",
choices=list(activations.keys()),
default="tanh")
parser.add_argument("--optimizer",
choices="sgdmomentum adam rmsprop",
default="rmsprop")
parser.add_argument("--continue-from")
parser.add_argument("--evaluate")
parser.add_argument("--dump-hiddens")
args = parser.parse_args()
np.random.seed(args.seed)
blocks.config.config.default_seed = args.seed
if args.continue_from:
from blocks.serialization import load
main_loop = load(args.continue_from)
main_loop.run()
sys.exit(0)
graphs, extensions, updates = construct_graphs(args, nclasses)
### optimization algorithm definition
if args.optimizer == "adam":
optimizer = Adam(learning_rate=args.learning_rate)
elif args.optimizer == "rmsprop":
optimizer = RMSProp(learning_rate=args.learning_rate, decay_rate=0.9)
elif args.optimizer == "sgdmomentum":
optimizer = Momentum(learning_rate=args.learning_rate, momentum=0.99)
step_rule = CompositeRule([
StepClipping(1.),
optimizer,
])
algorithm = GradientDescent(cost=graphs["training"].outputs[0],
parameters=graphs["training"].parameters,
step_rule=step_rule)
algorithm.add_updates(updates["training"])
model = Model(graphs["training"].outputs[0])
extensions = extensions["training"] + extensions["inference"]
# step monitor
step_channels = []
step_channels.extend([
algorithm.steps[param].norm(2).copy(name="step_norm:%s" % name)
for name, param in model.get_parameter_dict().items()
])
step_channels.append(
algorithm.total_step_norm.copy(name="total_step_norm"))
step_channels.append(
algorithm.total_gradient_norm.copy(name="total_gradient_norm"))
step_channels.extend(graphs["training"].outputs)
logger.warning("constructing training data monitor")
extensions.append(
TrainingDataMonitoring(step_channels,
prefix="iteration",
after_batch=True))
# parameter monitor
extensions.append(
DataStreamMonitoring([
param.norm(2).copy(name="parameter.norm:%s" % name)
for name, param in model.get_parameter_dict().items()
],
data_stream=None,
after_epoch=True))
validation_interval = 500
# performance monitor
for situation in "training inference".split():
if situation == "inference" and not args.evaluate:
# save time when we don't need the inference graph
continue
for which_set in "train valid test".split():
logger.warning("constructing %s %s monitor" %
(which_set, situation))
channels = list(graphs[situation].outputs)
extensions.append(
DataStreamMonitoring(channels,
prefix="%s_%s" % (which_set, situation),
every_n_batches=validation_interval,
data_stream=get_stream(
which_set=which_set,
batch_size=args.batch_size,
num_examples=10000,
length=args.length)))
extensions.extend([
TrackTheBest("valid_training_error_rate",
"best_valid_training_error_rate"),
DumpBest("best_valid_training_error_rate", "best.zip"),
FinishAfter(after_n_epochs=args.num_epochs),
#FinishIfNoImprovementAfter("best_valid_error_rate", epochs=50),
Checkpoint("checkpoint.zip",
on_interrupt=False,
every_n_epochs=1,
use_cpickle=True),
DumpLog("log.pkl", after_epoch=True)
])
if not args.cluster:
extensions.append(ProgressBar())
extensions.extend([
Timing(),
Printing(every_n_batches=validation_interval),
PrintingTo("log"),
])
main_loop = MainLoop(data_stream=get_stream(which_set="train",
batch_size=args.batch_size,
length=args.length,
augment=True),
algorithm=algorithm,
extensions=extensions,
model=model)
if args.dump_hiddens:
dump_hiddens(args, main_loop)
return
if args.evaluate:
evaluate(args, main_loop)
return
main_loop.run()
def main(save_to):
batch_size = 365
feature_maps = [6, 16]
mlp_hiddens = [120, 84]
conv_sizes = [5, 5]
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations, 1, image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='valid',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(
i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
x = tensor.tensor4('features')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
outs = VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(cg.variables)
# Create an interior activation model
model = Model([probs] + outs)
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
algorithm = MaximumActivationSearch(outputs=outs)
# Use the mnist test set, unshuffled
mnist_test = MNIST(("test",), sources=['features'])
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, batch_size))
extensions = [Timing(),
FinishAfter(after_n_epochs=1),
DataStreamMonitoring(
[],
mnist_test_stream,
prefix="test"),
Checkpoint("maxact.tar"),
ProgressBar(),
Printing()]
main_loop = MainLoop(
algorithm,
mnist_test_stream,
model=model,
extensions=extensions)
main_loop.run()
examples = mnist_test.get_example_stream()
example = examples.get_data(0)[0]
layers = convnet.layers
for output, record in algorithm.maximum_activations.items():
layer = get_brick(output)
activations, indices, snapshots = (
r.get_value() if r else None for r in record[1:])
filmstrip = Filmstrip(
example.shape[-2:], (indices.shape[1], indices.shape[0]),
background='blue')
if layer in layers:
fieldmap = layerarray_fieldmap(layers[0:layers.index(layer) + 1])
for unit in range(indices.shape[1]):
for index in range(100):
mask = make_mask(example.shape[-2:], fieldmap, numpy.clip(
snapshots[index, unit, :, :], 0, numpy.inf))
imagenum = indices[index, unit, 0]
filmstrip.set_image((unit, index),
examples.get_data(imagenum)[0], mask)
else:
for unit in range(indices.shape[1]):
for index in range(100):
imagenum = indices[index, unit]
filmstrip.set_image((unit, index),
examples.get_data(imagenum)[0])
filmstrip.save(layer.name + '_maxact.jpg')
def main(config, tr_stream, dev_stream, use_bokeh=False, the_task=None, the_track=None):
config['the_task'] = the_task
# Create Theano variables
logger.info('Creating theano variables')
source_sentence = tensor.lmatrix('source')
source_sentence_mask = tensor.matrix('source_mask')
target_sentence = tensor.lmatrix('target')
target_sentence_mask = tensor.matrix('target_mask')
sampling_input = tensor.lmatrix('input')
# Construct model
logger.info('Building RNN encoder-decoder')
encoder = BidirectionalEncoder(
# end_embed is dimension of word embedding matrix in encoder; enc_nhids number of hidden units in encoder GRU
config['src_vocab_size'], config['enc_embed'], config['enc_nhids'])
decoder = Decoder(
config['trg_vocab_size'], config['dec_embed'], config['dec_nhids'],
config['enc_nhids'] * 2, config['use_attention'], cost_type=config['error_fct'])
cost = decoder.cost(
encoder.apply(source_sentence, source_sentence_mask),
source_sentence_mask, target_sentence, target_sentence_mask)
testVar = decoder.getTestVar(
encoder.apply(source_sentence, source_sentence_mask),
source_sentence_mask, target_sentence, target_sentence_mask)
logger.info('Creating computational graph')
cg = ComputationGraph(cost)
# Initialize model
logger.info('Initializing model')
my_rng = numpy.random.RandomState(config['rng_value'])
if config['identity_init']:
encoder.weights_init = decoder.weights_init = Identity()
else:
encoder.weights_init = decoder.weights_init = IsotropicGaussian(
config['weight_scale'])
encoder.rng = decoder.rng = my_rng
encoder.biases_init = decoder.biases_init = Constant(0)
encoder.push_initialization_config()
decoder.push_initialization_config()
encoder.bidir.prototype.weights_init = Orthogonal()
encoder.bidir.prototype.rng = my_rng
decoder.transition.weights_init = Orthogonal()
decoder.transition.rng = my_rng
encoder.initialize()
decoder.initialize()
# apply dropout for regularization
if config['dropout'] < 1.0:
# dropout is applied to the output of maxout in ghog
logger.info('Applying dropout')
dropout_inputs = [x for x in cg.intermediary_variables
if x.name == 'maxout_apply_output']
cg = apply_dropout(cg, dropout_inputs, config['dropout'])
# Apply weight noise for regularization
if config['weight_noise_ff'] > 0.0:
logger.info('Applying weight noise to ff layers')
enc_params = Selector(encoder.lookup).get_params().values()
enc_params += Selector(encoder.fwd_fork).get_params().values()
enc_params += Selector(encoder.back_fork).get_params().values()
dec_params = Selector(
decoder.sequence_generator.readout).get_params().values()
dec_params += Selector(
decoder.sequence_generator.fork).get_params().values()
dec_params += Selector(decoder.state_init).get_params().values()
cg = apply_noise(cg, enc_params+dec_params, config['weight_noise_ff'], seed=my_rng)
cost = cg.outputs[0]
# Print shapes
shapes = [param.get_value().shape for param in cg.parameters]
logger.info("Parameter shapes: ")
for shape, count in Counter(shapes).most_common():
logger.info(' {:15}: {}'.format(shape, count))
logger.info("Total number of parameters: {}".format(len(shapes)))
# Print parameter names
enc_dec_param_dict = merge(Selector(encoder).get_parameters(),
Selector(decoder).get_parameters())
logger.info("Parameter names: ")
for name, value in enc_dec_param_dict.items():
logger.info(' {:15}: {}'.format(value.get_value().shape, name))
logger.info("Total number of parameters: {}"
.format(len(enc_dec_param_dict)))
# Set up training model
logger.info("Building model")
training_model = Model(cost)
# Set extensions
logger.info("Initializing extensions")
# this is ugly code and done, because I am not sure if the order of the extensions is important
if 'track2' in config['saveto']: # less epochs for track 2, because of more data
if config['early_stopping']:
extensions = [
FinishAfter(after_n_epochs=config['finish_after']/2),
#FinishAfter(after_n_batches=config['finish_after']),
TrainingDataMonitoring([cost], after_batch=True),
Printing(after_batch=True),
CheckpointNMT(config['saveto'],
every_n_batches=config['save_freq'])
]
else:
extensions = [
FinishAfter(after_n_epochs=config['finish_after']/2),
#FinishAfter(after_n_batches=config['finish_after']),
TrainingDataMonitoring([cost], after_batch=True),
Printing(after_batch=True),
CheckpointNMT(config['saveto'],
every_n_batches=config['save_freq'])
]
else:
if config['early_stopping']:
extensions = [
FinishAfter(after_n_epochs=config['finish_after']),
#FinishAfter(after_n_batches=config['finish_after']),
TrainingDataMonitoring([cost], after_batch=True),
Printing(after_batch=True),
CheckpointNMT(config['saveto'],
every_n_batches=config['save_freq'])
]
else:
extensions = [
FinishAfter(after_n_epochs=config['finish_after']),
#FinishAfter(after_n_batches=config['finish_after']),
TrainingDataMonitoring([cost], after_batch=True),
Printing(after_batch=True),
CheckpointNMT(config['saveto'],
every_n_batches=config['save_freq'])
]
# Set up beam search and sampling computation graphs if necessary
if config['hook_samples'] >= 1:
logger.info("Building sampling model")
sampling_representation = encoder.apply(
sampling_input, tensor.ones(sampling_input.shape))
generated = decoder.generate(sampling_input, sampling_representation)
search_model = Model(generated)
_, samples = VariableFilter(
bricks=[decoder.sequence_generator], name="outputs")(
ComputationGraph(generated[1])) # generated[1] is next_outputs
# Add sampling
if config['hook_samples'] >= 1:
logger.info("Building sampler")
extensions.append(
Sampler(model=search_model, data_stream=tr_stream,
hook_samples=config['hook_samples'],
#every_n_batches=1,
every_n_batches=config['sampling_freq'],
src_vocab_size=8))
#src_vocab_size=config['src_vocab_size']))
# Add early stopping based on bleu
if config['val_set'] is not None:
logger.info("Building accuracy validator")
extensions.append(
AccuracyValidator(sampling_input, samples=samples, config=config,
model=search_model, data_stream=dev_stream,
after_training=True,
#after_epoch=True))
every_n_epochs=5))
else:
logger.info("No validation set given for this language")
# Reload model if necessary
if config['reload']:
extensions.append(LoadNMT(config['saveto']))
# Set up training algorithm
logger.info("Initializing training algorithm")
algorithm = GradientDescent(
cost=cost, parameters=cg.parameters,
step_rule=CompositeRule([StepClipping(config['step_clipping']),
eval(config['step_rule'])()])
)
# Initialize main loop
logger.info("Initializing main loop")
main_loop = MainLoop(
model=training_model,
algorithm=algorithm,
data_stream=tr_stream,
extensions=extensions
)
# Train!
main_loop.run()
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
on_unused_sources='ignore',
step_rule=CompositeRule([
StepClipping(10.),
Adam(learning_rate),
]))
main_loop = MainLoop(model=Model(cost),
data_stream=data_stream_train,
algorithm=algorithm,
extensions=[
Timing(),
FinishAfter(after_n_epochs=n_epochs), monitor,
monitor_val, monitor_test,
saveSnapshot(
'/home/xuehongyang/checkpoints_read/snapshot',
save_main_loop=False,
after_epoch=True,
save_separately=['log', 'model']),
ProgressBar(),
Printing(every_n_batches=500),
Plot('videoqa_open_rereading',
channels=[['train_cost']],
every_n_batches=500,
after_batch=True)
])
print('starting...')
main_loop.run()
prefix='valid',
every_n_batches=config.valid_freq),
]
if plot_avail:
plot_channels = [
['train_' + v.name for v in lt] + ['valid_' + v.name for v in lv]
for lt, lv in zip(m.monitor_vars, m.monitor_vars_valid)
]
extensions += [
Plot(
document='deepmind_qa_' + model_name,
channels=plot_channels,
# server_url='http://localhost:5006/', # If you need, change this
every_n_batches=config.print_freq)
]
extensions += [
Printing(every_n_batches=config.print_freq, after_epoch=True),
ProgressBar()
]
main_loop = MainLoop(model=model,
data_stream=train_stream,
algorithm=algorithm,
extensions=extensions)
# Run the model !
main_loop.run()
main_loop.profile.report()
# vim: set sts=4 ts=4 sw=4 tw=0 et :
every_n_batches=n_batches,
prefix="valid")
extensions = [
ProgressBar(),
Timing(every_n_batches=n_batches), train_monitor, valid_monitor,
TrackTheBest('valid_nll', every_n_batches=n_batches),
Plot(save_dir + "progress/" + experiment_name + ".png",
[['train_nll', 'valid_nll'], ['valid_learning_rate']],
every_n_batches=n_batches,
email=False),
Checkpoint(save_dir + "pkl/best_" + experiment_name + ".pkl",
use_cpickle=True).add_condition(
['after_batch'],
predicate=OnLogRecord('valid_nll_best_so_far')),
Printing(every_n_batches=n_batches),
Flush(every_n_batches=n_batches, before_first_epoch=True),
LearningRateSchedule(lr,
'valid_nll',
path=save_dir + "pkl/best_" + experiment_name +
".pkl",
states=states.values(),
every_n_batches=n_batches,
before_first_epoch=True)
]
main_loop = MainLoop(model=model,
data_stream=train_stream,
algorithm=algorithm,
extensions=extensions)
def main(name, epochs, batch_size, learning_rate, attention, n_iter, enc_dim,
dec_dim, z_dim):
if name is None:
tag = "watt" if attention else "woatt"
name = "%s-t%d-enc%d-dec%d-z%d" % (tag, n_iter, enc_dim, dec_dim,
z_dim)
print("\nRunning experiment %s" % name)
print(" learning rate: %5.3f" % learning_rate)
print(" attention: %s" % attention)
print(" n_iterations: %d" % n_iter)
print(" encoder dimension: %d" % enc_dim)
print(" z dimension: %d" % z_dim)
print(" decoder dimension: %d" % dec_dim)
print()
#------------------------------------------------------------------------
x_dim = 28 * 28
img_height, img_width = (28, 28)
rnninits = {
'weights_init': Orthogonal(),
#'weights_init': IsotropicGaussian(0.001),
'biases_init': Constant(0.),
}
inits = {
'weights_init': Orthogonal(),
#'weights_init': IsotropicGaussian(0.01),
'biases_init': Constant(0.),
}
prior_mu = T.zeros([z_dim])
prior_log_sigma = T.zeros([z_dim])
if attention:
read_N = 4
write_N = 6
read_dim = 2 * read_N**2
reader = AttentionReader(x_dim=x_dim,
dec_dim=dec_dim,
width=img_width,
height=img_height,
N=read_N,
**inits)
writer = AttentionWriter(input_dim=dec_dim,
output_dim=x_dim,
width=img_width,
height=img_height,
N=read_N,
**inits)
else:
read_dim = 2 * x_dim
reader = Reader(x_dim=x_dim, dec_dim=dec_dim, **inits)
writer = Writer(input_dim=dec_dim, output_dim=x_dim, **inits)
encoder = LSTM(dim=enc_dim, name="RNN_enc", **rnninits)
decoder = LSTM(dim=dec_dim, name="RNN_dec", **rnninits)
encoder_mlp = MLP([Tanh()], [(read_dim + dec_dim), 4 * enc_dim],
name="MLP_enc",
**inits)
decoder_mlp = MLP([Tanh()], [z_dim, 4 * dec_dim], name="MLP_dec", **inits)
q_sampler = Qsampler(input_dim=enc_dim, output_dim=z_dim, **inits)
for brick in [
reader, writer, encoder, decoder, encoder_mlp, decoder_mlp,
q_sampler
]:
brick.allocate()
brick.initialize()
#------------------------------------------------------------------------
x = tensor.matrix('features')
# This is one iteration
def one_iteration(c, h_enc, c_enc, z_mean, z_log_sigma, z, h_dec, c_dec,
x):
x_hat = x - T.nnet.sigmoid(c)
r = reader.apply(x, x_hat, h_dec)
i_enc = encoder_mlp.apply(T.concatenate([r, h_dec], axis=1))
h_enc, c_enc = encoder.apply(states=h_enc,
cells=c_enc,
inputs=i_enc,
iterate=False)
z_mean, z_log_sigma, z = q_sampler.apply(h_enc)
i_dec = decoder_mlp.apply(z)
h_dec, c_dec = decoder.apply(states=h_dec,
cells=c_dec,
inputs=i_dec,
iterate=False)
c = c + writer.apply(h_dec)
return c, h_enc, c_enc, z_mean, z_log_sigma, z, h_dec, c_dec
outputs_info = [
T.zeros([batch_size, x_dim]), # c
T.zeros([batch_size, enc_dim]), # h_enc
T.zeros([batch_size, enc_dim]), # c_enc
T.zeros([batch_size, z_dim]), # z_mean
T.zeros([batch_size, z_dim]), # z_log_sigma
T.zeros([batch_size, z_dim]), # z
T.zeros([batch_size, dec_dim]), # h_dec
T.zeros([batch_size, dec_dim]), # c_dec
]
outputs, scan_updates = theano.scan(fn=one_iteration,
sequences=[],
outputs_info=outputs_info,
non_sequences=[x],
n_steps=n_iter)
c, h_enc, c_enc, z_mean, z_log_sigma, z, h_dec, c_dec = outputs
kl_terms = (prior_log_sigma - z_log_sigma + 0.5 *
(tensor.exp(2 * z_log_sigma) +
(z_mean - prior_mu)**2) / tensor.exp(2 * prior_log_sigma) -
0.5).sum(axis=-1)
x_recons = T.nnet.sigmoid(c[-1, :, :])
recons_term = BinaryCrossEntropy().apply(x, x_recons)
recons_term.name = "recons_term"
cost = recons_term + kl_terms.sum(axis=0).mean()
cost.name = "nll_bound"
#------------------------------------------------------------
cg = ComputationGraph([cost])
params = VariableFilter(roles=[PARAMETER])(cg.variables)
algorithm = GradientDescent(
cost=cost,
params=params,
step_rule=CompositeRule([
#StepClipping(3.),
Adam(learning_rate),
])
#step_rule=RMSProp(learning_rate),
#step_rule=Momentum(learning_rate=learning_rate, momentum=0.95)
)
algorithm.add_updates(scan_updates)
#------------------------------------------------------------------------
# Setup monitors
monitors = [cost]
for t in range(n_iter):
kl_term_t = kl_terms[t, :].mean()
kl_term_t.name = "kl_term_%d" % t
x_recons_t = T.nnet.sigmoid(c[t, :, :])
recons_term_t = BinaryCrossEntropy().apply(x, x_recons_t)
recons_term_t = recons_term_t.mean()
recons_term_t.name = "recons_term_%d" % t
monitors += [kl_term_t, recons_term_t]
train_monitors = monitors[:]
train_monitors += [aggregation.mean(algorithm.total_gradient_norm)]
train_monitors += [aggregation.mean(algorithm.total_step_norm)]
# Live plotting...
plot_channels = [["train_nll_bound", "test_nll_bound"],
["train_kl_term_%d" % t for t in range(n_iter)],
["train_recons_term_%d" % t for t in range(n_iter)],
["train_total_gradient_norm", "train_total_step_norm"]]
#------------------------------------------------------------
mnist_train = BinarizedMNIST("train", sources=['features'])
mnist_test = BinarizedMNIST("test", sources=['features'])
#mnist_train = MNIST("train", binary=True, sources=['features'])
#mnist_test = MNIST("test", binary=True, sources=['features'])
main_loop = MainLoop(
model=None,
data_stream=ForceFloatX(
DataStream(mnist_train,
iteration_scheme=SequentialScheme(
mnist_train.num_examples, batch_size))),
algorithm=algorithm,
extensions=[
Timing(),
FinishAfter(after_n_epochs=epochs),
DataStreamMonitoring(
monitors,
ForceFloatX(
DataStream(mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, batch_size))),
updates=scan_updates,
prefix="test"),
TrainingDataMonitoring(train_monitors,
prefix="train",
after_every_epoch=True),
SerializeMainLoop(name + ".pkl"),
Plot(name, channels=plot_channels),
ProgressBar(),
Printing()
])
main_loop.run()
def main(save_to, num_epochs,
regularization=0.0003, subset=None, num_batches=None,
histogram=None, resume=False):
batch_size = 500
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cost = (CategoricalCrossEntropy().apply(y.flatten(), probs)
.copy(name='cost'))
components = (ComponentwiseCrossEntropy().apply(y.flatten(), probs)
.copy(name='components'))
error_rate = (MisclassificationRate().apply(y.flatten(), probs)
.copy(name='error_rate'))
confusion = (ConfusionMatrix().apply(y.flatten(), probs)
.copy(name='confusion'))
confusion.tag.aggregation_scheme = Sum(confusion)
cg = ComputationGraph([cost, error_rate, components])
# Apply regularization to the cost
weights = VariableFilter(roles=[WEIGHT])(cg.variables)
l2_norm = sum([(W ** 2).sum() for W in weights])
l2_norm.name = 'l2_norm'
cost = cost + regularization * l2_norm
cost.name = 'cost_with_regularization'
if subset:
start = 30000 - subset // 2
mnist_train = MNIST(("train",), subset=slice(start, start+subset))
else:
mnist_train = MNIST(("train",))
mnist_train_stream = DataStream.default_stream(
mnist_train, iteration_scheme=ShuffledScheme(
mnist_train.num_examples, batch_size))
mnist_test = MNIST(("test",))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=ShuffledScheme(
mnist_test.num_examples, batch_size))
# Train with simple SGD
algorithm = GradientDescent(
cost=cost, parameters=cg.parameters,
step_rule=AdaDelta(decay_rate=0.99))
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = [Timing(),
FinishAfter(after_n_epochs=num_epochs,
after_n_batches=num_batches),
DataStreamMonitoring(
[cost, error_rate, confusion],
mnist_test_stream,
prefix="test"),
TrainingDataMonitoring(
[cost, error_rate, l2_norm,
aggregation.mean(algorithm.total_gradient_norm)],
prefix="train",
after_epoch=True),
Checkpoint(save_to),
ProgressBar(),
Printing()]
if histogram:
attribution = AttributionExtension(
components=components,
parameters=cg.parameters,
components_size=output_size,
after_batch=True)
extensions.insert(0, attribution)
if resume:
extensions.append(Load(save_to, True, True))
model = Model(cost)
main_loop = MainLoop(
algorithm,
mnist_train_stream,
model=model,
extensions=extensions)
main_loop.run()
if histogram:
save_attributions(attribution, filename=histogram)
with open('execution-log.json', 'w') as outfile:
json.dump(main_loop.log, outfile, cls=NumpyEncoder)
def start(self):
xx = T.matrix('features', config.floatX)
yy = T.imatrix('targets')
zm = BNUM*(xx.shape[0]//BNUM)
x = xx[:zm].reshape((BNUM, zm//BNUM, xx.shape[1])).dimshuffle(1, 0, 2)
y = yy[:zm].reshape((BNUM, zm//BNUM)).dimshuffle(1, 0)
# x = xx[:zm].reshape((zm//16, 16, xx.shape[1]))
# y = yy[:zm].reshape((zm//16, 16))
DIMS = [108*5, 200, 200, 200, LABEL]
NUMS = [1, 1, 1, 1, 1]
# DIMS = [108*5, 48]
# NUMS = [1, 1]
FUNCS = [
Rectifier,
Rectifier,
Rectifier,
# Rectifier,
# Rectifier,
# Maxout(num_pieces=5),
# Maxout(num_pieces=5),
# Maxout(num_pieces=5),
# SimpleRecurrent,
# SimpleRecurrent,
# SimpleRecurrent,
# LSTM,
# LSTM,
# LSTM,
# SequenceGenerator,
# Softmax,
None,
]
def lllistool(i, inp, func):
if func == LSTM:
NUMS[i+1] *= 4
sdim = DIMS[i]
if func == SimpleRecurrent or func == LSTM:
sdim = DIMS[i] + DIMS[i+1]
l = Linear(input_dim=DIMS[i], output_dim=DIMS[i+1] * NUMS[i+1],
weights_init=IsotropicGaussian(std=sdim**(-0.5)),
biases_init=IsotropicGaussian(std=sdim**(-0.5)),
name='Lin{}'.format(i))
l.initialize()
if func == SimpleRecurrent:
gong = func(dim=DIMS[i+1], activation=Rectifier(), weights_init=IsotropicGaussian(std=sdim**(-0.5)))
gong.initialize()
ret = gong.apply(l.apply(inp))
elif func == LSTM:
gong = func(dim=DIMS[i+1], activation=Tanh(), weights_init=IsotropicGaussian(std=sdim**(-0.5)))
gong.initialize()
print(inp)
ret, _ = gong.apply(
l.apply(inp),
T.zeros((inp.shape[1], DIMS[i+1])),
T.zeros((inp.shape[1], DIMS[i+1])),
)
elif func == SequenceGenerator:
gong = func(
readout=None,
transition=SimpleRecurrent(dim=100, activation=Rectifier(), weights_init=IsotropicGaussian(std=0.1)))
ret = None
elif func == None:
ret = l.apply(inp)
else:
gong = func()
ret = gong.apply(l.apply(inp))
return ret
oup = x
for i in range(len(DIMS)-1):
oup = lllistool(i, oup, FUNCS[i])
y_hat = oup
y_rsp = y.reshape((y.shape[0]*y.shape[1],))
y_dsf_rsp = y.dimshuffle(1, 0).reshape((y.shape[0]*y.shape[1],))
yh_rsp = y_hat.reshape((y_hat.shape[0]*y_hat.shape[1], y_hat.shape[2]))
yh_dsf_rsp = y_hat.dimshuffle(1, 0, 2).reshape((y_hat.shape[0]*y_hat.shape[1], y_hat.shape[2]))
sfmx = Softmax().apply(yh_rsp)
# cost = CategoricalCrossEntropy().apply(y, y_hat).astype(config.floatX)
# j, wlh = Yimumu(y_hat, y)
# cost = CategoricalCrossEntropy().apply(y_rsp, sfmx) + j
cost = CategoricalCrossEntropy().apply(y_rsp, sfmx)
# cost_p = cost_p.astype(config.floatX)
# cost = CTC_cost(y, y_hat)
cost = cost.astype(config.floatX)
cg = ComputationGraph(cost)
# cg_p = ComputationGraph(cost_p)
orig_cg = cg
ips = VariableFilter(roles=[INPUT])(cg.variables)
ops = VariableFilter(roles=[OUTPUT])(cg.variables)
# print(ips, ops)
# cg = apply_dropout(cg, ips[0:2:1], 0.2)
# cg = apply_dropout(cg, ips[2:-2:1], 0.5)
# cost = cg.outputs[0].astype(config.floatX)
cost.name = 'cost'
mps = theano.shared(np.array([ph2id(ph48239(id2ph(t))) for t in range(48)]))
# yh_dsf_rsp = theano.printing.Print('YapYapYap')(yh_dsf_rsp)
# z_hat = T.argmax(yh_dsf_rsp[:,:-1], axis=1)
z_hat = T.argmax(yh_dsf_rsp, axis=1)
# z_hat = theano.printing.Print('Yap')(z_hat)
# z_hat = Yimumu_Decode()(y_hat, wlh)
z_hat_hat = CTC_Decode()(y_hat)
y39,_ = scan(fn=lambda t: mps[t], outputs_info=None, sequences=[y_dsf_rsp])
y_hat39,_ = scan(fn=lambda t: mps[t], outputs_info=None, sequences=[z_hat])
y_hat_hat39 = y_hat39
# y_hat_hat39,_ = scan(fn=lambda t: mps[t], outputs_info=None, sequences=[z_hat_hat])
# trm = TrimOp()(y_hat_hat39)
# trm = trm[1:1+trm[0]]
# trm = theano.printing.Print('Trm')(trm)
lost01 = (T.sum(T.neq(y_hat39, y39)) / y39.shape[0]).astype(config.floatX)
lost01.name = '0/1 loss'
lost23 = (T.sum(T.neq(y_hat39, y39)) / y39.shape[0]).astype(config.floatX)
lost23.name = '2/3 loss'
edit01 = EditDistance()(y39, y_hat_hat39).astype(config.floatX) #+ T.sum(trm) * 1E-10
# edit01 = edit01.astype(config.floatX)
edit01.name = '0/1 edit'
edit23 = EditDistance()(y39, y_hat_hat39).astype(config.floatX)
edit23.name = '2/3 edit'
Ws = cg.parameters
# Ws = Ws + [wlh]
print(list(Ws))
norms = sum(w.norm(2) for w in Ws)
norms = norms.astype(config.floatX)
norms.name = 'norms'
path = pjoin(PATH['fuel'], 'train_train.hdf5')
data = H5PYDataset(path, which_set='train', load_in_memory=True, subset=slice(0, 100000))
# data = H5PYDataset(path, which_set='train', load_in_memory=True)
data_v = H5PYDataset(pjoin(PATH['fuel'], 'train_validate.hdf5'), which_set='validate', load_in_memory=True)
num = data.num_examples
data_stream = DataStream(data, iteration_scheme=ShuffledScheme(
num, batch_size=SLEN*BNUM))
data_stream_v = DataStream(data_v, iteration_scheme=SequentialScheme(
data_v.num_examples, batch_size=SLEN*BNUM))
algo = GradientDescent(cost=cost, params=Ws, step_rule=CompositeRule([
Momentum(0.005, 0.9)
# AdaDelta()
]))
# algo_p = GradientDescent(cost=cost_p, params=cg_p.parameters, step_rule=CompositeRule([
# Momentum(0.01, 0.9)
# # AdaDelta()
# ]))
monitor = DataStreamMonitoring( variables=[cost, lost01, edit01, norms],
data_stream=data_stream)
monitor_v = DataStreamMonitoring( variables=[lost23, edit23],
data_stream=data_stream_v)
plt = Plot('AlpYap', channels=[['0/1 loss', '2/3 loss'], ['0/1 edit', '2/3 edit']], after_epoch=True)
# main_loop_p = MainLoop(data_stream = data_stream,
# algorithm=algo_p,
# extensions=[monitor, monitor_v, FinishAfter(after_n_epochs=10), Printing(), plt])
# main_loop_p.run()
main_loop = MainLoop(data_stream = data_stream,
algorithm=algo,
extensions=[monitor, monitor_v, FinishAfter(after_n_epochs=2000), Printing(), plt])
main_loop.run()
pfile = open('zzz.pkl', 'wb')
pickle.dump(orig_cg, pfile)
# pickle.dump(wlh, pfile)
pfile.close()
################
test_feat = np.load(pjoin(PATH['numpy'], 'train_test_features.npy')).astype(config.floatX)
func = theano.function([xx], y_hat.astype(config.floatX))
test_hat = []
for i in range(19):
tmp = func(test_feat[i*10000:(i+1)*10000])
tmp = tmp.transpose((1, 0, 2)).reshape((tmp.shape[0]*tmp.shape[1], tmp.shape[2]))
test_hat.append(tmp)
test_hat = np.concatenate(test_hat, axis=0)
test_hat = np.concatenate((test_hat, np.zeros((2, LABEL))), axis=0)
alpha = T.tensor3(config.floatX)
beta = alpha.argmax(axis=2)
# beta = alpha[:,:,:-1].argmax(axis=2)
# beta = Yimumu_Decode()(alpha, wlh)
# beta = CTC_Decode()(alpha)
func2 = theano.function([alpha], beta)
lens = []
tags = []
with shelve.open(SHELVE['test']) as f:
names = f['names']
for n in names:
lens.append(len(f[n]))
for i in range(lens[-1]):
tags.append(n+'_'+str(i+1))
seq = []
seq2 = []
nowcnt = 0
for i in lens:
nxt = nowcnt + i
cur_hat = test_hat[nowcnt:nxt].reshape((i, 1, LABEL)).astype(config.floatX)
nowcnt = nxt
fc2 = func2(cur_hat).flatten()
fc3 = []
fc4 = []
for j in fc2:
fc3.append(ph48239(id2ph(j)))
fc4.append(ph2c(ph48239(id2ph(j))))
seq.append(fc3)
seq2.append(''.join(trim(fc4)))
seq_flat = np.concatenate(seq)
with open('hw1_outz.txt', 'w') as f:
f.write('id,prediction\n')
for t, i in zip(tags, seq_flat):
f.write(t+','+i+'\n')
with open('hw2_outz.txt', 'w') as f:
f.write('id,phone_sequence\n')
for n, i in zip(names, seq2):
f.write(n+','+i+'\n')
mnist_train = MNIST("train")
train_stream = Flatten(
DataStream.default_stream(dataset=mnist_train,
iteration_scheme=SequentialScheme(
mnist_train.num_examples, 128)), )
# load testing data
mnist_test = MNIST("test")
test_stream = Flatten(
DataStream.default_stream(dataset=mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, 1024)), )
# train the model
from blocks.model import Model
main_loop = MainLoop(model=Model(cost),
data_stream=train_stream,
algorithm=GradientDescent(
cost=cost,
params=ComputationGraph(cost).parameters,
step_rule=Scale(learning_rate=0.1)),
extensions=[
FinishAfter(after_n_epochs=5),
DataStreamMonitoring(variables=[cost, error_rate],
data_stream=test_stream,
prefix="test"),
Printing()
])
main_loop.run()
def main(nvis, nhid, encoding_lstm_dim, decoding_lstm_dim, T=1):
x = tensor.matrix('features')
# Construct and initialize model
encoding_mlp = MLP([Tanh()], [None, None])
decoding_mlp = MLP([Tanh()], [None, None])
encoding_lstm = LSTM(dim=encoding_lstm_dim)
decoding_lstm = LSTM(dim=decoding_lstm_dim)
draw = DRAW(nvis=nvis,
nhid=nhid,
T=T,
encoding_mlp=encoding_mlp,
decoding_mlp=decoding_mlp,
encoding_lstm=encoding_lstm,
decoding_lstm=decoding_lstm,
biases_init=Constant(0),
weights_init=Orthogonal())
draw.push_initialization_config()
encoding_lstm.weights_init = IsotropicGaussian(std=0.001)
decoding_lstm.weights_init = IsotropicGaussian(std=0.001)
draw.initialize()
# Compute cost
cost = -draw.log_likelihood_lower_bound(x).mean()
cost.name = 'nll_upper_bound'
model = Model(cost)
# Datasets and data streams
mnist_train = BinarizedMNIST('train')
train_loop_stream = ForceFloatX(
DataStream(dataset=mnist_train,
iteration_scheme=SequentialScheme(mnist_train.num_examples,
100)))
train_monitor_stream = ForceFloatX(
DataStream(dataset=mnist_train,
iteration_scheme=SequentialScheme(mnist_train.num_examples,
500)))
mnist_valid = BinarizedMNIST('valid')
valid_monitor_stream = ForceFloatX(
DataStream(dataset=mnist_valid,
iteration_scheme=SequentialScheme(mnist_valid.num_examples,
500)))
mnist_test = BinarizedMNIST('test')
test_monitor_stream = ForceFloatX(
DataStream(dataset=mnist_test,
iteration_scheme=SequentialScheme(mnist_test.num_examples,
500)))
# Get parameters and monitoring channels
computation_graph = ComputationGraph([cost])
params = VariableFilter(roles=[PARAMETER])(computation_graph.variables)
monitoring_channels = dict([
('avg_' + channel.tag.name, channel.mean())
for channel in VariableFilter(
name='.*term$')(computation_graph.auxiliary_variables)
])
for name, channel in monitoring_channels.items():
channel.name = name
monitored_quantities = monitoring_channels.values() + [cost]
# Training loop
step_rule = RMSProp(learning_rate=1e-3, decay_rate=0.95)
algorithm = GradientDescent(cost=cost, params=params, step_rule=step_rule)
algorithm.add_updates(computation_graph.updates)
main_loop = MainLoop(
model=model,
data_stream=train_loop_stream,
algorithm=algorithm,
extensions=[
Timing(),
SerializeMainLoop('vae.pkl', save_separately=['model']),
FinishAfter(after_n_epochs=200),
DataStreamMonitoring(monitored_quantities,
train_monitor_stream,
prefix="train",
updates=computation_graph.updates),
DataStreamMonitoring(monitored_quantities,
valid_monitor_stream,
prefix="valid",
updates=computation_graph.updates),
DataStreamMonitoring(monitored_quantities,
test_monitor_stream,
prefix="test",
updates=computation_graph.updates),
ProgressBar(),
Printing()
])
main_loop.run()
cost.name = "sequence_log_likelihood"
cg = ComputationGraph(cost)
model = Model(cost)
#################
# Algorithm
#################
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=CompositeRule(
[StepClipping(10.0),
Adam(lr)]))
train_monitor = TrainingDataMonitoring(variables=[cost],
after_epoch=True,
prefix="train")
extensions = extensions = [
train_monitor,
TrackTheBest('train_sequence_log_likelihood'),
Printing(after_epoch=True)
]
main_loop = MainLoop(model=model,
data_stream=data_stream,
algorithm=algorithm,
extensions=extensions)
main_loop.run()
step_rules = [RMSProp(learning_rate=learning_rate, decay_rate=decay_rate),
StepClipping(step_clipping)]
algorithm = GradientDescent(cost=cost, parameters=cg.parameters,
step_rule=CompositeRule(step_rules))
# Extensions
gradient_norm = aggregation.mean(algorithm.total_gradient_norm)
step_norm = aggregation.mean(algorithm.total_step_norm)
monitored_vars = [cost, gradient_norm, step_norm]
dev_monitor = DataStreamMonitoring(variables=[cost], after_epoch=True,
before_first_epoch=True, data_stream=dev_stream, prefix="dev")
train_monitor = TrainingDataMonitoring(variables=monitored_vars, after_batch=True,
before_first_epoch=True, prefix='tra')
extensions = [dev_monitor, train_monitor, Timing(), Printing(after_batch=True),
FinishAfter(after_n_epochs=nepochs),
saveload.Load(load_path),
saveload.Checkpoint(last_path),
] + track_best('dev_cost', save_path)
if learning_rate_decay not in (0, 1):
extensions.append(SharedVariableModifier(step_rules[0].learning_rate,
lambda n, lr: numpy.cast[theano.config.floatX](learning_rate_decay * lr), after_epoch=True, after_batch=False))
print('number of parameters in the model: ' + str(tensor.sum([p.size for p in cg.parameters]).eval()))
# Finally build the main loop and train the model
main_loop = MainLoop(data_stream=train_stream, algorithm=algorithm,
model=Model(cost), extensions=extensions)
main_loop.run()
