def create_main_loop(save_path):
model, bn_model, bn_updates = create_models()
ali, = bn_model.top_bricks
discriminator_loss, generator_loss = bn_model.outputs
step_rule = Adam(learning_rate=LEARNING_RATE, beta1=BETA1)
algorithm = ali_algorithm(discriminator_loss, ali.discriminator_parameters,
step_rule, generator_loss,
ali.generator_parameters, step_rule)
algorithm.add_updates(bn_updates)
streams = create_celeba_data_streams(BATCH_SIZE, MONITORING_BATCH_SIZE)
main_loop_stream, train_monitor_stream, valid_monitor_stream = streams
bn_monitored_variables = (
[v for v in bn_model.auxiliary_variables if 'norm' not in v.name] +
bn_model.outputs)
monitored_variables = (
[v for v in model.auxiliary_variables if 'norm' not in v.name] +
model.outputs)
extensions = [
Timing(),
FinishAfter(after_n_epochs=NUM_EPOCHS),
DataStreamMonitoring(
bn_monitored_variables, train_monitor_stream, prefix="train",
updates=bn_updates),
DataStreamMonitoring(
monitored_variables, valid_monitor_stream, prefix="valid"),
Checkpoint(save_path, after_epoch=True, after_training=True,
use_cpickle=True),
ProgressBar(),
Printing(),
]
main_loop = MainLoop(model=bn_model, data_stream=main_loop_stream,
algorithm=algorithm, extensions=extensions)
return main_loop
def create_main_loop():
model, bn_model, bn_updates = create_models()
ali, = bn_model.top_bricks
discriminator_loss, generator_loss = bn_model.outputs
step_rule = Adam(learning_rate=LEARNING_RATE, beta1=BETA1)
algorithm = ali_algorithm(discriminator_loss,
ali.discriminator_parameters, step_rule,
generator_loss, ali.generator_parameters,
step_rule)
algorithm.add_updates(bn_updates)
streams = create_gaussian_mixture_data_streams(
batch_size=BATCH_SIZE,
monitoring_batch_size=MONITORING_BATCH_SIZE,
means=MEANS,
variances=VARIANCES,
priors=PRIORS)
main_loop_stream, train_monitor_stream, valid_monitor_stream = streams
bn_monitored_variables = ([
v for v in bn_model.auxiliary_variables if 'norm' not in v.name
] + bn_model.outputs)
monitored_variables = (
[v
for v in model.auxiliary_variables if 'norm' not in v.name] +
model.outputs)
extensions = [
Timing(),
FinishAfter(after_n_epochs=NUM_EPOCHS),
DataStreamMonitoring(bn_monitored_variables,
train_monitor_stream,
prefix="train",
updates=bn_updates),
DataStreamMonitoring(monitored_variables,
valid_monitor_stream,
prefix="valid"),
Checkpoint(os.path.join(self._work_dir, "main_loop.tar"),
after_epoch=True,
after_training=True,
use_cpickle=True),
ProgressBar(),
Printing(),
#ModelLogger(folder=self._work_dir, after_epoch=True),
GraphLogger(num_modes=1,
num_samples=2500,
dimension=2,
r=0,
std=1,
folder=self._work_dir,
after_epoch=True,
after_training=True),
MetricLogger(means=MEANS,
variances=VARIANCES,
folder=self._work_dir,
after_epoch=True)
]
main_loop = MainLoop(model=bn_model,
data_stream=main_loop_stream,
algorithm=algorithm,
extensions=extensions)
return main_loop
def learning_algorithm(args):
name = args.algorithm
learning_rate = float(args.learning_rate)
momentum = args.momentum
clipping_threshold = args.clipping
clipping = StepClipping(threshold=np.cast[floatX](clipping_threshold))
if name == 'adam':
adam = Adam(learning_rate=learning_rate)
step_rule = CompositeRule([adam, clipping])
learning_rate = adam.learning_rate
elif name == 'rms_prop':
rms_prop = RMSProp(learning_rate=learning_rate)
step_rule = CompositeRule([clipping, rms_prop])
learning_rate = rms_prop.learning_rate
elif name == 'momentum':
sgd_momentum = Momentum(learning_rate=learning_rate, momentum=momentum)
step_rule = CompositeRule([clipping, sgd_momentum])
learning_rate = sgd_momentum.learning_rate
elif name == 'sgd':
sgd = Scale(learning_rate=learning_rate)
step_rule = CompositeRule([clipping, sgd])
learning_rate = sgd.learning_rate
else:
raise NotImplementedError
return step_rule, learning_rate
def prepare_opti(cost, test, *args):
model = Model(cost)
logger.info("Model created")
algorithm = GradientDescent(cost=cost,
parameters=model.parameters,
step_rule=Adam(learning_rate=0.0015),
on_unused_sources='ignore')
to_monitor = [algorithm.cost]
if args:
to_monitor.extend(args)
extensions = [
FinishAfter(after_n_epochs=nb_epoch),
FinishIfNoImprovementAfter(notification_name='loglikelihood_nat',
epochs=patience),
TrainingDataMonitoring(to_monitor, prefix="train", after_epoch=True),
DataStreamMonitoring(to_monitor, test_stream, prefix="test"),
Printing(),
ProgressBar(),
ApplyMask(before_first_epoch=True, after_batch=True),
Checkpoint(check, every_n_epochs=save_every),
SaveModel(name=path + '/' + 'pixelcnn_{}'.format(dataset),
every_n_epochs=save_every),
GenerateSamples(every_n_epochs=save_every),
#Checkpoint(path+'/'+'exp.log', save_separately=['log'],every_n_epochs=save_every),
]
if resume:
logger.info("Restoring from previous checkpoint")
extensions = [Load(path + '/' + check)]
return model, algorithm, extensions
def train_base_model(self, train_data, test_data, input_dim):
x = T.matrix('features')
y = T.matrix('targets')
mlp, cost, mis_cost = self.create_base_model(x, y, input_dim)
cg = ComputationGraph([cost])
inputs = VariableFilter(roles=[INPUT])(cg.variables)
cg = apply_dropout(cg, inputs, 0.2)
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Adam(learning_rate=0.001))
data_stream = train_data
data_stream_test = test_data
monitor = DataStreamMonitoring(variables=[mis_cost],
data_stream=data_stream_test,
prefix="test")
plot_ext = Plot('F1-measure',
channels=[['test_MisclassificationRate']],
after_batch=True)
main_loop = MainLoop(data_stream=data_stream,
algorithm=algorithm,
extensions=[
monitor,
FinishAfter(after_n_epochs=50),
Printing(), plot_ext
])
main_loop.run()
return mlp
def main():
x = tensor.matrix("features")
input_to_hidden1 = get_typical_layer(x, 784, 500)
#hidden1_to_hidden2 = get_typical_layer(input_to_hidden1, 500, 300)
hidden1_to_latent = get_typical_layer(input_to_hidden1, 500, 20)
latent_to_hidden2 = get_typical_layer(hidden1_to_latent, 20, 500)
#hidden3_to_hidden4 = get_typical_layer(latent_to_hidden3, 300, 500)
hidden2_to_output = get_typical_layer(latent_to_hidden2, 500, 784, Logistic())
hidden2_to_output.name = "last_before_output"
from blocks.bricks.cost import SquaredError, AbsoluteError, BinaryCrossEntropy
from blocks.graph import ComputationGraph
from blocks.algorithms import Adam, GradientDescent, Scale
from blocks.roles import WEIGHT
cost = BinaryCrossEntropy(name="error").apply(x, hidden2_to_output)
cg = ComputationGraph(cost)
weights = VariableFilter(roles=[WEIGHT]) (cg.variables)
# cost += 0.0001 * tensor.sum(map(lambda x: (x**2).sum(), weights))
# cost.name = "regularized error"
gd = GradientDescent(cost=cost, parameters=cg.parameters, step_rule=Adam())
from blocks.main_loop import MainLoop
from blocks.extensions import FinishAfter, Printing, ProgressBar
from blocks.extensions.monitoring import DataStreamMonitoring, TrainingDataMonitoring
monitor = TrainingDataMonitoring([cost], after_epoch=True)
main_loop = MainLoop(data_stream=get_data_stream(), algorithm=gd, extensions=[monitor, FinishAfter(after_n_epochs=5), ProgressBar(), Printing()])
main_loop.run()
showcase(cg, "last_before_output")
def construct_main_loop(name, task_name, patch_shape, batch_size,
n_spatial_dims, n_patches, n_epochs, learning_rate,
hyperparameters, **kwargs):
name = "%s_%s" % (name, task_name)
hyperparameters["name"] = name
task = get_task(**hyperparameters)
hyperparameters["n_channels"] = task.n_channels
x_uncentered, y = task.get_variables()
x = task.preprocess(x_uncentered)
# this is a theano variable; it may depend on the batch
hyperparameters["image_shape"] = x.shape[-n_spatial_dims:]
ram = construct_model(task=task, **hyperparameters)
ram.initialize()
hs = ram.compute(x, n_patches)
cost = ram.emitter.cost(hs, y, n_patches)
cost.name = "cost"
print "setting up main loop..."
graph = ComputationGraph(cost)
uselessflunky = Model(cost)
algorithm = GradientDescent(cost=cost,
parameters=graph.parameters,
step_rule=Adam(learning_rate=learning_rate))
monitors = construct_monitors(x=x,
x_uncentered=x_uncentered,
y=y,
hs=hs,
cost=cost,
algorithm=algorithm,
task=task,
model=uselessflunky,
ram=ram,
graph=graph,
**hyperparameters)
main_loop = MainLoop(
data_stream=task.get_stream("train"),
algorithm=algorithm,
extensions=(
monitors + [
FinishAfter(after_n_epochs=n_epochs),
DumpMinimum(name + '_best', channel_name='valid_error_rate'),
Dump(name + '_dump', every_n_epochs=10),
#Checkpoint(name+'_checkpoint.pkl', every_n_epochs=10, on_interrupt=False),
ProgressBar(),
Timing(),
Printing(),
PrintingTo(name + "_log")
]),
model=uselessflunky)
return main_loop
def test_adam():
a = shared_floatx([3, 4])
cost = (a ** 2).sum()
steps, updates = Adam().compute_steps(
OrderedDict([(a, tensor.grad(cost, a))]))
f = theano.function([], [steps[a]], updates=updates)
assert_allclose(f()[0], [0.0002, 0.0002], rtol=1e-5)
assert_allclose(f()[0], [0.00105263, 0.00105263], rtol=1e-5)
assert_allclose(f()[0], [0.00073801, 0.00073801], rtol=1e-5)
def run(model_name):
running_on_laptop = socket.gethostname() == 'yop'
X = tensor.tensor4('image_features', dtype='float32')
T = tensor.matrix('targets', dtype='float32')
image_border_size = 100
if running_on_laptop:
host_plot = 'http://*****:*****@ %s' %
(model_name, datetime.datetime.now(), socket.gethostname()),
channels=[['loss', 'valid_loss_test'], ['valid_error']],
after_epoch=True,
server_url=host_plot),
Printing(),
Checkpoint('train2')
]
main_loop = MainLoop(data_stream=train_stream,
algorithm=algorithm,
extensions=extensions)
main_loop.run()
def train(cost, error_rate, batch_size=100, num_epochs=150):
# Setting Loggesetr
timestr = time.strftime("%Y_%m_%d_at_%H_%M")
save_path = 'results/memory_' + timestr
log_path = os.path.join(save_path, 'log.txt')
os.makedirs(save_path)
fh = logging.FileHandler(filename=log_path)
fh.setLevel(logging.DEBUG)
logger.addHandler(fh)
# Training
blocks_model = Model(cost)
all_params = blocks_model.parameters
print "Number of found parameters:" + str(len(all_params))
print all_params
training_algorithm = GradientDescent(cost=cost,
parameters=all_params,
step_rule=Adam(learning_rate=0.001))
# training_algorithm = GradientDescent(
# cost=cost, params=all_params,
# step_rule=Scale(learning_rate=model.default_lr))
monitored_variables = [cost, error_rate]
# the rest is for validation
# train_data_stream, valid_data_stream = get_mnist_streams(
# 50000, batch_size)
train_data_stream, valid_data_stream = get_mnist_video_streams(batch_size)
train_monitoring = TrainingDataMonitoring(variables=monitored_variables,
prefix="train",
after_epoch=True)
valid_monitoring = DataStreamMonitoring(variables=monitored_variables,
data_stream=valid_data_stream,
prefix="valid",
after_epoch=True)
main_loop = MainLoop(
algorithm=training_algorithm,
data_stream=train_data_stream,
model=blocks_model,
extensions=[
train_monitoring, valid_monitoring,
FinishAfter(after_n_epochs=num_epochs),
SaveParams('valid_misclassificationrate_apply_error_rate',
blocks_model, save_path),
SaveLog(save_path, after_epoch=True),
ProgressBar(),
Printing()
])
main_loop.run()
def test_adam():
a = shared_floatx([3, 4])
cost = (a**2).sum()
steps, updates = Adam().compute_steps(
OrderedDict([(a, tensor.grad(cost, a))]))
f = theano.function([], [steps[a]], updates=updates)
rtol = 1e-4
assert_allclose(f()[0], [0.002, 0.002], rtol=rtol)
a.set_value([2, 3])
assert_allclose(f()[0], [0.0019407, 0.00196515], rtol=rtol)
a.set_value([1, 1.5])
assert_allclose(f()[0], [0.00178724, 0.0018223], rtol=rtol)
def create_main_loop(save_path):
model, bn_model, bn_updates = create_models()
ali, = bn_model.top_bricks
discriminator_loss, generator_loss = bn_model.outputs
step_rule = Adam(learning_rate=LEARNING_RATE, beta1=BETA1)
algorithm = ali_algorithm(discriminator_loss, ali.discriminator_parameters,
step_rule, generator_loss,
ali.generator_parameters, step_rule)
algorithm.add_updates(bn_updates)
streams = create_cifar10_data_streams(BATCH_SIZE, MONITORING_BATCH_SIZE)
main_loop_stream, train_monitor_stream, valid_monitor_stream = streams
for d in main_loop_stream.get_epoch_iterator(as_dict=True):
print(d.keys)
print(d['features'].shape, d['features'].dtype)
break
main_loop_stream = ShapesDataset(num_examples=600, img_size=32, min_diameter=3, seed=1234).create_stream(batch_size=BATCH_SIZE, is_train=True)
for d in main_loop_stream.get_epoch_iterator(as_dict=True):
print(d.keys)
print(d['features'].shape, d['features'].dtype)
break
train_monitor_stream = ShapesDataset(num_examples=100, img_size=32, min_diameter=3, seed=1234).create_stream(batch_size=BATCH_SIZE, is_train=False)
valid_monitor_stream = ShapesDataset(num_examples=100, img_size=32, min_diameter=3, seed=5678).create_stream(batch_size=BATCH_SIZE, is_train=False)
bn_monitored_variables = (
[v for v in bn_model.auxiliary_variables if 'norm' not in v.name] +
bn_model.outputs)
monitored_variables = (
[v for v in model.auxiliary_variables if 'norm' not in v.name] +
model.outputs)
extensions = [
Timing(),
FinishAfter(after_n_epochs=NUM_EPOCHS),
DataStreamMonitoring(
bn_monitored_variables, train_monitor_stream, prefix="train",
updates=bn_updates),
DataStreamMonitoring(
monitored_variables, valid_monitor_stream, prefix="valid"),
Checkpoint(save_path, after_epoch=True, after_training=True,
use_cpickle=True),
ProgressBar(),
Printing(),
]
main_loop = MainLoop(model=bn_model, data_stream=main_loop_stream,
algorithm=algorithm, extensions=extensions)
return main_loop
def main(save_to, num_epochs, batch_size):
mlp = MLP([Tanh(), Tanh(), Tanh(), Softmax()], [3072, 4096, 1024, 512, 10],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
mlp.initialize()
x = tt.tensor4('features', dtype='float32')
y = tt.vector('label', dtype='int32')
probs = mlp.apply(x.reshape((-1, 3072)))
cost = CategoricalCrossEntropy().apply(y, probs)
error_rate = MisclassificationRate().apply(y, probs)
cg = ComputationGraph([cost])
ws = VariableFilter(roles=[WEIGHT])(cg.variables)
cost = cost + .00005 * sum(([(w**2).sum() for w in ws]))
cost.name = 'final_cost'
train_dataset = Cifar10Dataset(data_dir='/home/belohlavek/data/cifar10',
is_train=True)
valid_dataset = Cifar10Dataset(data_dir='/home/belohlavek/data/cifar10',
is_train=False)
train_stream = train_dataset.get_stream(batch_size)
valid_stream = valid_dataset.get_stream(batch_size)
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Adam(learning_rate=0.001))
extensions = [
Timing(),
LogExtension('/home/belohlavek/ALI/mlp.log'),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring([cost, error_rate], valid_stream, prefix="test"),
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_epoch=True),
Checkpoint(save_to),
Printing()
]
main_loop = MainLoop(algorithm,
train_stream,
model=Model(cost),
extensions=extensions)
main_loop.run()
def run():
streams = create_celeba_streams(training_batch_size=100,
monitoring_batch_size=500,
include_targets=True)
main_loop_stream = streams[0]
train_monitor_stream = streams[1]
valid_monitor_stream = streams[2]
cg, bn_dropout_cg = create_training_computation_graphs()
# Compute parameter updates for the batch normalization population
# statistics. They are updated following an exponential moving average.
pop_updates = get_batch_normalization_updates(bn_dropout_cg)
decay_rate = 0.05
extra_updates = [(p, m * decay_rate + p * (1 - decay_rate))
for p, m in pop_updates]
# Prepare algorithm
step_rule = Adam()
algorithm = GradientDescent(cost=bn_dropout_cg.outputs[0],
parameters=bn_dropout_cg.parameters,
step_rule=step_rule)
algorithm.add_updates(extra_updates)
# Prepare monitoring
cost = bn_dropout_cg.outputs[0]
cost.name = 'cost'
train_monitoring = DataStreamMonitoring(
[cost], train_monitor_stream, prefix="train",
before_first_epoch=False, after_epoch=False, after_training=True,
updates=extra_updates)
cost, accuracy = cg.outputs
cost.name = 'cost'
accuracy.name = 'accuracy'
monitored_quantities = [cost, accuracy]
valid_monitoring = DataStreamMonitoring(
monitored_quantities, valid_monitor_stream, prefix="valid",
before_first_epoch=False, after_epoch=False, every_n_epochs=5)
# Prepare checkpoint
checkpoint = Checkpoint(
'celeba_classifier.zip', every_n_epochs=5, use_cpickle=True)
extensions = [Timing(), FinishAfter(after_n_epochs=50), train_monitoring,
valid_monitoring, checkpoint, Printing(), ProgressBar()]
main_loop = MainLoop(data_stream=main_loop_stream, algorithm=algorithm,
extensions=extensions)
main_loop.run()
def learning_algorithm(args):
name = args.algorithm
learning_rate = float(args.learning_rate)
momentum = args.momentum
clipping_threshold = args.clipping
if name == 'adam':
adam = Adam(learning_rate=learning_rate)
step_rule = adam
elif name == 'rms_prop':
rms_prop = RMSProp(learning_rate=learning_rate, decay_rate=0.9)
step_rule = CompositeRule([StepClipping(1.), rms_prop])
else:
sgd_momentum = Momentum(learning_rate=learning_rate, momentum=momentum)
step_rule = sgd_momentum
return step_rule
def prepare_opti(cost, test):
model = Model(cost)
algorithm = GradientDescent(cost=cost,
parameters=model.parameters,
step_rule=Adam(),
on_unused_sources='ignore')
extensions = [
FinishAfter(after_n_epochs=nb_epoch),
FinishIfNoImprovementAfter(notification_name='test_vae_cost',
epochs=patience),
TrainingDataMonitoring([algorithm.cost], after_epoch=True),
DataStreamMonitoring([algorithm.cost], test, prefix="test"),
Printing(),
ProgressBar(),
#SaveModel(name='vae', after_n_epochs=save_every)
]
return model, algorithm, extensions
def learning_algorithm(args):
name = args.algorithm
learning_rate = float(args.learning_rate)
momentum = args.momentum
clipping_threshold = args.clipping
if name == 'adam':
clipping = StepClipping(threshold=np.cast[floatX](clipping_threshold))
adam = Adam(learning_rate=learning_rate)
step_rule = CompositeRule([adam, clipping])
elif name == 'rms_prop':
clipping = StepClipping(threshold=np.cast[floatX](clipping_threshold))
rms_prop = RMSProp(learning_rate=learning_rate)
step_rule = CompositeRule([clipping, rms_prop])
else:
clipping = StepClipping(threshold=np.cast[floatX](clipping_threshold))
sgd_momentum = Momentum(learning_rate=learning_rate, momentum=momentum)
step_rule = CompositeRule([clipping, sgd_momentum])
return step_rule
def learning_algorithm(args):
name = args.algorithm
learning_rate = float(args.learning_rate)
momentum = args.momentum
clipping_threshold = args.clipping
if name == 'adam':
clipping = StepClipping(threshold=np.cast[floatX](clipping_threshold))
adam = Adam(learning_rate=learning_rate)
# [adam, clipping] means 'step clipping'
# [clipping, adam] means 'gradient clipping'
step_rule = CompositeRule([adam, clipping])
elif name == 'rms_prop':
clipping = StepClipping(threshold=np.cast[floatX](clipping_threshold))
rms_prop = RMSProp(learning_rate=learning_rate)
rm_non_finite = RemoveNotFinite()
step_rule = CompositeRule([clipping, rms_prop, rm_non_finite])
else:
clipping = StepClipping(threshold=np.cast[floatX](clipping_threshold))
sgd_momentum = Momentum(learning_rate=learning_rate, momentum=momentum)
rm_non_finite = RemoveNotFinite()
step_rule = CompositeRule([clipping, sgd_momentum, rm_non_finite])
return step_rule
def learning_algorithm(learning_rate,
momentum=0.0,
clipping_threshold=100,
algorithm='sgd'):
if algorithm == 'adam':
clipping = StepClipping(threshold=np.cast[floatX](clipping_threshold))
adam = Adam(learning_rate=learning_rate)
# [adam, clipping] means 'step clipping'
# [clipping, adam] means 'gradient clipping'
step_rule = CompositeRule([adam, clipping])
elif algorithm == 'rms_prop':
clipping = StepClipping(threshold=np.cast[floatX](clipping_threshold))
rms_prop = RMSProp(learning_rate=learning_rate)
rm_non_finite = RemoveNotFinite()
step_rule = CompositeRule([clipping, rms_prop, rm_non_finite])
elif algorithm == 'sgd':
clipping = StepClipping(threshold=np.cast[floatX](clipping_threshold))
sgd_momentum = Momentum(learning_rate=learning_rate, momentum=momentum)
rm_non_finite = RemoveNotFinite()
step_rule = CompositeRule([clipping, sgd_momentum, rm_non_finite])
else:
raise NotImplementedError
return step_rule
def train(self, training_data):
step_rules = [Adam(), StepClipping(1.0)]
algorithm = GradientDescent(
cost=self.Cost,
parameters=self.ComputationGraph.parameters,
step_rule=CompositeRule(step_rules))
train_stream = DataStream.default_stream(
training_data,
iteration_scheme=SequentialScheme(training_data.num_examples,
batch_size=20))
main = MainLoop(model=Model(self.Cost),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
FinishAfter(),
Printing(),
Checkpoint('trainingdata.tar', every_n_epochs=10)
])
main.run()
attended=mlp_context.apply(context))
cost = cost_matrix.sum() / x_mask.sum()
cost.name = "sequence_log_likelihood"
cg = ComputationGraph(cost)
model = Model(cost)
#################
# Algorithm
#################
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=CompositeRule(
[StepClipping(10.0),
Adam(lr)]))
train_monitor = TrainingDataMonitoring(variables=[cost],
after_epoch=True,
prefix="train")
extensions = extensions = [
train_monitor,
TrackTheBest('train_sequence_log_likelihood'),
Printing(after_epoch=True)
]
main_loop = MainLoop(model=model,
data_stream=data_stream,
algorithm=algorithm,
extensions=extensions)
def main(name, dataset, epochs, batch_size, learning_rate, attention, n_iter,
enc_dim, dec_dim, z_dim, oldmodel):
image_size, data_train, data_valid, data_test = datasets.get_data(dataset)
train_stream = Flatten(
DataStream(data_train,
iteration_scheme=SequentialScheme(data_train.num_examples,
batch_size)))
valid_stream = Flatten(
DataStream(data_valid,
iteration_scheme=SequentialScheme(data_valid.num_examples,
batch_size)))
test_stream = Flatten(
DataStream(data_test,
iteration_scheme=SequentialScheme(data_test.num_examples,
batch_size)))
if name is None:
name = dataset
img_height, img_width = image_size
x_dim = img_height * img_width
rnninits = {
#'weights_init': Orthogonal(),
'weights_init': IsotropicGaussian(0.01),
'biases_init': Constant(0.),
}
inits = {
#'weights_init': Orthogonal(),
'weights_init': IsotropicGaussian(0.01),
'biases_init': Constant(0.),
}
if attention != "":
read_N, write_N = attention.split(',')
read_N = int(read_N)
write_N = int(write_N)
read_dim = 2 * read_N**2
reader = AttentionReader(x_dim=x_dim,
dec_dim=dec_dim,
width=img_width,
height=img_height,
N=read_N,
**inits)
writer = AttentionWriter(input_dim=dec_dim,
output_dim=x_dim,
width=img_width,
height=img_height,
N=write_N,
**inits)
attention_tag = "r%d-w%d" % (read_N, write_N)
else:
read_dim = 2 * x_dim
reader = Reader(x_dim=x_dim, dec_dim=dec_dim, **inits)
writer = Writer(input_dim=dec_dim, output_dim=x_dim, **inits)
attention_tag = "full"
#----------------------------------------------------------------------
# Learning rate
def lr_tag(value):
""" Convert a float into a short tag-usable string representation. E.g.:
0.1 -> 11
0.01 -> 12
0.001 -> 13
0.005 -> 53
"""
exp = np.floor(np.log10(value))
leading = ("%e" % value)[0]
return "%s%d" % (leading, -exp)
lr_str = lr_tag(learning_rate)
subdir = time.strftime("%Y%m%d-%H%M%S") + "-" + name
longname = "%s-%s-t%d-enc%d-dec%d-z%d-lr%s" % (
dataset, attention_tag, n_iter, enc_dim, dec_dim, z_dim, lr_str)
pickle_file = subdir + "/" + longname + ".pkl"
print("\nRunning experiment %s" % longname)
print(" dataset: %s" % dataset)
print(" subdirectory: %s" % subdir)
print(" learning rate: %g" % learning_rate)
print(" attention: %s" % attention)
print(" n_iterations: %d" % n_iter)
print(" encoder dimension: %d" % enc_dim)
print(" z dimension: %d" % z_dim)
print(" decoder dimension: %d" % dec_dim)
print(" batch size: %d" % batch_size)
print(" epochs: %d" % epochs)
print()
#----------------------------------------------------------------------
encoder_rnn = LSTM(dim=enc_dim, name="RNN_enc", **rnninits)
decoder_rnn = LSTM(dim=dec_dim, name="RNN_dec", **rnninits)
encoder_mlp = MLP([Identity()], [(read_dim + dec_dim), 4 * enc_dim],
name="MLP_enc",
**inits)
decoder_mlp = MLP([Identity()], [z_dim, 4 * dec_dim],
name="MLP_dec",
**inits)
q_sampler = Qsampler(input_dim=enc_dim, output_dim=z_dim, **inits)
draw = DrawModel(n_iter,
reader=reader,
encoder_mlp=encoder_mlp,
encoder_rnn=encoder_rnn,
sampler=q_sampler,
decoder_mlp=decoder_mlp,
decoder_rnn=decoder_rnn,
writer=writer)
draw.initialize()
#------------------------------------------------------------------------
x = tensor.matrix('features')
#x_recons = 1. + x
x_recons, kl_terms = draw.reconstruct(x)
#x_recons, _, _, _, _ = draw.silly(x, n_steps=10, batch_size=100)
#x_recons = x_recons[-1,:,:]
#samples = draw.sample(100)
#x_recons = samples[-1, :, :]
#x_recons = samples[-1, :, :]
recons_term = BinaryCrossEntropy().apply(x, x_recons)
recons_term.name = "recons_term"
cost = recons_term + kl_terms.sum(axis=0).mean()
cost.name = "nll_bound"
#------------------------------------------------------------
cg = ComputationGraph([cost])
params = VariableFilter(roles=[PARAMETER])(cg.variables)
algorithm = GradientDescent(
cost=cost,
params=params,
step_rule=CompositeRule([
StepClipping(10.),
Adam(learning_rate),
])
#step_rule=RMSProp(learning_rate),
#step_rule=Momentum(learning_rate=learning_rate, momentum=0.95)
)
#algorithm.add_updates(scan_updates)
#------------------------------------------------------------------------
# Setup monitors
monitors = [cost]
for t in range(n_iter):
kl_term_t = kl_terms[t, :].mean()
kl_term_t.name = "kl_term_%d" % t
#x_recons_t = T.nnet.sigmoid(c[t,:,:])
#recons_term_t = BinaryCrossEntropy().apply(x, x_recons_t)
#recons_term_t = recons_term_t.mean()
#recons_term_t.name = "recons_term_%d" % t
monitors += [kl_term_t]
train_monitors = monitors[:]
train_monitors += [aggregation.mean(algorithm.total_gradient_norm)]
train_monitors += [aggregation.mean(algorithm.total_step_norm)]
# Live plotting...
plot_channels = [
["train_nll_bound", "test_nll_bound"],
["train_kl_term_%d" % t for t in range(n_iter)],
#["train_recons_term_%d" % t for t in range(n_iter)],
["train_total_gradient_norm", "train_total_step_norm"]
]
#------------------------------------------------------------
if not os.path.exists(subdir):
os.makedirs(subdir)
main_loop = MainLoop(
model=Model(cost),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
Timing(),
FinishAfter(after_n_epochs=epochs),
TrainingDataMonitoring(train_monitors,
prefix="train",
after_epoch=True),
# DataStreamMonitoring(
# monitors,
# valid_stream,
## updates=scan_updates,
# prefix="valid"),
DataStreamMonitoring(
monitors,
test_stream,
# updates=scan_updates,
prefix="test"),
Checkpoint(name,
before_training=False,
after_epoch=True,
save_separately=['log', 'model']),
#Checkpoint(image_size=image_size, save_subdir=subdir, path=pickle_file, before_training=False, after_epoch=True, save_separately=['log', 'model']),
Plot(name, channels=plot_channels),
ProgressBar(),
Printing()
])
if oldmodel is not None:
print("Initializing parameters with old model %s" % oldmodel)
with open(oldmodel, "rb") as f:
oldmodel = pickle.load(f)
main_loop.model.set_param_values(oldmodel.get_param_values())
del oldmodel
main_loop.run()
def test_adam_broadcastable():
verify_broadcastable_handling(Adam())
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=0.2)
convnet.initialize()
#########################################################
#Generate output and error signal
predict = convnet.apply(x)
cost = CategoricalCrossEntropy().apply(y.flatten(), predict).copy(name='cost')
error = MisclassificationRate().apply(y.flatten(), predict)
#Little trick to plot the error rate in two different plots (We can't use two time the same data in the plot for a unknow reason)
error_rate = error.copy(name='error_rate')
error_rate2 = error.copy(name='error_rate2')
cg = ComputationGraph([cost, error_rate])
########### ALGORITHM of training#############
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Adam(learning_rate=0.0005))
extensions = [
Timing(),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring([cost, error_rate, error_rate2],
stream_valid,
prefix="valid"),
TrainingDataMonitoring(
[cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)],
prefix="train",
after_epoch=True),
Checkpoint("catsVsDogs256.pkl"),
ProgressBar(),
Printing()
]
emit = gmm_emitter.emit(h[-2])
emit.name = 'emitter'
cg = ComputationGraph(cost)
model = Model(cost)
#################
# Algorithm
#################
n_batches = 139*16
algorithm = GradientDescent(
cost=cost, parameters=cg.parameters,
step_rule=CompositeRule([StepClipping(10.0), Adam(lr)]))
train_monitor = TrainingDataMonitoring(
variables=[cost],
every_n_batches = n_batches,
prefix="train")
valid_monitor = DataStreamMonitoring(
[cost],
valid_stream,
after_epoch = True,
#before_first_epoch = False,
prefix="valid")
extensions = extensions=[
Timing(every_n_batches = n_batches),
prefix='validation',
after_epoch=True)
monitor_test = DataStreamMonitoring(variables=[error],
data_stream=data_stream_test,
prefix='test',
after_epoch=True)
learning_rate = 0.00008
n_epochs = 100
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
on_unused_sources='ignore',
step_rule=CompositeRule([
StepClipping(10.),
Adam(learning_rate),
]))
main_loop = MainLoop(model=Model(cost),
data_stream=data_stream_train,
algorithm=algorithm,
extensions=[
Timing(),
FinishAfter(after_n_epochs=n_epochs), monitor,
monitor_val, monitor_test,
saveSnapshot(
'/home/xuehongyang/checkpoints_read/snapshot',
save_main_loop=False,
after_epoch=True,
save_separately=['log', 'model']),
ProgressBar(),
Printing(every_n_batches=500),
def construct_main_loop(name, task_name, patch_shape, batch_size,
n_spatial_dims, n_patches, max_epochs, patience_epochs,
learning_rate, gradient_limiter, hyperparameters,
**kwargs):
task = tasks.get_task(**hyperparameters)
hyperparameters["n_channels"] = task.n_channels
extensions = []
# let theta noise decay as training progresses
for key in "location_std scale_std".split():
hyperparameters[key] = theano.shared(hyperparameters[key], name=key)
extensions.append(
util.ExponentialDecay(hyperparameters[key],
hyperparameters["%s_decay" % key],
after_batch=True))
print "constructing graphs..."
graphs, outputs, updates = construct_graphs(task=task, **hyperparameters)
print "setting up main loop..."
from blocks.model import Model
model = Model(outputs["train"]["cost"])
from blocks.algorithms import GradientDescent, CompositeRule, StepClipping, Adam, RMSProp
from extensions import Compressor
if gradient_limiter == "clip":
limiter = StepClipping(1.)
elif gradient_limiter == "compress":
limiter = Compressor()
else:
raise ValueError()
algorithm = GradientDescent(
cost=outputs["train"]["cost"],
parameters=graphs["train"].parameters,
step_rule=CompositeRule([limiter,
Adam(learning_rate=learning_rate)]))
algorithm.add_updates(updates["train"])
extensions.extend(
construct_monitors(algorithm=algorithm,
task=task,
model=model,
graphs=graphs,
outputs=outputs,
updates=updates,
**hyperparameters))
from blocks.extensions import FinishAfter, Printing, ProgressBar, Timing
from blocks.extensions.stopping import FinishIfNoImprovementAfter
from blocks.extensions.training import TrackTheBest
from blocks.extensions.saveload import Checkpoint
from dump import DumpBest, LightCheckpoint, PrintingTo, DumpGraph, DumpLog
extensions.extend([
TrackTheBest("valid_error_rate", "best_valid_error_rate"),
FinishIfNoImprovementAfter("best_valid_error_rate",
epochs=patience_epochs),
FinishAfter(after_n_epochs=max_epochs),
DumpBest("best_valid_error_rate", name + "_best.zip"),
Checkpoint(hyperparameters["checkpoint_save_path"],
on_interrupt=False,
every_n_epochs=10,
use_cpickle=True),
DumpLog("log.pkl", after_epoch=True),
ProgressBar(),
Timing(),
Printing(),
PrintingTo(name + "_log"),
DumpGraph(name + "_grad_graph")
])
from blocks.main_loop import MainLoop
main_loop = MainLoop(data_stream=task.get_stream("train"),
algorithm=algorithm,
extensions=extensions,
model=model)
from tabulate import tabulate
print "parameter sizes:"
print tabulate(
(key, "x".join(map(str,
value.get_value().shape)), value.get_value().size)
for key, value in main_loop.model.get_parameter_dict().items())
return main_loop
def main(args):
"""Run experiment. """
lr_tag = float_tag(args.learning_rate)
x_dim, train_stream, valid_stream, test_stream = datasets.get_streams(
args.data, args.batch_size)
#------------------------------------------------------------
# Setup model
deterministic_act = Tanh
deterministic_size = 1.
if args.method == 'vae':
sizes_tag = args.layer_spec.replace(",", "-")
layer_sizes = [int(i) for i in args.layer_spec.split(",")]
layer_sizes, z_dim = layer_sizes[:-1], layer_sizes[-1]
name = "%s-%s-%s-lr%s-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.n_samples, sizes_tag)
if args.activation == "tanh":
hidden_act = Tanh()
elif args.activation == "logistic":
hidden_act = Logistic()
elif args.activation == "relu":
hidden_act = Rectifier()
else:
raise "Unknown hidden nonlinearity %s" % args.hidden_act
model = VAE(x_dim=x_dim,
hidden_layers=layer_sizes,
hidden_act=hidden_act,
z_dim=z_dim,
batch_norm=args.batch_normalization)
model.initialize()
elif args.method == 'dvae':
sizes_tag = args.layer_spec.replace(",", "-")
layer_sizes = [int(i) for i in args.layer_spec.split(",")]
layer_sizes, z_dim = layer_sizes[:-1], layer_sizes[-1]
name = "%s-%s-%s-lr%s-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.n_samples, sizes_tag)
if args.activation == "tanh":
hidden_act = Tanh()
elif args.activation == "logistic":
hidden_act = Logistic()
elif args.activation == "relu":
hidden_act = Rectifier()
else:
raise "Unknown hidden nonlinearity %s" % args.hidden_act
model = DVAE(x_dim=x_dim,
hidden_layers=layer_sizes,
hidden_act=hidden_act,
z_dim=z_dim,
batch_norm=args.batch_normalization)
model.initialize()
elif args.method == 'rws':
sizes_tag = args.layer_spec.replace(",", "-")
qbase = "" if not args.no_qbaseline else "noqb-"
name = "%s-%s-%s-%slr%s-dl%d-spl%d-%s" % \
(args.data, args.method, args.name, qbase, lr_tag, args.deterministic_layers, args.n_samples, sizes_tag)
p_layers, q_layers = create_layers(args.layer_spec, x_dim,
args.deterministic_layers,
deterministic_act,
deterministic_size)
model = ReweightedWakeSleep(
p_layers,
q_layers,
qbaseline=(not args.no_qbaseline),
)
model.initialize()
elif args.method == 'bihm-rws':
sizes_tag = args.layer_spec.replace(",", "-")
name = "%s-%s-%s-lr%s-dl%d-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.deterministic_layers, args.n_samples, sizes_tag)
p_layers, q_layers = create_layers(args.layer_spec, x_dim,
args.deterministic_layers,
deterministic_act,
deterministic_size)
model = BiHM(
p_layers,
q_layers,
l1reg=args.l1reg,
l2reg=args.l2reg,
)
model.initialize()
elif args.method == 'continue':
import cPickle as pickle
from os.path import basename, splitext
with open(args.model_file, 'rb') as f:
m = pickle.load(f)
if isinstance(m, MainLoop):
m = m.model
model = m.get_top_bricks()[0]
while len(model.parents) > 0:
model = model.parents[0]
assert isinstance(model, (BiHM, ReweightedWakeSleep, VAE))
mname, _, _ = basename(args.model_file).rpartition("_model.pkl")
name = "%s-cont-%s-lr%s-spl%s" % (mname, args.name, lr_tag,
args.n_samples)
else:
raise ValueError("Unknown training method '%s'" % args.method)
#------------------------------------------------------------
x = tensor.matrix('features')
#------------------------------------------------------------
# Testset monitoring
train_monitors = []
valid_monitors = []
test_monitors = []
for s in [1, 10, 100, 1000]:
log_p, log_ph = model.log_likelihood(x, s)
log_p = -log_p.mean()
log_ph = -log_ph.mean()
log_p.name = "log_p_%d" % s
log_ph.name = "log_ph_%d" % s
#train_monitors += [log_p, log_ph]
#valid_monitors += [log_p, log_ph]
test_monitors += [log_p, log_ph]
#------------------------------------------------------------
# Z estimation
#for s in [100000]:
# z2 = tensor.exp(model.estimate_log_z2(s)) / s
# z2.name = "z2_%d" % s
#
# valid_monitors += [z2]
# test_monitors += [z2]
#------------------------------------------------------------
# Gradient and training monitoring
if args.method in ['vae', 'dvae']:
log_p_bound, gradients = model.get_gradients(x, args.n_samples)
log_p_bound = -log_p_bound.mean()
log_p_bound.name = "log_p_bound"
cost = log_p_bound
train_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
valid_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
test_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
else:
log_p, log_ph, gradients = model.get_gradients(x, args.n_samples)
log_p = -log_p.mean()
log_ph = -log_ph.mean()
log_p.name = "log_p"
log_ph.name = "log_ph"
cost = log_ph
train_monitors += [log_p, log_ph]
valid_monitors += [log_p, log_ph]
#------------------------------------------------------------
# Detailed monitoring
"""
n_layers = len(p_layers)
log_px, w, log_p, log_q, samples = model.log_likelihood(x, n_samples)
exp_samples = []
for l in xrange(n_layers):
e = (w.dimshuffle(0, 1, 'x')*samples[l]).sum(axis=1)
e.name = "inference_h%d" % l
e.tag.aggregation_scheme = aggregation.TakeLast(e)
exp_samples.append(e)
s1 = samples[1]
sh1 = s1.shape
s1_ = s1.reshape([sh1[0]*sh1[1], sh1[2]])
s0, _ = model.p_layers[0].sample_expected(s1_)
s0 = s0.reshape([sh1[0], sh1[1], s0.shape[1]])
s0 = (w.dimshuffle(0, 1, 'x')*s0).sum(axis=1)
s0.name = "inference_h0^"
s0.tag.aggregation_scheme = aggregation.TakeLast(s0)
exp_samples.append(s0)
# Draw P-samples
p_samples, _, _ = model.sample_p(100)
#weights = model.importance_weights(samples)
#weights = weights / weights.sum()
for i, s in enumerate(p_samples):
s.name = "psamples_h%d" % i
s.tag.aggregation_scheme = aggregation.TakeLast(s)
#
samples = model.sample(100, oversample=100)
for i, s in enumerate(samples):
s.name = "samples_h%d" % i
s.tag.aggregation_scheme = aggregation.TakeLast(s)
"""
cg = ComputationGraph([cost])
#------------------------------------------------------------
if args.step_rule == "momentum":
step_rule = Momentum(args.learning_rate, 0.95)
elif args.step_rule == "rmsprop":
step_rule = RMSProp(args.learning_rate)
elif args.step_rule == "adam":
step_rule = Adam(args.learning_rate)
else:
raise "Unknown step_rule %s" % args.step_rule
#parameters = cg.parameters[:4] + cg.parameters[5:]
parameters = cg.parameters
algorithm = GradientDescent(
cost=cost,
parameters=parameters,
gradients=gradients,
step_rule=CompositeRule([
#StepClipping(25),
step_rule,
#RemoveNotFinite(1.0),
]))
#------------------------------------------------------------
train_monitors += [
aggregation.mean(algorithm.total_gradient_norm),
aggregation.mean(algorithm.total_step_norm)
]
#------------------------------------------------------------
# Live plotting?
plotting_extensions = []
if args.live_plotting:
plotting_extensions = [
PlotManager(
name,
[
Plotter(channels=[[
"valid_%s" % cost.name, "valid_log_p"
], ["train_total_gradient_norm", "train_total_step_norm"]],
titles=[
"validation cost",
"norm of training gradient and step"
]),
DisplayImage(
[
WeightDisplay(model.p_layers[0].mlp.
linear_transformations[0].W,
n_weights=100,
image_shape=(28, 28))
]
#ImageDataStreamDisplay(test_stream, image_shape=(28,28))]
)
])
]
main_loop = MainLoop(
model=Model(cost),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
Timing(),
ProgressBar(),
TrainingDataMonitoring(
train_monitors, prefix="train", after_epoch=True),
DataStreamMonitoring(
valid_monitors, data_stream=valid_stream, prefix="valid"),
DataStreamMonitoring(test_monitors,
data_stream=test_stream,
prefix="test",
after_epoch=False,
after_training=True,
every_n_epochs=10),
#SharedVariableModifier(
# algorithm.step_rule.components[0].learning_rate,
# half_lr_func,
# before_training=False,
# after_epoch=False,
# after_batch=False,
# every_n_epochs=half_lr),
TrackTheBest('valid_%s' % cost.name),
Checkpoint(name + ".pkl", save_separately=['log', 'model']),
FinishIfNoImprovementAfter('valid_%s_best_so_far' % cost.name,
epochs=args.patience),
FinishAfter(after_n_epochs=args.max_epochs),
Printing()
] + plotting_extensions)
main_loop.run()
def main(name, epochs, batch_size, learning_rate, attention, n_iter, enc_dim,
dec_dim, z_dim):
if name is None:
tag = "watt" if attention else "woatt"
name = "%s-t%d-enc%d-dec%d-z%d" % (tag, n_iter, enc_dim, dec_dim,
z_dim)
print("\nRunning experiment %s" % name)
print(" learning rate: %5.3f" % learning_rate)
print(" attention: %s" % attention)
print(" n_iterations: %d" % n_iter)
print(" encoder dimension: %d" % enc_dim)
print(" z dimension: %d" % z_dim)
print(" decoder dimension: %d" % dec_dim)
print()
#------------------------------------------------------------------------
x_dim = 28 * 28
img_height, img_width = (28, 28)
rnninits = {
'weights_init': Orthogonal(),
#'weights_init': IsotropicGaussian(0.001),
'biases_init': Constant(0.),
}
inits = {
'weights_init': Orthogonal(),
#'weights_init': IsotropicGaussian(0.01),
'biases_init': Constant(0.),
}
prior_mu = T.zeros([z_dim])
prior_log_sigma = T.zeros([z_dim])
if attention:
read_N = 4
write_N = 6
read_dim = 2 * read_N**2
reader = AttentionReader(x_dim=x_dim,
dec_dim=dec_dim,
width=img_width,
height=img_height,
N=read_N,
**inits)
writer = AttentionWriter(input_dim=dec_dim,
output_dim=x_dim,
width=img_width,
height=img_height,
N=read_N,
**inits)
else:
read_dim = 2 * x_dim
reader = Reader(x_dim=x_dim, dec_dim=dec_dim, **inits)
writer = Writer(input_dim=dec_dim, output_dim=x_dim, **inits)
encoder = LSTM(dim=enc_dim, name="RNN_enc", **rnninits)
decoder = LSTM(dim=dec_dim, name="RNN_dec", **rnninits)
encoder_mlp = MLP([Tanh()], [(read_dim + dec_dim), 4 * enc_dim],
name="MLP_enc",
**inits)
decoder_mlp = MLP([Tanh()], [z_dim, 4 * dec_dim], name="MLP_dec", **inits)
q_sampler = Qsampler(input_dim=enc_dim, output_dim=z_dim, **inits)
for brick in [
reader, writer, encoder, decoder, encoder_mlp, decoder_mlp,
q_sampler
]:
brick.allocate()
brick.initialize()
#------------------------------------------------------------------------
x = tensor.matrix('features')
# This is one iteration
def one_iteration(c, h_enc, c_enc, z_mean, z_log_sigma, z, h_dec, c_dec,
x):
x_hat = x - T.nnet.sigmoid(c)
r = reader.apply(x, x_hat, h_dec)
i_enc = encoder_mlp.apply(T.concatenate([r, h_dec], axis=1))
h_enc, c_enc = encoder.apply(states=h_enc,
cells=c_enc,
inputs=i_enc,
iterate=False)
z_mean, z_log_sigma, z = q_sampler.apply(h_enc)
i_dec = decoder_mlp.apply(z)
h_dec, c_dec = decoder.apply(states=h_dec,
cells=c_dec,
inputs=i_dec,
iterate=False)
c = c + writer.apply(h_dec)
return c, h_enc, c_enc, z_mean, z_log_sigma, z, h_dec, c_dec
outputs_info = [
T.zeros([batch_size, x_dim]), # c
T.zeros([batch_size, enc_dim]), # h_enc
T.zeros([batch_size, enc_dim]), # c_enc
T.zeros([batch_size, z_dim]), # z_mean
T.zeros([batch_size, z_dim]), # z_log_sigma
T.zeros([batch_size, z_dim]), # z
T.zeros([batch_size, dec_dim]), # h_dec
T.zeros([batch_size, dec_dim]), # c_dec
]
outputs, scan_updates = theano.scan(fn=one_iteration,
sequences=[],
outputs_info=outputs_info,
non_sequences=[x],
n_steps=n_iter)
c, h_enc, c_enc, z_mean, z_log_sigma, z, h_dec, c_dec = outputs
kl_terms = (prior_log_sigma - z_log_sigma + 0.5 *
(tensor.exp(2 * z_log_sigma) +
(z_mean - prior_mu)**2) / tensor.exp(2 * prior_log_sigma) -
0.5).sum(axis=-1)
x_recons = T.nnet.sigmoid(c[-1, :, :])
recons_term = BinaryCrossEntropy().apply(x, x_recons)
recons_term.name = "recons_term"
cost = recons_term + kl_terms.sum(axis=0).mean()
cost.name = "nll_bound"
#------------------------------------------------------------
cg = ComputationGraph([cost])
params = VariableFilter(roles=[PARAMETER])(cg.variables)
algorithm = GradientDescent(
cost=cost,
params=params,
step_rule=CompositeRule([
#StepClipping(3.),
Adam(learning_rate),
])
#step_rule=RMSProp(learning_rate),
#step_rule=Momentum(learning_rate=learning_rate, momentum=0.95)
)
algorithm.add_updates(scan_updates)
#------------------------------------------------------------------------
# Setup monitors
monitors = [cost]
for t in range(n_iter):
kl_term_t = kl_terms[t, :].mean()
kl_term_t.name = "kl_term_%d" % t
x_recons_t = T.nnet.sigmoid(c[t, :, :])
recons_term_t = BinaryCrossEntropy().apply(x, x_recons_t)
recons_term_t = recons_term_t.mean()
recons_term_t.name = "recons_term_%d" % t
monitors += [kl_term_t, recons_term_t]
train_monitors = monitors[:]
train_monitors += [aggregation.mean(algorithm.total_gradient_norm)]
train_monitors += [aggregation.mean(algorithm.total_step_norm)]
# Live plotting...
plot_channels = [["train_nll_bound", "test_nll_bound"],
["train_kl_term_%d" % t for t in range(n_iter)],
["train_recons_term_%d" % t for t in range(n_iter)],
["train_total_gradient_norm", "train_total_step_norm"]]
#------------------------------------------------------------
mnist_train = BinarizedMNIST("train", sources=['features'])
mnist_test = BinarizedMNIST("test", sources=['features'])
#mnist_train = MNIST("train", binary=True, sources=['features'])
#mnist_test = MNIST("test", binary=True, sources=['features'])
main_loop = MainLoop(
model=None,
data_stream=ForceFloatX(
DataStream(mnist_train,
iteration_scheme=SequentialScheme(
mnist_train.num_examples, batch_size))),
algorithm=algorithm,
extensions=[
Timing(),
FinishAfter(after_n_epochs=epochs),
DataStreamMonitoring(
monitors,
ForceFloatX(
DataStream(mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, batch_size))),
updates=scan_updates,
prefix="test"),
TrainingDataMonitoring(train_monitors,
prefix="train",
after_every_epoch=True),
SerializeMainLoop(name + ".pkl"),
Plot(name, channels=plot_channels),
ProgressBar(),
Printing()
])
main_loop.run()
features_nocar_cat = tensor.dmatrix('features_nocar_cat')
features_nocar_int = tensor.dmatrix('features_nocar_int')
features_cp = tensor.imatrix('codepostal')
features_hascar = tensor.imatrix('features_hascar')
labels = tensor.dmatrix('labels')
prediction, cost, params, valid_cost = build_mlp(
features_car_cat, features_car_int, features_nocar_cat, features_nocar_int,
features_cp, features_hascar, means, labels)
model = Model([cost, prediction])
# load algorithm
from blocks.algorithms import Adam
algorithm = GradientDescent(cost=cost,
parameters=params,
step_rule=Adam(),
on_unused_sources='ignore')
from blocks.extensions import Printing, Timing, FinishAfter
from blocks.extensions.training import TrackTheBest
from blocks.extensions.monitoring import TrainingDataMonitoring, DataStreamMonitoring
from blocks.extensions.stopping import FinishIfNoImprovementAfter
from blocks.extensions.saveload import Checkpoint
from blocks_extras.extensions.plot import Plot
import socket
import datetime
import time
host_plot = 'http://tfjgeorge.com:5006'
cost.name = 'cost'
valid_cost.name = 'valid_cost'
args = parser.parse_args()
np.random.seed(args.seed)
blocks.config.config.default_seed = args.seed
if args.continue_from:
from blocks.serialization import load
main_loop = load(args.continue_from)
main_loop.run()
sys.exit(0)
graphs, extensions, updates = construct_graphs(args, nclasses)
### optimization algorithm definition
if args.optimizer == "adam":
optimizer = Adam(learning_rate=args.learning_rate)
# zzzz
optimizer.learning_rate = theano.shared(np.asarray(optimizer.learning_rate, dtype=theano.config.floatX))
elif args.optimizer == "rmsprop":
optimizer = RMSProp(learning_rate=args.learning_rate, decay_rate=0.9)
elif args.optimizer == "sgdmomentum":
optimizer = Momentum(learning_rate=args.learning_rate, momentum=0.99)
step_rule = CompositeRule([
StepClipping(1.),
optimizer,
])
algorithm = GradientDescent(cost=graphs["training"].outputs[0],
parameters=graphs["training"].parameters,
step_rule=step_rule)
algorithm.add_updates(updates["training"])
model = Model(graphs["training"].outputs[0])