def construct_graphs(task, video_shape, hyperparameters, **kwargs):
x, x_shape, y = task.get_variables()
convnet = construct_model(task=task, **hyperparameters)
convnet.initialize()
emitter = task.get_emitter(input_dim=np.prod(convnet.get_dim("output")),
**hyperparameters)
emitter.initialize()
# shape (duration, batch, features)
hs = [
convnet.apply(x[:, :, i]).flatten(ndim=2)
for i in range(video_shape[0])
]
dists = [emitter.emit_distribution(h) for h in hs]
ps = T.stack([dist.probabilities for dist in dists]).mean(axis=0)
costs = emitter.emit_costs(probabilities=ps, targets=y)
cost = costs.cross_entropy.copy(name="cost")
# gather all the outputs we could possibly care about for training
# *and* monitoring; prepare_graphs will do graph transformations
# after which we may *only* use these to access *any* variables.
outputs_by_name = OrderedDict()
for key in "x x_shape cost".split():
outputs_by_name[key] = locals()[key]
for key in task.monitor_outputs():
outputs_by_name[key] = costs[key]
outputs = list(outputs_by_name.values())
# construct training and inference graphs
mode_by_set = OrderedDict([("train", "training"), ("valid", "inference"),
("test", "inference")])
outputs_by_mode, updates_by_mode = OrderedDict(), OrderedDict()
for mode in "training inference".split():
(outputs_by_mode[mode],
updates_by_mode[mode]) = prepare_mode(mode,
outputs,
convnet=convnet,
emitter=emitter,
**hyperparameters)
# inference updates may make sense at some point but don't know
# where to put them now
assert not updates_by_mode["inference"]
# assign by set for convenience
graphs_by_set = OrderedDict([(which_set,
ComputationGraph(outputs_by_mode[mode]))
for which_set, mode in mode_by_set.items()])
outputs_by_set = OrderedDict([(which_set,
OrderedDict(
util.equizip(outputs_by_name.keys(),
outputs_by_mode[mode])))
for which_set, mode in mode_by_set.items()])
updates_by_set = OrderedDict([(which_set, updates_by_mode[mode])
for which_set, mode in mode_by_set.items()])
return graphs_by_set, outputs_by_set, updates_by_set
def construct_graphs(task, hyperparameters, **kwargs):
x, x_shape, y = task.get_variables()
convnet = construct_model(task=task, **hyperparameters)
convnet.initialize()
h = convnet.apply(x)
h = h.flatten(ndim=2)
emitter = task.get_emitter(input_dim=np.prod(convnet.get_dim("output")),
**hyperparameters)
emitter.initialize()
emitter_outputs = emitter.emit(h, y)
cost = emitter_outputs.cost.copy(name="cost")
# gather all the outputs we could possibly care about for training
# *and* monitoring; prepare_graphs will do graph transformations
# after which we may *only* use these to access *any* variables.
outputs_by_name = OrderedDict()
for key in "x x_shape cost".split():
outputs_by_name[key] = locals()[key]
for key in task.monitor_outputs():
outputs_by_name[key] = emitter_outputs[key]
outputs = list(outputs_by_name.values())
# construct training and inference graphs
mode_by_set = OrderedDict([("train", "training"), ("valid", "inference"),
("test", "inference")])
outputs_by_mode, updates_by_mode = OrderedDict(), OrderedDict()
for mode in "training inference".split():
(outputs_by_mode[mode],
updates_by_mode[mode]) = prepare_mode(mode,
outputs,
convnet=convnet,
emitter=emitter,
**hyperparameters)
# inference updates may make sense at some point but don't know
# where to put them now
assert not updates_by_mode["inference"]
# assign by set for convenience
graphs_by_set = OrderedDict([(which_set,
ComputationGraph(outputs_by_mode[mode]))
for which_set, mode in mode_by_set.items()])
outputs_by_set = OrderedDict([(which_set,
OrderedDict(
util.equizip(outputs_by_name.keys(),
outputs_by_mode[mode])))
for which_set, mode in mode_by_set.items()])
updates_by_set = OrderedDict([(which_set, updates_by_mode[mode])
for which_set, mode in mode_by_set.items()])
return graphs_by_set, outputs_by_set, updates_by_set
def construct_graphs(task, hyperparameters, **kwargs):
x, x_shape, y = task.get_variables()
convnet = construct_model(task=task, **hyperparameters)
convnet.initialize()
h = convnet.apply(x)
h = h.flatten(ndim=2)
emitter = task.get_emitter(
input_dim=np.prod(convnet.get_dim("output")),
**hyperparameters)
emitter.initialize()
emitter_outputs = emitter.emit(h, y)
cost = emitter_outputs.cost.copy(name="cost")
# gather all the outputs we could possibly care about for training
# *and* monitoring; prepare_graphs will do graph transformations
# after which we may *only* use these to access *any* variables.
outputs_by_name = OrderedDict()
for key in "x x_shape cost".split():
outputs_by_name[key] = locals()[key]
for key in task.monitor_outputs():
outputs_by_name[key] = emitter_outputs[key]
outputs = list(outputs_by_name.values())
# construct training and inference graphs
mode_by_set = OrderedDict([
("train", "training"),
("valid", "inference"),
("test", "inference")])
outputs_by_mode, updates_by_mode = OrderedDict(), OrderedDict()
for mode in "training inference".split():
(outputs_by_mode[mode],
updates_by_mode[mode]) = prepare_mode(
mode, outputs, convnet=convnet, emitter=emitter, **hyperparameters)
# inference updates may make sense at some point but don't know
# where to put them now
assert not updates_by_mode["inference"]
# assign by set for convenience
graphs_by_set = OrderedDict([
(which_set, ComputationGraph(outputs_by_mode[mode]))
for which_set, mode in mode_by_set.items()])
outputs_by_set = OrderedDict([
(which_set, OrderedDict(util.equizip(outputs_by_name.keys(),
outputs_by_mode[mode])))
for which_set, mode in mode_by_set.items()])
updates_by_set = OrderedDict([
(which_set, updates_by_mode[mode])
for which_set, mode in mode_by_set.items()])
return graphs_by_set, outputs_by_set, updates_by_set
def make_graph(data, model, config, fold="valid"):
h = H()
h.inputs = data.get_variables([data.get_tfrecord_path(fold)],
config.hp.batch_size)
h.mask = config.masker.get_variable(tf.shape(h.inputs.image)[0])
h.global_step = config.global_step
h.model = model(image=h.inputs.image,
mask=h.mask,
caption=h.inputs.caption,
caption_length=h.inputs.caption_length)
if D.train:
h.lr = tf.Variable(config.hp.lr.init,
name="learning_rate",
trainable=False,
dtype=tf.float32)
tf.summary.scalar("learning_rate", h.lr)
h.lr_decay_op = tf.assign(h.lr, config.hp.lr.decay * h.lr)
h.loss = h.model.loss
h.parameters = tf.trainable_variables()
h.gradients = tf.gradients(h.loss, h.parameters)
h.optimizer = tf.train.AdamOptimizer(h.lr)
h.train_op = h.optimizer.apply_gradients(util.equizip(
h.gradients, h.parameters),
global_step=h.global_step)
h.summaries = []
if D.train:
for k, v in h.Narrow(
"model.loss model.loss_given model.loss_asked").Items():
tf.summary.scalar(k, v)
else:
tf.summary.image("real", h.model.x, max_outputs=3)
tf.summary.image("fake", h.model.xhat, max_outputs=3)
tf.summary.image("fake_asked",
tf.cast(
(1 - h.mask) * tf.cast(h.model.xhat, tf.float32),
tf.uint8),
max_outputs=3)
tf.summary.image("realfake",
tf.cast(
h.mask * tf.cast(h.model.x, tf.float32) +
(1 - h.mask) * tf.cast(h.model.xhat, tf.float32),
tf.uint8),
max_outputs=3)
tf.summary.image("mask", h.mask, max_outputs=3)
tf.summary.image("entropies", h.model.entropies, max_outputs=3)
return h
def make_graph(self):
with tf.name_scope("train"):
graph = make_training_graph(self.data,
self.model,
self.config,
fold="train")
summaries = [
var for var in tf.get_collection(tf.GraphKeys.SUMMARIES)
if var.name.startswith("train")
]
for parameter, gradient in util.equizip(graph.parameters,
graph.gradients):
summaries.append(
tf.summary.scalar(
"gradmla_%s" % parameter.name.replace(":", "_"),
tfutil.meanlogabs(gradient)))
for parameter in tf.trainable_variables():
summaries.append(
tf.summary.scalar("mla_%s" % parameter.name.replace(":", "_"),
tfutil.meanlogabs(parameter)))
graph.summary_op = tf.summary.merge(summaries)
return graph
def construct_graphs(task, n_patches, hyperparameters, **kwargs):
x, x_shape, y = task.get_variables()
ram = construct_model(task=task, **hyperparameters)
ram.initialize()
scopes = []
scopes.append(ram.apply(util.Scope(x=x, x_shape=x_shape), initial=True))
n_steps = n_patches - 1
for i in xrange(n_steps):
scopes.append(
ram.apply(
util.Scope(x=x,
x_shape=x_shape,
previous_states=scopes[-1].rnn_outputs)))
emitter = task.get_emitter(input_dim=ram.get_dim("states"),
**hyperparameters)
emitter.initialize()
emitter_outputs = emitter.emit(scopes[-1].rnn_outputs["states"], y)
emitter_cost = emitter_outputs.cost.copy(name="emitter_cost")
excursion_cost = (T.stack([scope.excursion for scope in scopes
]).mean().copy(name="excursion_cost"))
cost = (emitter_cost + excursion_cost).copy(name="cost")
# gather all the outputs we could possibly care about for training
# *and* monitoring; prepare_graphs will do graph transformations
# after which we may *only* use these to access *any* variables.
outputs_by_name = OrderedDict()
for key in "x x_shape emitter_cost excursion_cost cost".split():
outputs_by_name[key] = locals()[key]
for key in task.monitor_outputs():
outputs_by_name[key] = emitter_outputs[key]
for key in "true_location true_scale raw_location raw_scale patch savings".split(
):
outputs_by_name[key] = T.stack([scope[key] for scope in scopes])
outputs = list(outputs_by_name.values())
# construct training and inference graphs
mode_by_set = OrderedDict([("train", "training"), ("valid", "inference"),
("test", "inference")])
outputs_by_mode, updates_by_mode = OrderedDict(), OrderedDict()
for mode in "training inference".split():
(outputs_by_mode[mode],
updates_by_mode[mode]) = prepare_mode(mode,
outputs,
ram=ram,
emitter=emitter,
**hyperparameters)
# inference updates may make sense at some point but don't know
# where to put them now
assert not updates_by_mode["inference"]
# assign by set for convenience
graphs_by_set = OrderedDict([(which_set,
ComputationGraph(outputs_by_mode[mode]))
for which_set, mode in mode_by_set.items()])
outputs_by_set = OrderedDict([(which_set,
OrderedDict(
util.equizip(outputs_by_name.keys(),
outputs_by_mode[mode])))
for which_set, mode in mode_by_set.items()])
updates_by_set = OrderedDict([(which_set, updates_by_mode[mode])
for which_set, mode in mode_by_set.items()])
return graphs_by_set, outputs_by_set, updates_by_set
def construct_graphs(task, n_patches, hyperparameters, **kwargs):
x, x_shape, y = task.get_variables()
ram = construct_model(task=task, **hyperparameters)
ram.initialize()
scopes = []
scopes.append(ram.apply(util.Scope(x=x, x_shape=x_shape), initial=True))
n_steps = n_patches - 1
for i in xrange(n_steps):
scopes.append(ram.apply(util.Scope(
x=x, x_shape=x_shape,
previous_states=scopes[-1].rnn_outputs)))
emitter = task.get_emitter(
input_dim=ram.get_dim("states"),
**hyperparameters)
emitter.initialize()
emitter_outputs = emitter.emit(scopes[-1].rnn_outputs["states"], y)
emitter_cost = emitter_outputs.cost.copy(name="emitter_cost")
excursion_cost = (T.stack([scope.excursion for scope in scopes])
.mean().copy(name="excursion_cost"))
cost = (emitter_cost + excursion_cost).copy(name="cost")
# gather all the outputs we could possibly care about for training
# *and* monitoring; prepare_graphs will do graph transformations
# after which we may *only* use these to access *any* variables.
outputs_by_name = OrderedDict()
for key in "x x_shape emitter_cost excursion_cost cost".split():
outputs_by_name[key] = locals()[key]
for key in task.monitor_outputs():
outputs_by_name[key] = emitter_outputs[key]
for key in "true_location true_scale raw_location raw_scale patch savings".split():
outputs_by_name[key] = T.stack([scope[key] for scope in scopes])
outputs = list(outputs_by_name.values())
# construct training and inference graphs
mode_by_set = OrderedDict([
("train", "training"),
("valid", "inference"),
("test", "inference")])
outputs_by_mode, updates_by_mode = OrderedDict(), OrderedDict()
for mode in "training inference".split():
(outputs_by_mode[mode],
updates_by_mode[mode]) = prepare_mode(
mode, outputs, ram=ram, emitter=emitter, **hyperparameters)
# inference updates may make sense at some point but don't know
# where to put them now
assert not updates_by_mode["inference"]
# assign by set for convenience
graphs_by_set = OrderedDict([
(which_set, ComputationGraph(outputs_by_mode[mode]))
for which_set, mode in mode_by_set.items()])
outputs_by_set = OrderedDict([
(which_set, OrderedDict(util.equizip(outputs_by_name.keys(),
outputs_by_mode[mode])))
for which_set, mode in mode_by_set.items()])
updates_by_set = OrderedDict([
(which_set, updates_by_mode[mode])
for which_set, mode in mode_by_set.items()])
return graphs_by_set, outputs_by_set, updates_by_set
def FlatCall(self, fn, *args, **kwargs):
return Holster(
util.equizip(self.Keys(), fn(list(self.Values()), *args,
**kwargs)))
def main(argv=()):
assert not argv[1:]
checkpoint_dir = os.path.dirname(FLAGS.checkpoint)
hp_string = open(os.path.join(checkpoint_dir, "hp.conf")).read()
config = H(data_dir="/Tmp/cooijmat/mscoco",
basename=FLAGS.basename,
num_samples=FLAGS.num_samples,
temperature=FLAGS.temperature)
config.hp = H(util.parse_hp(hp_string))
print str(config.hp)
dirname = "sample_%s_%s_%s_T%s" % (config.basename, FLAGS.strategy,
datetime.datetime.now().isoformat(),
config.temperature)
dirname = dirname[:255] # >:-(((((((((((((((((((((((((((((((((((((((((
config.output_dir = dirname
if not tf.gfile.Exists(config.output_dir):
tf.gfile.MakeDirs(config.output_dir)
data = datasets.MscocoNP(config)
config.hp.caption.depth = data.caption_depth
model = models.Model(config.hp)
config.hp.masker.image = H(config.hp.image) # -_-
config.masker = maskers.make(config.hp.masker.kind, hp=config.hp.masker)
with D.Bind(train=False):
graph = make_graph(data, model, config)
saver = tf.train.Saver()
session = tf.Session()
saver.restore(session, FLAGS.checkpoint)
def predictor(image, mask):
feed_dict = {
graph.inputs.image: image,
graph.inputs.caption: original.caption,
graph.inputs.caption_length: original.caption_length,
graph.mask: mask,
}
values = graph.Narrow("model.pxhat").FlatCall(session.run,
feed_dict=feed_dict)
return values.model.pxhat
config.predictor = predictor
sampler = Strategy.make(FLAGS.strategy, config)
original = next(
data.get_batches(data.get_filenames("valid"),
batch_size=config.num_samples,
shuffle=False))
xs = original.image
with bamboo.scope("original"):
masks = np.ones(xs.shape).astype(np.float32)
bamboo.log(x=xs, mask=masks)
masks = (maskers.ContiguousMasker(H(
image=config.hp.masker.image, size=32)).get_value(config.num_samples))
xs = masks * xs
with bamboo.scope("masked"):
bamboo.log(x=xs, mask=masks)
xhats, masks = sampler(xs, masks)
with bamboo.scope("final"):
bamboo.log(x=xhats, mask=masks)
for i, (caption, x,
xhat) in enumerate(util.equizip(original.caption, xs, xhats)):
scipy.misc.imsave(
os.path.join(config.output_dir, "%i_original.png" % i), x)
scipy.misc.imsave(os.path.join(config.output_dir, "%i_sample.png" % i),
xhat)
with open(os.path.join(config.output_dir, "%i_caption.txt" % i),
"w") as file:
file.write(data.tokenizer.decode(caption))
bamboo.dump(os.path.join(config.output_dir, "log.npz"))