def __init__(self, hp):
children = [("conv", ConvReader(H(radius=hp.radius,
size=3 * hp.size))),
("rnn",
RecurrentReader(
H(bidir=hp.bidir,
cell=H(kind="quasi",
size=hp.size,
normalize=hp.normalize))))]
super(QuasiReader, self).__init__(hp, children=children)
def test_regression2(self):
h = H(k=H())
h.k.l = 2
g = H()
h = H(k=H())
g.h = h
g.h.k.l = 2
i = h.Map(lambda x: x + 1)
self.assertEqual(i.Size(), 0)
i.k.l = 2
def get_variables(self, filenames, batch_size, num_epochs=None):
with tf.name_scope('input'):
reader = tf.TFRecordReader()
filename_queue = tf.train.string_input_producer(
list(filenames), num_epochs=num_epochs, name="filequeue")
record_queue_capacity = 10000
record_queue = tf.RandomShuffleQueue(
capacity=record_queue_capacity,
min_after_dequeue=record_queue_capacity // 2,
dtypes=[tf.string],
name="shufflequeue")
enqueue_ops = []
for _ in range(2):
_, record = reader.read(filename_queue)
enqueue_ops.append(record_queue.enqueue([record]))
tf.train.queue_runner.add_queue_runner(
tf.train.queue_runner.QueueRunner(record_queue, enqueue_ops))
tf.summary.scalar(
"queue/%s/fraction_of_%d_full" %
(record_queue.name, record_queue_capacity),
tf.cast(record_queue.size(), tf.float32) /
record_queue_capacity)
serialized_example = record_queue.dequeue()
context, sequence = tf.parse_single_sequence_example(
serialized_example,
context_features={
"image/data": tf.FixedLenFeature([], dtype=tf.string)
},
sequence_features={
"image/caption_characters":
tf.FixedLenSequenceFeature([], dtype=tf.int64),
"image/caption_words":
tf.FixedLenSequenceFeature([], dtype=tf.int64),
})
image = tf.image.decode_jpeg(context["image/data"],
channels=Mscoco.IMAGE_DEPTH)
image.set_shape(
[Mscoco.IMAGE_HEIGHT, Mscoco.IMAGE_WIDTH, Mscoco.IMAGE_DEPTH])
caption = sequence["image/caption_%ss" %
self.config.hp.caption.token]
caption_length = tf.shape(caption)[0]
singular = H(image=image,
caption=caption,
caption_length=caption_length)
plural = singular.FlatCall(tf.train.batch,
batch_size=batch_size,
num_threads=2,
capacity=30 * batch_size,
dynamic_pad=True,
allow_smaller_final_batch=True)
return plural
def __call__(self, x, merger, mergee):
hp = self.hp
for i in range(hp.profundity):
x = tfutil.conv_layer(x,
depth=hp.depth,
radius=hp.radius,
scope="conv%i" % i)
x = merger(i, x, mergee)
return H(output=x)
def __call__(self, x, merger, mergee):
hp = self.hp
for i in range(0, hp.profundity, 2):
x = tfutil.residual_block(x,
depth=hp.depth,
radius=hp.radius,
scope="res%i" % i)
x = merger(i, x, mergee)
return H(output=x)
def load_batch(self, filenames):
images, captions = zip(*list(map(self.load_file, filenames)))
h = H()
h.image = np.array(images)
h.caption_length = np.array(list(map(len, captions)))
h.caption = np.array([
util.padto(np.array(caption, dtype=int), max(h.caption_length))
for caption in captions
])
return h
def get_variables(self):
h = H()
h.image = tf.placeholder(tf.uint8, [
None, Mscoco.IMAGE_HEIGHT, Mscoco.IMAGE_WIDTH, Mscoco.IMAGE_DEPTH
],
name="image")
h.caption = tf.placeholder(tf.int32, [None, None], name="caption")
h.caption_length = tf.placeholder(tf.int32, [None],
name="caption_length")
return h
def test(self):
h = H([("c.d", H(e=3)), ("c.f.g c.f.h", 4), ("c.f.i", [5])], a=1, b=2)
self.assertEqual(h.a, 1)
self.assertEqual(h.b, 2)
self.assertEqual(h.c.d.e, 3)
self.assertEqual(h.c.f.g, 4)
self.assertEqual(h.c.f.h, 4)
self.assertEqual(h.c.f.i, [5])
g = h.Narrow("c.d.e c.f.i")
with self.assertRaises(KeyError): g.a
with self.assertRaises(KeyError): g.c.f.g
self.assertEqual(g.c.d.e, 3)
self.assertEqual(g.c.f.i, [5])
g = h.c.f
self.assertEqual(dict(g.Items()),
dict(g=4, h=4, i=[5]))
self.assertEqual(set(h.Keys()), set("c.d.e c.f.g c.f.h c.f.i a b".split()))
self.assertEqual(h.FlatCall(lambda x: x), h)
self.assertEqual(h.c.FlatCall(lambda x: x), h.c)
def __call__(self, x, length):
hp = self.hp
h = H()
h.input = x
h.input_length = length
for key, child in self.children.Items():
h[key] = child(x, length)
x, length, z = h[key].output, h[key].output_length, h[key].summary
h.output = x
h.output_length = length
h.summary = z
return h
def __call__(self, x, length):
hp = self.hp
h = H()
h.input = x
h.input_length = length
w = tf.get_variable("w",
shape=[hp.radius, h.input.shape[-1], hp.size],
initializer=tf.uniform_unit_scaling_initializer())
h.output = tf.nn.conv1d(h.input, w, stride=1, padding="VALID")
h.output_length = h.input_length - (hp.radius - 1)
# summarize by global average pooling
h.summary = tf.reduce_mean(h.output, axis=1)
return h
def main():
datasource = datasets.MscocoNP(H(data_dir="/Tmp/cooijmat/mscoco", hp=H(caption=H(token="character"))))
tfrecord_dir = os.environ["MSCOCO_TFRECORD_DIR"]
tf.gfile.MakeDirs(tfrecord_dir)
tokenizers = ordict((token, datasource.get_tokenizer(token))
for token in "character word".split())
for token, tokenizer in tokenizers.items():
tokenmap_path = os.path.join(tfrecord_dir, "tokenmap_%s.pkl" % token)
print "writing to", tokenmap_path
pkl.dump(tokenizer.tokenmap, open(tokenmap_path, "wb"))
print "done"
def _to_sequence_example(image_path):
identifier = os.path.splitext(os.path.basename(image_path))[0]
caption_words = tokenizers["word"].process(datasource.get_caption_string(identifier))
caption_characters = tokenizers["character"].process(datasource.get_caption_string(identifier))
with tf.gfile.FastGFile(image_path, "rb") as f:
jpeg = f.read()
return tf.train.SequenceExample(
context=tf.train.Features(feature={
"image/identifier": _bytes_feature(identifier),
"image/data": _bytes_feature(jpeg),
}),
feature_lists=tf.train.FeatureLists(feature_list={
"image/caption_characters": _int64_feature_list(caption_characters),
"image/caption_words": _int64_feature_list(caption_words),
}))
for fold in "train valid".split():
output_path = os.path.join(tfrecord_dir, fold + ".tfrecords")
print "writing to", output_path
writer = tf.python_io.TFRecordWriter(output_path)
for filename in datasource.get_filenames(fold):
example = _to_sequence_example(filename)
writer.write(example.SerializeToString())
writer.close()
def make_graph(data, model, config, fold="valid"):
h = H()
h.inputs = data.get_variables()
h.mask = tf.placeholder(tf.float32, [
None, config.hp.image.size, config.hp.image.size,
config.hp.masker.image.depth
],
name="mask")
h.model = model(image=h.inputs.image,
mask=h.mask,
caption=h.inputs.caption,
caption_length=h.inputs.caption_length)
return h
def make_graph(data, model, config, fold="valid"):
h = H()
h.inputs = data.get_variables([data.get_tfrecord_path(fold)],
config.hp.batch_size)
h.mask = config.masker.get_variable(tf.shape(h.inputs.image)[0])
h.global_step = config.global_step
h.model = model(image=h.inputs.image,
mask=h.mask,
caption=h.inputs.caption,
caption_length=h.inputs.caption_length)
if D.train:
h.lr = tf.Variable(config.hp.lr.init,
name="learning_rate",
trainable=False,
dtype=tf.float32)
tf.summary.scalar("learning_rate", h.lr)
h.lr_decay_op = tf.assign(h.lr, config.hp.lr.decay * h.lr)
h.loss = h.model.loss
h.parameters = tf.trainable_variables()
h.gradients = tf.gradients(h.loss, h.parameters)
h.optimizer = tf.train.AdamOptimizer(h.lr)
h.train_op = h.optimizer.apply_gradients(util.equizip(
h.gradients, h.parameters),
global_step=h.global_step)
h.summaries = []
if D.train:
for k, v in h.Narrow(
"model.loss model.loss_given model.loss_asked").Items():
tf.summary.scalar(k, v)
else:
tf.summary.image("real", h.model.x, max_outputs=3)
tf.summary.image("fake", h.model.xhat, max_outputs=3)
tf.summary.image("fake_asked",
tf.cast(
(1 - h.mask) * tf.cast(h.model.xhat, tf.float32),
tf.uint8),
max_outputs=3)
tf.summary.image("realfake",
tf.cast(
h.mask * tf.cast(h.model.x, tf.float32) +
(1 - h.mask) * tf.cast(h.model.xhat, tf.float32),
tf.uint8),
max_outputs=3)
tf.summary.image("mask", h.mask, max_outputs=3)
tf.summary.image("entropies", h.model.entropies, max_outputs=3)
return h
def __call__(self, image, mask, caption, caption_length):
hp = self.hp
h = H()
with tf.variable_scope("reader") as scope:
h.reader = self.reader(tf.one_hot(caption, hp.caption.depth),
length=caption_length)
h.x = image
h.px = tf.one_hot(h.x, hp.image.levels)
h.mask = mask
assert tfutil.get_depth(h.mask) == hp.masker.image.depth
h.context = tf.concat([
tfutil.collapse(h.px * h.mask[:, :, :, :, None],
[0, 1, 2, [3, 4]]), h.mask
],
axis=3)
with tf.variable_scope("convnet") as scope:
h.convnet = self.convnet(h.context, self.merger, h.reader)
h.exhat = tf.reshape(
tfutil.conv_layer(h.convnet.output,
radius=1,
depth=3 * hp.image.levels,
fn=lambda x: x,
scope="exhat"), tf.shape(h.px))
h.pxhat = tf.nn.softmax(h.exhat)
h.xhat = tf.cast(tf.argmax(h.exhat, axis=4), tf.uint8)
lossfn = (dict(xent=tfutil.softmax_xent,
emd=ft.partial(tfutil.softmax_emd, distance=tf.abs),
emd2=ft.partial(tfutil.softmax_emd,
distance=tf.square))[hp.loss])
h.losses = lossfn(labels=h.px, logits=h.exhat)
h.loss_total = tf.reduce_mean(h.losses)
h.loss_given = tf.reduce_sum(h.losses * h.mask / tf.reduce_sum(h.mask))
h.loss_asked = tf.reduce_sum(h.losses * (1 - h.mask) /
tf.reduce_sum(1 - h.mask))
h.loss = h.loss_total if hp.optimize_given else h.loss_asked
h.entropies = -tf.reduce_sum(tfutil.softmax_xent(labels=h.pxhat,
logits=h.exhat),
axis=3,
keep_dims=True)
return h
def __call__(self, session, supervisor):
aggregates = H({
"model.loss": util.MeanAggregate(),
"model.loss_given": util.MeanAggregate(),
"model.loss_asked": util.MeanAggregate(),
"summary_op": util.LastAggregate()
})
batch_size = 10 * self.config.hp.batch_size
# spend at most 1/16 of the time validating
max_num_batches = self.config.hp.validate.interval // 16
for _, feed_dict in zip(
range(max_num_batches),
self.data.get_feed_dicts(self.graph.inputs,
"valid",
batch_size=batch_size,
shuffle=False)):
feed_dict = dict(feed_dict)
feed_dict.update(self.config.masker.get_feed_dict(batch_size))
values = self.graph.Narrow(
"model.loss model.loss_given model.loss_asked summary_op"
).FlatCall(session.run, feed_dict=feed_dict)
for aggregate, value in aggregates.Zip(values):
aggregate(value)
values = H(
(key, aggregate.value) for key, aggregate in aggregates.Items())
supervisor.summary_computed(session, values.summary_op)
for key, value in values.Items():
# summary_ops return strings?? the plot thickens -__-
if not isinstance(value, basestring):
value = tf.Summary(value=[
tf.Summary.Value(tag="valid/%s" % key, simple_value=value)
])
supervisor.summary_computed(session, value)
return values
def __call__(self, x, length):
hp = self.hp
h = H()
h.input = x
h.input_length = length
if hp.bidir:
h.cell_fw = cells.make(hp.cell.kind,
num_units=hp.cell.size,
normalize=hp.cell.normalize,
scope="fw")
h.cell_bw = cells.make(hp.cell.kind,
num_units=hp.cell.size,
normalize=hp.cell.normalize,
scope="bw")
batch_size = tf.shape(x)[0]
h.output, h.state = tf.nn.bidirectional_dynamic_rnn(
h.cell_fw,
h.cell_bw,
x,
sequence_length=length,
initial_state_fw=[
tf.tile(s[None, :], [batch_size, 1])
for s in h.cell_fw.initial_state_parameters
],
initial_state_bw=[
tf.tile(s[None, :], [batch_size, 1])
for s in h.cell_bw.initial_state_parameters
],
scope="birnn")
h.output = tf.concat(h.output, axis=2)
else:
h.cell = cells.make(hp.cell.kind,
num_units=hp.cell.size,
normalize=hp.cell.normalize,
scope="fw")
batch_size = tf.shape(x)[0]
h.output, h.state = tf.nn.dynamic_rnn(
h.cell,
x,
sequence_length=length,
initial_state=[
tf.tile(s[None, :], [batch_size, 1])
for s in h.cell.initial_state_parameters
],
scope="rnn")
h.output_length = length
h.summary = h.output[:, -1]
return h
def __call__(self, x, merger, mergee):
hp = self.hp
# try to do the right thing
h, w = x.get_shape().as_list()[1], x.get_shape().as_list()[2]
assert h == w
ndilations = int(round(np.log2(h) - 1))
dilation_interval = int(round(hp.profundity / ndilations))
dilation = 1
def batch_to_space(x):
if dilation == 1: return x
return tf.batch_to_space_nd(x, [dilation, dilation],
tf.zeros([2, 2], dtype=tf.int32))
def space_to_batch(x):
if dilation == 1: return x
return tf.space_to_batch_nd(x, [dilation, dilation],
tf.zeros([2, 2], dtype=tf.int32))
for i in range(hp.profundity):
if i != 0 and i % dilation_interval == 0:
x = batch_to_space(x)
dilation *= 2
x = space_to_batch(x)
x = tfutil.conv_layer(x,
depth=hp.depth,
radius=hp.radius,
scope="conv%i" % i)
if merger.should_merge(i):
x = batch_to_space(x)
x = merger(i, x, mergee)
x = space_to_batch(x)
x = batch_to_space(x)
return H(output=x)
def sample_hp():
hp = H(defaults)
hp.batch_size = uniform(5, 30)
hp.optimize_given = boolean()
hp["reader.radius"] = uniform(4, 32)
hp["reader.size"] = uniform(32, 256)
hp["reader.bidir"] = boolean()
hp["reader.normalize"] = boolean()
hp["convnet.kind"] = categorical(
"straight straight_residual straight_dilated".split())
hp["convnet.depth"] = uniform(64, 512)
hp["convnet.profundity"] = uniform(4, 48)
hp["convnet.radius"] = uniform(2, 7)
hp["merger.kind"] = categorical("attention conv".split())
hp.merger.kind = "conv"
if hp.merger.kind == "conv":
hp.merger.depth = uniform(4, 64)
nmerges = uniform(1, min(5, hp.convnet.profundity))
layers = np.random.choice(hp.convnet.profundity,
size=(nmerges, ),
replace=False)
hp["merger.layers"] = ",".join(map(str, sorted(layers)))
return hp
def boolean():
return np.random.rand() < 0.5
def categorical(xs):
return np.random.choice(xs)
defaults = H(**util.parse_hp("""
lr.init=0.001
lr.decay=0.1
lr.patience=1000
validate.interval=100
num_steps=100000
masker.kind=orderless
image.size=64
image.depth=3
image.levels=256
caption.token=word
reader.kind=quasi
"""))
def sample_hp():
hp = H(defaults)
hp.batch_size = uniform(5, 30)
hp.optimize_given = boolean()
hp["reader.radius"] = uniform(4, 32)
hp["reader.size"] = uniform(32, 256)
hp["reader.bidir"] = boolean()
def main(argv=()):
if argv[1:]:
raise ValueError("leftover arguments: %r" % argv[1:])
config = H(data_dir=FLAGS.data_dir,
base_output_dir=FLAGS.base_output_dir,
basename=FLAGS.basename,
resume=FLAGS.resume,
trace_fraction=FLAGS.trace_fraction)
if FLAGS.hpfile:
with open(FLAGS.hpfile) as hpfile:
hpstring = hpfile.read()
else:
hpstring = FLAGS.hp
config.hp = H(util.parse_hp(hpstring))
print str(config.hp)
config.label = util.make_label(config)
dirname = "%s_%s" % (datetime.datetime.now().isoformat(), config.label)
dirname = dirname[:255] # >:-(((((((((((((((((((((((((((((((((((((((((
config.output_dir = os.path.join(config.base_output_dir, dirname)
data = datasets.MscocoTF(config)
config.hp.caption.depth = data.caption_depth
# NOTE: all hyperparameters must be set at this point
prepare_run_directory(config)
model = models.Model(config.hp)
config.hp.masker.image = config.hp.image # -_-
config.masker = maskers.make(config.hp.masker.kind, hp=config.hp.masker)
config.global_step = tf.Variable(0, name="global_step", trainable=False)
trainer = Trainer(data, model, config)
tf.get_variable_scope().reuse_variables()
evaluator = Evaluator(data, model, config)
earlystopper = EarlyStopper(config, trainer.graph.lr_decay_op)
supervisor = tf.train.Supervisor(logdir=config.output_dir, summary_op=None)
with supervisor.managed_session() as session:
while True:
global_step = tf.train.global_step(session, config.global_step)
if supervisor.should_stop():
print "supervisor says should stop"
break
if earlystopper.should_stop(global_step):
print "earlystopper says should stop"
break
trainer(session, supervisor)
sys.stdout.write("\r%i " % global_step)
sys.stdout.flush()
if global_step % config.hp.validate.interval == 0:
values = evaluator(session, supervisor)
earlystopper.track(global_step, values.model.loss, session)
print "%5i loss:%10f loss asked:%10f loss given:%10f lr:%6g best:%10f age:%6i" % (
global_step,
values.model.loss,
values.model.loss_asked,
values.model.loss_given,
# NOTE: earlystopper.best_loss is best of median filtered losses
trainer.graph.lr.eval(session),
earlystopper.best_loss,
earlystopper.age)
if global_step >= config.hp.num_steps:
print "hp.num_steps reached"
break
def main(argv=()):
assert not argv[1:]
checkpoint_dir = os.path.dirname(FLAGS.checkpoint)
hp_string = open(os.path.join(checkpoint_dir, "hp.conf")).read()
config = H(data_dir="/Tmp/cooijmat/mscoco",
basename=FLAGS.basename,
num_samples=FLAGS.num_samples,
temperature=FLAGS.temperature)
config.hp = H(util.parse_hp(hp_string))
print str(config.hp)
dirname = "sample_%s_%s_%s_T%s" % (config.basename, FLAGS.strategy,
datetime.datetime.now().isoformat(),
config.temperature)
dirname = dirname[:255] # >:-(((((((((((((((((((((((((((((((((((((((((
config.output_dir = dirname
if not tf.gfile.Exists(config.output_dir):
tf.gfile.MakeDirs(config.output_dir)
data = datasets.MscocoNP(config)
config.hp.caption.depth = data.caption_depth
model = models.Model(config.hp)
config.hp.masker.image = H(config.hp.image) # -_-
config.masker = maskers.make(config.hp.masker.kind, hp=config.hp.masker)
with D.Bind(train=False):
graph = make_graph(data, model, config)
saver = tf.train.Saver()
session = tf.Session()
saver.restore(session, FLAGS.checkpoint)
def predictor(image, mask):
feed_dict = {
graph.inputs.image: image,
graph.inputs.caption: original.caption,
graph.inputs.caption_length: original.caption_length,
graph.mask: mask,
}
values = graph.Narrow("model.pxhat").FlatCall(session.run,
feed_dict=feed_dict)
return values.model.pxhat
config.predictor = predictor
sampler = Strategy.make(FLAGS.strategy, config)
original = next(
data.get_batches(data.get_filenames("valid"),
batch_size=config.num_samples,
shuffle=False))
xs = original.image
with bamboo.scope("original"):
masks = np.ones(xs.shape).astype(np.float32)
bamboo.log(x=xs, mask=masks)
masks = (maskers.ContiguousMasker(H(
image=config.hp.masker.image, size=32)).get_value(config.num_samples))
xs = masks * xs
with bamboo.scope("masked"):
bamboo.log(x=xs, mask=masks)
xhats, masks = sampler(xs, masks)
with bamboo.scope("final"):
bamboo.log(x=xhats, mask=masks)
for i, (caption, x,
xhat) in enumerate(util.equizip(original.caption, xs, xhats)):
scipy.misc.imsave(
os.path.join(config.output_dir, "%i_original.png" % i), x)
scipy.misc.imsave(os.path.join(config.output_dir, "%i_sample.png" % i),
xhat)
with open(os.path.join(config.output_dir, "%i_caption.txt" % i),
"w") as file:
file.write(data.tokenizer.decode(caption))
bamboo.dump(os.path.join(config.output_dir, "log.npz"))
def test_regression1(self):
h = H()
h["graph.train"] = 0
h.graph.valid = 1
self.assertEqual(set(h.graph.Keys()), set("train valid".split()))
self.assertEqual(set(h.graph.Narrow("train valid").Keys()), set("train valid".split()))
from holster import H # global variable store D = H()
def __init__(self, hp, children=()):
self.hp = hp
self.children = H(children)
def test_regression1(self):
h = H()
h["g.t"] = 0
h.g.v = 1
self.assertEqual(set(h.g.Keys()), set("tv"))
self.assertEqual(set(h.g.Narrow("t v").Keys()), set("tv"))