def _make(self, hp, global_step=None):
ts = NS()
ts.global_step = global_step
ts.x = tf.placeholder(dtype=tf.int32, name="x")
ts.seq = self.model.make_training_graph(x=ts.x,
length=self.segment_length)
ts.final_state = ts.seq.final_state
ts.loss = ts.seq.loss
ts.error = ts.seq.error
ts.learning_rate = tf.Variable(hp.initial_learning_rate,
dtype=tf.float32,
trainable=False,
name="learning_rate")
ts.decay_op = tf.assign(ts.learning_rate,
ts.learning_rate * hp.decay_rate)
ts.optimizer = tf.train.AdamOptimizer(ts.learning_rate)
ts.params = tf.trainable_variables()
print[param.name for param in ts.params]
ts.gradients = tf.gradients(
ts.loss,
ts.params,
# secret memory-conserving sauce
aggregation_method=tf.AggregationMethod.EXPERIMENTAL_TREE)
loose_params = [
param for param, gradient in util.equizip(ts.params, ts.gradients)
if gradient is None
]
if loose_params:
raise ValueError("loose parameters: %s" %
" ".join(param.name for param in loose_params))
# tensorflow fails miserably to compute gradient for these
for reg_var in tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES):
ts.gradients[ts.params.index(reg_var)] += (
hp.weight_decay *
tf.gradients(tf.sqrt(tf.reduce_sum(reg_var**2)), [reg_var])[0])
ts.clipped_gradients, _ = tf.clip_by_global_norm(
ts.gradients, hp.clip_norm)
ts.training_op = ts.optimizer.apply_gradients(
util.equizip(ts.clipped_gradients, ts.params),
global_step=ts.global_step)
ts.summaries = [
tf.summary.scalar("loss_train", ts.loss),
tf.summary.scalar("error_train", ts.error),
tf.summary.scalar("learning_rate", ts.learning_rate)
]
for parameter, gradient in util.equizip(ts.params, ts.gradients):
ts.summaries.append(
tf.summary.scalar("meanlogabs_%s" % parameter.name,
tfutil.meanlogabs(parameter)))
ts.summaries.append(
tf.summary.scalar("meanlogabsgrad_%s" % parameter.name,
tfutil.meanlogabs(gradient)))
return ts
def test_equizip(self):
self.assertRaises(
ValueError,
lambda: list(util.equizip(range(2), range(3), range(5))))
self.assertRaises(
ValueError,
lambda: list(util.equizip(range(5), range(3), range(5))))
self.assertEqual([(2, 0), (1, 1), (0, 2)],
list(util.equizip(reversed(range(3)), range(3))))
def to_examples(segmented_examples):
segments_by_example = [[[
comparablearray(list(s.strip()), dtype="|S1")
] for s in e.split("|")] for e in segmented_examples]
examples_by_segment = list(
map(list, util.equizip(*segments_by_example)))
return examples_by_segment
def main(argv):
primer_paths = argv[1:]
assert FLAGS.model_ckpt
model_dir = os.path.dirname(FLAGS.model_ckpt)
if not FLAGS.model_hyperparameters:
FLAGS.model_hyperparameters = os.path.join(model_dir,
"hyperparameters.yaml")
if not FLAGS.base_output_path:
FLAGS.base_output_path = model_dir + "/"
hp = hyperparameters.load(FLAGS.model_hyperparameters)
assert primer_paths
dataset = datasets.construct(FLAGS.data_type,
paths=primer_paths,
frequency=hp.sampling_frequency,
bit_depth=hp.bit_depth)
primers = dataset.examples
primers = preprocess_primers(primers, hp=hp)
model = models.construct(hp)
sampler = sampling.Sampler(model, hp=hp)
saver = tf.train.Saver()
session = tf.Session()
saver.restore(session, FLAGS.model_ckpt)
sample_length = hp.sampling_frequency * FLAGS.sample_duration
xhat = sampler.run(primers=primers,
length=sample_length,
temperature=FLAGS.temperature,
session=session,
hp=hp)
x, = util.examples_as_arrays(primers)
for i, (p, s) in enumerate(util.equizip(x, xhat)):
output_path = FLAGS.base_output_path + "temp_%s_%i" % (
FLAGS.temperature, i)
print output_path
dataset.dump(output_path, [np.concatenate([p, s], axis=0)])
output_path = FLAGS.base_output_path + "samples.npz"
print "writing raw sample data to %s" % output_path
np.savez_compressed(output_path, x=x, xhat=xhat)
def test_segmented_batches(self):
length = np.random.randint(2, 100)
segment_length = np.random.randint(1, length)
example_count = np.random.randint(2, 100)
batch_size = np.random.randint(1, example_count)
feature_shapes = [
np.random.randint(1, 10, size=np.random.randint(1, 4))
for _ in range(np.random.randint(1, 4))
]
examples = [[
np.random.rand(length, *shape) for shape in feature_shapes
] for _ in range(example_count)]
for batch in util.batches(examples, batch_size, augment=False):
for segment in util.segments(examples, segment_length):
self.assertEqual(batch_size, len(batch))
for features in segment:
self.assertEqual(len(feature_shapes), len(features))
for feature, feature_shape in util.equizip(
features, feature_shapes):
self.assertLessEqual(len(feature), segment_length)
self.assertEqual(tuple(feature_shape),
feature.shape[1:])
def make_transition_graph(state,
transition,
x=None,
context=None,
temperature=1.0,
hp=None):
"""Make the graph that processes a single sequence element.
Args:
state: `_make_sequence_graph` loop state.
transition: Model transition function mapping (xchunk, model_state,
context) to (output, new_model_state).
x: Sequence of integer (categorical) inputs. Axes [time, batch].
context: Optional Tensor denoting context, shaped [batch, ?].
temperature: Softmax temperature to use for sampling.
hp: Model hyperparameters.
Returns:
Updated loop state.
"""
state = NS.Copy(state)
xchunk = _get_flat_chunk(state.xhats if x is None else x,
state.i * hp.chunk_size,
hp.chunk_size,
depth=hp.data_dim)
embedding = tfutil.layers([xchunk], sizes=hp.io_sizes, use_bn=hp.use_bn)
h, state.model = transition(embedding, state.model, context=context)
# predict the next chunk
exhats = []
with tf.variable_scope("xhat") as scope:
for j in range(hp.chunk_size):
if j > 0:
scope.reuse_variables()
xchunk = _get_flat_chunk(state.xhats if x is None else x,
state.i * hp.chunk_size + j,
hp.chunk_size,
depth=hp.data_dim)
embedding = tfutil.layers([h, xchunk],
sizes=hp.io_sizes,
use_bn=hp.use_bn)
exhat = tfutil.project(embedding, output_dim=hp.data_dim)
exhats.append(exhat)
state.xhats = state.xhats.write((state.i + 1) * hp.chunk_size + j,
tfutil.sample(exhat, temperature))
if x is not None:
targets = tf.unpack(_get_1hot_chunk(x, (state.i + 1) * hp.chunk_size,
hp.chunk_size,
depth=hp.data_dim),
num=hp.chunk_size,
axis=1)
state.losses = _put_chunk(state.losses, state.i * hp.chunk_size, [
tf.nn.softmax_cross_entropy_with_logits(exhat, target)
for exhat, target in util.equizip(exhats, targets)
])
state.errors = _put_chunk(state.errors, state.i * hp.chunk_size, [
tf.not_equal(tf.nn.top_k(exhat)[1],
tf.nn.top_k(target)[1])
for exhat, target in util.equizip(exhats, targets)
])
state.exhats = _put_chunk(state.exhats, state.i * hp.chunk_size,
exhats)
state.i += 1
return state