def composed_embed_blk():
leaf_case = direct_embed_blk()
nonleaf_case = td.Composition(name='composed_embed_nonleaf')
with nonleaf_case.scope():
children = td.GetItem('children').reads(nonleaf_case.input)
clen = td.Scalar().reads(td.GetItem('clen').reads(nonleaf_case.input))
cclens = td.Map(td.GetItem('clen') >> td.Scalar()).reads(children)
fchildren = td.Map(direct_embed_blk()).reads(children)
initial_state = td.Composition()
with initial_state.scope():
initial_state.output.reads(
td.FromTensor(tf.zeros(hyper.word_dim)),
td.FromTensor(tf.zeros([])),
)
summed = td.Zip().reads(fchildren, cclens, td.Broadcast().reads(clen))
summed = td.Fold(continous_weighted_add_blk(),
initial_state).reads(summed)[0]
added = td.Function(tf.add, name='add_bias').reads(
summed, td.FromTensor(param.get('B')))
normed = clip_by_norm_blk().reads(added)
act_fn = tf.nn.relu if hyper.use_relu else tf.nn.tanh
relu = td.Function(act_fn).reads(normed)
nonleaf_case.output.reads(relu)
return td.OneOf(lambda node: node['clen'] == 0, {
True: leaf_case,
False: nonleaf_case
})
def l2loss_blk():
# rewrite using metric
leaf_case = td.Composition()
with leaf_case.scope():
leaf_case.output.reads(td.FromTensor(tf.constant(1.)))
nonleaf_case = td.Composition()
with nonleaf_case.scope():
direct = direct_embed_blk().reads(nonleaf_case.input)
com = composed_embed_blk().reads(nonleaf_case.input)
loss = td.Function(batch_nn_l2loss).reads(direct, com)
nonleaf_case.output.reads(loss)
return td.OneOf(lambda node: node['clen'] != 0, {
False: leaf_case,
True: nonleaf_case
})
def build_train_graph_for_RVAE(rvae_block, look_behind_length=0):
token_emb_size = get_size_of_input_vecotrs(rvae_block)
c = td.Composition()
with c.scope():
padded_input_sequence = td.Map(td.Vector(token_emb_size)).reads(
c.input)
network_output = rvae_block
network_output.reads(padded_input_sequence)
un_normalised_token_probs = td.GetItem(0).reads(network_output)
mus_and_log_sigs = td.GetItem(1).reads(network_output)
input_sequence = td.Slice(
start=look_behind_length).reads(padded_input_sequence)
# TODO: metric that output of rnn is the same as input sequence
cross_entropy_loss = td.ZipWith(
td.Function(softmax_crossentropy)) >> td.Mean()
cross_entropy_loss.reads(un_normalised_token_probs, input_sequence)
kl_loss = td.Function(kl_divergence)
kl_loss.reads(mus_and_log_sigs)
td.Metric('cross_entropy_loss').reads(cross_entropy_loss)
td.Metric('kl_loss').reads(kl_loss)
c.output.reads(td.Void())
return c
def build_decoder_block_for_analysis(z_size, token_emb_size, decoder_cell,
input_size):
c = td.Composition()
c.set_input_type(
td.TupleType(td.TensorType((z_size, )),
td.SequenceType(td.TensorType((input_size, )))))
with c.scope():
hidden_state = td.GetItem(0).reads(c.input)
rnn_input = td.GetItem(1).reads(c.input)
# decoder_output = build_program_decoder_for_analysis(
# token_emb_size, default_gru_cell(z_size)
# )
decoder_output = decoder_cell
decoder_output.reads(rnn_input, hidden_state)
decoder_rnn_output = td.GetItem(1).reads(decoder_output)
un_normalised_token_probs = td.GetItem(0).reads(decoder_output)
# get the first output (meant to only compute one interation)
c.output.reads(
td.GetItem(0).reads(un_normalised_token_probs),
td.GetItem(0).reads(decoder_rnn_output))
return td.Record((td.Vector(z_size), td.Map(td.Vector(input_size)))) >> c
def add_metrics(is_root, is_neutral):
"""A block that adds metrics for loss and hits; output is the LSTM state."""
c = td.Composition(name='predict(is_root=%s, is_neutral=%s)' %
(is_root, is_neutral))
with c.scope():
# destructure the input; (labels, (logits, state))
labels = c.input[0]
logits = td.GetItem(0).reads(c.input[1])
state = td.GetItem(1).reads(c.input[1])
# calculate loss
loss = td.Function(tf_node_loss)
td.Metric('all_loss').reads(loss.reads(logits, labels))
if is_root: td.Metric('root_loss').reads(loss)
# calculate fine-grained hits
hits = td.Function(tf_fine_grained_hits)
td.Metric('all_hits').reads(hits.reads(logits, labels))
if is_root: td.Metric('root_hits').reads(hits)
# calculate binary hits, if the label is not neutral
if not is_neutral:
binary_hits = td.Function(tf_binary_hits).reads(logits, labels)
td.Metric('all_binary_hits').reads(binary_hits)
if is_root: td.Metric('root_binary_hits').reads(binary_hits)
# output the state, which will be read by our by parent's LSTM cell
c.output.reads(state)
return c
def add_metrics(is_root):
c = td.Composition(name='predict(is_root=%s)' % (is_root))
with c.scope():
labels = c.input[0]
logits = td.GetItem(0).reads(c.input[1])
state = td.GetItem(1).reads(c.input[1])
loss = td.Function(tf_node_loss)
td.Metric('all_loss').reads(loss.reads(logits, labels))
if is_root:
td.Metric('root_loss').reads(loss)
result_logits = td.Function(tf_logits)
td.Metric('all_logits').reads(result_logits.reads(logits))
if is_root:
td.Metric('root_logits').reads(result_logits)
# reserve pred and labels
pred = td.Function(tf_pred)
td.Metric('all_pred').reads(pred.reads(logits))
if is_root:
td.Metric('root_pred').reads(pred)
answer = td.Function(tf_label)
td.Metric('all_labels').reads(answer.reads(labels))
if is_root:
td.Metric('root_label').reads(answer)
c.output.reads(state)
return c
def feature_detector_blk(max_depth=2):
"""Input: node dict
Output: TensorType([hyper.conv_dim, ])
Single patch of the conv. Depth is max_depth
"""
blk = td.Composition()
with blk.scope():
nodes_in_patch = collect_node_for_conv_patch_blk(
max_depth=max_depth).reads(blk.input)
# map from python object to tensors
mapped = td.Map(
td.Record((coding_blk(), td.Scalar(), td.Scalar(), td.Scalar(),
td.Scalar()))).reads(nodes_in_patch)
# mapped = [(feature, idx, depth, max_depth), (...)]
# compute weighted feature for each elem
weighted = td.Map(weighted_feature_blk()).reads(mapped)
# weighted = [fea, fea, fea, ...]
# add together
added = td.Reduce(td.Function(tf.add)).reads(weighted)
# added = TensorType([hyper.conv_dim, ])
# add bias
biased = td.Function(tf.add).reads(added,
td.FromTensor(param.get('Bconv')))
# biased = TensorType([hyper.conv_dim, ])
# tanh
tanh = td.Function(tf.nn.tanh).reads(biased)
# tanh = TensorType([hyper.conv_dim, ])
blk.output.reads(tanh)
return blk
def tree_sum_blk(loss_blk):
# traverse the tree to sum up the loss
tree_sum_fwd = td.ForwardDeclaration(td.PyObjectType(), td.TensorType([]))
tree_sum = td.Composition()
with tree_sum.scope():
myloss = loss_blk().reads(tree_sum.input)
children = td.GetItem('children').reads(tree_sum.input)
mapped = td.Map(tree_sum_fwd()).reads(children)
summed = td.Reduce(td.Function(tf.add)).reads(mapped)
summed = td.Function(tf.add).reads(summed, myloss)
tree_sum.output.reads(summed)
tree_sum_fwd.resolve_to(tree_sum)
return tree_sum
def weighted_feature_blk():
"""Input: (feature , idx , pclen, depth, max_depth)
(TensorType([hyper.word_dim, ]), Scalar, Scalar, Scalar, Scalar)
Output: weighted_feature
TensorType([hyper.conv_dim, ])
"""
blk = td.Composition()
with blk.scope():
fea = blk.input[0]
Wi = tri_combined_blk().reads(blk.input[1], blk.input[2], blk.input[3],
blk.input[4])
weighted_fea = td.Function(embedding.batch_mul).reads(fea, Wi)
blk.output.reads(weighted_fea)
return blk
def set_metrics(self, train=True):
"""A block that adds metrics for loss and hits;
output is the LSTM state."""
c = td.Composition(
name='predict')
with c.scope():
# destructure the input; (labels, logits)
labels = c.input[0]
logits = c.input[1]
# calculate loss
loss = td.Function(self.tf_node_loss)
td.Metric('root_loss').reads(loss.reads(logits, labels))
hits = td.Function(self.tf_fine_grained_hits)
td.Metric('root_hits').reads(hits.reads(logits, labels))
c.output.reads(logits)
return c
def build_token_level_RVAE(z_size, token_emb_size, look_behind_length):
c = td.Composition()
c.set_input_type(
td.SequenceType(td.TensorType(([token_emb_size]), 'float32')))
with c.scope():
padded_input_sequence = c.input
# build encoder block
encoder_rnn_cell = build_program_encoder(default_gru_cell(2 * z_size))
output_sequence = td.RNN(encoder_rnn_cell) >> td.GetItem(0)
mus_and_log_sigs = output_sequence >> td.GetItem(-1)
reparam_z = resampling_block(z_size)
if look_behind_length > 0:
decoder_input_sequence = (
td.Slice(stop=-1) >> td.NGrams(look_behind_length) >> td.Map(
td.Concat()))
else:
decoder_input_sequence = td.Map(
td.Void() >> td.FromTensor(tf.zeros((0, ))))
# build decoder block
un_normalised_token_probs = build_program_decoder(
token_emb_size, default_gru_cell(z_size), just_tokens=True)
# remove padding for input sequence
input_sequence = td.Slice(start=look_behind_length)
input_sequence.reads(padded_input_sequence)
mus_and_log_sigs.reads(input_sequence)
reparam_z.reads(mus_and_log_sigs)
decoder_input_sequence.reads(padded_input_sequence)
td.Metric('encoder_sequence_length').reads(
td.Length().reads(input_sequence))
td.Metric('decoder_sequence_length').reads(
td.Length().reads(decoder_input_sequence))
un_normalised_token_probs.reads(decoder_input_sequence, reparam_z)
c.output.reads(un_normalised_token_probs, mus_and_log_sigs)
return c
def coding_blk():
"""Input: node dict
Output: TensorType([1, hyper.word_dim])
"""
Wcomb1 = param.get('Wcomb1')
Wcomb2 = param.get('Wcomb2')
blk = td.Composition()
with blk.scope():
direct = embedding.direct_embed_blk().reads(blk.input)
composed = embedding.composed_embed_blk().reads(blk.input)
Wcomb1 = td.FromTensor(param.get('Wcomb1'))
Wcomb2 = td.FromTensor(param.get('Wcomb2'))
direct = td.Function(embedding.batch_mul).reads(direct, Wcomb1)
composed = td.Function(embedding.batch_mul).reads(composed, Wcomb2)
added = td.Function(tf.add).reads(direct, composed)
blk.output.reads(added)
return blk
def bidirectional_dynamic_FC(fw_cell, bw_cell, hidden):
bidir_conv_lstm = td.Composition()
with bidir_conv_lstm.scope():
fw_seq = td.Identity().reads(bidir_conv_lstm.input[0])
labels = (
td.GetItem(1) >> td.Map(td.Metric("labels")) >> td.Void()).reads(
bidir_conv_lstm.input)
bw_seq = td.Slice(step=-1).reads(fw_seq)
forward_dir = (td.RNN(fw_cell) >> td.GetItem(0)).reads(fw_seq)
back_dir = (td.RNN(bw_cell) >> td.GetItem(0)).reads(bw_seq)
back_to_leftright = td.Slice(step=-1).reads(back_dir)
output_transform = td.FC(1, activation=None)
bidir_common = (td.ZipWith(
td.Concat() >> output_transform >> td.Metric('logits'))).reads(
forward_dir, back_to_leftright)
bidir_conv_lstm.output.reads(bidir_common)
return bidir_conv_lstm
def dynamic_pooling_blk():
"""Input: root node dic
Output: pooled, TensorType([hyper.conv_dim, ])
"""
leaf_case = feature_detector_blk()
pool_fwd = td.ForwardDeclaration(td.PyObjectType(),
td.TensorType([
hyper.conv_dim,
]))
pool = td.Composition()
with pool.scope():
cur_fea = feature_detector_blk().reads(pool.input)
children = td.GetItem('children').reads(pool.input)
mapped = td.Map(pool_fwd()).reads(children)
summed = td.Reduce(td.Function(tf.maximum)).reads(mapped)
summed = td.Function(tf.maximum).reads(summed, cur_fea)
pool.output.reads(summed)
pool = td.OneOf(lambda x: x['clen'] == 0, {True: leaf_case, False: pool})
pool_fwd.resolve_to(pool)
return pool
def continous_weighted_add_blk():
block = td.Composition(name='continous_weighted_add')
with block.scope():
initial = td.GetItem(0).reads(block.input)
cur = td.GetItem(1).reads(block.input)
last = td.GetItem(0).reads(initial)
idx = td.GetItem(1).reads(initial)
cur_fea = td.GetItem(0).reads(cur)
cur_clen = td.GetItem(1).reads(cur)
pclen = td.GetItem(2).reads(cur)
Wi = linear_combine_blk().reads(cur_clen, pclen, idx)
weighted_fea = td.Function(batch_mul).reads(cur_fea, Wi)
block.output.reads(
td.Function(tf.add, name='add_last_weighted_fea').reads(
last, weighted_fea),
# XXX: rewrite using tf.range
td.Function(tf.add, name='add_idx_1').reads(
idx, td.FromTensor(tf.constant(1.))))
return block
def kl_divergence(mus_and_log_sigs):
halfway = int(mus_and_log_sigs.get_shape()[1].value / 2) # HACK: make this cleaner
mus = mus_and_log_sigs[:, :halfway]
log_sigs = mus_and_log_sigs[:, halfway:]
kl_loss_term = -0.5 * tf.reduce_mean(
1 + log_sigs - tf.square(mus) - tf.exp(log_sigs),
axis=1
)
return kl_loss_term
c = td.Composition()
with c.scope():
input_sequence = td.Map(td.Vector(54)).reads(c.input)
# net = build_VAE(Z_SIZE, 54)
# un_normalised_token_probs, mus_and_log_sigs = input_sequence >> build_VAE(Z_SIZE, 54)
network_output = build_VAE(Z_SIZE, 54)
network_output.reads(input_sequence)
un_normalised_token_probs = td.GetItem(0).reads(network_output)
mus_and_log_sigs = td.GetItem(1).reads(network_output)
cross_entropy_loss = td.ZipWith(td.Function(softmax_crossentropy)) >> td.Mean()
cross_entropy_loss.reads(
un_normalised_token_probs,
def build_VAE(z_size, token_emb_size):
c = td.Composition()
c.set_input_type(td.SequenceType(td.TensorType(([token_emb_size]), 'float32')))
with c.scope():
# input_sequence = td.Map(td.Vector(token_emb_size)).reads(c.input)
input_sequence = c.input
# encoder composition TODO: refactor this out
# rnn_cell = td.ScopedLayer(
# tf.contrib.rnn.LSTMCell(
# num_units=2*z_size,
# initializer=tf.contrib.layers.xavier_initializer(),
# activation=tf.tanh
# ),
# 'encoder'
# )
encoder_rnn_cell = td.ScopedLayer(
tf.contrib.rnn.GRUCell(
num_units=2*z_size,
# initializer=tf.contrib.layers.xavier_initializer(),
activation=tf.tanh
),
'encoder'
)
output_sequence = td.RNN(encoder_rnn_cell) >> td.GetItem(0)
mus_and_log_sigs = output_sequence >> td.GetItem(-1)
# reparam_z = mus_and_log_sigs >> td.Function(resampling)
reparam_z = td.Function(resampling, name='resampling')
reparam_z.set_input_type(td.TensorType((2 * z_size,)))
reparam_z.set_output_type(td.TensorType((z_size,)))
# A list of same length of input_sequence, but with empty values
# this is used for the decoder to map over
list_of_nothing = td.Map(
td.Void() >> td.FromTensor(tf.zeros((0,)))
)
# decoder composition
# TODO: refactor this out
# decoder_rnn = td.ScopedLayer(
# tf.contrib.rnn.LSTMCell(
# num_units=z_size,
# initializer=tf.contrib.layers.xavier_initializer(),
# activation=tf.tanh
# ),
# 'decoder'
# )
decoder_rnn = td.ScopedLayer(
tf.contrib.rnn.GRUCell(
num_units=z_size,
# initializer=tf.contrib.layers.xavier_initializer(),
activation=tf.tanh
),
'decoder'
)
decoder_rnn_output = td.RNN(
decoder_rnn,
initial_state_from_input=True
) >> td.GetItem(0)
fc_layer = td.FC(
token_emb_size,
activation=tf.nn.relu,
initializer=tf.contrib.layers.xavier_initializer()
)
un_normalised_token_probs = decoder_rnn_output >> td.Map(fc_layer)
# reparam_z.reads(input_sequence)
mus_and_log_sigs.reads(input_sequence)
reparam_z.reads(mus_and_log_sigs)
list_of_nothing.reads(input_sequence)
un_normalised_token_probs.reads(list_of_nothing, reparam_z)
c.output.reads(un_normalised_token_probs, mus_and_log_sigs)
return c
