def get_score(self, statement, hyps, f):
in_string, in_parents, in_left, in_right, in_params, depths, \
parent_arity, leaf_position, arity = self.parse_statement_and_hyps(
statement, hyps, f)
#print in_string
to_middle = self.gru_block(self.v.forward_start,
in_string,
in_params,
hs_backward=self.v.backward_start,
parents=in_parents,
left_siblings=in_left,
right_siblings=in_right,
bidirectional=self.config.p.bidirectional,
structure_data=list(
zip(depths, parent_arity, leaf_position,
arity)),
feed_to_attention=False)
h = nn.ConcatNode(to_middle, self.g)
h = nn.DropoutNode(h, self.dropout, self.g)
h = nn.RELUDotAdd(h, self.v.main_first_W, self.v.main_first_b, self.g)
h = nn.DropoutNode(h, self.dropout, self.g)
for i in range(self.config.p.out_layers):
h = nn.RELUDotAdd(h, self.v.main_Ws[i], self.v.main_bs[i], self.g)
h = nn.DropoutNode(h, self.dropout, self.g)
h = nn.DotNode(h, self.v.last_W, self.g)
h = nn.AddNode([h, self.v.last_b], self.g)
return h
def x_to_predictions(self, x):
x = nn.DropoutNode(x, self.dropout, self.g)
for W, b in zip(self.v.out_Ws, self.v.out_bs):
x = nn.RELUDotAdd(x, W, b, self.g)
x = nn.DropoutNode(x, self.dropout, self.g)
logits = nn.RELUDotAdd(x, self.v.last_W, self.v.last_b, self.g)
#print logits.value
#print
return logits
def __init__(self, variables, config, tree, context, fit, this_ua, prop):
train = False
DefaultModel.__init__(self, config, variables, train=train)
self.g = None # don't remember all of the stuff in the graph.
# we don't know the output, so don't try to determine it
self.parse_and_augment_proof_step(tree, context, fit, this_ua, prop)
# merge the inputs together so that we can bidirection it
in_string, in_parents, in_left, in_right, in_params, depths, parent_arity, leaf_position, arity = merge_graph_structures(
[self.known_graph_structure, self.to_prove_graph_structure],
[self.v.known_gru_block, self.v.to_prove_gru_block])
# do the left side gru blocks
to_middle = self.gru_block(self.v.forward_start, in_string, in_params,
hs_backward=self.v.backward_start, parents=in_parents,
left_siblings=in_left, right_siblings=in_right,
bidirectional=self.config.p.bidirectional,
structure_data = zip(depths, parent_arity, leaf_position, arity),
feed_to_attention=self.config.p.attention)
# set up the attentional model
if self.config.p.attention:
self.set_up_attention()
# process the middle
from_middle = [nn.RELUDotAdd(x, W, b, self.g)
for x, W, b in zip(to_middle, self.v.middle_W, self.v.middle_b)]
self.initial_h = from_middle
def get_vector(self, statement, hyps, f, statement_gru, hyps_gru,
forward_start, backward_start, first_W, first_b, Ws, bs):
in_string, in_parents, in_left, in_right, in_params, depths, \
parent_arity, leaf_position, arity = self.parse_statement_and_hyps(
statement, hyps, f, statement_gru, hyps_gru)
#print in_string
to_middle = self.gru_block(forward_start, in_string, in_params,
hs_backward=backward_start, parents=in_parents,
left_siblings=in_left, right_siblings=in_right,
bidirectional=self.config.p.bidirectional,
structure_data = zip(depths, parent_arity, leaf_position, arity),
feed_to_attention=False)
h = nn.ConcatNode(to_middle, self.g)
h = nn.DropoutNode(h, self.dropout, self.g)
h = nn.RELUDotAdd(h, first_W, first_b, self.g)
h = nn.DropoutNode(h, self.dropout, self.g)
for i in range(self.config.p.out_layers):
h = nn.RELUDotAdd(h, Ws[i], bs[i], self.g)
h = nn.DropoutNode(h, self.dropout, self.g)
return h
def __init__(self,
variables,
config,
proof_step,
train=False,
target_index=None):
''' this is the model. As a single pass, it processes the
inputs, and computes the losses, and runs a training step if
train.
'''
DefaultModel.__init__(self, config, variables, train=train)
if not self.train:
np.random.seed(proof_step.context.number +
+proof_step.prop.number + proof_step.tree.size())
self.parse_and_augment_proof_step(proof_step,
target_index=target_index)
# merge the inputs together so that we can bidirection it
in_string, in_parents, in_left, in_right, in_params, depths, parent_arity, leaf_position, arity = merge_graph_structures(
[self.known_graph_structure, self.to_prove_graph_structure],
[self.v.known_gru_block, self.v.to_prove_gru_block])
# print
# print in_string
# print in_parents
# print in_left
# print in_right
# print depths
# print parent_arity
# print leaf_position
# print arity
# do the left side gru blocks
to_middle = self.gru_block(self.v.forward_start,
in_string,
in_params,
hs_backward=self.v.backward_start,
parents=in_parents,
left_siblings=in_left,
right_siblings=in_right,
bidirectional=self.config.p.bidirectional,
structure_data=list(
zip(depths, parent_arity, leaf_position,
arity)),
feed_to_attention=self.config.p.attention)
# set up the attentional model
if self.config.p.attention:
self.set_up_attention()
# process the middle
from_middle = [
nn.RELUDotAdd(x, W, b, self.g)
for x, W, b in zip(to_middle, self.v.middle_W, self.v.middle_b)
]
# process the right side
out_string = self.out_graph_structure.string
out_parents = self.out_graph_structure.parents
out_left = self.out_graph_structure.left_sibling
arity = self.out_graph_structure.arity
leaf_position = self.out_graph_structure.leaf_position
parent_arity = self.out_graph_structure.parent_arity
depths = self.out_graph_structure.depth
structure_data = list(zip(depths, parent_arity, leaf_position, arity))
out_length = len(out_string)
out_xs = []
all_hs = []
hs = from_middle
for i in range(out_length):
# figure out the augmentation stuff.
if self.config.p.augmentation:
parent = out_parents[i]
parent_hs = [x.no_parent for x in self.v.out_gru_block.aug
] if parent == -1 else all_hs[parent]
left = out_left[i]
left_hs = [
x.no_left_sibling for x in self.v.out_gru_block.aug
] if left == -1 else all_hs[left]
else:
parent_hs = None
left_hs = None
hs, x = self.forward_vertical_slice(
hs,
parent_hs,
left_hs,
out_string[i],
self.v.out_gru_block.forward,
structure_data[i],
takes_attention=self.config.p.attention)
all_hs.append(hs)
out_xs.append(x)
# test
# calculate logits and score
self.correct_string = out_string[1:] + ['END_OF_SECTION']
#out_xs = [nn.ZerosNode([64], self.g) for token in correct_string]
self.all_correct = True
self.num_correct = 0
self.all_logits = [self.x_to_predictions(x) for x in out_xs]
self.prediction = []
all_costs = [
self.score(logits, c_token)
for logits, c_token in zip(self.all_logits, self.correct_string)
]
perplexity = nn.AddNode(all_costs, self.g)
#self.logit_matrix = np.concat([l.value for l in all_logits])
self.loss = nn.AddNode([perplexity, self.loss], self.g)
# and train
if train:
# print 'training2'
# print len(self.v.vs), len(self.g.nodes)
self.g.backprop(self.loss)
# for v in self.v.vs:
# print v.name, np.mean(v.value ** 2)
#self.v.optimizer.minimize()
# put the outputs in the standard training format
self.outputs = [
perplexity.value, self.num_correct, 1 * self.all_correct
]
self.output_counts = [out_length, out_length, 1]