def l2_normalize(vector):
square_sum = dy.sqrt(
dy.bmax(
dy.sum_elems(dy.square(vector)),
np.finfo(float).eps * dy.ones((1))[0],
))
return dy.cdiv(vector, square_sum)
def do_one_sequence(rnn, params, sequence):
# setup the sequence
dy.renew_cg()
s0 = rnn.initial_state()
R = params["R"]
bias = params["bias"]
lookup = params["lookup"]
input_sequence = [input_token2int[t] for (t, _) in sequence]
output_sequence = [output_token2int[t] for (_, t) in sequence]
s = s0
loss = []
for input_token, output_token in zip(input_sequence, output_sequence):
s = s.add_input(lookup[input_token])
probs = dy.softmax(R * s.output() + bias)
# MinMax to avoid undefined 0/1 probabilities
# 1e-15 is arbitrary
min_val = dy.constant((OUTPUT_VOCAB_SIZE, ), 1e-15)
max_val = dy.constant((OUTPUT_VOCAB_SIZE, ), 1 - 1e-15)
probs = dy.bmax(dy.bmin(probs, max_val), min_val)
loss.append(-dy.log(dy.pick(probs, output_token)))
loss = dy.esum(loss)
return loss
def leaky_relu(x): return dy.bmax(.1*x, x)
def leaky_relu(x, a):
return dy.bmax(a*x, x)
def leaky_ReLu(self, inputvec, alpha=0.1):
return dy.bmax(alpha * inputvec, inputvec)
def l2_normalize(x):
square_sum = dynet.sqrt(dynet.bmax(dynet.sum_elems(dynet.square(x)), np.finfo(float).eps * dynet.ones((1))[0]))
return dynet.cdiv(x, square_sum)
def l2_normalize(x):
epsilon = np.finfo(float).eps * dy.ones(pred.dim()[0])
norm = dy.sqrt(dy.sum_elems(dy.square(x)))
sign = dy.cdiv(x, dy.bmax(dy.abs(x), epsilon))
return dy.cdiv(dy.cmult(sign, dy.bmax(dy.abs(x), epsilon)), dy.bmax(norm, epsilon[0]))
def leaky_relu(x):
"""Leaky ReLU implementation."""
return dy.bmax(.1*x, x)
def penalized_tanh(x):
alpha = 0.25
tanh_x = dy.tanh(x)
return dy.bmax(tanh_x, alpha * tanh_x)
def get_next_score_expressions(self, legit_act_idxs):
res_list = []
meta_list = []
'''
print ("debug: self.action_history", self.action_history)
print ("debug: self.is_end()", self.is_end())
print ("debug: self.qinfo.seq_qid", self.qinfo.seq_qid)
print ("debug: legit_act_idxs", legit_act_idxs)
'''
qwVecs = self.attend_question_coverage()
qwAvgVec = self.ques_avg_emb
qLSTMVec = dt.tanh(self.H * dt.concatenate(
[self.fw[-1], self.bw[0]])) # question words LSTM embedding
if self.resinfo != None:
prev_qwVecs = self.prev_ques_emb
prev_qwAvgVec = self.prev_ques_avg_emb
prev_qLSTMVec = dt.tanh(
self.prev_H *
dt.concatenate([self.prev_fw[-1], self.prev_bw[0]]))
for act_idx in legit_act_idxs:
action = self.af.actions[act_idx]
act_type = action.type
#print("act_type", act_type)
col = action.col
colnameVec = self.headers_embs[col]
colWdVecs = self.colname_embs[col]
r = action.row
if self.action_history != []:
# for condition check, assuming the last action of the current state is ActWhereCol
c = self.af.actions[self.action_history[-1]].col
condCellVec = self.entries_embs[r][c]
if act_type == ActionType.Stop:
# use the average after mask
actScore = dt.dot_product(dt.average(qwVecs), self.NulW)
coverageMap = dt.inputVector(np.zeros(len(qwVecs)))
elif act_type == ActionType.Select:
lstScores = self.QCMatch.score_expression(
qwVecs, qwAvgVec, qLSTMVec, colnameVec, colWdVecs)
scoreVec = dt.concatenate(lstScores)
actScore = self.SelColFF.score_expression(scoreVec)
coverageMap = self.determine_coverage_by_name(
qwVecs, colnameVec)
elif act_type == ActionType.WhereCol: # same as ActionType.ActSelect, but with different coefficients in weighted sum
# column type embedding # TODO: MAY BE WRONG IMPLEMENTATION HERE
if self.qinfo.values_in_ques:
colTypeScore = self.ColTypeN
else:
colTypeScore = self.ColTypeW
lstScores = self.QCMatch.score_expression(
qwVecs, qwAvgVec, qLSTMVec, colnameVec, colWdVecs)
scoreVec = dt.concatenate(lstScores + [colTypeScore])
actScore = self.WhereColFF.score_expression(scoreVec)
coverageMap = self.determine_coverage_by_name(
qwVecs, colnameVec)
elif act_type == ActionType.CondEqRow:
actScore = nnmod.MaxScore(qwVecs, condCellVec)
coverageMap = self.determine_coverage_by_name(
qwVecs, condCellVec)
elif act_type == ActionType.CondNeRow:
entScore = nnmod.MaxScore(qwVecs, condCellVec)
negScore = self.Negate.score_expression(qwAvgVec)
scoreVec = dt.concatenate([entScore, negScore])
actScore = self.NegFF.score_expression(scoreVec)
coverageMap = self.determine_coverage_by_name(
qwVecs, condCellVec)
elif act_type == ActionType.CondGT or act_type == ActionType.FpCondGT:
actScore = self.CondGT.score_expression(qwVecs, action.val[0])
coverageMap = self.determine_coverage_by_name(
qwVecs, self.CondGT.OpW)
elif act_type == ActionType.CondGE or act_type == ActionType.FpCondGE:
actScore = self.CondGE.score_expression(qwVecs, action.val[0])
coverageMap = self.determine_coverage_by_name(
qwVecs, self.CondGE.OpW)
elif act_type == ActionType.CondLT or act_type == ActionType.FpCondLT:
actScore = self.CondLT.score_expression(qwVecs, action.val[0])
coverageMap = self.determine_coverage_by_name(
qwVecs, self.CondLT.OpW)
elif act_type == ActionType.CondLE or act_type == ActionType.FpCondLE:
actScore = self.CondLE.score_expression(qwVecs, action.val[0])
coverageMap = self.determine_coverage_by_name(
qwVecs, self.CondLE.OpW)
elif act_type == ActionType.ArgMin or act_type == ActionType.FpArgMin:
actScore = self.ArgMin.score_expression(qwVecs)
coverageMap = self.determine_coverage_by_name(
qwVecs, self.ArgMin.OpW)
elif act_type == ActionType.ArgMax or act_type == ActionType.FpArgMax:
actScore = self.ArgMax.score_expression(qwVecs)
coverageMap = self.determine_coverage_by_name(
qwVecs, self.ArgMax.OpW)
elif act_type == ActionType.FpWhereCol: # similar to ActionType.WhereCol
# column type embedding
if self.qinfo.values_in_ques:
colTypeScore = self.ColTypeN
else:
colTypeScore = self.ColTypeW
lstScores = self.QCMatch.score_expression(
qwVecs, qwAvgVec, qLSTMVec, colnameVec, colWdVecs)
lstPrevScores = self.QCMatch.score_expression(
prev_qwVecs, prev_qwAvgVec, prev_qLSTMVec, colnameVec,
colWdVecs)
scoreVec = dt.concatenate(lstScores + [colTypeScore] +
lstPrevScores)
actScore = self.FpWhereColFF.score_expression(scoreVec)
coverageMap = self.determine_coverage_by_name(
qwVecs, colnameVec)
elif act_type == ActionType.FpCondEqRow:
entScore = nnmod.MaxScore(qwVecs, condCellVec)
prev_entScore = nnmod.MaxScore(prev_qwVecs, condCellVec)
actScore = dt.bmax(entScore, prev_entScore)
coverageMap = self.determine_coverage_by_name(
qwVecs, condCellVec)
elif act_type == ActionType.FpCondNeRow:
entScore = nnmod.MaxScore(qwVecs, condCellVec)
prev_entScore = nnmod.MaxScore(prev_qwVecs, condCellVec)
negScore = self.Negate.score_expression(qwAvgVec)
scoreVec = dt.concatenate(
[dt.bmax(entScore, prev_entScore), negScore])
actScore = self.NegFF.score_expression(scoreVec)
coverageMap = self.determine_coverage_by_name(
qwVecs, condCellVec)
elif act_type == ActionType.SameAsPrevious:
quesLSTMScore = dt.dot_product(prev_qLSTMVec, qLSTMVec)
quesAvgScore = dt.dot_product(prev_qwAvgVec, qwAvgVec)
actScore = dt.dot_product(
self.SameAsPreviousW,
dt.concatenate([quesLSTMScore, quesAvgScore]))
coverageMap = dt.inputVector(np.zeros(len(qwVecs)))
else:
assert False, "Error! Unknown act_type: %d" % act_type
res_list.append(actScore)
meta_list.append(coverageMap)
return dt.concatenate(res_list), meta_list
def __init__(self):
# first initialize a computation graph container (or model).
self.nnmodel = dynet.Model()
# assign the algorithm for backpropagation updates.
self.updater = dynet.AdamTrainer(self.nnmodel)
num_words, num_tags, num_labels = 4808, 46, 46
word_embed_dim, pos_embed_dim, label_embed_dim = 100, 32, 32
hidden_layer1_dim, hidden_layer2_dim = 600, 600
num_actions = 93
self.minibatch_size = 1000
# create embeddings for words and tag features.
self.word_embedding = self.nnmodel.add_lookup_parameters(
(num_words, word_embed_dim))
glove_word_embeddings_dict = {
} # key is the word, value is the list of 100 embeddings
embed_lines = open("glove.6B.100d.txt", 'r').read().splitlines()
for line in embed_lines:
word = line.split()[0]
values = line.split()
del values[0]
glove_word_embeddings_dict[word] = values
vocab_words = open("./data/vocabs.word", 'r').read().splitlines()
i = 0
for word_line in vocab_words:
word = word_line.split()[0]
if (word in glove_word_embeddings_dict):
self.word_embedding[i] = np.asarray(
glove_word_embeddings_dict[word])
self.pos_embedding = self.nnmodel.add_lookup_parameters(
(num_tags, pos_embed_dim))
self.label_embedding = self.nnmodel.add_lookup_parameters(
(num_labels, label_embed_dim))
# mbda x: dynet.bmax(.1 * x, x))assign transfer function
self.transfer = (lambda x: dynet.bmax(.1 * x, x))
self.input_dim = 20 * (word_embed_dim +
pos_embed_dim) + 12 * label_embed_dim
self.hidden_layer1 = self.nnmodel.add_parameters(
(hidden_layer1_dim, self.input_dim))
self.hidden_layer1_bias = self.nnmodel.add_parameters(
hidden_layer1_dim, init=dynet.ConstInitializer(0.2))
self.hidden_layer2 = self.nnmodel.add_parameters(
(hidden_layer2_dim, hidden_layer1_dim))
self.hidden_layer2_bias = self.nnmodel.add_parameters(
hidden_layer2_dim, init=dynet.ConstInitializer(0.2))
# define the output weight.
self.output_layer = self.nnmodel.add_parameters(
(num_actions, hidden_layer2_dim))
# define the bias vector and initialize it as zero.
self.output_bias = self.nnmodel.add_parameters(
num_actions, init=dynet.ConstInitializer(0))
self.dropout_prob = 0.2
'''
You can add more arguments for examples actions and model paths.
You need to load your model here.
actions: provides indices for actions.
it has the same order as the data/vocabs.actions file.
'''
# if you prefer to have your own index for actions, change this.
self.actions = [
'SHIFT', 'LEFT-ARC:prep', 'LEFT-ARC:dobj', 'LEFT-ARC:poss',
'LEFT-ARC:amod', 'LEFT-ARC:xcomp', 'LEFT-ARC:mark',
'LEFT-ARC:conj', 'LEFT-ARC:nn', 'LEFT-ARC:rcmod', 'LEFT-ARC:advcl',
'LEFT-ARC:cc', 'LEFT-ARC:pcomp', 'LEFT-ARC:expl', 'LEFT-ARC:tmod',
'LEFT-ARC:csubj', 'LEFT-ARC:number', 'LEFT-ARC:iobj',
'LEFT-ARC:<null>', 'LEFT-ARC:preconj', 'LEFT-ARC:nsubj',
'LEFT-ARC:appos', 'LEFT-ARC:infmod', 'LEFT-ARC:partmod',
'LEFT-ARC:ccomp', 'LEFT-ARC:aux', 'LEFT-ARC:auxpass',
'LEFT-ARC:parataxis', 'LEFT-ARC:det', 'LEFT-ARC:punct',
'LEFT-ARC:discourse', 'LEFT-ARC:dep', 'LEFT-ARC:cop',
'LEFT-ARC:pobj', 'LEFT-ARC:num', 'LEFT-ARC:prt',
'LEFT-ARC:possessive', 'LEFT-ARC:rroot', 'LEFT-ARC:npadvmod',
'LEFT-ARC:mwe', 'LEFT-ARC:neg', 'LEFT-ARC:predet',
'LEFT-ARC:nsubjpass', 'LEFT-ARC:quantmod', 'LEFT-ARC:root',
'LEFT-ARC:acomp', 'LEFT-ARC:advmod', 'RIGHT-ARC:prep',
'RIGHT-ARC:dobj', 'RIGHT-ARC:poss', 'RIGHT-ARC:amod',
'RIGHT-ARC:xcomp', 'RIGHT-ARC:mark', 'RIGHT-ARC:conj',
'RIGHT-ARC:nn', 'RIGHT-ARC:rcmod', 'RIGHT-ARC:advcl',
'RIGHT-ARC:cc', 'RIGHT-ARC:pcomp', 'RIGHT-ARC:expl',
'RIGHT-ARC:tmod', 'RIGHT-ARC:csubj', 'RIGHT-ARC:number',
'RIGHT-ARC:iobj', 'RIGHT-ARC:<null>', 'RIGHT-ARC:preconj',
'RIGHT-ARC:nsubj', 'RIGHT-ARC:appos', 'RIGHT-ARC:infmod',
'RIGHT-ARC:partmod', 'RIGHT-ARC:ccomp', 'RIGHT-ARC:aux',
'RIGHT-ARC:auxpass', 'RIGHT-ARC:parataxis', 'RIGHT-ARC:det',
'RIGHT-ARC:punct', 'RIGHT-ARC:discourse', 'RIGHT-ARC:dep',
'RIGHT-ARC:cop', 'RIGHT-ARC:pobj', 'RIGHT-ARC:num',
'RIGHT-ARC:prt', 'RIGHT-ARC:possessive', 'RIGHT-ARC:rroot',
'RIGHT-ARC:npadvmod', 'RIGHT-ARC:mwe', 'RIGHT-ARC:neg',
'RIGHT-ARC:predet', 'RIGHT-ARC:nsubjpass', 'RIGHT-ARC:quantmod',
'RIGHT-ARC:root', 'RIGHT-ARC:acomp', 'RIGHT-ARC:advmod'
]
