def l2_normalize(vector): square_sum = dy.sqrt( dy.bmax( dy.sum_elems(dy.square(vector)), np.finfo(float).eps * dy.ones((1))[0], )) return dy.cdiv(vector, square_sum)
def do_one_sequence(rnn, params, sequence): # setup the sequence dy.renew_cg() s0 = rnn.initial_state() R = params["R"] bias = params["bias"] lookup = params["lookup"] input_sequence = [input_token2int[t] for (t, _) in sequence] output_sequence = [output_token2int[t] for (_, t) in sequence] s = s0 loss = [] for input_token, output_token in zip(input_sequence, output_sequence): s = s.add_input(lookup[input_token]) probs = dy.softmax(R * s.output() + bias) # MinMax to avoid undefined 0/1 probabilities # 1e-15 is arbitrary min_val = dy.constant((OUTPUT_VOCAB_SIZE, ), 1e-15) max_val = dy.constant((OUTPUT_VOCAB_SIZE, ), 1 - 1e-15) probs = dy.bmax(dy.bmin(probs, max_val), min_val) loss.append(-dy.log(dy.pick(probs, output_token))) loss = dy.esum(loss) return loss
def leaky_relu(x): return dy.bmax(.1*x, x)
def leaky_relu(x, a): return dy.bmax(a*x, x)
def leaky_ReLu(self, inputvec, alpha=0.1): return dy.bmax(alpha * inputvec, inputvec)
def l2_normalize(x): square_sum = dynet.sqrt(dynet.bmax(dynet.sum_elems(dynet.square(x)), np.finfo(float).eps * dynet.ones((1))[0])) return dynet.cdiv(x, square_sum)
def l2_normalize(x): epsilon = np.finfo(float).eps * dy.ones(pred.dim()[0]) norm = dy.sqrt(dy.sum_elems(dy.square(x))) sign = dy.cdiv(x, dy.bmax(dy.abs(x), epsilon)) return dy.cdiv(dy.cmult(sign, dy.bmax(dy.abs(x), epsilon)), dy.bmax(norm, epsilon[0]))
def leaky_relu(x): """Leaky ReLU implementation.""" return dy.bmax(.1*x, x)
def penalized_tanh(x): alpha = 0.25 tanh_x = dy.tanh(x) return dy.bmax(tanh_x, alpha * tanh_x)
def get_next_score_expressions(self, legit_act_idxs): res_list = [] meta_list = [] ''' print ("debug: self.action_history", self.action_history) print ("debug: self.is_end()", self.is_end()) print ("debug: self.qinfo.seq_qid", self.qinfo.seq_qid) print ("debug: legit_act_idxs", legit_act_idxs) ''' qwVecs = self.attend_question_coverage() qwAvgVec = self.ques_avg_emb qLSTMVec = dt.tanh(self.H * dt.concatenate( [self.fw[-1], self.bw[0]])) # question words LSTM embedding if self.resinfo != None: prev_qwVecs = self.prev_ques_emb prev_qwAvgVec = self.prev_ques_avg_emb prev_qLSTMVec = dt.tanh( self.prev_H * dt.concatenate([self.prev_fw[-1], self.prev_bw[0]])) for act_idx in legit_act_idxs: action = self.af.actions[act_idx] act_type = action.type #print("act_type", act_type) col = action.col colnameVec = self.headers_embs[col] colWdVecs = self.colname_embs[col] r = action.row if self.action_history != []: # for condition check, assuming the last action of the current state is ActWhereCol c = self.af.actions[self.action_history[-1]].col condCellVec = self.entries_embs[r][c] if act_type == ActionType.Stop: # use the average after mask actScore = dt.dot_product(dt.average(qwVecs), self.NulW) coverageMap = dt.inputVector(np.zeros(len(qwVecs))) elif act_type == ActionType.Select: lstScores = self.QCMatch.score_expression( qwVecs, qwAvgVec, qLSTMVec, colnameVec, colWdVecs) scoreVec = dt.concatenate(lstScores) actScore = self.SelColFF.score_expression(scoreVec) coverageMap = self.determine_coverage_by_name( qwVecs, colnameVec) elif act_type == ActionType.WhereCol: # same as ActionType.ActSelect, but with different coefficients in weighted sum # column type embedding # TODO: MAY BE WRONG IMPLEMENTATION HERE if self.qinfo.values_in_ques: colTypeScore = self.ColTypeN else: colTypeScore = self.ColTypeW lstScores = self.QCMatch.score_expression( qwVecs, qwAvgVec, qLSTMVec, colnameVec, colWdVecs) scoreVec = dt.concatenate(lstScores + [colTypeScore]) actScore = self.WhereColFF.score_expression(scoreVec) coverageMap = self.determine_coverage_by_name( qwVecs, colnameVec) elif act_type == ActionType.CondEqRow: actScore = nnmod.MaxScore(qwVecs, condCellVec) coverageMap = self.determine_coverage_by_name( qwVecs, condCellVec) elif act_type == ActionType.CondNeRow: entScore = nnmod.MaxScore(qwVecs, condCellVec) negScore = self.Negate.score_expression(qwAvgVec) scoreVec = dt.concatenate([entScore, negScore]) actScore = self.NegFF.score_expression(scoreVec) coverageMap = self.determine_coverage_by_name( qwVecs, condCellVec) elif act_type == ActionType.CondGT or act_type == ActionType.FpCondGT: actScore = self.CondGT.score_expression(qwVecs, action.val[0]) coverageMap = self.determine_coverage_by_name( qwVecs, self.CondGT.OpW) elif act_type == ActionType.CondGE or act_type == ActionType.FpCondGE: actScore = self.CondGE.score_expression(qwVecs, action.val[0]) coverageMap = self.determine_coverage_by_name( qwVecs, self.CondGE.OpW) elif act_type == ActionType.CondLT or act_type == ActionType.FpCondLT: actScore = self.CondLT.score_expression(qwVecs, action.val[0]) coverageMap = self.determine_coverage_by_name( qwVecs, self.CondLT.OpW) elif act_type == ActionType.CondLE or act_type == ActionType.FpCondLE: actScore = self.CondLE.score_expression(qwVecs, action.val[0]) coverageMap = self.determine_coverage_by_name( qwVecs, self.CondLE.OpW) elif act_type == ActionType.ArgMin or act_type == ActionType.FpArgMin: actScore = self.ArgMin.score_expression(qwVecs) coverageMap = self.determine_coverage_by_name( qwVecs, self.ArgMin.OpW) elif act_type == ActionType.ArgMax or act_type == ActionType.FpArgMax: actScore = self.ArgMax.score_expression(qwVecs) coverageMap = self.determine_coverage_by_name( qwVecs, self.ArgMax.OpW) elif act_type == ActionType.FpWhereCol: # similar to ActionType.WhereCol # column type embedding if self.qinfo.values_in_ques: colTypeScore = self.ColTypeN else: colTypeScore = self.ColTypeW lstScores = self.QCMatch.score_expression( qwVecs, qwAvgVec, qLSTMVec, colnameVec, colWdVecs) lstPrevScores = self.QCMatch.score_expression( prev_qwVecs, prev_qwAvgVec, prev_qLSTMVec, colnameVec, colWdVecs) scoreVec = dt.concatenate(lstScores + [colTypeScore] + lstPrevScores) actScore = self.FpWhereColFF.score_expression(scoreVec) coverageMap = self.determine_coverage_by_name( qwVecs, colnameVec) elif act_type == ActionType.FpCondEqRow: entScore = nnmod.MaxScore(qwVecs, condCellVec) prev_entScore = nnmod.MaxScore(prev_qwVecs, condCellVec) actScore = dt.bmax(entScore, prev_entScore) coverageMap = self.determine_coverage_by_name( qwVecs, condCellVec) elif act_type == ActionType.FpCondNeRow: entScore = nnmod.MaxScore(qwVecs, condCellVec) prev_entScore = nnmod.MaxScore(prev_qwVecs, condCellVec) negScore = self.Negate.score_expression(qwAvgVec) scoreVec = dt.concatenate( [dt.bmax(entScore, prev_entScore), negScore]) actScore = self.NegFF.score_expression(scoreVec) coverageMap = self.determine_coverage_by_name( qwVecs, condCellVec) elif act_type == ActionType.SameAsPrevious: quesLSTMScore = dt.dot_product(prev_qLSTMVec, qLSTMVec) quesAvgScore = dt.dot_product(prev_qwAvgVec, qwAvgVec) actScore = dt.dot_product( self.SameAsPreviousW, dt.concatenate([quesLSTMScore, quesAvgScore])) coverageMap = dt.inputVector(np.zeros(len(qwVecs))) else: assert False, "Error! Unknown act_type: %d" % act_type res_list.append(actScore) meta_list.append(coverageMap) return dt.concatenate(res_list), meta_list
def __init__(self): # first initialize a computation graph container (or model). self.nnmodel = dynet.Model() # assign the algorithm for backpropagation updates. self.updater = dynet.AdamTrainer(self.nnmodel) num_words, num_tags, num_labels = 4808, 46, 46 word_embed_dim, pos_embed_dim, label_embed_dim = 100, 32, 32 hidden_layer1_dim, hidden_layer2_dim = 600, 600 num_actions = 93 self.minibatch_size = 1000 # create embeddings for words and tag features. self.word_embedding = self.nnmodel.add_lookup_parameters( (num_words, word_embed_dim)) glove_word_embeddings_dict = { } # key is the word, value is the list of 100 embeddings embed_lines = open("glove.6B.100d.txt", 'r').read().splitlines() for line in embed_lines: word = line.split()[0] values = line.split() del values[0] glove_word_embeddings_dict[word] = values vocab_words = open("./data/vocabs.word", 'r').read().splitlines() i = 0 for word_line in vocab_words: word = word_line.split()[0] if (word in glove_word_embeddings_dict): self.word_embedding[i] = np.asarray( glove_word_embeddings_dict[word]) self.pos_embedding = self.nnmodel.add_lookup_parameters( (num_tags, pos_embed_dim)) self.label_embedding = self.nnmodel.add_lookup_parameters( (num_labels, label_embed_dim)) # mbda x: dynet.bmax(.1 * x, x))assign transfer function self.transfer = (lambda x: dynet.bmax(.1 * x, x)) self.input_dim = 20 * (word_embed_dim + pos_embed_dim) + 12 * label_embed_dim self.hidden_layer1 = self.nnmodel.add_parameters( (hidden_layer1_dim, self.input_dim)) self.hidden_layer1_bias = self.nnmodel.add_parameters( hidden_layer1_dim, init=dynet.ConstInitializer(0.2)) self.hidden_layer2 = self.nnmodel.add_parameters( (hidden_layer2_dim, hidden_layer1_dim)) self.hidden_layer2_bias = self.nnmodel.add_parameters( hidden_layer2_dim, init=dynet.ConstInitializer(0.2)) # define the output weight. self.output_layer = self.nnmodel.add_parameters( (num_actions, hidden_layer2_dim)) # define the bias vector and initialize it as zero. self.output_bias = self.nnmodel.add_parameters( num_actions, init=dynet.ConstInitializer(0)) self.dropout_prob = 0.2 ''' You can add more arguments for examples actions and model paths. You need to load your model here. actions: provides indices for actions. it has the same order as the data/vocabs.actions file. ''' # if you prefer to have your own index for actions, change this. self.actions = [ 'SHIFT', 'LEFT-ARC:prep', 'LEFT-ARC:dobj', 'LEFT-ARC:poss', 'LEFT-ARC:amod', 'LEFT-ARC:xcomp', 'LEFT-ARC:mark', 'LEFT-ARC:conj', 'LEFT-ARC:nn', 'LEFT-ARC:rcmod', 'LEFT-ARC:advcl', 'LEFT-ARC:cc', 'LEFT-ARC:pcomp', 'LEFT-ARC:expl', 'LEFT-ARC:tmod', 'LEFT-ARC:csubj', 'LEFT-ARC:number', 'LEFT-ARC:iobj', 'LEFT-ARC:<null>', 'LEFT-ARC:preconj', 'LEFT-ARC:nsubj', 'LEFT-ARC:appos', 'LEFT-ARC:infmod', 'LEFT-ARC:partmod', 'LEFT-ARC:ccomp', 'LEFT-ARC:aux', 'LEFT-ARC:auxpass', 'LEFT-ARC:parataxis', 'LEFT-ARC:det', 'LEFT-ARC:punct', 'LEFT-ARC:discourse', 'LEFT-ARC:dep', 'LEFT-ARC:cop', 'LEFT-ARC:pobj', 'LEFT-ARC:num', 'LEFT-ARC:prt', 'LEFT-ARC:possessive', 'LEFT-ARC:rroot', 'LEFT-ARC:npadvmod', 'LEFT-ARC:mwe', 'LEFT-ARC:neg', 'LEFT-ARC:predet', 'LEFT-ARC:nsubjpass', 'LEFT-ARC:quantmod', 'LEFT-ARC:root', 'LEFT-ARC:acomp', 'LEFT-ARC:advmod', 'RIGHT-ARC:prep', 'RIGHT-ARC:dobj', 'RIGHT-ARC:poss', 'RIGHT-ARC:amod', 'RIGHT-ARC:xcomp', 'RIGHT-ARC:mark', 'RIGHT-ARC:conj', 'RIGHT-ARC:nn', 'RIGHT-ARC:rcmod', 'RIGHT-ARC:advcl', 'RIGHT-ARC:cc', 'RIGHT-ARC:pcomp', 'RIGHT-ARC:expl', 'RIGHT-ARC:tmod', 'RIGHT-ARC:csubj', 'RIGHT-ARC:number', 'RIGHT-ARC:iobj', 'RIGHT-ARC:<null>', 'RIGHT-ARC:preconj', 'RIGHT-ARC:nsubj', 'RIGHT-ARC:appos', 'RIGHT-ARC:infmod', 'RIGHT-ARC:partmod', 'RIGHT-ARC:ccomp', 'RIGHT-ARC:aux', 'RIGHT-ARC:auxpass', 'RIGHT-ARC:parataxis', 'RIGHT-ARC:det', 'RIGHT-ARC:punct', 'RIGHT-ARC:discourse', 'RIGHT-ARC:dep', 'RIGHT-ARC:cop', 'RIGHT-ARC:pobj', 'RIGHT-ARC:num', 'RIGHT-ARC:prt', 'RIGHT-ARC:possessive', 'RIGHT-ARC:rroot', 'RIGHT-ARC:npadvmod', 'RIGHT-ARC:mwe', 'RIGHT-ARC:neg', 'RIGHT-ARC:predet', 'RIGHT-ARC:nsubjpass', 'RIGHT-ARC:quantmod', 'RIGHT-ARC:root', 'RIGHT-ARC:acomp', 'RIGHT-ARC:advmod' ]