def __call__(self, query, options, gold, lengths, query_no): if len(options) == 1: return None, 0 final = [] if args.word_vectors: qvecs = [dy.lookup(self.pEmbedding, w) for w in query] qvec_max = dy.emax(qvecs) qvec_mean = dy.average(qvecs) for otext, features in options: if not args.no_features: inputs = dy.inputTensor(features) if args.word_vectors: ovecs = [dy.lookup(self.pEmbedding, w) for w in otext] ovec_max = dy.emax(ovecs) ovec_mean = dy.average(ovecs) if args.no_features: inputs = dy.concatenate( [qvec_max, qvec_mean, ovec_max, ovec_mean]) else: inputs = dy.concatenate( [inputs, qvec_max, qvec_mean, ovec_max, ovec_mean]) if args.drop > 0: inputs = dy.dropout(inputs, args.drop) h = inputs for pH, pB in zip(self.hidden, self.bias): h = dy.affine_transform([pB, pH, h]) if args.nonlin == "linear": pass elif args.nonlin == "tanh": h = dy.tanh(h) elif args.nonlin == "cube": h = dy.cube(h) elif args.nonlin == "logistic": h = dy.logistic(h) elif args.nonlin == "relu": h = dy.rectify(h) elif args.nonlin == "elu": h = dy.elu(h) elif args.nonlin == "selu": h = dy.selu(h) elif args.nonlin == "softsign": h = dy.softsign(h) elif args.nonlin == "swish": h = dy.cmult(h, dy.logistic(h)) final.append(dy.sum_dim(h, [0])) final = dy.concatenate(final) nll = -dy.log_softmax(final) dense_gold = [] for i in range(len(options)): dense_gold.append(1.0 / len(gold) if i in gold else 0.0) answer = dy.inputTensor(dense_gold) loss = dy.transpose(answer) * nll predicted_link = np.argmax(final.npvalue()) return loss, predicted_link
def perceptron_loss(scores, reference): if use_cost_augmented: predictions = hamming_augmented_decode(scores, reference) else: predictions = [np.argmax(score.npvalue()) for score in scores] margin = dy.scalarInput(-2) if predictions != reference: reference_score = calc_sequence_score(scores, reference) prediction_score = calc_sequence_score(scores, predictions) if use_cost_augmented: # One could actually get the hamming augmented value during decoding, but we didn't do it here for # demonstration purpose. hamming = dy.scalarInput(hamming_cost(predictions, reference)) loss = prediction_score + hamming - reference_score else: loss = prediction_score - reference_score if use_hinge: loss = dy.emax([dy.scalarInput(0), loss - margin]) return loss else: return dy.scalarInput(0)
def compose( self, embeds: Union[dy.Expression, List[dy.Expression]]) -> dy.Expression: if type(embeds) != list: embeds = [ dy.pick_batch_elem(embeds, i) for i in range(embeds.dim()[1]) ] return dy.emax(embeds)
def exprseq_pooling(self, exprseq): # Reduce to vector if exprseq.expr_tensor != None: if len(exprseq.expr_tensor.dim()[0]) > 1: return dy.max_dim(exprseq.expr_tensor, d=1) else: return exprseq.expr_tensor else: return dy.emax(exprseq.expr_list)
def get_activation(self, name): if name == "Sigmoid": return dy.logistic elif name == "Average": return dy.average elif name == "Maximum": return dy.emax() elif name == "ReLu": return dy.rectify
def exprseq_pooling(self, exprseq): # Reduce to vector exprseq = ExpressionSequence(expr_tensor=exprseq.mask.add_to_tensor_expr(exprseq.as_tensor(),-1e10), mask=exprseq.mask) if exprseq.expr_tensor != None: if len(exprseq.expr_tensor.dim()[0]) > 1: return dy.max_dim(exprseq.expr_tensor, d=1) else: return exprseq.expr_tensor else: return dy.emax(exprseq.expr_list)
def encode_batch(self, input_seq_batch): # TODO: test with batching encoded_inputs, masks = self.bilstm.encode_batch(input_seq_batch) max_output = dn.emax(encoded_inputs) # one mask per step, all are [1]'s since only one state from max pool encoder max_masks = [[1] * len(input_seq_batch)] # vizualize how many entries are selected from each input self.values_per_input(encoded_inputs, input_seq_batch, max_output) return [max_output], max_masks
def dynet_logsumexp(xs): emacs = dy.emax(xs) return dy.log(dy.esum([dy.exp(x - emacs) for x in xs])) + emacs
def pool(input_, _): return dy.emax(input_)
def MaxScore(qwVecs, vec): ret = dt.emax([dt.dot_product(qwVec, vec) for qwVec in qwVecs]) return ret
def AvgMaxScore(qwVecs, colWdVecs): ret = dt.average([ dt.emax([dt.dot_product(qwVec, colWdVec) for qwVec in qwVecs]) for colWdVec in colWdVecs ]) return ret
def get_next_score_expressions(self, legit_actions): res_list = [] for act in legit_actions: act_type = self.act2type[act] qwVecs = self.ques_emb qwAvgVec = dt.average(qwVecs) i_repr = dt.concatenate([self.fw[-1], self.bw[0]]) qLSTMVec = dt.tanh(self.H * i_repr) # question words LSTM embedding if act_type == SqaState.ActSelect: # question_embedding x column_name_embedding col = self.selectAct2Col(act) colnameVec = self.headers_embs[col] colPriorScore = dt.dot_product(self.ColW, colnameVec) colMaxScore = dt.emax( [dt.dot_product(qwVec, colnameVec) for qwVec in qwVecs]) colAvgScore = dt.dot_product(qwAvgVec, colnameVec) colQLSTMScore = dt.dot_product(qLSTMVec, colnameVec) colScore = dt.dot_product( self.SelColW, dt.concatenate([ colPriorScore, colMaxScore, colAvgScore, colQLSTMScore ])) res_list.append(colScore) elif act_type == SqaState.ActWhereCol: # same as SqaState.ActSelect # question_embedding x column_name_embedding col = self.whereColAct2Col(act) colnameVec = self.headers_embs[col] colPriorScore = dt.dot_product(self.ColW, colnameVec) colMaxScore = dt.emax( [dt.dot_product(qwVec, colnameVec) for qwVec in qwVecs]) colAvgScore = dt.dot_product(qwAvgVec, colnameVec) colQLSTMScore = dt.dot_product(qLSTMVec, colnameVec) colScore = dt.dot_product( self.SelColWhereW, dt.concatenate([ colPriorScore, colMaxScore, colAvgScore, colQLSTMScore ])) res_list.append(colScore) elif act_type == SqaState.ActWhereEqRow: r = self.whereEqRowAct2Row(act) c = self.whereColAct2Col( self.action_history[-1] ) # assuming the last action of the curren state is ActWhereCol entryVec = self.entries_embs[r][c] # max_w sim(w,entry) entScore = dt.emax( [dt.dot_product(qwVec, entryVec) for qwVec in qwVecs]) res_list.append(entScore) elif act_type == SqaState.ActWhereNul: res_list.append(dt.dot_product(dt.average(qwVecs), self.NulW)) return dt.concatenate(res_list)