def train(self, dataset, dev_dataset, step_size=0.1, max_epochs=50): model_acc = None self.Y = list(set([y for (y, x) in dataset])) # pour le moment, nous n'avons pas encore profité du corpus # de développement pour l'amélioration -> à ajouter for e in range(max_epochs): loss = 0.0 e = 0 for y, x in dataset: ypred = self.tag(x) if y != ypred: loss += 1.0 delta_ref = SparseWeightVector.code_phi(x, y) delta_pred = SparseWeightVector.code_phi(x, ypred) self.model += step_size * (delta_ref - delta_pred) # la moyenne du passé accumulé des poids if not model_acc: self.model_acc = self.model e += 1 else: model_acc += self.model e += 1 self.model = model_acc * (1 / float(e)) print("Loss (#errors) = ", loss) if loss == 0.0: return
def train(self, dataset, step_size=0.1, max_epochs=50): model_acc = None self.Y = list(set([y for (y, x) in dataset])) for e in range(max_epochs): loss = 0.0 e = 0 for y, x in dataset: ypred = self.tag(x) if y != ypred: loss += 1.0 delta_ref = SparseWeightVector.code_phi(x, y) delta_pred = SparseWeightVector.code_phi(x, ypred) self.model += step_size * (delta_ref - delta_pred) if not model_acc: self.model_acc = self.model e += 1 else: model_acc += self.model e += 1 self.model = model_acc * (1 / float(e)) print("Loss (#errors) = ", loss) if loss == 0.0: return
def train(self,dataset,step_size=1.0,max_epochs=100): """ @param dataset: a list of couples (y_tags,x_words) """ self.Y = list(set([y for (ytags,xwords) in dataset for y in ytags])) N = len(dataset) for e in range(max_epochs): loss = 0.0 for ytags,xwords in dataset: ypreds = self.tag(xwords) if ypreds != ytags: loss += 1.0 ytags_bigrams = list(zip([self.source_tag]+ytags,ytags)) ypreds_bigrams= list(zip([self.source_tag]+ypreds,ypreds)) delta_pred = SparseWeightVector() for y,x in zip(ypreds_bigrams,xwords): delta_pred += SparseWeightVector.code_phi(x,y) delta_ref = SparseWeightVector() for y,x in zip(ytags_bigrams,xwords): delta_ref += SparseWeightVector.code_phi(x,y) self.model += step_size*(delta_ref-delta_pred) print('Loss = ',loss, "Sequence accurracy = ",(N-loss)/N) if loss == 0: return
def train(self, dataset, step_size=0.1, max_epochs=100): #Maximizes log likelihood self.Y = list(set([y for (y, x) in dataset])) for e in range(max_epochs): #Batch gradient ascent delta_ref = SparseWeightVector() delta_pred = SparseWeightVector() loss = 0.0 for y, x in dataset: delta_ref += SparseWeightVector.code_phi(x, y) preds = self.predict(x) for idx, c in enumerate(self.Y): delta_pred += SparseWeightVector.code_phi(x, c) * preds[idx] loss += log(preds[self.Y.index(y)]) self.model += step_size * (delta_ref - delta_pred) print('Loss (log likelihood) = ', loss)
def train(self, dataset, step_size=1.0, max_epochs=100): """ @param dataset : a list of dependency trees """ N = len(dataset) for e in range(max_epochs): loss = 0.0 for ref_tree in dataset: tokens = ref_tree.tokens pred_tree = self.parse_one(tokens) if ref_tree.accurracy(pred_tree) != 1.0: loss += 1.0 delta_ref = SparseWeightVector() for gov_idx, dep_idx in ref_tree.edges: x_repr = self.__make_arc_representation( gov_idx, dep_idx, tokens) ylabel = ArcFactoredParser.RIGHTARC if gov_idx < dep_idx else ArcFactoredParser.LEFTARC delta_ref += SparseWeightVector.code_phi( x_repr, ylabel) delta_pred = SparseWeightVector() for gov_idx, dep_idx in pred_tree.edges: x_repr = self.__make_arc_representation( gov_idx, dep_idx, tokens) ylabel = ArcFactoredParser.RIGHTARC if gov_idx < dep_idx else ArcFactoredParser.LEFTARC delta_pred += SparseWeightVector.code_phi( x_repr, ylabel) self.model += step_size * (delta_ref - delta_pred) print('Loss = ', loss, "%Exact match = ", (N - loss) / N) if loss == 0.0: return
def static_train(self, treebank, step_size=1.0, max_epochs=100): """ Trains a model with a static oracle @param treebank : a list of dependency trees """ dataset = [] for dtree in treebank: dataset.extend(self.static_oracle_derivation(dtree)) N = len(dataset) for e in range(max_epochs): loss = 0.0 for ref_config, ref_action, tokens in dataset: pred_config, pred_action = self.predict_local( ref_config, tokens) if ref_action != pred_action: loss += 1.0 delta_ref = SparseWeightVector() S, B, A, score = ref_config x_repr = self.__make_config_representation(S, B, tokens) delta_ref += SparseWeightVector.code_phi( x_repr, ref_action) delta_pred = SparseWeightVector() S, B, A, score = ref_config x_repr = self.__make_config_representation(S, B, tokens) delta_pred += SparseWeightVector.code_phi( x_repr, pred_action) self.model += step_size * (delta_ref - delta_pred) print('Loss = ', loss, "%Local accurracy = ", (N - loss) / N) if loss == 0.0: return
def dynamic_train(self, treebank, step_size=1.0, max_epochs=100): ACTIONS = [ArcEagerTransitionParser.LEFTARC,ArcEagerTransitionParser.RIGHTARC,\ ArcEagerTransitionParser.SHIFT,ArcEagerTransitionParser.REDUCE,\ ArcEagerTransitionParser.TERMINATE] N = len(treebank) for e in range(max_epochs): loss, total = 0, 0 for dtree in treebank: ref_arcs = set(dtree.edges) n = len(dtree.tokens) C = ((0, ), tuple(range(1, n)), tuple(), 0.0 ) #A config is a hashable quadruple with score action = None while action != ArcEagerTransitionParser.TERMINATE: pred_config, pred_action = self.predict_local( C, dtree.tokens) optimal_actions = list([ a for a in ACTIONS if self.dynamic_oracle(C, a, ref_arcs) ]) total += 1 if pred_action not in optimal_actions: loss += 1 optimal_config, optimal_action = self.predict_local( C, dtree.tokens, allowed=optimal_actions) delta_ref = SparseWeightVector() S, B, A, score = C x_repr = self.__make_config_representation( S, B, dtree.tokens) delta_ref += SparseWeightVector.code_phi( x_repr, optimal_action) delta_pred = SparseWeightVector() S, B, A, score = C x_repr = self.__make_config_representation( S, B, dtree.tokens) delta_pred += SparseWeightVector.code_phi( x_repr, pred_action) self.model += step_size * (delta_ref - delta_pred) action = self.choose(pred_action, optimal_actions) if action == ArcEagerTransitionParser.SHIFT: C = self.shift(C, dtree.tokens) elif action == ArcEagerTransitionParser.LEFTARC: C = self.leftarc(C, dtree.tokens) elif action == ArcEagerTransitionParser.RIGHTARC: C = self.rightarc(C, dtree.tokens) elif action == ArcEagerTransitionParser.REDUCE: C = self.reduce_config(C, dtree.tokens) elif action == ArcEagerTransitionParser.TERMINATE: C = self.terminate(C, dtree.tokens) print('Loss = ', loss, "%Local accurracy = ", (total - loss) / total) if loss == 0.0: return
def __init__(self): """ @param weights: a SparseWeightVector @param non_terminals: an ordered list of non terminals """ self.model = SparseWeightVector() self.nonterminals_decode = [] #maps integers to symbols self.nonterminals_code = {} #maps symbols to integers
def train(self, treebank, step_size=1.0, max_epochs=100, beam_size=4, left_markov=True): """ @param treebank a list of ConsTrees @param left_markov: if true -> left markovization else right markovization """ self.transform(treebank, left_markov) dataset = list([(tree.tokens(), self.reference_derivation(tree)) for tree in treebank]) N = len(dataset) for e in range(max_epochs): loss = 0.0 for sentence, ref_derivation in dataset: pred_beam = (self.parse_one(sentence, get_beam=True)) (update, ref_prefix, pred_prefix) = self.early_prefix(ref_derivation, pred_beam) if update: loss += 1.0 delta_ref = SparseWeightVector() current_config = ref_prefix[0][1] for action, config in ref_prefix[1:]: S, B, score = current_config x_repr = self.__make_config_representation( S, B, sentence) delta_ref += SparseWeightVector.code_phi( x_repr, action) current_config = config delta_pred = SparseWeightVector() current_config = pred_prefix[0][1] for action, config in pred_prefix[1:]: S, B, score = current_config x_repr = self.__make_config_representation( S, B, sentence) delta_pred += SparseWeightVector.code_phi( x_repr, action) current_config = config self.model += step_size * (delta_ref - delta_pred) print('Loss = ', loss, "%Exact match = ", (N - loss) / N) if loss == 0.0: return
def train(self, dataset, step_size=1.0, max_epochs=100, beam_size=4): """ @param dataset : a list of dependency trees """ N = len(dataset) sequences = list([(dtree.tokens, self.oracle_derivation(dtree)) for dtree in dataset]) for e in range(max_epochs): loss = 0.0 for tokens, ref_derivation in sequences: pred_beam = self.parse_one(tokens, beam_size, get_beam=True) (update, ref_prefix, pred_prefix) = self.early_prefix(ref_derivation, pred_beam) #print('R',ref_derivation) #print('P',pred_prefix) #self.test(dataset,beam_size) if update: #print (pred_prefix) loss += 1.0 delta_ref = SparseWeightVector() current_config = ref_prefix[0][1] for action, config in ref_prefix: S, B, A, score = current_config x_repr = self.__make_config_representation( S, B, tokens) delta_ref += SparseWeightVector.code_phi( x_repr, action) current_config = config delta_pred = SparseWeightVector() current_config = pred_prefix[0][1] for action, config in pred_prefix: S, B, A, score = current_config x_repr = self.__make_config_representation( S, B, tokens) delta_pred += SparseWeightVector.code_phi( x_repr, action) current_config = config self.model += step_size * (delta_ref - delta_pred) print('Loss = ', loss, "%Exact match = ", (N - loss) / N) if loss == 0.0: return
def train(self, dataset, step_size=0.1, max_epochs=100): """ @param dataset: a list of couples (y_tags,x_words) """ self.Y = list(set([y for (ytags, xwords) in dataset for y in ytags])) #pre-computes delta_ref (first term of the gradient is constant) delta_ref = SparseWeightVector() for ytags, xwords in dataset: ytags_bigrams = list(zip([self.source_tag] + ytags, ytags)) for x, y in zip(xwords, ytags_bigrams): delta_ref += SparseWeightVector.code_phi(x, y) for e in range(max_epochs): loss = 0.0 delta_pred = SparseWeightVector() for ytags, xwords in dataset: N = len(xwords) K = len(self.Y) alphas, Z = self.forward(xwords) betas, _ = self.backward(xwords) #forward-backward at init for ytag in range(K): prob = (self.score(self.source_tag, self.Y[ytag], xwords[0]) * betas[0, ytag]) / Z delta_pred += prob * SparseWeightVector.code_phi( xwords[0], (self.source_tag, self.Y[ytag])) #forward-backward loop for i in range(1, N): for yprev in range(K): for ytag in range(K): prob = (alphas[i - 1, yprev] * self.score( self.Y[yprev], self.Y[ytag], xwords[i]) * betas[i, ytag]) / Z delta_pred += prob * SparseWeightVector.code_phi( xwords[i], (self.Y[yprev], self.Y[ytag])) loss += log(self.sequence_score(ytags, xwords) / Z) print('Log likelihood(D) = ', loss) self.model += step_size * (delta_ref - delta_pred)
def train(self, dataset, dev, step_size=0.1, max_epochs=20): self.Y = list(set([y for (y, x) in dataset])) dev_accs = [] train_accs = [] train_losses = [] T = 1. avg_cumul = SparseWeightVector() for e in range(max_epochs): loss = 0.0 random.shuffle(dataset) for y, x in dataset: ypred = self.tag(x) if y != ypred: loss += 1.0 delta_ref = SparseWeightVector.code_phi(x, y) delta_pred = SparseWeightVector.code_phi(x, ypred) update = step_size * (delta_ref - delta_pred) self.model += update avg_cumul += T * update T += 1 # Calculate accuracy if len(dataset) != 0: acc = (len(dataset) - loss) / len(dataset) else: acc = (len(dataset) - loss) train_accs.append(acc) # Stock Loss train_losses.append(loss) # Update Avg Perceptron self.model_avg = self.model - avg_cumul / T # Calculate acc for dev corpus dev_acc = self.test(dev, True) dev_accs.append(dev_acc) # Print loss et acc print("Epoch = " + str(e) + ", Loss (#errors) = " + str(loss) + ", Accuracy = " + str(acc * 100) + ", Dev acc = " + str(dev_acc * 100)) # Stop if loss null if loss == 0.0: return train_losses, train_accs, dev_accs return train_losses, train_accs, dev_accs
class AvgPerceptron: def __init__(self): self.model = SparseWeightVector() self.Y = [] #classes def train(self, dataset, dev_dataset, step_size=0.1, max_epochs=50): model_acc = None self.Y = list(set([y for (y, x) in dataset])) # pour le moment, nous n'avons pas encore profité du corpus # de développement pour l'amélioration -> à ajouter for e in range(max_epochs): loss = 0.0 e = 0 for y, x in dataset: ypred = self.tag(x) if y != ypred: loss += 1.0 delta_ref = SparseWeightVector.code_phi(x, y) delta_pred = SparseWeightVector.code_phi(x, ypred) self.model += step_size * (delta_ref - delta_pred) # la moyenne du passé accumulé des poids if not model_acc: self.model_acc = self.model e += 1 else: model_acc += self.model e += 1 self.model = model_acc * (1 / float(e)) print("Loss (#errors) = ", loss) if loss == 0.0: return def predict(self, dataline): return list([self.model.dot(dataline, c) for c in self.Y]) def tag(self, dataline): scores = self.predict(dataline) imax = scores.index(max(scores)) return self.Y[imax] def test(self, dataset): result = list([(y == self.tag(x)) for y, x in dataset]) return sum(result) / len(result)
class MultinomialLogistic: def __init__(self): self.model = SparseWeightVector() self.Y = [] #classes def train(self, dataset, step_size=0.1, max_epochs=100): #Maximizes log likelihood self.Y = list(set([y for (y, x) in dataset])) for e in range(max_epochs): #Batch gradient ascent delta_ref = SparseWeightVector() delta_pred = SparseWeightVector() loss = 0.0 for y, x in dataset: delta_ref += SparseWeightVector.code_phi(x, y) preds = self.predict(x) for idx, c in enumerate(self.Y): delta_pred += SparseWeightVector.code_phi(x, c) * preds[idx] loss += log(preds[self.Y.index(y)]) self.model += step_size * (delta_ref - delta_pred) print('Loss (log likelihood) = ', loss) def predict(self, dataline): probs = list([exp(self.model.dot(dataline, c)) for c in self.Y]) Z = sum(probs) probs = list([p / Z for p in probs]) return probs def tag(self, dataline): probs = self.predict(dataline) imax = probs.index(max(probs)) return self.Y[imax] def test(self, dataset): result = list([(y == self.tag(x)) for y, x in dataset]) return sum(result) / len(result)
class MultiClassPerceptron: def __init__(self): self.model = SparseWeightVector() self.Y = [] #classes def train(self, dataset, step_size=0.1, max_epochs=50): model_acc = None self.Y = list(set([y for (y, x) in dataset])) for e in range(max_epochs): loss = 0.0 e = 0 for y, x in dataset: ypred = self.tag(x) if y != ypred: loss += 1.0 delta_ref = SparseWeightVector.code_phi(x, y) delta_pred = SparseWeightVector.code_phi(x, ypred) self.model += step_size * (delta_ref - delta_pred) if not model_acc: self.model_acc = self.model e += 1 else: model_acc += self.model e += 1 self.model = model_acc * (1 / float(e)) print("Loss (#errors) = ", loss) if loss == 0.0: return def predict(self, dataline): return list([self.model.dot(dataline, c) for c in self.Y]) def tag(self, dataline): scores = self.predict(dataline) imax = scores.index(max(scores)) return self.Y[imax] def test(self, dataset): result = list([(y == self.tag(x)) for y, x in dataset]) return sum(result) / len(result)
def train(self,treebank,step_size=1.0,max_epochs=100,left_markov=True): """ Trains the parser with a structured perceptron @param: treebank a list of ConsTrees @param: left_markov :uses left markovization or right markovization """ self.transform(treebank,left_markov) #binarizes the treebank #makes a (x,y) pattern for the data set dataset = list( [(ViterbiCKY.ngram_tokens(tree.tokens()),self.tree_as_edges(tree)) for tree in treebank] ) N = len(dataset) for e in range(max_epochs): loss = 0.0 for sentence,ref_edges in dataset: pred_edges = set(self.parse_one(sentence,edges=True)) ref_edges = set(ref_edges) if pred_edges != ref_edges: #update loss += 1.0 delta_ref = SparseWeightVector() for r_edge in ref_edges: if len(r_edge) == 3: (i,j,Nroot),(i,k,Nl),(k,j,Nr) = r_edge x_repr = self.__make_edge_representation(i,j,k,sentence) delta_ref += SparseWeightVector.code_phi(x_repr,(Nroot,Nl,Nr)) elif len(r_edge) == 2: (i,j,pos),widx = r_edge x_repr = self.__make_unary_representation(widx,sentence) delta_ref += SparseWeightVector.code_phi(x_repr,pos) delta_pred = SparseWeightVector() for p_edge in pred_edges: if len(p_edge) == 3: (i,j,Nroot),(i,k,Nl),(k,j,Nr) = p_edge x_repr = self.__make_edge_representation(i,j,k,sentence) delta_pred += SparseWeightVector.code_phi(x_repr,(Nroot,Nl,Nr)) elif len(p_edge) == 2: (i,j,pos),widx = p_edge x_repr = self.__make_unary_representation(widx,sentence) delta_pred += SparseWeightVector.code_phi(x_repr,pos) self.model += step_size * (delta_ref-delta_pred) print('Loss = ',loss, "%Local accurracy = ",(N-loss)/N) if loss == 0.0: return
class ArcFactoredParser: LEFTARC = "L" RIGHTARC = "R" def __init__(self): self.model = SparseWeightVector() def _argmax(self, x, y, argmax, argvalue): """ computes m = max(x,y) and updates prev argmax by argvalue if max(x,y) = y @param x: number (current max) @param y: number @param argmax: the current argmax @param argvalue: the potential argmax @return (max,argmax) a couple with the current max and argmax """ if y > x: return (y, argvalue) return (x, argmax) def parse_one(self, sentence): """ @param sentence: a list of tokens as encoded in a dependency tree (first token is a dummy root token) @return : a DependencyTree Object """ COMPLETE, INCOMPLETE = 1, 0 LEFTARROW, RIGHTARROW = 0, 1 N = len(sentence) chart = np.zeros((N, N, 2, 2)) history = {} #recurrence for span_length in range(1, N): for i in range(N - span_length): j = i + span_length #incomplete max_left, max_right = -inf, -inf amax_left, amax_right = i, i for k in range(i, j): tmp_score = chart[i][k][RIGHTARROW][COMPLETE] \ + chart[k+1][j][LEFTARROW][COMPLETE] \ + self.score(j,i,sentence) max_left, amax_left = self._argmax(max_left, tmp_score, amax_left, k) tmp_score = chart[i][k][RIGHTARROW][COMPLETE] \ + chart[k+1][j][LEFTARROW][COMPLETE] \ + self.score(i,j,sentence) max_right, amax_right = self._argmax( max_right, tmp_score, amax_right, k) chart[i][j][LEFTARROW][INCOMPLETE] = max_left chart[i][j][RIGHTARROW][INCOMPLETE] = max_right history[(i, j, LEFTARROW, INCOMPLETE)] = [ (i, amax_left, RIGHTARROW, COMPLETE), (amax_left + 1, j, LEFTARROW, COMPLETE) ] history[(i, j, RIGHTARROW, INCOMPLETE)] = [ (i, amax_right, RIGHTARROW, COMPLETE), (amax_right + 1, j, LEFTARROW, COMPLETE) ] #complete max_left, max_right = -inf, -inf amax_left, amax_right = i, i for k in range(i, j): max_left, amax_left = self._argmax( max_left, chart[i][k][LEFTARROW][COMPLETE] + chart[k][j][LEFTARROW][INCOMPLETE], amax_left, k) for k in range(i + 1, j + 1): max_right, amax_right = self._argmax( max_right, chart[i][k][RIGHTARROW][INCOMPLETE] + chart[k][j][RIGHTARROW][COMPLETE], amax_right, k) chart[i][j][LEFTARROW][COMPLETE] = max_left chart[i][j][RIGHTARROW][COMPLETE] = max_right history[(i, j, LEFTARROW, COMPLETE)] = [(i, amax_left, LEFTARROW, COMPLETE), (amax_left, j, LEFTARROW, INCOMPLETE)] history[(i, j, RIGHTARROW, COMPLETE)] = [(i, amax_right, RIGHTARROW, INCOMPLETE), (amax_right, j, RIGHTARROW, COMPLETE)] #backtrace (collects edges of the dependency tree) edges = [] agenda = [(0, N - 1, RIGHTARROW, COMPLETE)] while agenda: current_item = agenda.pop() (i, j, direction, c) = current_item if c == INCOMPLETE and i != j: if direction == LEFTARROW: edges.append((j, i)) elif direction == RIGHTARROW: edges.append((i, j)) if current_item in history: agenda.extend(history[current_item]) return DependencyTree(tokens=sentence, edges=edges) def score(self, gov_idx, dep_idx, toklist): """ @param gov_idx,dep_idx : the indexes of the governor and dependant in the sentence @toklist: the list of tokens of the sentence @return : a float (score) """ dep_repr = self.__make_arc_representation(gov_idx, dep_idx, toklist) ylabel = ArcFactoredParser.RIGHTARC if gov_idx < dep_idx else ArcFactoredParser.LEFTARC return self.model.dot(dep_repr, ylabel) def __make_arc_representation(self, gov_idx, dep_idx, toklist): """ Creates a list of values from which to code a dependency arc as binary features Inserts the interactions between the words for coding the dependency in the representation. @param gov_idx,dep_idx : the indexes of the governor and dependant in the sentence @toklist: the list of tokens of the sentence @return : a list of tuples ready to be binarized and scored. """ interaction1 = ( toklist[gov_idx][1], toklist[dep_idx][1], ) interaction2 = ( toklist[gov_idx][0], toklist[dep_idx][0], ) #add more interactions here to improve the parser return toklist[gov_idx] + toklist[dep_idx] + (interaction1, ) + ( interaction2, ) def train(self, dataset, step_size=1.0, max_epochs=100): """ @param dataset : a list of dependency trees """ N = len(dataset) for e in range(max_epochs): loss = 0.0 for ref_tree in dataset: tokens = ref_tree.tokens pred_tree = self.parse_one(tokens) if ref_tree.accurracy(pred_tree) != 1.0: loss += 1.0 delta_ref = SparseWeightVector() for gov_idx, dep_idx in ref_tree.edges: x_repr = self.__make_arc_representation( gov_idx, dep_idx, tokens) ylabel = ArcFactoredParser.RIGHTARC if gov_idx < dep_idx else ArcFactoredParser.LEFTARC delta_ref += SparseWeightVector.code_phi( x_repr, ylabel) delta_pred = SparseWeightVector() for gov_idx, dep_idx in pred_tree.edges: x_repr = self.__make_arc_representation( gov_idx, dep_idx, tokens) ylabel = ArcFactoredParser.RIGHTARC if gov_idx < dep_idx else ArcFactoredParser.LEFTARC delta_pred += SparseWeightVector.code_phi( x_repr, ylabel) self.model += step_size * (delta_ref - delta_pred) print('Loss = ', loss, "%Exact match = ", (N - loss) / N) if loss == 0.0: return def test(self, dataset): N = len(dataset) sum_acc = 0.0 for ref_tree in dataset: tokens = ref_tree.tokens pred_tree = self.parse_one(tokens) print(pred_tree) print() sum_acc += ref_tree.accurracy(pred_tree) return sum_acc / N
def __init__(self): self.model = SparseWeightVector() self.Y = [] #classes
def __init__(self): self.model = SparseWeightVector()
class ViterbiCKY: """ This implements a CKY parser with perceptron scoring. """ def __init__(self): """ @param weights: a SparseWeightVector @param non_terminals: an ordered list of non terminals """ self.model = SparseWeightVector() self.nonterminals_decode = [] #maps integers to symbols self.nonterminals_code = {} #maps symbols to integers def transform(self,dataset,left_markov = True): """ In place (destructive) conversion of a treebank to Chomsky Normal Form. Builds the list of the parser nonterminals as a side effect and indexes references trees. @param dataset a list of ConsTrees @param left_markov: if true -> left markovization else right markovization """ all_nonterminals = set() for tree in dataset: tree.close_unaries() if left_markov: tree.left_markovize() else: tree.right_markovize() all_nonterminals.update(tree.collect_nonterminals()) self.nonterminals_decode = list(all_nonterminals) self.nonterminals_code = dict(zip(self.nonterminals_decode,range(len(self.nonterminals_decode)))) for tree in dataset: tree.index_leaves() @staticmethod def ngram_tokens(toklist): """ Returns the input with additional fields for ngrams based features """ BOL = '@@@' EOL = '$$$' wordlist = [BOL] + toklist + [EOL] word_trigrams = list(zip(wordlist,wordlist[1:],wordlist[2:])) return list(zip(wordlist[1:-1],word_trigrams)) def score_edge(self,Nroot,Nleft,Nright,i,j,k,sentence): """ Scores an edge. @param Nroot,Nleft,Nright : the root node, the left node and the right node @param i,j,k: the i,j,k indexes of the clique to be scored @param sentence: the list of words to be parsed @return a perceptron score """ edge_repr = self.__make_edge_representation(i,j,k,sentence) return self.model.dot(edge_repr,(Nroot,Nleft,Nright)) def score_unary(self,Nroot,word_idx,sentence): """ Scores an unary edge @param Nroot: the pos tag of the word @param word_idx: the index of the word in the sentence @param sentence: the sentence """ unary_repr = self.__make_unary_representation(word_idx,sentence) return self.model.dot(unary_repr,Nroot) def __make_edge_representation(self,i,j,k,sentence): """ Builds features for an edge. @param i,j,k: the i,j,k indexes of the clique to be scored """ return [sentence[i],sentence[k-1],sentence[k],sentence[j-1]] def __make_unary_representation(self,word_idx,sentence): """ Builds features for an unary reduction. """ return [sentence[word_idx]] def __build_tree(self,root_vertex,tree_root,history,sentence): """ Builds a parse tree from a chart history @param root_vertex: the tuple (i,j,label) encoding a vertex @param tree_root: the current ConsTree root @param history: the parse forest @param sentence: the list of words to be parsed @return the root of the tree """ (i,k,labelL),(k,j,labelR) = history[root_vertex] left = ConsTree(self.nonterminals_decode[labelL]) right = ConsTree(self.nonterminals_decode[labelR]) tree_root.children = [left,right] if k-i > 1: self.__build_tree((i,k,labelL),left,history,sentence) else: left.add_child(ConsTree(sentence[i][0])) if j-k > 1: self.__build_tree((k,j,labelR),right,history,sentence) else: right.add_child(ConsTree(sentence[k][0])) return tree_root def __build_edges(self,root_vertex,history): """ Extracts the hyperedges of the best tree from the chart and history (method useful for running the update while training) """ left,right = history[root_vertex] result = [(root_vertex,left,right)] i,k,lblA = left k,j,lblB = right if k-i == 1: result.append((left,i)) else: result.extend(self.__build_edges(left,history)) if j-k == 1: result.append((right,k)) else: result.extend(self.__build_edges(right,history)) return result def parse_one(self,sentence,untransform=True,edges=False): """ Parses a sentence with the cky viterbi algorithm @param sentence: a list of word strings @param untransform: if true, untransforms the result @param edges: if true returns hyperedges instead of a parse tree @return a ConsTree """ N = len(sentence) G = len(self.nonterminals_decode) #num nonterminal symbols chart = np.empty([N,N+1,G]) chart.fill(-inf) # 0 for perceptron history = {} for i in range(N):#init (= part of speech tagging) for Nt in range(G): chart[i,i+1,Nt] = self.score_unary(Nt,i,sentence) for span in range(2,N+1):#recurrence for i in range(N+1-span): j = i+span for Nt in range(G): for k in range(i+1,j): for Nt1 in range(G): for Nt2 in range(G): score = chart[i,k,Nt1]+chart[k,j,Nt2]+self.score_edge(Nt,Nt1,Nt2,i,j,k,sentence) if score > chart[i,j,Nt]: chart[i,j,Nt] = score history[(i,j,Nt)] = ((i,k,Nt1),(k,j,Nt2)) #Finds the max max_succ,argmax_succ = chart[0,N,0],(0,N,0) for Nt in range(1,G): if chart[0,N,Nt] > max_succ: max_succ,argmax_succ = chart[0,N,Nt],(0,N,Nt) i,j,label = argmax_succ if edges:#returns a list of hyperedges, useful for updates return self.__build_edges(argmax_succ,history) else:#builds a ConsTree result = self.__build_tree(argmax_succ,ConsTree(self.nonterminals_decode[label]),history,sentence) if untransform: result.unbinarize() result.expand_unaries() return result def tree_as_edges(self,tree_root): """ Returns a list of hyperedges from a *binary* Constree Supposes that leaves a indexed by a field 'idx' @param tree_root: a constree @return : a list of hyperedges @see ConsTree.index_leaves(...) """ assert(len(tree_root.children) <= 2) if len(tree_root.children) == 1: #unary edges idx = tree_root.get_child().idx jdx = idx + 1 return [((idx,jdx,self.nonterminals_code[tree_root.label]),idx)] elif len(tree_root.children) == 2:#binary edges left_edges = self.tree_as_edges(tree_root.children[0]) right_edges = self.tree_as_edges(tree_root.children[1]) i,k,lblA = left_edges[0][0] k,j,lblB = right_edges[0][0] result = [((i,j,self.nonterminals_code[tree_root.label]),(i,k,lblA),(k,j,lblB))] result.extend(left_edges) result.extend(right_edges) return result def test(self,treebank): """ Tests a model against a treebank and returns the average F-score. """ Fscore = [] for ref_tree in treebank: xinput = ViterbiCKY.ngram_tokens(ref_tree.tokens()) pred_tree = self.parse_one(xinput) print(pred_tree) P,R,F = ref_tree.compare(pred_tree) Fscore.append(F) return sum(Fscore)/len(Fscore) def train(self,treebank,step_size=1.0,max_epochs=100,left_markov=True): """ Trains the parser with a structured perceptron @param: treebank a list of ConsTrees @param: left_markov :uses left markovization or right markovization """ self.transform(treebank,left_markov) #binarizes the treebank #makes a (x,y) pattern for the data set dataset = list( [(ViterbiCKY.ngram_tokens(tree.tokens()),self.tree_as_edges(tree)) for tree in treebank] ) N = len(dataset) for e in range(max_epochs): loss = 0.0 for sentence,ref_edges in dataset: pred_edges = set(self.parse_one(sentence,edges=True)) ref_edges = set(ref_edges) if pred_edges != ref_edges: #update loss += 1.0 delta_ref = SparseWeightVector() for r_edge in ref_edges: if len(r_edge) == 3: (i,j,Nroot),(i,k,Nl),(k,j,Nr) = r_edge x_repr = self.__make_edge_representation(i,j,k,sentence) delta_ref += SparseWeightVector.code_phi(x_repr,(Nroot,Nl,Nr)) elif len(r_edge) == 2: (i,j,pos),widx = r_edge x_repr = self.__make_unary_representation(widx,sentence) delta_ref += SparseWeightVector.code_phi(x_repr,pos) delta_pred = SparseWeightVector() for p_edge in pred_edges: if len(p_edge) == 3: (i,j,Nroot),(i,k,Nl),(k,j,Nr) = p_edge x_repr = self.__make_edge_representation(i,j,k,sentence) delta_pred += SparseWeightVector.code_phi(x_repr,(Nroot,Nl,Nr)) elif len(p_edge) == 2: (i,j,pos),widx = p_edge x_repr = self.__make_unary_representation(widx,sentence) delta_pred += SparseWeightVector.code_phi(x_repr,pos) self.model += step_size * (delta_ref-delta_pred) print('Loss = ',loss, "%Local accurracy = ",(N-loss)/N) if loss == 0.0: return
class StructuredPerceptron: def __init__(self): self.model = SparseWeightVector() self.Y = [] #classes self.source_tag = "@@@" def tag(self,sentence): """ Viterbi + backtrace """ N = len(sentence) K = len(self.Y) viterbi = np.zeros((N,K)) history = np.zeros((N,K)) #init for j in range(K): viterbi[0,j] = self.score(self.source_tag,self.Y[j],sentence[0]) #Recurrence for i in range(1,N): for j in range(K): smax,amax = -inf,-inf for pred in range(K): score = viterbi[i-1,pred] + self.score(self.Y[pred],self.Y[j],sentence[i]) if score > smax: smax,amax = score,pred viterbi[i,j],history[i,j] = smax,amax #End state smax,amax = -inf,-inf for pred in range(K): score = viterbi[N-1,pred] if score > smax: smax,amax = score,pred #Backtrace rev_tag_sequence = [] for i in range(N-1,-1,-1): rev_tag_sequence.append(self.Y[ amax ]) amax = int(history[i,amax]) return list(reversed(rev_tag_sequence)) def score(self,y_pred,y,word_repr): """ Scores a structured perceptron clique @param y_pred : prev tag @param y : current tag @word_repr : a word data representation (a list of hashable symbols) @return a real value """ return self.model.dot(word_repr,(y_pred,y)) def train(self,dataset,step_size=1.0,max_epochs=100): """ @param dataset: a list of couples (y_tags,x_words) """ self.Y = list(set([y for (ytags,xwords) in dataset for y in ytags])) N = len(dataset) for e in range(max_epochs): loss = 0.0 for ytags,xwords in dataset: ypreds = self.tag(xwords) if ypreds != ytags: loss += 1.0 ytags_bigrams = list(zip([self.source_tag]+ytags,ytags)) ypreds_bigrams= list(zip([self.source_tag]+ypreds,ypreds)) delta_pred = SparseWeightVector() for y,x in zip(ypreds_bigrams,xwords): delta_pred += SparseWeightVector.code_phi(x,y) delta_ref = SparseWeightVector() for y,x in zip(ytags_bigrams,xwords): delta_ref += SparseWeightVector.code_phi(x,y) self.model += step_size*(delta_ref-delta_pred) print('Loss = ',loss, "Sequence accurracy = ",(N-loss)/N) if loss == 0: return def test(self,dataset): N = 0.0 correct = 0.0 for ytags,xwords in dataset: N += len(ytags) ypreds = self.tag(xwords) correct += sum([ref == pred for ref,pred in zip(ytags,ypreds)]) return correct / N
class AvgPerceptron: """ Averaged Perceptron """ def __init__(self): self.model = SparseWeightVector() self.Y = [] # classes self.model_avg = SparseWeightVector() def train(self, dataset, dev, step_size=0.1, max_epochs=20): self.Y = list(set([y for (y, x) in dataset])) dev_accs = [] train_accs = [] train_losses = [] T = 1. avg_cumul = SparseWeightVector() for e in range(max_epochs): loss = 0.0 random.shuffle(dataset) for y, x in dataset: ypred = self.tag(x) if y != ypred: loss += 1.0 delta_ref = SparseWeightVector.code_phi(x, y) delta_pred = SparseWeightVector.code_phi(x, ypred) update = step_size * (delta_ref - delta_pred) self.model += update avg_cumul += T * update T += 1 # Calculate accuracy if len(dataset) != 0: acc = (len(dataset) - loss) / len(dataset) else: acc = (len(dataset) - loss) train_accs.append(acc) # Stock Loss train_losses.append(loss) # Update Avg Perceptron self.model_avg = self.model - avg_cumul / T # Calculate acc for dev corpus dev_acc = self.test(dev, True) dev_accs.append(dev_acc) # Print loss et acc print("Epoch = " + str(e) + ", Loss (#errors) = " + str(loss) + ", Accuracy = " + str(acc * 100) + ", Dev acc = " + str(dev_acc * 100)) # Stop if loss null if loss == 0.0: return train_losses, train_accs, dev_accs return train_losses, train_accs, dev_accs def predict(self, dataline, avg=False): if avg: return list([self.model_avg.dot(dataline, c) for c in self.Y]) else: return list([self.model.dot(dataline, c) for c in self.Y]) def tag(self, dataline, avg=False): scores = self.predict(dataline, avg) imax = scores.index(max(scores)) return self.Y[imax] def test(self, dataset, avg=False): result = list([(y == self.tag(x, avg)) for y, x in dataset]) return sum(result) / len(result)
class ConstituentTransitionParser: SHIFT = "S" REDUCE = "R" STOP = "!" def __init__(self): self.model = SparseWeightVector() self.nonterminals = [] def static_oracle(self, stack, buffer, ref_triples): """ Returns the action to do given a configuration and a ref parse tree @param ref_triples : the triples from the reference tree @param stack: the config stack @param buffer: a list of integers @return a couple (parse action, action param) """ if len(stack) >= 2: (i, k, X1), (k, j, X2) = stack[-2], stack[-1] for X in self.nonterminals: if (i, j, X) in ref_triples: return (ConstituentTransitionParser.REDUCE, X) if buffer: idx = buffer[0] for tag in self.nonterminals: if (idx, idx + 1, tag) in ref_triples: return (ConstituentTransitionParser.SHIFT, tag) return (ConstituentTransitionParser.STOP, ConstituentTransitionParser.STOP) def reference_derivation(self, ref_tree): """ Returns a reference derivation given a reference tree @param ref_tree: a ConsTree """ ref_tree.index_leaves() ref_triples = set(ref_tree.triples()) sentence = ref_tree.tokens() N = len(sentence) action = (None, None) c = (tuple(), tuple(range(N)), 0.0) derivation = [(action, c)] for t in range(2 * N): #because 2N-1+terminate S, B, score = c action, param = self.static_oracle(S, B, ref_triples) if action == ConstituentTransitionParser.REDUCE: c = self.reduce(c, param, sentence) elif action == ConstituentTransitionParser.SHIFT: c = self.shift(c, param, sentence) else: c = self.terminate(c, sentence) derivation.append(((action, param), c)) return derivation def build_tree(self, derivation, sentence): """ Builds a ConsTree from a parse derivation @param derivation: a parse derivation @param sentence: a list of tokens @return a ConsTree """ tree_stack = [] for (action, param), C in derivation: S, B, score = C if action == ConstituentTransitionParser.SHIFT: i, j, lbl = S[-1] tag_node = ConsTree(param) leaf_node = ConsTree(sentence[i]) tag_node.add_child(leaf_node) tree_stack.append(tag_node) elif action == ConstituentTransitionParser.REDUCE: root_node = ConsTree(param) rnode = tree_stack.pop() lnode = tree_stack.pop() root_node.children = [lnode, rnode] tree_stack.append(root_node) return tree_stack[-1] def reduce(self, C, param, sentence): """ Performs a reduction from the current configuration and returns the result @param S: a stack @param B: a buffer @param param: the category for reduction @return a configuration """ S, B, score = C i, k, _ = S[-2] k, j, _ = S[-1] return ( S[:-2] + ((i, j, param), ), B, score + self.score(C, (ConstituentTransitionParser.REDUCE, param), sentence)) def shift(self, C, param, sentence): """ Performs a reduction from the current configuration and returns the result @param S: a stack @param B: a buffer @param param: the category for reduction @return a configuration """ S, B, score = C idx = S[-1][1] if S else 0 return ( S + ((idx, idx + 1, param), ), B[1:], score + self.score(C, (ConstituentTransitionParser.SHIFT, param), sentence)) def terminate(self, C, sentence): """ Performs a stop action returns the result """ S, B, score = C return (S, B, score + self.score(C, (ConstituentTransitionParser.STOP, ConstituentTransitionParser.STOP), sentence)) def score(self, configuration, action, tokens): """ Computes the prefix score of a derivation @param configuration : a triple (S,B,score) @param action: an action label @param tokens: the x-sequence of tokens to be parsed @return a prefix score """ S, B, old_score = configuration config_repr = self.__make_config_representation(S, B, tokens) return old_score + self.model.dot(config_repr, action) def __make_config_representation(self, S, B, tokens): """ This gathers the information for coding the configuration as a feature vector. @param S: a configuration stack @param B a configuration buffer @return an ordered list of tuples """ #default values for inaccessible positions s0cat, s1cat, s0l, s0r, s1l, s1r, b0, b1, b2 = "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_" if len(S) > 0: i, j, lbl = S[-1] s0l, s0r, s0cat = tokens[i], tokens[j - 1], lbl if len(S) > 1: i, j, lbl = S[-2] s1l, s1r, s1cat = tokens[i], tokens[j - 1], lbl if len(B) > 0: b0 = tokens[B[0]] if len(B) > 1: b1 = tokens[B[1]] if len(B) > 2: b2 = tokens[B[2]] wordlist = [s0l, s0r, s1l, s1r, b0, b1, b2] catlist = [s0cat, s1cat, b0] word_bigrams = list(zip(wordlist, wordlist[1:])) word_trigrams = list(zip(wordlist, wordlist[1:], wordlist[2:])) cat_bigrams = list(zip(catlist, catlist[1:])) return word_bigrams + word_trigrams + cat_bigrams def transform(self, dataset, left_markov=True): """ In place (destructive) conversion of a treebank to Chomsky Normal Form. Builds the list of the parser nonterminals as a side effect and indexes references trees. @param dataset a list of ConsTrees @param left_markov: if true -> left markovization else right markovization """ all_nonterminals = set() for tree in dataset: tree.close_unaries() if left_markov: tree.left_markovize() else: tree.right_markovize() all_nonterminals.update(tree.collect_nonterminals()) self.nonterminals = list(all_nonterminals) def parse_one(self, sentence, beam_size=4, get_beam=False, deriv=False, untransform=True): """ @param sentence: a list of strings @param beam_size: size of the beam @param get_beam : returns the beam instead of tree like structures @param deriv: returns the derivation instead of the parse tree @param untransform: bool if true unbinarizes the resulting tree. """ actions = [ConstituentTransitionParser.SHIFT,\ ConstituentTransitionParser.REDUCE,\ ConstituentTransitionParser.STOP] all_actions = list([(a, p) for a in actions for p in self.nonterminals]) N = len(sentence) init = (tuple(), tuple(range(N)), 0.0 ) #A config is a hashable triple with score current_beam = [(-1, (None, None), init)] beam = [current_beam] for i in range(2 * N): #because 2*N-1+terminate next_beam = [] for idx, (_, action, config) in enumerate(current_beam): S, B, score = config for (a, p) in all_actions: if a == ConstituentTransitionParser.SHIFT: if B: newconfig = self.shift(config, p, sentence) next_beam.append((idx, (a, p), newconfig)) elif a == ConstituentTransitionParser.REDUCE: if len(S) >= 2: newconfig = self.reduce(config, p, sentence) next_beam.append((idx, (a, p), newconfig)) elif a == ConstituentTransitionParser.STOP: if len(S) < 2 and not B: newconfig = self.terminate(config, sentence) next_beam.append((idx, (a, a), newconfig)) next_beam.sort(key=lambda x: x[2][2], reverse=True) next_beam = next_beam[:beam_size] beam.append(next_beam) current_beam = next_beam if get_beam: return beam else: #Backtrace for derivation idx = 1 prev_jdx = 0 derivation = [] while prev_jdx != -1: current = beam[-idx][prev_jdx] prev_jdx, prev_action, C = current derivation.append((prev_action, C)) idx += 1 derivation.reverse() if deriv: return derivation else: result = self.build_tree(derivation, sentence) if untransform: result.unbinarize() result.expand_unaries() return result def early_prefix(self, ref_parse, beam): """ Finds the prefix for early update, that is the prefix where the ref parse fall off the beam. @param ref_parse: a parse derivation @param beam: a beam output by the parse_one function @return (bool, ref parse prefix, best in beam prefix) the bool is True if update required false otherwise """ idx = 0 for (actionR, configR), (beamCol) in zip(ref_parse, beam): found = False for source_idx, action, configTarget in beamCol: if action == actionR and configTarget[:-1] == configR[:-1]: #-1 -> does not test score equality found = True break if not found: #backtrace jdx = idx source_idx = 0 early_prefix = [] while jdx >= 0: new_source_idx, action, config = beam[jdx][source_idx] early_prefix.append((action, config)) source_idx = new_source_idx jdx -= 1 early_prefix.reverse() return (True, ref_parse[:idx + 1], early_prefix) idx += 1 #if no error found check that the best in beam is the ref parse last_ref_action, last_ref_config = ref_parse[-1] _, last_pred_action, last_pred_config = beam[-1][0] if last_pred_config[:-1] == last_ref_config[:-1]: return (False, None, None) #returns a no update message else: #backtrace jdx = len(beam) - 1 source_idx = 0 early_prefix = [] while jdx >= 0: new_source_idx, action, config = beam[jdx][source_idx] early_prefix.append((action, config)) source_idx = new_source_idx jdx -= 1 early_prefix.reverse() return (True, ref_parse, early_prefix) def test(self, treebank, beam_size=4): """ @param treebank a list of ConsTrees @param left_markov: if true -> left markovization else right markovization @return the avg f-score """ Fscores = [] for tree in treebank: result = self.parse_one(tree.tokens(), beam_size) print(result) P, R, F = tree.compare(result) Fscores.append(F) return sum(Fscores) / len(Fscores) def train(self, treebank, step_size=1.0, max_epochs=100, beam_size=4, left_markov=True): """ @param treebank a list of ConsTrees @param left_markov: if true -> left markovization else right markovization """ self.transform(treebank, left_markov) dataset = list([(tree.tokens(), self.reference_derivation(tree)) for tree in treebank]) N = len(dataset) for e in range(max_epochs): loss = 0.0 for sentence, ref_derivation in dataset: pred_beam = (self.parse_one(sentence, get_beam=True)) (update, ref_prefix, pred_prefix) = self.early_prefix(ref_derivation, pred_beam) if update: loss += 1.0 delta_ref = SparseWeightVector() current_config = ref_prefix[0][1] for action, config in ref_prefix[1:]: S, B, score = current_config x_repr = self.__make_config_representation( S, B, sentence) delta_ref += SparseWeightVector.code_phi( x_repr, action) current_config = config delta_pred = SparseWeightVector() current_config = pred_prefix[0][1] for action, config in pred_prefix[1:]: S, B, score = current_config x_repr = self.__make_config_representation( S, B, sentence) delta_pred += SparseWeightVector.code_phi( x_repr, action) current_config = config self.model += step_size * (delta_ref - delta_pred) print('Loss = ', loss, "%Exact match = ", (N - loss) / N) if loss == 0.0: return
class LinearChainCRF: def __init__(self): self.model = SparseWeightVector() self.Y = [] #classes self.source_tag = "@@@" def tag(self, sentence): """ Viterbi + backtrace """ N = len(sentence) K = len(self.Y) viterbi = np.zeros((N, K)) history = np.zeros((N, K)) #init for j in range(K): viterbi[0, j] = self.score(self.source_tag, self.Y[j], sentence[0]) #Recurrence for i in range(1, N): for j in range(K): smax, amax = 0, 0 for pred in range(K): score = viterbi[i - 1, pred] * self.score( self.Y[pred], self.Y[j], sentence[i]) if score > smax: smax, amax = score, pred viterbi[i, j], history[i, j] = smax, amax #End state smax, amax = 0, 0 for pred in range(K): score = viterbi[N - 1, pred] if score > smax: smax, amax = score, pred #Backtrace rev_tag_sequence = [] for i in range(N - 1, -1, -1): rev_tag_sequence.append(self.Y[amax]) amax = int(history[i, amax]) return list(reversed(rev_tag_sequence)) def sequence_score(self, ytags, xwords): """ Returns the unnormalized exp(dot product) score of a tag sequence given words and model parameters. @param ytags : a tag sequence @param xwords: a sequence of word representations """ ytags_bigrams = list(zip([self.source_tag] + ytags, ytags)) score = 1 for x, y in zip(xwords, ytags_bigrams): score *= self.score(y[0], y[1], x) return score def score(self, y_pred, y, word_repr): """ Scores a CRF clique (psi value for y-1,y,x) @param y_pred : prev tag @param y : current tag @word_repr : a word data representation (a list of hashable symbols) @return a psi (potential) positive value """ return exp(self.model.dot(word_repr, (y_pred, y))) def forward(self, sentence): """ @param sentence: a list of xwords @return a forward matrix and Z (norm constant) """ N = len(sentence) K = len(self.Y) forward = np.zeros((N, K)) #init for j in range(K): forward[0, j] = self.score(self.source_tag, self.Y[j], sentence[0]) #recurrence for i in range(1, N): for j in range(K): for pred in range(K): forward[i, j] += forward[i - 1, pred] * self.score( self.Y[pred], self.Y[j], sentence[i]) return (forward, forward[N - 1, :].sum()) def backward(self, sentence): """ @param sentence: a list of xwords @return a backward matrix and Z (norm constant) """ N = len(sentence) K = len(self.Y) backward = np.zeros((N, K)) backward[N - 1, :] = 1.0 #recurrence for i in range(N - 2, -1, -1): for j in range(K): for succ in range(K): backward[i, j] += backward[i + 1, succ] * self.score( self.Y[j], self.Y[succ], sentence[i + 1]) Z = sum([ self.score(self.source_tag, self.Y[succ], sentence[0]) * backward[0, succ] for succ in range(K) ]) return (backward, Z) def train(self, dataset, step_size=0.1, max_epochs=100): """ @param dataset: a list of couples (y_tags,x_words) """ self.Y = list(set([y for (ytags, xwords) in dataset for y in ytags])) #pre-computes delta_ref (first term of the gradient is constant) delta_ref = SparseWeightVector() for ytags, xwords in dataset: ytags_bigrams = list(zip([self.source_tag] + ytags, ytags)) for x, y in zip(xwords, ytags_bigrams): delta_ref += SparseWeightVector.code_phi(x, y) for e in range(max_epochs): loss = 0.0 delta_pred = SparseWeightVector() for ytags, xwords in dataset: N = len(xwords) K = len(self.Y) alphas, Z = self.forward(xwords) betas, _ = self.backward(xwords) #forward-backward at init for ytag in range(K): prob = (self.score(self.source_tag, self.Y[ytag], xwords[0]) * betas[0, ytag]) / Z delta_pred += prob * SparseWeightVector.code_phi( xwords[0], (self.source_tag, self.Y[ytag])) #forward-backward loop for i in range(1, N): for yprev in range(K): for ytag in range(K): prob = (alphas[i - 1, yprev] * self.score( self.Y[yprev], self.Y[ytag], xwords[i]) * betas[i, ytag]) / Z delta_pred += prob * SparseWeightVector.code_phi( xwords[i], (self.Y[yprev], self.Y[ytag])) loss += log(self.sequence_score(ytags, xwords) / Z) print('Log likelihood(D) = ', loss) self.model += step_size * (delta_ref - delta_pred) def test(self, dataset): N = 0.0 correct = 0.0 for ytags, xwords in dataset: N += len(ytags) ypreds = self.tag(xwords) correct += sum([ref == pred for ref, pred in zip(ytags, ypreds)]) return correct / N
def __init__(self): self.model = SparseWeightVector() self.nonterminals = []
class ArcStandardTransitionParser: #actions LEFTARC = "L" RIGHTARC = "R" SHIFT = "S" TERMINATE = "T" def __init__(self): self.model = SparseWeightVector() @staticmethod def static_oracle(configuration, reference_arcs, N): """ @param configuration: a parser configuration @param reference arcs: a set of dependency arcs @param N: the length of the input sequence """ S, B, A, score = configuration all_words = range(N) if len(S) >= 2: i, j = S[-2], S[-1] if j != 0 and (i, j) in reference_arcs and all( [(j, k) in A for k in all_words if (j, k) in reference_arcs]): return ArcStandardTransitionParser.RIGHTARC elif i != 0 and (j, i) in reference_arcs and all( [(i, k) in A for k in all_words if (i, k) in reference_arcs]): return ArcStandardTransitionParser.LEFTARC if B: return ArcStandardTransitionParser.SHIFT return ArcStandardTransitionParser.TERMINATE def oracle_derivation(self, ref_parse): """ This generates an oracle reference derivation from a sentence @param ref_parse: a DependencyTree object @return : the oracle derivation """ sentence = ref_parse.tokens edges = set(ref_parse.edges) N = len(sentence) C = (tuple(), tuple(range(len(sentence))), tuple(), 0.0 ) #A config is a hashable quadruple with score action = None derivation = [(action, C)] while action != ArcStandardTransitionParser.TERMINATE: action = ArcStandardTransitionParser.static_oracle(C, edges, N) if action == ArcStandardTransitionParser.SHIFT: C = self.shift(C, sentence) elif action == ArcStandardTransitionParser.LEFTARC: C = self.leftarc(C, sentence) elif action == ArcStandardTransitionParser.RIGHTARC: C = self.rightarc(C, sentence) elif action == ArcStandardTransitionParser.TERMINATE: C = self.terminate(C, sentence) derivation.append((action, C)) return derivation def shift(self, configuration, tokens): """ Performs the shift action and returns a new configuration """ S, B, A, score = configuration w0 = B[0] return (S + (w0, ), B[1:], A, score + self.score( configuration, ArcStandardTransitionParser.SHIFT, tokens)) def leftarc(self, configuration, tokens): """ Performs the left arc action and returns a new configuration """ S, B, A, score = configuration i, j = S[-2], S[-1] return (S[:-2] + (j, ), B, A + ((j, i), ), score + self.score( configuration, ArcStandardTransitionParser.LEFTARC, tokens)) def rightarc(self, configuration, tokens): S, B, A, score = configuration i, j = S[-2], S[-1] return (S[:-1], B, A + ((i, j), ), score + self.score( configuration, ArcStandardTransitionParser.RIGHTARC, tokens)) def terminate(self, configuration, tokens): S, B, A, score = configuration return (S, B, A, score + self.score( configuration, ArcStandardTransitionParser.TERMINATE, tokens)) def parse_one(self, sentence, beam_size=4, get_beam=False): actions = [ArcStandardTransitionParser.LEFTARC,\ ArcStandardTransitionParser.RIGHTARC,\ ArcStandardTransitionParser.SHIFT,\ ArcStandardTransitionParser.TERMINATE] N = len(sentence) init = (tuple(), tuple(range(N)), tuple(), 0.0 ) #A config is a hashable quadruple with score current_beam = [(-1, None, init)] beam = [current_beam] for i in range(2 * N): #because 2N-1+terminate next_beam = [] for idx, (_, action, config) in enumerate(current_beam): S, B, A, score = config for a in actions: if a == ArcStandardTransitionParser.SHIFT: if B: newconfig = self.shift(config, sentence) next_beam.append((idx, a, newconfig)) elif a == ArcStandardTransitionParser.LEFTARC: if len(S) >= 2 and S[ -2] != 0: #a word cannot dominate the dummy root newconfig = self.leftarc(config, sentence) next_beam.append((idx, a, newconfig)) elif a == ArcStandardTransitionParser.RIGHTARC: if len(S) >= 2: newconfig = self.rightarc(config, sentence) next_beam.append((idx, a, newconfig)) elif a == ArcStandardTransitionParser.TERMINATE: if len(S) < 2 and not B: newconfig = self.terminate(config, sentence) next_beam.append((idx, a, newconfig)) next_beam.sort(key=lambda x: x[2][3], reverse=True) next_beam = next_beam[:beam_size] beam.append(next_beam) current_beam = next_beam if get_beam: return beam else: succ = beam[-1][0][ 2] #success in last beam, top position, newconfig print(beam[-1][0][1], succ) return DependencyTree(tokens=sentence, edges=succ[2]) def early_prefix(self, ref_parse, beam): """ Finds the prefix for early update, that is the prefix where the ref parse fall off the beam. @param ref_parse: a parse derivation @param beam: a beam output by the parse_one function @return (bool, ref parse prefix, best in beam prefix) the bool is True if update required false otherwise """ idx = 0 for (actionR, configR), (beamCol) in zip(ref_parse, beam): found = False #print("seeking",configR, "at index",idx) for source_idx, action, configTarget in beamCol: #print(" ",configTarget) if action == actionR and configTarget[:-1] == configR[:-1]: #-1 -> does not test score equality found = True #print(" => found") break if not found: #print(" => not found") #backtrace jdx = idx source_idx = 0 early_prefix = [] while jdx >= 0: new_source_idx, action, config = beam[jdx][source_idx] early_prefix.append((action, config)) source_idx = new_source_idx jdx -= 1 early_prefix.reverse() return (True, ref_parse[:idx + 1], early_prefix) idx += 1 #if no error found check that the best in beam is the ref parse last_ref_action, last_ref_config = ref_parse[-1] _, last_pred_action, last_pred_config = beam[-1][0] if last_pred_config[:-1] == last_ref_config[:-1]: return (False, None, None) #returns a no update message else: #backtrace jdx = len(beam) - 1 source_idx = 0 early_prefix = [] while jdx >= 0: new_source_idx, action, config = beam[jdx][source_idx] early_prefix.append((action, config)) source_idx = new_source_idx jdx -= 1 early_prefix.reverse() return (True, ref_parse, early_prefix) def score(self, configuration, action, tokens): """ Computes the prefix score of a derivation @param configuration : a quintuple (S,B,A,score,history) @param action: an action label in {LEFTARC,RIGHTARC,TERMINATE,SHIFT} @param tokens: the x-sequence of tokens to be parsed @return a prefix score """ S, B, A, old_score = configuration config_repr = self.__make_config_representation(S, B, tokens) return old_score + self.model.dot(config_repr, action) def __make_config_representation(self, S, B, tokens): """ This gathers the information for coding the configuration as a feature vector. @param S: a configuration stack @param B a configuration buffer @return an ordered list of tuples """ #default values for inaccessible positions s0w, s1w, s0t, s1t, b0w, b1w, b0t, b1t = "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_" if len(S) > 0: s0w, s0t = tokens[S[-1]][0], tokens[S[-1]][1] if len(S) > 1: s1w, s1t = tokens[S[-2]][0], tokens[S[-2]][1] if len(B) > 0: b0w, b0t = tokens[B[0]][0], tokens[B[0]][1] if len(B) > 1: b1w, b1t = tokens[B[1]][0], tokens[B[1]][1] wordlist = [s0w, s1w, b0w, b1w] taglist = [s0t, s1t, b0t, b1t] word_bigrams = list(zip(wordlist, wordlist[1:])) tag_bigrams = list(zip(taglist, taglist[1:])) word_trigrams = list(zip(wordlist, wordlist[1:], wordlist[2:])) tag_trigrams = list(zip(taglist, taglist[1:], taglist[2:])) return word_bigrams + tag_bigrams + word_trigrams + tag_trigrams def test(self, dataset, beam_size=4): """ @param dataset: a list of DependencyTrees @param beam_size: size of the beam """ N = len(dataset) sum_acc = 0.0 for ref_tree in dataset: tokens = ref_tree.tokens pred_tree = self.parse_one(tokens, beam_size) print(pred_tree) print() sum_acc += ref_tree.accurracy(pred_tree) return sum_acc / N def train(self, dataset, step_size=1.0, max_epochs=100, beam_size=4): """ @param dataset : a list of dependency trees """ N = len(dataset) sequences = list([(dtree.tokens, self.oracle_derivation(dtree)) for dtree in dataset]) for e in range(max_epochs): loss = 0.0 for tokens, ref_derivation in sequences: pred_beam = self.parse_one(tokens, beam_size, get_beam=True) (update, ref_prefix, pred_prefix) = self.early_prefix(ref_derivation, pred_beam) #print('R',ref_derivation) #print('P',pred_prefix) #self.test(dataset,beam_size) if update: #print (pred_prefix) loss += 1.0 delta_ref = SparseWeightVector() current_config = ref_prefix[0][1] for action, config in ref_prefix: S, B, A, score = current_config x_repr = self.__make_config_representation( S, B, tokens) delta_ref += SparseWeightVector.code_phi( x_repr, action) current_config = config delta_pred = SparseWeightVector() current_config = pred_prefix[0][1] for action, config in pred_prefix: S, B, A, score = current_config x_repr = self.__make_config_representation( S, B, tokens) delta_pred += SparseWeightVector.code_phi( x_repr, action) current_config = config self.model += step_size * (delta_ref - delta_pred) print('Loss = ', loss, "%Exact match = ", (N - loss) / N) if loss == 0.0: return
def __init__(self): self.model = SparseWeightVector() self.Y = [] #classes self.source_tag = "@@@"
class ArcEagerTransitionParser: #actions LEFTARC = "LA" RIGHTARC = "RA" SHIFT = "S" REDUCE = "R" TERMINATE = "T" def __init__(self): self.model = SparseWeightVector() @staticmethod def static_oracle(configuration, reference_arcs, N): """ @param configuration: a parser configuration @param reference arcs: a set of dependency arcs @param N: the length of the input sequence @return the action to execute given config and reference arcs """ S, B, A, score = configuration all_words = range(N) if S and B: i, j = S[-1], B[0] if i != 0 and (j, i) in reference_arcs: return ArcEagerTransitionParser.LEFTARC if (i, j) in reference_arcs: return ArcEagerTransitionParser.RIGHTARC if S and any([(k,S[-1]) in A for k in all_words])\ and all ([(S[-1],k) in A for k in all_words if (S[-1],k) in reference_arcs]): return ArcEagerTransitionParser.REDUCE if B: return ArcEagerTransitionParser.SHIFT return ArcEagerTransitionParser.TERMINATE @staticmethod def dynamic_oracle(configuration, action, reference_arcs): """ Computes the cost of an action given a configuration and a reference tree @param configuration: a parser configuration tuple @param reference_arcs a set of dependencies @return a bool set to true if cost = 0 , false otherwise (cost > 0 or impossible action) """ S, B, A, score = configuration if S and B: i, j = S[-1], B[0] if action == ArcEagerTransitionParser.LEFTARC: if any([(k, i) in reference_arcs for k in B[1:]]): return False if any([(i, k) in reference_arcs for k in B]): return False return True elif action == ArcEagerTransitionParser.RIGHTARC: if any([(k, j) in reference_arcs for k in B]): return False if any([(k, j) in reference_arcs for k in S[:-1]]): return False if any([(j, k) in reference_arcs for k in S]): return False return True if S: if action == ArcEagerTransitionParser.REDUCE: if any([(i, k) in reference_arcs for k in B]): return False return True if B: if action == ArcEagerTransitionParser.SHIFT: if any([(j, k) in reference_arcs or (k, j) in reference_arcs for k in S]): return False return True if not B and action == ArcEagerTransitionParser.TERMINATE: return True return False def static_oracle_derivation(self, ref_parse): """ This generates a static oracle reference derivation from a sentence @param ref_parse: a DependencyTree object @return : the oracle derivation as a list of (Configuration,action,toklist) triples """ sentence = ref_parse.tokens edges = set(ref_parse.edges) N = len(sentence) C = ((0, ), tuple(range(1, len(sentence))), tuple(), 0.0 ) #A config is a hashable quadruple with score action = ArcEagerTransitionParser.static_oracle(C, edges, N) derivation = [(C, action, sentence)] while C[1] and action != ArcEagerTransitionParser.TERMINATE: #print(C,action) if action == ArcEagerTransitionParser.SHIFT: C = self.shift(C, sentence) elif action == ArcEagerTransitionParser.LEFTARC: C = self.leftarc(C, sentence) elif action == ArcEagerTransitionParser.RIGHTARC: C = self.rightarc(C, sentence) elif action == ArcEagerTransitionParser.REDUCE: C = self.reduce_config(C, sentence) elif action == ArcEagerTransitionParser.TERMINATE: C = self.terminate(C, sentence) action = ArcEagerTransitionParser.static_oracle(C, edges, N) derivation.append((C, action, sentence)) return derivation def shift(self, configuration, tokens): """ Performs the shift action and returns a new configuration """ S, B, A, score = configuration w0 = B[0] return ( S + (w0, ), B[1:], A, score + self.score(configuration, ArcEagerTransitionParser.SHIFT, tokens)) def leftarc(self, configuration, tokens): """ Performs the left arc action and returns a new configuration """ S, B, A, score = configuration i, j = S[-1], B[0] return (S[:-1], B, A + ((j, i), ), score + self.score( configuration, ArcEagerTransitionParser.LEFTARC, tokens)) def rightarc(self, configuration, tokens): S, B, A, score = configuration i, j = S[-1], B[0] return (S + (j, ), B[1:], A + ((i, j), ), score + self.score( configuration, ArcEagerTransitionParser.RIGHTARC, tokens)) def reduce_config(self, configuration, tokens): S, B, A, score = configuration return ( S[:-1], B, A, score + self.score(configuration, ArcEagerTransitionParser.REDUCE, tokens)) def terminate(self, configuration, tokens): S, B, A, score = configuration return (S, B, A, score + self.score( configuration, ArcEagerTransitionParser.TERMINATE, tokens)) def predict_local(self, configuration, sentence, allowed=None): """ Statistical prediction of an action given a configuration @param configuration: a tuple (S,B,A,score) @param sentence: a list of tokens @param allowed: a list of allowed actions @return (new_config,action_performed) """ action_set = set([ArcEagerTransitionParser.LEFTARC,ArcEagerTransitionParser.RIGHTARC,\ ArcEagerTransitionParser.SHIFT,ArcEagerTransitionParser.REDUCE,\ ArcEagerTransitionParser.TERMINATE]) if allowed: action_set = set(allowed) N = len(sentence) S, B, A, score = configuration candidates = [] if B and ArcEagerTransitionParser.SHIFT in action_set: candidates.append( (self.shift(configuration, sentence), ArcEagerTransitionParser.SHIFT)) if S and ArcEagerTransitionParser.REDUCE in action_set: i = S[-1] if any([(k, i) in A for k in range(N)]): candidates.append((self.reduce_config(configuration, sentence), ArcEagerTransitionParser.REDUCE)) if S and B: if ArcEagerTransitionParser.LEFTARC in action_set: i = S[-1] if i != 0 and not any([(k, i) in A for k in range(N)]): candidates.append((self.leftarc(configuration, sentence), ArcEagerTransitionParser.LEFTARC)) if ArcEagerTransitionParser.RIGHTARC in action_set: j = B[0] if not any([(k, j) in A for k in range(N)]): candidates.append((self.rightarc(configuration, sentence), ArcEagerTransitionParser.RIGHTARC)) if not B and ArcEagerTransitionParser.TERMINATE in action_set: candidates.append( (self.terminate(configuration, sentence), ArcEagerTransitionParser.TERMINATE)) if candidates: candidates.sort(key=lambda x: x[0][3], reverse=True) return candidates[0] else: #emergency exit when we have no candidate return (C, ArcEagerTransitionParser.TERMINATE) def parse_one(self, sentence): """ Greedy parsing @param sentence: a list of tokens """ N = len(sentence) C = ((0, ), tuple(range(1, N)), tuple(), 0.0 ) #A config is a hashable quadruple with score action = None while action != ArcEagerTransitionParser.TERMINATE: C, action = self.predict_local(C, sentence) #Connects any remaining dummy root to 0 S, B, A, score = C Aset = set(A) for s in S: if s != 0 and not any([(k, s) in Aset for k in range(N)]): Aset.add((0, s)) return DependencyTree(tokens=sentence, edges=list(Aset)) def score(self, configuration, action, tokens): """ Computes the prefix score of a derivation @param configuration : a quintuple (S,B,A,score,history) @param action: an action label in {LEFTARC,RIGHTARC,REDUCE,TERMINATE,SHIFT} @param tokens: the x-sequence of tokens to be parsed @return a prefix score """ S, B, A, old_score = configuration config_repr = self.__make_config_representation(S, B, tokens) return old_score + self.model.dot(config_repr, action) def __make_config_representation(self, S, B, tokens): """ This gathers the information for coding the configuration as a feature vector. @param S: a configuration stack @param B a configuration buffer @return an ordered list of tuples """ #default values for inaccessible positions s0w, s1w, s0t, s1t, b0w, b1w, b0t, b1t = "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_", "_UNDEF_" if len(S) > 0: s0w, s0t = tokens[S[-1]][0], tokens[S[-1]][1] if len(S) > 1: s1w, s1t = tokens[S[-2]][0], tokens[S[-2]][1] if len(B) > 0: b0w, b0t = tokens[B[0]][0], tokens[B[0]][1] if len(B) > 1: b1w, b1t = tokens[B[1]][0], tokens[B[1]][1] wordlist = [s0w, s1w, b0w, b1w] taglist = [s0t, s1t, b0t, b1t] word_bigrams = list(zip(wordlist, wordlist[1:])) tag_bigrams = list(zip(taglist, taglist[1:])) word_trigrams = list(zip(wordlist, wordlist[1:], wordlist[2:])) tag_trigrams = list(zip(taglist, taglist[1:], taglist[2:])) return word_bigrams + tag_bigrams + word_trigrams + tag_trigrams def test(self, dataset): """ @param dataset: a list of DependencyTrees @param beam_size: size of the beam """ N = len(dataset) sum_acc = 0.0 for ref_tree in dataset: tokens = ref_tree.tokens pred_tree = self.parse_one(tokens) print(pred_tree) print() sum_acc += ref_tree.accurracy(pred_tree) return sum_acc / N def choose(self, pred_action, optimal_actions): """ Choice function for dynamic oracle. Chooses next action in case of ambiguity. (does not perform exploration) """ if pred_action in optimal_actions: return pred_action else: return optimal_actions[randrange(0, len(optimal_actions))] def dynamic_train(self, treebank, step_size=1.0, max_epochs=100): ACTIONS = [ArcEagerTransitionParser.LEFTARC,ArcEagerTransitionParser.RIGHTARC,\ ArcEagerTransitionParser.SHIFT,ArcEagerTransitionParser.REDUCE,\ ArcEagerTransitionParser.TERMINATE] N = len(treebank) for e in range(max_epochs): loss, total = 0, 0 for dtree in treebank: ref_arcs = set(dtree.edges) n = len(dtree.tokens) C = ((0, ), tuple(range(1, n)), tuple(), 0.0 ) #A config is a hashable quadruple with score action = None while action != ArcEagerTransitionParser.TERMINATE: pred_config, pred_action = self.predict_local( C, dtree.tokens) optimal_actions = list([ a for a in ACTIONS if self.dynamic_oracle(C, a, ref_arcs) ]) total += 1 if pred_action not in optimal_actions: loss += 1 optimal_config, optimal_action = self.predict_local( C, dtree.tokens, allowed=optimal_actions) delta_ref = SparseWeightVector() S, B, A, score = C x_repr = self.__make_config_representation( S, B, dtree.tokens) delta_ref += SparseWeightVector.code_phi( x_repr, optimal_action) delta_pred = SparseWeightVector() S, B, A, score = C x_repr = self.__make_config_representation( S, B, dtree.tokens) delta_pred += SparseWeightVector.code_phi( x_repr, pred_action) self.model += step_size * (delta_ref - delta_pred) action = self.choose(pred_action, optimal_actions) if action == ArcEagerTransitionParser.SHIFT: C = self.shift(C, dtree.tokens) elif action == ArcEagerTransitionParser.LEFTARC: C = self.leftarc(C, dtree.tokens) elif action == ArcEagerTransitionParser.RIGHTARC: C = self.rightarc(C, dtree.tokens) elif action == ArcEagerTransitionParser.REDUCE: C = self.reduce_config(C, dtree.tokens) elif action == ArcEagerTransitionParser.TERMINATE: C = self.terminate(C, dtree.tokens) print('Loss = ', loss, "%Local accurracy = ", (total - loss) / total) if loss == 0.0: return def static_train(self, treebank, step_size=1.0, max_epochs=100): """ Trains a model with a static oracle @param treebank : a list of dependency trees """ dataset = [] for dtree in treebank: dataset.extend(self.static_oracle_derivation(dtree)) N = len(dataset) for e in range(max_epochs): loss = 0.0 for ref_config, ref_action, tokens in dataset: pred_config, pred_action = self.predict_local( ref_config, tokens) if ref_action != pred_action: loss += 1.0 delta_ref = SparseWeightVector() S, B, A, score = ref_config x_repr = self.__make_config_representation(S, B, tokens) delta_ref += SparseWeightVector.code_phi( x_repr, ref_action) delta_pred = SparseWeightVector() S, B, A, score = ref_config x_repr = self.__make_config_representation(S, B, tokens) delta_pred += SparseWeightVector.code_phi( x_repr, pred_action) self.model += step_size * (delta_ref - delta_pred) print('Loss = ', loss, "%Local accurracy = ", (N - loss) / N) if loss == 0.0: return