def get_words(expn, parent, lmk=None, rel=None): words = [] probs = [] entropy = [] for n in expn.split(): if n in NONTERMINALS: if n == parent == 'LANDMARK-PHRASE': # we need to move to the parent landmark lmk = parent_landmark(lmk) # we need to keep expanding expansion, exp_prob, exp_ent = get_expansion(n, parent, lmk, rel) w, w_prob, w_ent = get_words(expansion, n, lmk, rel) words.append(w) probs.append(exp_prob * w_prob) entropy.append(exp_ent + w_ent) else: # get word for POS w_db = Word.get_words(pos=n, lmk=lmk_id(lmk), rel=rel_type(rel)) counter = collections.Counter(w_db) keys, counts = zip(*counter.items()) counts = np.array(counts) counts /= counts.sum() w, w_prob, w_entropy = categorical_sample(keys, counts) words.append(w.word) probs.append(w.prob) entropy.append(w_entropy) p, H = np.prod(probs), np.sum(entropy) print 'expanding %s to %s (p: %f, H: %f)' % (expn, words, p, H) return words, p, H
def train_rec( tree, parent=None, lmk=None, rel=None, prev_word='<no prev word>', update=1, printing=False): lhs = tree.node if isinstance(tree[0], ParentedTree): rhs = ' '.join(n.node for n in tree) else: rhs = ' '.join(n for n in tree) # check if this version of nltk uses a function for parent if hasattr( tree.parent, '__call__' ): parent = tree.parent().node if tree.parent() else None else: parent = tree.parent.node if tree.parent else None if lhs == 'RELATION': # everything under a RELATION node should ignore the landmark lmk = None if lhs == 'LANDMARK-PHRASE': # everything under a LANDMARK-PHRASE node should ignore the relation rel = None if lhs == parent == 'LANDMARK-PHRASE': # we need to move to the parent landmark lmk = parent_landmark(lmk) lmk_class = (lmk.object_class if lmk and lhs != 'LOCATION-PHRASE' else None) lmk_ori_rels = get_lmk_ori_rels_str(lmk) if lhs != 'LOCATION-PHRASE' else None lmk_color = (lmk.color if lmk and lhs != 'LOCATION-PHRASE' else None) if lhs in NONTERMINALS: update_expansion_counts(update=update, lhs=lhs, rhs=rhs, parent=parent, lmk_class=lmk_class, lmk_ori_rels=lmk_ori_rels, lmk_color=lmk_color, rel=rel) for subtree in tree: prev_word = train_rec(tree=subtree, parent=parent, lmk=lmk, rel=rel, prev_word=prev_word, printing=printing) else: update_word_counts(update=update, pos=lhs, word=rhs, prev_word=prev_word, lmk_class=lmk_class, lmk_ori_rels=lmk_ori_rels, lmk_color=lmk_color, rel=rel) return rhs
def get_tree_prob(tree, lmk=None, rel=None): prob = 1.0 if len(tree.productions()) == 1: # if this tree only has one production # it means that its child is a terminal (word) word = tree[0] pos = tree.node p = WordCPT.probability(word=word, pos=pos, lmk=lmk_id(lmk), rel=rel_type(rel)) print p, pos, '->', word, m2s(lmk,rel) prob *= p else: lhs = tree.node rhs = ' '.join(n.node for n in tree) parent = tree.parent().node if tree.parent() else None if lhs == 'RELATION': # everything under a RELATION node should ignore the landmark lmk = None elif lhs == 'LANDMARK-PHRASE': # everything under a LANDMARK-PHRASE node should ignore the relation rel = None if parent == 'LANDMARK-PHRASE': # if the current node is a LANDMARK-PHRASE and the parent node # is also a LANDMARK-PHRASE then we should move to the parent # of the current landmark lmk = parent_landmark(lmk) if not parent: # LOCATION-PHRASE has no parent and is not related to lmk and rel p = ExpansionCPT.probability(rhs=rhs, lhs=lhs) print p, repr(lhs), '->', repr(rhs) else: p = ExpansionCPT.probability(rhs=rhs, lhs=lhs, parent=parent, lmk=lmk_id(lmk), rel=rel_type(rel)) print p, repr(lhs), '->', repr(rhs), 'parent=%r'%parent, m2s(lmk,rel) prob *= p # call get_tree_prob recursively for each subtree for subtree in tree: prob *= get_tree_prob(subtree, lmk, rel) return prob
def save_tree(tree, loc, rel, lmk, parent=None): if len(tree.productions()) == 1: # if this tree only has one production # it means that its child is a terminal (word) word = Word() word.word = tree[0] word.pos = tree.node word.parent = parent word.location = loc else: prod = Production() prod.lhs = tree.node prod.rhs = ' '.join(n.node for n in tree) prod.parent = parent prod.location = loc # some productions are related to semantic representation if prod.lhs == 'RELATION': prod.relation = rel_type(rel) if hasattr(rel, 'measurement'): prod.relation_distance_class = rel.measurement.best_distance_class prod.relation_degree_class = rel.measurement.best_degree_class elif prod.lhs == 'LANDMARK-PHRASE': prod.landmark = lmk_id(lmk) prod.landmark_class = lmk.object_class prod.landmark_orientation_relations = get_lmk_ori_rels_str(lmk) prod.landmark_color = lmk.color # next landmark phrase will need the parent landmark lmk = parent_landmark(lmk) elif prod.lhs == 'LANDMARK': # LANDMARK has the same landmark as its parent LANDMARK-PHRASE prod.landmark = parent.landmark prod.landmark_class = parent.landmark_class prod.landmark_orientation_relations = parent.landmark_orientation_relations prod.landmark_color = parent.landmark_color # save subtrees, keeping track of parent for subtree in tree: save_tree(subtree, loc, rel, lmk, prod)
def get_expansion(lhs, parent=None, lmk=None, rel=None): lhs_rhs_parent_chain = [] prob_chain = [] entropy_chain = [] terminals = [] landmarks = [] for n in lhs.split(): if n in NONTERMINALS: if n == parent == 'LANDMARK-PHRASE': # we need to move to the parent landmark lmk = parent_landmark(lmk) lmk_class = (lmk.object_class if lmk else None) lmk_ori_rels = get_lmk_ori_rels_str(lmk) lmk_color = (lmk.color if lmk else None) rel_class = rel_type(rel) dist_class = (rel.measurement.best_distance_class if hasattr(rel, 'measurement') else None) deg_class = (rel.measurement.best_degree_class if hasattr(rel, 'measurement') else None) cp_db = CProduction.get_production_counts(lhs=n, parent=parent, lmk_class=lmk_class, lmk_ori_rels=lmk_ori_rels, lmk_color=lmk_color, rel=rel_class, dist_class=dist_class, deg_class=deg_class) if cp_db.count() <= 0: logger('Could not expand %s (parent: %s, lmk_class: %s, lmk_ori_rels: %s, lmk_color: %s, rel: %s, dist_class: %s, deg_class: %s)' % (n, parent, lmk_class, lmk_ori_rels, lmk_color, rel_class, dist_class, deg_class)) terminals.append( n ) continue ckeys, ccounts = zip(*[(cprod.rhs,cprod.count) for cprod in cp_db.all()]) ccounter = {} for cprod in cp_db.all(): if cprod.rhs in ccounter: ccounter[cprod.rhs] += cprod.count else: ccounter[cprod.rhs] = cprod.count ckeys, ccounts = zip(*ccounter.items()) # print 'ckeys', ckeys # print 'ccounts', ccounts ccounts = np.array(ccounts, dtype=float) ccounts /= ccounts.sum() cprod, cprod_prob, cprod_entropy = categorical_sample(ckeys, ccounts) # print cprod, cprod_prob, cprod_entropy lhs_rhs_parent_chain.append( ( n,cprod,parent,lmk ) ) prob_chain.append( cprod_prob ) entropy_chain.append( cprod_entropy ) lrpc, pc, ec, t, ls = get_expansion( lhs=cprod, parent=n, lmk=lmk, rel=rel ) lhs_rhs_parent_chain.extend( lrpc ) prob_chain.extend( pc ) entropy_chain.extend( ec ) terminals.extend( t ) landmarks.extend( ls ) else: terminals.append( n ) landmarks.append( lmk ) return lhs_rhs_parent_chain, prob_chain, entropy_chain, terminals, landmarks
def get_tree_probs(tree, lmk=None, rel=None): lhs_rhs_parent_chain = [] prob_chain = [] entropy_chain = [] term_prods = [] lhs = tree.node if isinstance(tree[0], ParentedTree): rhs = ' '.join(n.node for n in tree) else: rhs = ' '.join(n for n in tree) parent = tree.parent.node if tree.parent else None if lhs == 'RELATION': # everything under a RELATION node should ignore the landmark lmk = None if lhs == 'LANDMARK-PHRASE': # everything under a LANDMARK-PHRASE node should ignore the relation rel = None if lhs == parent == 'LANDMARK-PHRASE': # we need to move to the parent landmark lmk = parent_landmark(lmk) lmk_class = (lmk.object_class if lmk and lhs != 'LOCATION-PHRASE' else None) lmk_ori_rels = get_lmk_ori_rels_str(lmk) if lhs != 'LOCATION-PHRASE' else None lmk_color = (lmk.color if lmk and lhs != 'LOCATION-PHRASE' else None) rel_class = rel_type(rel) if lhs != 'LOCATION-PHRASE' else None dist_class = (rel.measurement.best_distance_class if hasattr(rel, 'measurement') and lhs != 'LOCATION-PHRASE' else None) deg_class = (rel.measurement.best_degree_class if hasattr(rel, 'measurement') and lhs != 'LOCATION-PHRASE' else None) if lhs in NONTERMINALS: cp_db = CProduction.get_production_counts(lhs=lhs, parent=parent, lmk_class=lmk_class, lmk_ori_rels=lmk_ori_rels, lmk_color=lmk_color, rel=rel_class, dist_class=dist_class, deg_class=deg_class) if cp_db.count() <= 0: logger('Could not expand %s (parent: %s, lmk_class: %s, lmk_ori_rels: %s, lmk_color: %s, rel: %s, dist_class: %s, deg_class: %s)' % (lhs, parent, lmk_class, lmk_ori_rels, lmk_color, rel_class, dist_class, deg_class)) else: ckeys, ccounts = zip(*[(cprod.rhs,cprod.count) for cprod in cp_db.all()]) ccounter = {} for cprod in cp_db.all(): if cprod.rhs in ccounter: ccounter[cprod.rhs] += cprod.count else: ccounter[cprod.rhs] = cprod.count + 1 # we have never seen this RHS in this context before if rhs not in ccounter: ccounter[rhs] = 1 ckeys, ccounts = zip(*ccounter.items()) # add 1 smoothing ccounts = np.array(ccounts, dtype=float) ccount_probs = ccounts / ccounts.sum() cprod_entropy = -np.sum( (ccount_probs * np.log(ccount_probs)) ) cprod_prob = ccounter[rhs]/ccounts.sum() # logger('ckeys: %s' % str(ckeys)) # logger('ccounts: %s' % str(ccounts)) # logger('rhs: %s, cprod_prob: %s, cprod_entropy: %s' % (rhs, cprod_prob, cprod_entropy)) prob_chain.append( cprod_prob ) entropy_chain.append( cprod_entropy ) lhs_rhs_parent_chain.append( ( lhs, rhs, parent, lmk, rel ) ) for subtree in tree: pc, ec, lrpc, tps = get_tree_probs(subtree, lmk, rel) prob_chain.extend( pc ) entropy_chain.extend( ec ) lhs_rhs_parent_chain.extend( lrpc ) term_prods.extend( tps ) else: cw_db = CWord.get_word_counts(pos=lhs, lmk_class=lmk_class, lmk_ori_rels=lmk_ori_rels, lmk_color=lmk_color, rel=rel_class, rel_dist_class=dist_class, rel_deg_class=deg_class) if cw_db.count() <= 0: # we don't know the probability or entropy values for the context we have never seen before # we just update the term_prods list logger('Could not expand %s (lmk_class: %s, lmk_ori_rels: %s, lmk_color: %s, rel: %s, dist_class: %s, deg_class: %s)' % (lhs, lmk_class, lmk_ori_rels, lmk_color, rel_class, dist_class, deg_class)) else: ckeys, ccounts = zip(*[(cword.word,cword.count) for cword in cw_db.all()]) ccounter = {} for cword in cw_db.all(): if cword.word in ccounter: ccounter[cword.word] += cword.count else: ccounter[cword.word] = cword.count + 1 # we have never seen this RHS in this context before if rhs not in ccounter: ccounter[rhs] = 1 ckeys, ccounts = zip(*ccounter.items()) # logger('ckeys: %s' % str(ckeys)) # logger('ccounts: %s' % str(ccounts)) # add 1 smoothing ccounts = np.array(ccounts, dtype=float) ccount_probs = ccounts/ccounts.sum() w_prob = ccounter[rhs]/ccounts.sum() w_entropy = -np.sum( (ccount_probs * np.log(ccount_probs)) ) prob_chain.append(w_prob) entropy_chain.append(w_entropy) term_prods.append( (lhs, rhs, lmk, rel) ) return prob_chain, entropy_chain, lhs_rhs_parent_chain, term_prods
def get_expansion(lhs, parent=None, lmk=None, rel=None): lhs_rhs_parent_chain = [] prob_chain = [] entropy_chain = [] terminals = [] landmarks = [] for n in lhs.split(): if n in NONTERMINALS: if n == parent == 'LANDMARK-PHRASE': # we need to move to the parent landmark lmk = parent_landmark(lmk) lmk_class = (lmk.object_class if lmk else None) lmk_ori_rels = get_lmk_ori_rels_str(lmk) lmk_color = (lmk.color if lmk else None) rel_class = rel_type(rel) dist_class = (rel.measurement.best_distance_class if hasattr( rel, 'measurement') else None) deg_class = (rel.measurement.best_degree_class if hasattr( rel, 'measurement') else None) cp_db = CProduction.get_production_counts( lhs=n, parent=parent, lmk_class=lmk_class, lmk_ori_rels=lmk_ori_rels, lmk_color=lmk_color, rel=rel_class, dist_class=dist_class, deg_class=deg_class) if cp_db.count() <= 0: logger( 'Could not expand %s (parent: %s, lmk_class: %s, lmk_ori_rels: %s, lmk_color: %s, rel: %s, dist_class: %s, deg_class: %s)' % (n, parent, lmk_class, lmk_ori_rels, lmk_color, rel_class, dist_class, deg_class)) terminals.append(n) continue ckeys, ccounts = zip(*[(cprod.rhs, cprod.count) for cprod in cp_db.all()]) ccounter = {} for cprod in cp_db.all(): if cprod.rhs in ccounter: ccounter[cprod.rhs] += cprod.count else: ccounter[cprod.rhs] = cprod.count ckeys, ccounts = zip(*ccounter.items()) # print 'ckeys', ckeys # print 'ccounts', ccounts ccounts = np.array(ccounts, dtype=float) ccounts /= ccounts.sum() cprod, cprod_prob, cprod_entropy = categorical_sample( ckeys, ccounts) # print cprod, cprod_prob, cprod_entropy lhs_rhs_parent_chain.append((n, cprod, parent, lmk)) prob_chain.append(cprod_prob) entropy_chain.append(cprod_entropy) lrpc, pc, ec, t, ls = get_expansion(lhs=cprod, parent=n, lmk=lmk, rel=rel) lhs_rhs_parent_chain.extend(lrpc) prob_chain.extend(pc) entropy_chain.extend(ec) terminals.extend(t) landmarks.extend(ls) else: terminals.append(n) landmarks.append(lmk) return lhs_rhs_parent_chain, prob_chain, entropy_chain, terminals, landmarks