def get_words(expn, parent, lmk=None, rel=None):
words = []
probs = []
entropy = []
for n in expn.split():
if n in NONTERMINALS:
if n == parent == 'LANDMARK-PHRASE':
# we need to move to the parent landmark
lmk = parent_landmark(lmk)
# we need to keep expanding
expansion, exp_prob, exp_ent = get_expansion(n, parent, lmk, rel)
w, w_prob, w_ent = get_words(expansion, n, lmk, rel)
words.append(w)
probs.append(exp_prob * w_prob)
entropy.append(exp_ent + w_ent)
else:
# get word for POS
w_db = Word.get_words(pos=n, lmk=lmk_id(lmk), rel=rel_type(rel))
counter = collections.Counter(w_db)
keys, counts = zip(*counter.items())
counts = np.array(counts)
counts /= counts.sum()
w, w_prob, w_entropy = categorical_sample(keys, counts)
words.append(w.word)
probs.append(w.prob)
entropy.append(w_entropy)
p, H = np.prod(probs), np.sum(entropy)
print 'expanding %s to %s (p: %f, H: %f)' % (expn, words, p, H)
return words, p, H
def train_rec( tree, parent=None, lmk=None, rel=None, prev_word='<no prev word>', update=1, printing=False):
lhs = tree.node
if isinstance(tree[0], ParentedTree): rhs = ' '.join(n.node for n in tree)
else: rhs = ' '.join(n for n in tree)
# check if this version of nltk uses a function for parent
if hasattr( tree.parent, '__call__' ):
parent = tree.parent().node if tree.parent() else None
else:
parent = tree.parent.node if tree.parent else None
if lhs == 'RELATION':
# everything under a RELATION node should ignore the landmark
lmk = None
if lhs == 'LANDMARK-PHRASE':
# everything under a LANDMARK-PHRASE node should ignore the relation
rel = None
if lhs == parent == 'LANDMARK-PHRASE':
# we need to move to the parent landmark
lmk = parent_landmark(lmk)
lmk_class = (lmk.object_class if lmk and lhs != 'LOCATION-PHRASE' else None)
lmk_ori_rels = get_lmk_ori_rels_str(lmk) if lhs != 'LOCATION-PHRASE' else None
lmk_color = (lmk.color if lmk and lhs != 'LOCATION-PHRASE' else None)
if lhs in NONTERMINALS:
update_expansion_counts(update=update,
lhs=lhs,
rhs=rhs,
parent=parent,
lmk_class=lmk_class,
lmk_ori_rels=lmk_ori_rels,
lmk_color=lmk_color,
rel=rel)
for subtree in tree:
prev_word = train_rec(tree=subtree, parent=parent, lmk=lmk, rel=rel, prev_word=prev_word, printing=printing)
else:
update_word_counts(update=update,
pos=lhs,
word=rhs,
prev_word=prev_word,
lmk_class=lmk_class,
lmk_ori_rels=lmk_ori_rels,
lmk_color=lmk_color,
rel=rel)
return rhs
def get_tree_prob(tree, lmk=None, rel=None):
prob = 1.0
if len(tree.productions()) == 1:
# if this tree only has one production
# it means that its child is a terminal (word)
word = tree[0]
pos = tree.node
p = WordCPT.probability(word=word, pos=pos,
lmk=lmk_id(lmk), rel=rel_type(rel))
print p, pos, '->', word, m2s(lmk,rel)
prob *= p
else:
lhs = tree.node
rhs = ' '.join(n.node for n in tree)
parent = tree.parent().node if tree.parent() else None
if lhs == 'RELATION':
# everything under a RELATION node should ignore the landmark
lmk = None
elif lhs == 'LANDMARK-PHRASE':
# everything under a LANDMARK-PHRASE node should ignore the relation
rel = None
if parent == 'LANDMARK-PHRASE':
# if the current node is a LANDMARK-PHRASE and the parent node
# is also a LANDMARK-PHRASE then we should move to the parent
# of the current landmark
lmk = parent_landmark(lmk)
if not parent:
# LOCATION-PHRASE has no parent and is not related to lmk and rel
p = ExpansionCPT.probability(rhs=rhs, lhs=lhs)
print p, repr(lhs), '->', repr(rhs)
else:
p = ExpansionCPT.probability(rhs=rhs, lhs=lhs, parent=parent,
lmk=lmk_id(lmk), rel=rel_type(rel))
print p, repr(lhs), '->', repr(rhs), 'parent=%r'%parent, m2s(lmk,rel)
prob *= p
# call get_tree_prob recursively for each subtree
for subtree in tree:
prob *= get_tree_prob(subtree, lmk, rel)
return prob
def save_tree(tree, loc, rel, lmk, parent=None):
if len(tree.productions()) == 1:
# if this tree only has one production
# it means that its child is a terminal (word)
word = Word()
word.word = tree[0]
word.pos = tree.node
word.parent = parent
word.location = loc
else:
prod = Production()
prod.lhs = tree.node
prod.rhs = ' '.join(n.node for n in tree)
prod.parent = parent
prod.location = loc
# some productions are related to semantic representation
if prod.lhs == 'RELATION':
prod.relation = rel_type(rel)
if hasattr(rel, 'measurement'):
prod.relation_distance_class = rel.measurement.best_distance_class
prod.relation_degree_class = rel.measurement.best_degree_class
elif prod.lhs == 'LANDMARK-PHRASE':
prod.landmark = lmk_id(lmk)
prod.landmark_class = lmk.object_class
prod.landmark_orientation_relations = get_lmk_ori_rels_str(lmk)
prod.landmark_color = lmk.color
# next landmark phrase will need the parent landmark
lmk = parent_landmark(lmk)
elif prod.lhs == 'LANDMARK':
# LANDMARK has the same landmark as its parent LANDMARK-PHRASE
prod.landmark = parent.landmark
prod.landmark_class = parent.landmark_class
prod.landmark_orientation_relations = parent.landmark_orientation_relations
prod.landmark_color = parent.landmark_color
# save subtrees, keeping track of parent
for subtree in tree:
save_tree(subtree, loc, rel, lmk, prod)
def save_tree(tree, loc, rel, lmk, parent=None):
if len(tree.productions()) == 1:
# if this tree only has one production
# it means that its child is a terminal (word)
word = Word()
word.word = tree[0]
word.pos = tree.node
word.parent = parent
word.location = loc
else:
prod = Production()
prod.lhs = tree.node
prod.rhs = ' '.join(n.node for n in tree)
prod.parent = parent
prod.location = loc
# some productions are related to semantic representation
if prod.lhs == 'RELATION':
prod.relation = rel_type(rel)
if hasattr(rel, 'measurement'):
prod.relation_distance_class = rel.measurement.best_distance_class
prod.relation_degree_class = rel.measurement.best_degree_class
elif prod.lhs == 'LANDMARK-PHRASE':
prod.landmark = lmk_id(lmk)
prod.landmark_class = lmk.object_class
prod.landmark_orientation_relations = get_lmk_ori_rels_str(lmk)
prod.landmark_color = lmk.color
# next landmark phrase will need the parent landmark
lmk = parent_landmark(lmk)
elif prod.lhs == 'LANDMARK':
# LANDMARK has the same landmark as its parent LANDMARK-PHRASE
prod.landmark = parent.landmark
prod.landmark_class = parent.landmark_class
prod.landmark_orientation_relations = parent.landmark_orientation_relations
prod.landmark_color = parent.landmark_color
# save subtrees, keeping track of parent
for subtree in tree:
save_tree(subtree, loc, rel, lmk, prod)
def get_expansion(lhs, parent=None, lmk=None, rel=None):
lhs_rhs_parent_chain = []
prob_chain = []
entropy_chain = []
terminals = []
landmarks = []
for n in lhs.split():
if n in NONTERMINALS:
if n == parent == 'LANDMARK-PHRASE':
# we need to move to the parent landmark
lmk = parent_landmark(lmk)
lmk_class = (lmk.object_class if lmk else None)
lmk_ori_rels = get_lmk_ori_rels_str(lmk)
lmk_color = (lmk.color if lmk else None)
rel_class = rel_type(rel)
dist_class = (rel.measurement.best_distance_class if hasattr(rel, 'measurement') else None)
deg_class = (rel.measurement.best_degree_class if hasattr(rel, 'measurement') else None)
cp_db = CProduction.get_production_counts(lhs=n,
parent=parent,
lmk_class=lmk_class,
lmk_ori_rels=lmk_ori_rels,
lmk_color=lmk_color,
rel=rel_class,
dist_class=dist_class,
deg_class=deg_class)
if cp_db.count() <= 0:
logger('Could not expand %s (parent: %s, lmk_class: %s, lmk_ori_rels: %s, lmk_color: %s, rel: %s, dist_class: %s, deg_class: %s)' % (n, parent, lmk_class, lmk_ori_rels, lmk_color, rel_class, dist_class, deg_class))
terminals.append( n )
continue
ckeys, ccounts = zip(*[(cprod.rhs,cprod.count) for cprod in cp_db.all()])
ccounter = {}
for cprod in cp_db.all():
if cprod.rhs in ccounter: ccounter[cprod.rhs] += cprod.count
else: ccounter[cprod.rhs] = cprod.count
ckeys, ccounts = zip(*ccounter.items())
# print 'ckeys', ckeys
# print 'ccounts', ccounts
ccounts = np.array(ccounts, dtype=float)
ccounts /= ccounts.sum()
cprod, cprod_prob, cprod_entropy = categorical_sample(ckeys, ccounts)
# print cprod, cprod_prob, cprod_entropy
lhs_rhs_parent_chain.append( ( n,cprod,parent,lmk ) )
prob_chain.append( cprod_prob )
entropy_chain.append( cprod_entropy )
lrpc, pc, ec, t, ls = get_expansion( lhs=cprod, parent=n, lmk=lmk, rel=rel )
lhs_rhs_parent_chain.extend( lrpc )
prob_chain.extend( pc )
entropy_chain.extend( ec )
terminals.extend( t )
landmarks.extend( ls )
else:
terminals.append( n )
landmarks.append( lmk )
return lhs_rhs_parent_chain, prob_chain, entropy_chain, terminals, landmarks
def get_tree_probs(tree, lmk=None, rel=None):
lhs_rhs_parent_chain = []
prob_chain = []
entropy_chain = []
term_prods = []
lhs = tree.node
if isinstance(tree[0], ParentedTree): rhs = ' '.join(n.node for n in tree)
else: rhs = ' '.join(n for n in tree)
parent = tree.parent.node if tree.parent else None
if lhs == 'RELATION':
# everything under a RELATION node should ignore the landmark
lmk = None
if lhs == 'LANDMARK-PHRASE':
# everything under a LANDMARK-PHRASE node should ignore the relation
rel = None
if lhs == parent == 'LANDMARK-PHRASE':
# we need to move to the parent landmark
lmk = parent_landmark(lmk)
lmk_class = (lmk.object_class if lmk and lhs != 'LOCATION-PHRASE' else None)
lmk_ori_rels = get_lmk_ori_rels_str(lmk) if lhs != 'LOCATION-PHRASE' else None
lmk_color = (lmk.color if lmk and lhs != 'LOCATION-PHRASE' else None)
rel_class = rel_type(rel) if lhs != 'LOCATION-PHRASE' else None
dist_class = (rel.measurement.best_distance_class if hasattr(rel, 'measurement') and lhs != 'LOCATION-PHRASE' else None)
deg_class = (rel.measurement.best_degree_class if hasattr(rel, 'measurement') and lhs != 'LOCATION-PHRASE' else None)
if lhs in NONTERMINALS:
cp_db = CProduction.get_production_counts(lhs=lhs,
parent=parent,
lmk_class=lmk_class,
lmk_ori_rels=lmk_ori_rels,
lmk_color=lmk_color,
rel=rel_class,
dist_class=dist_class,
deg_class=deg_class)
if cp_db.count() <= 0:
logger('Could not expand %s (parent: %s, lmk_class: %s, lmk_ori_rels: %s, lmk_color: %s, rel: %s, dist_class: %s, deg_class: %s)' % (lhs, parent, lmk_class, lmk_ori_rels, lmk_color, rel_class, dist_class, deg_class))
else:
ckeys, ccounts = zip(*[(cprod.rhs,cprod.count) for cprod in cp_db.all()])
ccounter = {}
for cprod in cp_db.all():
if cprod.rhs in ccounter: ccounter[cprod.rhs] += cprod.count
else: ccounter[cprod.rhs] = cprod.count + 1
# we have never seen this RHS in this context before
if rhs not in ccounter: ccounter[rhs] = 1
ckeys, ccounts = zip(*ccounter.items())
# add 1 smoothing
ccounts = np.array(ccounts, dtype=float)
ccount_probs = ccounts / ccounts.sum()
cprod_entropy = -np.sum( (ccount_probs * np.log(ccount_probs)) )
cprod_prob = ccounter[rhs]/ccounts.sum()
# logger('ckeys: %s' % str(ckeys))
# logger('ccounts: %s' % str(ccounts))
# logger('rhs: %s, cprod_prob: %s, cprod_entropy: %s' % (rhs, cprod_prob, cprod_entropy))
prob_chain.append( cprod_prob )
entropy_chain.append( cprod_entropy )
lhs_rhs_parent_chain.append( ( lhs, rhs, parent, lmk, rel ) )
for subtree in tree:
pc, ec, lrpc, tps = get_tree_probs(subtree, lmk, rel)
prob_chain.extend( pc )
entropy_chain.extend( ec )
lhs_rhs_parent_chain.extend( lrpc )
term_prods.extend( tps )
else:
cw_db = CWord.get_word_counts(pos=lhs,
lmk_class=lmk_class,
lmk_ori_rels=lmk_ori_rels,
lmk_color=lmk_color,
rel=rel_class,
rel_dist_class=dist_class,
rel_deg_class=deg_class)
if cw_db.count() <= 0:
# we don't know the probability or entropy values for the context we have never seen before
# we just update the term_prods list
logger('Could not expand %s (lmk_class: %s, lmk_ori_rels: %s, lmk_color: %s, rel: %s, dist_class: %s, deg_class: %s)' % (lhs, lmk_class, lmk_ori_rels, lmk_color, rel_class, dist_class, deg_class))
else:
ckeys, ccounts = zip(*[(cword.word,cword.count) for cword in cw_db.all()])
ccounter = {}
for cword in cw_db.all():
if cword.word in ccounter: ccounter[cword.word] += cword.count
else: ccounter[cword.word] = cword.count + 1
# we have never seen this RHS in this context before
if rhs not in ccounter: ccounter[rhs] = 1
ckeys, ccounts = zip(*ccounter.items())
# logger('ckeys: %s' % str(ckeys))
# logger('ccounts: %s' % str(ccounts))
# add 1 smoothing
ccounts = np.array(ccounts, dtype=float)
ccount_probs = ccounts/ccounts.sum()
w_prob = ccounter[rhs]/ccounts.sum()
w_entropy = -np.sum( (ccount_probs * np.log(ccount_probs)) )
prob_chain.append(w_prob)
entropy_chain.append(w_entropy)
term_prods.append( (lhs, rhs, lmk, rel) )
return prob_chain, entropy_chain, lhs_rhs_parent_chain, term_prods
def get_expansion(lhs, parent=None, lmk=None, rel=None):
lhs_rhs_parent_chain = []
prob_chain = []
entropy_chain = []
terminals = []
landmarks = []
for n in lhs.split():
if n in NONTERMINALS:
if n == parent == 'LANDMARK-PHRASE':
# we need to move to the parent landmark
lmk = parent_landmark(lmk)
lmk_class = (lmk.object_class if lmk else None)
lmk_ori_rels = get_lmk_ori_rels_str(lmk)
lmk_color = (lmk.color if lmk else None)
rel_class = rel_type(rel)
dist_class = (rel.measurement.best_distance_class if hasattr(
rel, 'measurement') else None)
deg_class = (rel.measurement.best_degree_class if hasattr(
rel, 'measurement') else None)
cp_db = CProduction.get_production_counts(
lhs=n,
parent=parent,
lmk_class=lmk_class,
lmk_ori_rels=lmk_ori_rels,
lmk_color=lmk_color,
rel=rel_class,
dist_class=dist_class,
deg_class=deg_class)
if cp_db.count() <= 0:
logger(
'Could not expand %s (parent: %s, lmk_class: %s, lmk_ori_rels: %s, lmk_color: %s, rel: %s, dist_class: %s, deg_class: %s)'
% (n, parent, lmk_class, lmk_ori_rels, lmk_color,
rel_class, dist_class, deg_class))
terminals.append(n)
continue
ckeys, ccounts = zip(*[(cprod.rhs, cprod.count)
for cprod in cp_db.all()])
ccounter = {}
for cprod in cp_db.all():
if cprod.rhs in ccounter: ccounter[cprod.rhs] += cprod.count
else: ccounter[cprod.rhs] = cprod.count
ckeys, ccounts = zip(*ccounter.items())
# print 'ckeys', ckeys
# print 'ccounts', ccounts
ccounts = np.array(ccounts, dtype=float)
ccounts /= ccounts.sum()
cprod, cprod_prob, cprod_entropy = categorical_sample(
ckeys, ccounts)
# print cprod, cprod_prob, cprod_entropy
lhs_rhs_parent_chain.append((n, cprod, parent, lmk))
prob_chain.append(cprod_prob)
entropy_chain.append(cprod_entropy)
lrpc, pc, ec, t, ls = get_expansion(lhs=cprod,
parent=n,
lmk=lmk,
rel=rel)
lhs_rhs_parent_chain.extend(lrpc)
prob_chain.extend(pc)
entropy_chain.extend(ec)
terminals.extend(t)
landmarks.extend(ls)
else:
terminals.append(n)
landmarks.append(lmk)
return lhs_rhs_parent_chain, prob_chain, entropy_chain, terminals, landmarks