file_contents = "\n".join(output)
f.write(file_contents)
f.close()
print "Finished writing %d scores to output.txt" % (len(output) - 1,)
elif score_stdin or parse_stdin:
import tree_utils
trees = parsers.parse(cmd_utils.get_stdin())
for tree in trees:
print tree
if score_stdin:
sentence_transitions = tree_utils.transitions_in_tree(tree)
sentence_probs = []
for transition in sentence_transitions:
print "Transitions: %s" % (transition)
probs = hmm_utils.prob_of_all_transitions(transition, counts, gram_size=3)
print "Probs: %s" % (probs)
sentence_probs += probs
total = 1
for prob in sentence_probs:
total *= prob
print "Total: %f" % (total,)
elif sentence_parse_stdin:
import sentence_tokenizer
sentences = sentence_tokenizer.parse_sentences(cmd_utils.get_stdin(), use_cache=False)
print sentences
elif word_order_parse_stdin:
import sentence_tokenizer
import word_order
lines = cmd_utils.get_stdin_lines()
issues_in_text = []
f.write(file_contents)
f.close()
print "Finished writing %d scores to output.txt" % (len(output) - 1, )
elif score_stdin or parse_stdin:
import tree_utils
trees = parsers.parse(cmd_utils.get_stdin())
for tree in trees:
print tree
if score_stdin:
sentence_transitions = tree_utils.transitions_in_tree(tree)
sentence_probs = []
for transition in sentence_transitions:
print "Transitions: %s" % (transition)
probs = hmm_utils.prob_of_all_transitions(transition,
counts,
gram_size=3)
print "Probs: %s" % (probs)
sentence_probs += probs
total = 1
for prob in sentence_probs:
total *= prob
print "Total: %f" % (total, )
elif sentence_parse_stdin:
import sentence_tokenizer
sentences = sentence_tokenizer.parse_sentences(cmd_utils.get_stdin(),
use_cache=False)
print sentences
elif word_order_parse_stdin:
import sentence_tokenizer
import word_order
def parse_sentences(line, use_cache=True, include_prob=False):
log("Working on: %s" % (line,), 2)
if use_cache:
correct_parse = cache_get("sentence_tokenizer", line)
if correct_parse:
log("Cache Hit: %s" % (correct_parse[0],), 4)
log("-------------\n", 4)
return correct_parse if include_prob else correct_parse[0]
all_possible_sentences = _possible_sentences_in_line(line)
all_possible_sentence_probs = []
invalid_possible_sentences = []
stored_probs = {}
for possible_sentences in all_possible_sentences:
log("Examining: %s" % (possible_sentences,), 1)
prob_for_sentences = []
sent_is_impossible = False
for possible_sentence in possible_sentences:
if use_cache:
possible_sentence_prob = cache_get('possible_sentences', possible_sentence)
if possible_sentence_prob is not None:
log("Cache Hit: %s (from %s)" % (possible_sentence, 'possible sentences'), 4)
prob_for_sentences.append(possible_sentence_prob)
continue
if contains_any_invalid_setences(possible_sentences, invalid_possible_sentences) or sent_is_impossible:
prob_for_sentences.append(0)
continue
elif possible_sentence in stored_probs:
prob_for_sentences.append(stored_probs[possible_sentence])
continue
sentence_trees = parsers.parse(possible_sentence)
if len(sentence_trees) == 0:
log("Wasn't able to parse input %s" % (possible_sentence,), 0)
prob_for_sentences.append(0)
invalid_possible_sentences.append(possible_sentence)
sent_is_impossible = True
continue
else:
sentence_tree = sentence_trees[0]
if cmd_log_level() >= 4:
print "--------"
print "Pre Simplified Tree"
print sentence_tree
tree_utils.simplify_tree(sentence_tree,
remove_starting_cc=possible_sentences.index(possible_sentence) == 0)
if cmd_log_level() >= 4:
print "--------"
print "Post Simplified Tree"
print sentence_tree
sentence_transitions = tree_utils.transitions_in_tree(sentence_tree)
if not is_possible_sentence(sentence_tree):
log("%s" % (sentence_transitions,), 2)
log("Invalid parse", 2)
prob_for_sentences.append(0)
invalid_possible_sentences.append(possible_sentence)
sent_is_impossible = True
if use_cache:
cache_set('possible_sentences', possible_sentence, 0)
else:
log("%s" % (sentence_transitions,), 2)
sentence_probs = []
for transition in sentence_transitions:
try:
probs = hmm_utils.prob_of_all_transitions(transition, counts, gram_size=3)
except KeyError, e:
log("'Imposible' Tag order", 2, sep=' ** ')
log("%s" % (e,), 2, sep=' ** ')
probs = [0]
sentence_probs += probs
log("Transitions: %s" % (transition,), 3)
log("Probabilities: %s" % (probs,), 3)
attempt_sentence_prob = prod(sentence_probs)
sentence_prob_boost = boost_for_sentence_tree(sentence_tree)
attempt_sentence_prob *= sentence_prob_boost
prob_for_sentences.append(attempt_sentence_prob)
stored_probs[possible_sentence] = attempt_sentence_prob
if use_cache:
cache_set('possible_sentences', possible_sentence, attempt_sentence_prob)
weighted_score = prod(prob_for_sentences) * (weight ** (len(possible_sentences) - 1))
if weighted_score > 0:
log("Valid Parse: %s" % (possible_sentences,), 2)
log(weighted_score, 2)
all_possible_sentence_probs.append(weighted_score)
def parse_sentences(line, use_cache=True, include_prob=False):
log("Working on: %s" % (line, ), 2)
if use_cache:
correct_parse = cache_get("sentence_tokenizer", line)
if correct_parse:
log("Cache Hit: %s" % (correct_parse[0], ), 4)
log("-------------\n", 4)
return correct_parse if include_prob else correct_parse[0]
all_possible_sentences = _possible_sentences_in_line(line)
all_possible_sentence_probs = []
invalid_possible_sentences = []
stored_probs = {}
for possible_sentences in all_possible_sentences:
log("Examining: %s" % (possible_sentences, ), 1)
prob_for_sentences = []
sent_is_impossible = False
for possible_sentence in possible_sentences:
if use_cache:
possible_sentence_prob = cache_get('possible_sentences',
possible_sentence)
if possible_sentence_prob is not None:
log(
"Cache Hit: %s (from %s)" %
(possible_sentence, 'possible sentences'), 4)
prob_for_sentences.append(possible_sentence_prob)
continue
if contains_any_invalid_setences(
possible_sentences,
invalid_possible_sentences) or sent_is_impossible:
prob_for_sentences.append(0)
continue
elif possible_sentence in stored_probs:
prob_for_sentences.append(stored_probs[possible_sentence])
continue
sentence_trees = parsers.parse(possible_sentence)
if len(sentence_trees) == 0:
log("Wasn't able to parse input %s" % (possible_sentence, ), 0)
prob_for_sentences.append(0)
invalid_possible_sentences.append(possible_sentence)
sent_is_impossible = True
continue
else:
sentence_tree = sentence_trees[0]
if cmd_log_level() >= 4:
print "--------"
print "Pre Simplified Tree"
print sentence_tree
tree_utils.simplify_tree(
sentence_tree,
remove_starting_cc=possible_sentences.index(
possible_sentence) == 0)
if cmd_log_level() >= 4:
print "--------"
print "Post Simplified Tree"
print sentence_tree
sentence_transitions = tree_utils.transitions_in_tree(
sentence_tree)
if not is_possible_sentence(sentence_tree):
log("%s" % (sentence_transitions, ), 2)
log("Invalid parse", 2)
prob_for_sentences.append(0)
invalid_possible_sentences.append(possible_sentence)
sent_is_impossible = True
if use_cache:
cache_set('possible_sentences', possible_sentence, 0)
else:
log("%s" % (sentence_transitions, ), 2)
sentence_probs = []
for transition in sentence_transitions:
try:
probs = hmm_utils.prob_of_all_transitions(transition,
counts,
gram_size=3)
except KeyError, e:
log("'Imposible' Tag order", 2, sep=' ** ')
log("%s" % (e, ), 2, sep=' ** ')
probs = [0]
sentence_probs += probs
log("Transitions: %s" % (transition, ), 3)
log("Probabilities: %s" % (probs, ), 3)
attempt_sentence_prob = prod(sentence_probs)
sentence_prob_boost = boost_for_sentence_tree(sentence_tree)
attempt_sentence_prob *= sentence_prob_boost
prob_for_sentences.append(attempt_sentence_prob)
stored_probs[possible_sentence] = attempt_sentence_prob
if use_cache:
cache_set('possible_sentences', possible_sentence,
attempt_sentence_prob)
weighted_score = prod(prob_for_sentences) * (weight**(
len(possible_sentences) - 1))
if weighted_score > 0:
log("Valid Parse: %s" % (possible_sentences, ), 2)
log(weighted_score, 2)
all_possible_sentence_probs.append(weighted_score)
