def test_oracle(self):
for passage in self.load_passages():
oracle = Oracle(passage)
state = State(passage)
actions_taken = []
while True:
actions = oracle.get_actions(state)
action = next(iter(actions))
state.transition(action)
actions_taken.append("%s\n" % action)
if state.finished:
break
with open("test_files/standard3.oracle_actions.txt") as f:
self.assertSequenceEqual(actions_taken, f.readlines())
def parse(self, passages, mode=ParseMode.test, evaluate=False):
"""
Parse given passages
:param passages: iterable of passages to parse
:param mode: ParseMode value.
If train, use oracle to train on given passages.
Otherwise, just parse with classifier.
:param evaluate: whether to evaluate parsed passages with respect to given ones.
Only possible when given passages are annotated.
:return: generator of parsed passages (or in train mode, the original ones),
or, if evaluate=True, of pairs of (Passage, Scores).
"""
assert mode in ParseMode, "Invalid parse mode: %s" % mode
train = (mode is ParseMode.train)
if not train and not self.trained:
self.train()
passage_word = "sentence" if Config().args.sentences else \
"paragraph" if Config().args.paragraphs else \
"passage"
self.total_actions = 0
self.total_correct = 0
total_duration = 0
total_tokens = 0
passage_index = 0
if not hasattr(passages, "__iter__"): # Single passage given
passages = (passages, )
for passage_index, passage in enumerate(passages):
l0 = passage.layer(layer0.LAYER_ID)
num_tokens = len(l0.all)
l1 = passage.layer(layer1.LAYER_ID)
labeled = len(l1.all) > 1
assert not train or labeled, "Cannot train on unannotated passage: %s" % passage.ID
assert not evaluate or labeled, "Cannot evaluate on unannotated passage: %s" % passage.ID
print("%s %-7s" % (passage_word, passage.ID),
end=Config().line_end,
flush=True)
started = time.time()
self.action_count = 0
self.correct_count = 0
textutil.annotate(passage, verbose=Config().args.verbose
) # tag POS and parse dependencies
self.state = State(passage)
self.state_hash_history = set()
self.oracle = Oracle(passage) if train else None
failed = False
if ClassifierProperty.require_init_features in self.model.model.get_classifier_properties(
):
self.model.init_features(self.state, train)
try:
self.parse_passage(
train) # This is where the actual parsing takes place
except ParserException as e:
if train:
raise
Config().log("%s %s: %s" % (passage_word, passage.ID, e))
failed = True
predicted_passage = self.state.create_passage(assert_proper=Config().args.verify) \
if not train or Config().args.verify else passage
duration = time.time() - started
total_duration += duration
num_tokens -= len(self.state.buffer)
total_tokens += num_tokens
if train: # We have an oracle to verify by
if not failed and Config().args.verify:
self.verify_passage(passage, predicted_passage, train)
if self.action_count:
print("%-16s" %
("%d%% (%d/%d)" %
(100 * self.correct_count / self.action_count,
self.correct_count, self.action_count)),
end=Config().line_end)
print("%0.3fs" % duration, end="")
print("%-15s" % (" (failed)" if failed else " (%d tokens/s)" %
(num_tokens / duration)),
end="")
print(Config().line_end, end="")
if train:
print(Config().line_end, flush=True)
self.model.model.finished_item(train)
self.total_correct += self.correct_count
self.total_actions += self.action_count
if train and Config().args.save_every and (
passage_index + 1) % Config().args.save_every == 0:
self.eval_and_save()
self.eval_index += 1
yield (predicted_passage,
evaluate_passage(
predicted_passage,
passage)) if evaluate else predicted_passage
if passages:
print("Parsed %d %ss" % (passage_index + 1, passage_word))
if self.oracle and self.total_actions:
print("Overall %d%% correct transitions (%d/%d) on %s" %
(100 * self.total_correct / self.total_actions,
self.total_correct, self.total_actions, mode.name))
print(
"Total time: %.3fs (average time/%s: %.3fs, average tokens/s: %d)"
% (total_duration, passage_word, total_duration /
(passage_index + 1), total_tokens / total_duration),
flush=True)
