def test_intent_evaluation_report(tmpdir_factory):
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
report_filename = os.path.join(report_folder, "intent_report.json")
utils.create_dir(report_folder)
intent_results = [
IntentEvaluationResult("", "restaurant_search",
"I am hungry", 0.12345),
IntentEvaluationResult("greet", "greet",
"hello", 0.98765)]
result = evaluate_intents(intent_results,
report_folder,
successes_filename=None,
errors_filename=None,
confmat_filename=None,
intent_hist_filename=None)
report = json.loads(utils.read_file(report_filename))
greet_results = {"precision": 1.0,
"recall": 1.0,
"f1-score": 1.0,
"support": 1}
prediction = {'text': 'hello',
'intent': 'greet',
'predicted': 'greet',
'confidence': 0.98765}
assert len(report.keys()) == 4
assert report["greet"] == greet_results
assert result["predictions"][0] == prediction
def test_intent_evaluation_report(tmpdir_factory):
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
report_filename = os.path.join(report_folder, "intent_report.json")
utils.create_dir(report_folder)
intent_results = [
IntentEvaluationResult("", "restaurant_search",
"I am hungry", 0.12345),
IntentEvaluationResult("greet", "greet",
"hello", 0.98765)]
result = evaluate_intents(intent_results,
report_folder,
successes_filename=None,
errors_filename=None,
confmat_filename=None,
intent_hist_filename=None)
report = json.loads(utils.read_file(report_filename))
greet_results = {"precision": 1.0,
"recall": 1.0,
"f1-score": 1.0,
"support": 1}
prediction = {'text': 'hello',
'intent': 'greet',
'predicted': 'greet',
'confidence': 0.98765}
assert len(report.keys()) == 4
assert report["greet"] == greet_results
assert result["predictions"][0] == prediction
def test_entity_evaluation_report(tmpdir_factory):
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
mock_extractors = ["A", "B"]
report_filename_a = os.path.join(report_folder, "A_report.json")
report_filename_b = os.path.join(report_folder, "B_report.json")
utils.create_dir(report_folder)
result = evaluate_entities([EN_targets], [EN_predicted], [EN_tokens],
mock_extractors, report_folder)
report_a = json.loads(utils.read_file(report_filename_a))
report_b = json.loads(utils.read_file(report_filename_b))
assert len(report_a) == 8
assert report_a["datetime"]["support"] == 1.0
assert report_b["macro avg"]["recall"] == 0.2
assert result["A"]["accuracy"] == 0.75
def test_entity_evaluation_report(tmpdir_factory):
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
mock_extractors = ["A", "B"]
report_filename_a = os.path.join(report_folder, "A_report.json")
report_filename_b = os.path.join(report_folder, "B_report.json")
utils.create_dir(report_folder)
result = evaluate_entities([EN_targets],
[EN_predicted],
[EN_tokens],
mock_extractors,
report_folder)
report_a = json.loads(utils.read_file(report_filename_a))
report_b = json.loads(utils.read_file(report_filename_b))
assert len(report_a) == 8
assert report_a["datetime"]["support"] == 1.0
assert report_b["macro avg"]["recall"] == 0.2
assert result["A"]["accuracy"] == 0.75
def _guess_format(filename: Text) -> Text:
"""Applies heuristics to guess the data format of a file."""
guess = UNK
content = utils.read_file(filename)
try:
js = json.loads(content)
except ValueError:
if any([marker in content for marker in _markdown_section_markers]):
guess = MARKDOWN
else:
for fformat, format_heuristic in _json_format_heuristics.items():
if format_heuristic(js, filename):
guess = fformat
break
return guess
def _guess_format(filename: Text) -> Text:
"""Applies heuristics to guess the data format of a file."""
guess = UNK
content = utils.read_file(filename)
try:
js = json.loads(content)
except ValueError:
if any([marker in content for marker in _markdown_section_markers]):
guess = MARKDOWN
else:
for fformat, format_heuristic in _json_format_heuristics.items():
if format_heuristic(js, filename):
guess = fformat
break
return guess
def _guess_format(filename):
# type: (Text) -> Text
"""Applies heuristics to guess the data format of a file."""
guess = UNK
content = utils.read_file(filename)
try:
js = json.loads(content)
except ValueError:
if "## intent:" in content:
guess = MARKDOWN
else:
for fformat, format_heuristic in _json_format_heuristics.items():
if format_heuristic(js, filename):
guess = fformat
return guess
def _guess_format(filename):
# type: (Text) -> Text
"""Applies heuristics to guess the data format of a file."""
guess = UNK
content = utils.read_file(filename)
try:
js = json.loads(content)
except ValueError:
if "## intent:" in content:
guess = MARKDOWN
else:
for fformat, format_heuristic in _json_format_heuristics.items():
if format_heuristic(js, filename):
guess = fformat
return guess
def load(
cls,
model_dir=None, # type: Optional[Text]
model_metadata=None, # type: Optional[Metadata]
cached_component=None, # type: Optional[RegexFeaturizer]
**kwargs # type: **Any
):
# type: (...) -> RegexFeaturizer
meta = model_metadata.for_component(cls.name)
file_name = meta.get("ner_dicts")
regex_file = file_name
if os.path.exists(regex_file):
known_patterns = utils.read_file(regex_file)
return NerdictFeaturizer(meta, known_patterns=known_patterns)
else:
return NerdictFeaturizer(meta)
def read(self, filename, **kwargs):
"""Reads TrainingData from a file."""
return self.reads(utils.read_file(filename), **kwargs)
def read(self, filename, **kwargs):
"""Reads TrainingData from a file."""
return self.reads(utils.read_file(filename), **kwargs)
