def test_empty_actual(schema, entity_id):
"""Everything should be reported as missing."""
fixture: Dict = {
"some_multiple_text": ["foo"],
"outer": {
"some_multiple_text": ["bar", "baq"],
"inner": {
"some_multiple_text": ["baz"]
}
}
}
observed: Dict = {}
expected: Outcome = Outcome()
expected.missings.append(
MissingValue(entity_id, "immutable", "/some_multiple_text",
"MultipleText", json.dumps(["foo"])))
expected.missings.append(
MissingValue(entity_id, "immutable", "/outer/some_multiple_text",
"MultipleText", json.dumps(["bar", "baq"])))
expected.missings.append(
MissingValue(entity_id, "immutable", "/outer/inner/some_multiple_text",
"MultipleText", json.dumps(["baz"])))
actual: Outcome = Outcome()
crawl: CrawlImmutable = CrawlImmutable(entity_id, schema, fixture,
observed, actual)
crawl()
assert expected == actual
def test_null_actual(schema, period, entity_id):
"""Everything should be reported as missing."""
fixture: Dict = {
"some_multiple_text": ["x", "y"],
"outer": {
"some_multiple_text": ["xx", "yy"],
"inner": {
"some_multiple_text": ["xxx", "yyy"],
}
}
}
expected: Outcome = Outcome()
expected.missings.append(
MissingValue(entity_id, period, "/some_multiple_text", "MultipleText",
json.dumps(fixture["some_multiple_text"])))
expected.missings.append(
MissingValue(entity_id, period, "/outer/some_multiple_text",
"MultipleText",
json.dumps(fixture["outer"]["some_multiple_text"])))
expected.missings.append(
MissingValue(
entity_id, period, "/outer/inner/some_multiple_text",
"MultipleText",
json.dumps(fixture["outer"]["inner"]["some_multiple_text"])))
actual: Outcome = Outcome()
crawl: CrawlPeriod = CrawlPeriod(entity_id, schema, fixture, None, actual,
period)
crawl()
assert expected == actual
def test_null_actual(schema, period, entity_id):
"""Everything should be reported as missing."""
fixture: Dict = {
"some_text": "foo",
"outer": {
"some_text": "bar",
"inner": {
"some_text": "baz"
}
}
}
expected: Outcome = Outcome()
expected.missings.append(
MissingValue(entity_id, period, "/some_text", "Text", "foo"))
expected.missings.append(
MissingValue(entity_id, period, "/outer/some_text", "Text", "bar"))
expected.missings.append(
MissingValue(entity_id, period, "/outer/inner/some_text", "Text",
"baz"))
actual: Outcome = Outcome()
crawl: CrawlPeriod = CrawlPeriod(entity_id, schema, fixture, None, actual,
period)
crawl()
assert expected == actual
def test_almost_same_not_equal():
p: Outcome = Outcome()
p_match: ValueMatch = ValueMatch("the_entity_id", "the_period",
"/path/to/var", "Text", "foo")
p.matches.append(p_match)
p_mismatch: ValueMismatch = ValueMismatch("the_entity_id", "the_period",
"/path/to/var", "Text", "foo",
"bar")
p.mismatches.append(p_mismatch)
p_missing: MissingValue = MissingValue("the_entity_id", "the_period",
"/path/to/var", "Text", None)
p.missings.append(p_missing)
q: Outcome = Outcome()
q_mismatch: ValueMismatch = ValueMismatch("the_entity_id", "the_period",
"/path/to/var", "Text", "foo",
"bar")
q.mismatches.append(q_mismatch)
q_missing: MissingValue = MissingValue("the_entity_id", "the_period",
"/path/to/var", "Text", None)
q.missings.append(q_missing)
assert p != q
def test_missing_values(outcomes):
expected: List[MissingValue] = [
MissingValue("entity_2", "period_2", "/temporal_root_text", "Text",
"JKL"),
MissingValue("entity_2", "period_2",
"/temporal_folder/temporal_folder_text", "Text", "111"),
MissingValue("entity_2", "immutable",
"/immutable_folder/immutable_folder_text", "Text", "222")
]
actual: List[MissingValue] = list(outcomes.missing_values)
assert actual == expected
def test_each_has_different_missing_not_equal():
p: Outcome = Outcome()
p_missing: MissingValue = MissingValue("the_entity_id", "the_period",
"/path/to/var", "Text", None)
p.missings.append(p_missing)
q: Outcome = Outcome()
q_missing: MissingValue = MissingValue("the_entity_id", "the_period",
"/path/to/var", "Text", "Blah")
q.missings.append(q_missing)
assert p != q
def test_both_have_same_missing_equal():
p: Outcome = Outcome()
p_missing: MissingValue = MissingValue("the_entity_id", "the_period",
"/path/to/var", "Text", None)
p.missings.append(p_missing)
q: Outcome = Outcome()
q_missing: MissingValue = MissingValue("the_entity_id", "the_period",
"/path/to/var", "Text", None)
q.missings.append(q_missing)
assert p == q
def test_missing_complex(schema, entity_id, period):
"""All same except for one missing item."""
fixture: Dict = {
"outer": [
{
"the_folder": {
"inner": [{
"some_text": "foo"
}]
}
},
{
"the_folder": {
"inner": [{
"some_text": "bar"
}]
}
},
]
}
observed: Dict = {}
expected: Outcome = Outcome()
expected.missings.append(
MissingValue(entity_id, period, "/outer", "List",
json.dumps(fixture["outer"])))
actual: Outcome = Outcome()
crawl: CrawlPeriod = CrawlPeriod(entity_id, schema, fixture, observed,
actual, period)
crawl()
assert expected == actual
def test_missing_mismatch_simple(schema, period, entity_id):
"""One mismatch, one missing item."""
fixture: Dict = {
"some_text": "foo",
"outer": {
"some_text": "bar",
"inner": {
"some_text": "baz"
}
}
}
observation: Dict = {
"some_text": "foo",
"outer": {
"inner": {
"some_text": "123"
}
}
}
# 1 match, 1 mismatch, 1 missing
expected: Outcome = Outcome()
expected.matches.append(ValueMatch(entity_id, period, "/some_text", "Text", "foo"))
expected.missings.append(MissingValue(entity_id, period, "/outer/some_text", "Text", "bar"))
expected.mismatches.append(ValueMismatch(entity_id, period, "/outer/inner/some_text", "Text", "baz", "123"))
actual: Outcome = Outcome()
crawl: CrawlPeriod = CrawlPeriod(entity_id, schema, fixture, observation, actual, period)
crawl()
assert expected == actual
def _record_missing(self, path: ListType, data_type: str,
value: Optional[Any]) -> None:
if not _is_simple_value(value):
value = json.dumps(value, sort_keys=True)
path_str = nesteddicts.path_to_str(path)
missing: MissingValue = MissingValue(self.entity_id, self.label,
path_str, data_type, value)
self.outcome.missings.append(missing)
def test_one_has_missing_not_equal():
p: Outcome = Outcome()
p_missing: MissingValue = MissingValue("the_entity_id", "the_period",
"/path/to/var", "Text", None)
p.missings.append(p_missing)
q: Outcome = Outcome()
assert p != q
def test_explicit_null_missing(schema, period, entity_id):
fixture: Dict = {"some_text": None}
observation: Dict = {}
expected: Outcome = Outcome()
expected.missings.append(MissingValue(entity_id, period, "/some_text", "Text", None))
actual: Outcome = Outcome()
crawl: CrawlPeriod = CrawlPeriod(entity_id, schema, fixture, observation, actual, period)
crawl()
assert expected == actual
def _record_all_as_missing(self, f_subtree: Optional[Any], path: ListType[str]) -> None:
"""Recursively find all non-folders in the subtree, recording them as missing variables."""
data_type: str
if len(path) == 0:
data_type = "Folder"
else:
var: Optional[Variable] = self.schema.lookup(path)
assert var is not None
data_type = var.data_type
if data_type == "Folder":
assert f_subtree is not None
for key, subfolder in f_subtree.items():
self._record_all_as_missing(subfolder, path + [key])
else:
var_path: str = nesteddicts.path_to_str(path)
missing: MissingValue = MissingValue(self.entity_id, self.label, var_path, data_type, f_subtree)
self.outcome.missings.append(missing)
def test_missing_mismatch_simple(schema, period, entity_id):
"""One mismatch, one missing item."""
fixture: Dict = {
"some_multiple_text": ["x", "y"],
"outer": {
"some_multiple_text": ["xx", "yy"],
"inner": {
"some_multiple_text": ["xxx", "yyy"],
}
}
}
observation: Dict = {
"some_multiple_text": ["x", "y"],
"outer": {
"inner": {
"some_multiple_text": ["aaa", "bbb", "ccc"],
}
}
}
# 1 match, 1 mismatch, 1 missing
expected: Outcome = Outcome()
expected.matches.append(
ValueMatch(entity_id, period, "/some_multiple_text", "MultipleText",
json.dumps(fixture["some_multiple_text"])))
expected.missings.append(
MissingValue(entity_id, period, "/outer/some_multiple_text",
"MultipleText",
json.dumps(fixture["outer"]["some_multiple_text"])))
expected.mismatches.append(
ValueMismatch(
entity_id, period, "/outer/inner/some_multiple_text",
"MultipleText",
json.dumps(fixture["outer"]["inner"]["some_multiple_text"]),
json.dumps(["aaa", "bbb", "ccc"])))
actual: Outcome = Outcome()
crawl: CrawlPeriod = CrawlPeriod(entity_id, schema, fixture, observation,
actual, period)
crawl()
assert expected == actual
def _record_missing(self, path: ListType, data_type: str, value: Optional[Any]) -> None:
path_str = nesteddicts.path_to_str(path)
missing: MissingValue = MissingValue(self.entity_id, self.label, path_str, data_type, value)
self.outcome.missings.append(missing)