def _do_cast_error_test(data_type: str, raw: Optional[Any]):
spec: Dict = {
"var": {
"name": "the_var",
"data_type": data_type,
"sort_order": 0
}
}
immutable: Track = Track.build(spec, None, "immutable")
temporal: Track = Track.build({}, None, "temporal")
schema: Schema = Schema(temporal, immutable)
content: Dict = {"immutable": {"the_var": raw}}
composite: Composite = Composite(schema, content)
cast: Cast = Cast(schema, {})
cast(composite)
expected: Dict = {
"immutable": {
"qc": {
"_exceptions": {
"cast_errors": {
"the_var": raw
}
}
}
}
}
actual: Dict = composite.content
assert actual == expected
def test_underscore_folders_ignored():
spec: Dict = {
"binary_in_root": {
"name": "the_binary",
"data_type": "Binary",
"sort_order": 0
}
}
immutable: Track = Track.build(spec, None, "immutable")
temporal: Track = Track.build({}, None, "temporal")
schema: Schema = Schema(temporal, immutable)
content: Dict = {
"immutable": {
"the_binary": "true",
"_folder": {
"foo": "shouldn't matter",
"bar": "also shouldn't matter"
}
}
}
expected: Dict = {
"immutable": {
"the_binary": True,
"_folder": {
"foo": "shouldn't matter",
"bar": "also shouldn't matter"
}
}
}
composite: Composite = Composite(schema, content)
cast: Cast = Cast(schema, {})
cast(composite)
assert composite.content == expected
def test_cast_temporal(schema, composite, period: str, var_id: str,
expected: Optional[Any]):
cast: Cast = Cast(schema, {})
cast(composite)
actual: Optional[Any] = composite.get_observation(var_id, period)
if actual != expected:
print("breakpoint")
assert actual == expected
def test_cast_ignores_valid_multiple_text(schema, value):
content: Dict = {
"immutable": {
"the_var": value
}
}
expected: Dict = copy.deepcopy(content)
composite: Composite = Composite(schema, content)
cast: Cast = Cast(schema, {})
cast(composite)
assert composite.content == expected
def test_bare_str_multitext_gets_embedded_in_list(schema):
content: Dict = {
"immutable": {
"the_var": "should be in a list"
}
}
expected: Dict = {
"immutable": {
"the_var": ["should be in a list"]
}
}
composite: Composite = Composite(schema, content)
cast: Cast = Cast(schema, {})
cast(composite)
assert composite.content == expected
def test_cast_nested_illegal_value(nested_track):
"""If a primitive contains an unintelligible value, it is deleted from the dataset, since we don't want to falsely
report explicit null and its presence could break downstream steps. The value is recorded as an exception. If the
exception occurs inside a list, only the last example is recorded."""
temporal: Track = Track.build({}, None, "temporal")
schema: Schema = Schema(temporal, nested_track)
content: Dict = {
"immutable": {
"folder": {
"some_int":
"foo",
"list": [{
"some_int": "bar",
}, {
"some_int": "baz",
}, {
"some_int": "75"
}]
}
}
}
expected: Dict = {
"immutable": {
"folder": {
"list": [{}, {}, {
"some_int": 75
}]
},
"qc": {
"_exceptions": {
"cast_errors": {
"folder": {
"some_int": "foo",
"list": {
"some_int": "baz"
}
}
}
}
}
}
}
composite: Composite = Composite(schema, content)
cast: Cast = Cast(schema, {})
cast(composite)
assert composite.content == expected
def test_cast_nonexistent(nested_track):
"""If Cast encounters an unknown variable, it records an exception and then leaves it unaltered. If the unknown
variable is nested inside one or more lists or keyed lists, only the last example of the error is recorded as an
exception."""
temporal: Track = Track.build({}, None, "temporal")
schema: Schema = Schema(temporal, nested_track)
content: Dict = {
"immutable": {
"fake_in_root": "0",
"folder": {
"some_int":
"5",
"fake_in_folder":
"7",
"list": [
{
"some_int": "42",
"fake_in_list":
"this will not appear" # When there's a list, only the last example is recorded
},
{
"some_int": "24",
"fake_in_list": "this will appear in the exceptions"
}
]
}
}
}
expected: Dict = {
"immutable": {
"fake_in_root": "0",
"folder": {
"some_int":
5,
"fake_in_folder":
"7",
"list": [
{
"some_int": 42,
"fake_in_list":
"this will not appear" # When there's a list, only the last example is recorded
},
{
"some_int": 24,
"fake_in_list": "this will appear in the exceptions"
}
]
},
"qc": {
"_exceptions": {
"unknown_vars": {
"fake_in_root": "0",
"folder": {
"fake_in_folder": "7",
"list": {
"fake_in_list":
"this will appear in the exceptions"
}
}
}
}
}
}
}
composite: Composite = Composite(schema, content)
cast: Cast = Cast(schema, {})
cast(composite)
assert composite.content == expected
def test_cast_complex():
spec: Dict = {
"root_list": {
"name": "the_list",
"data_type": "List",
"sort_order": 0
},
"int_in_list": {
"name": "some_int",
"data_type": "Integer",
"parent": "root_list",
"sort_order": 0
},
"folder_in_list": {
"name": "the_folder",
"data_type": "Folder",
"parent": "root_list",
"sort_order": 1
},
"binary_in_folder_in_list": {
"name": "some_binary",
"data_type": "Binary",
"parent": "folder_in_list",
"sort_order": 0
},
"keyed_list_in_folder_in_list": {
"name": "the_keyed_list",
"data_type": "KeyedList",
"parent": "folder_in_list",
"sort_order": 1
},
"date_in_keyed_list_in_folder_in_list": {
"name": "some_date",
"data_type": "Date",
"parent": "keyed_list_in_folder_in_list",
"sort_order": 0
}
}
immutable: Track = Track.build(spec, None, "immutable")
temporal: Track = Track.build({}, None, "temporal")
schema: Schema = Schema(temporal, immutable)
content: Dict = {
"immutable": {
"the_list": [{
"some_int": "-0",
"the_folder": {
"some_binary": "1",
"the_keyed_list": {
"bob": {
"some_date": "2001-01-01"
},
"susan": {
"some_date": "000000"
}
}
}
}, {
"some_int": "74",
"the_folder": {
"some_binary": "FaLsE",
"the_keyed_list": {
"bob": {
"some_date": "201712"
},
"susan": {}
}
}
}]
}
}
expected: Dict = {
"immutable": {
"the_list": [{
"some_int": 0,
"the_folder": {
"some_binary": True,
"the_keyed_list": {
"bob": {
"some_date": "2001-01-01"
},
"susan": {
"some_date": None
}
}
}
}, {
"some_int": 74,
"the_folder": {
"some_binary": False,
"the_keyed_list": {
"bob": {
"some_date": "2017-12-01"
},
"susan": {}
}
}
}]
}
}
composite: Composite = Composite(schema, content)
cast: Cast = Cast(schema, {})
cast(composite)
assert composite.content == expected
def test_cast_immutable(schema, composite, var_id: str,
expected: Optional[Any]):
cast: Cast = Cast(schema, {})
cast(composite)
assert composite.get_immutable(var_id) == expected
def _make_cast(na_values: Set[str]) -> Cast:
return Cast(schema, {}, na_values)