def _validate_records_structure(
records: List[AirbyteRecordMessage],
configured_catalog: ConfiguredAirbyteCatalog):
"""
Check object structure simmilar to one expected by schema. Sometimes
just running schema validation is not enough case schema could have
additionalProperties parameter set to true and no required fields
therefore any arbitrary object would pass schema validation.
This method is here to catch those cases by extracting all the pathes
from the object and compare it to pathes expected from jsonschema. If
there no common pathes then raise an alert.
:param records: List of airbyte record messages gathered from connector instances.
:param configured_catalog: SAT testcase parameters parsed from yaml file
"""
schemas: Dict[str, Set] = {}
for stream in configured_catalog.streams:
schemas[stream.stream.name] = set(
get_expected_schema_structure(stream.stream.json_schema))
for record in records:
schema_pathes = schemas.get(record.stream)
if not schema_pathes:
continue
record_fields = set(get_object_structure(record.data))
common_fields = set.intersection(record_fields, schema_pathes)
assert common_fields, f" Record from {record.stream} stream should have some fields mentioned by json schema, {schema_pathes}"
def _validate_field_appears_at_least_once_in_stream(
self, records: List, schema: Dict):
"""
Get all possible schema paths, then diff with existing record paths.
In case of `oneOf` or `anyOf` schema props, compare only choice which is present in records.
"""
expected_paths = get_expected_schema_structure(schema,
annotate_one_of=True)
expected_paths = set(flatten(tuple(expected_paths)))
for record in records:
record_paths = set(get_object_structure(record))
paths_to_remove = {
path
for path in expected_paths
if re.sub(r"\([0-9]*\)", "", path) in record_paths
}
for path in paths_to_remove:
path_parts = re.split(r"\([0-9]*\)", path)
if len(path_parts) > 1:
expected_paths -= {
path
for path in expected_paths if path_parts[0] in path
}
expected_paths -= paths_to_remove
return sorted(list(expected_paths))
def test_get_object_strucutre(object, pathes):
assert get_object_structure(object) == pathes