def test_visit(self, catalog):
executor = MetadataExecutor(
[
MetadataRule("UniqueEntitiesName", [], "replication-key", "created_at"),
MetadataRule(
"UniqueEntitiesName",
["properties", "created_at"],
"is-replication-key",
True,
),
]
)
visit(catalog, executor)
stream_node = next(
s for s in catalog["streams"] if s["tap_stream_id"] == "UniqueEntitiesName"
)
stream_metadata_node = next(
m for m in stream_node["metadata"] if len(m["breadcrumb"]) == 0
)
property_metadata_node = next(
m
for m in stream_node["metadata"]
if m["breadcrumb"] == ["properties", "created_at"]
)
assert stream_node["replication_key"] == "created_at"
assert stream_metadata_node["metadata"]["replication-key"] == "created_at"
assert property_metadata_node["metadata"]["is-replication-key"] == True
def test_select_negated(self, catalog, attrs):
selector = SelectExecutor(
["*.*", "!UniqueEntitiesName.code", "!UniqueEntitiesName.name"])
visit(catalog, selector)
lister = ListSelectedExecutor()
visit(catalog, lister)
assert lister.selected_properties["UniqueEntitiesName"] == attrs
def assert_properties(properties):
with properties_file.open() as catalog:
schema = json.load(catalog)
lister = ListSelectedExecutor()
visit(schema, lister)
assert lister.selected_properties[
"UniqueEntitiesName"] == properties
def test_select(self, catalog, attrs):
selector = SelectExecutor(
["UniqueEntitiesName.name", "UniqueEntitiesName.code", "*.payload.content"]
)
visit(catalog, selector)
lister = ListSelectedExecutor()
visit(catalog, lister)
assert lister.selected_properties["UniqueEntitiesName"] == attrs
def test_select_all(self, catalog, select_all_executor):
visit(catalog, select_all_executor)
self.assert_catalog_is_selected(catalog)
streams = {stream["tap_stream_id"]: stream for stream in catalog["streams"]}
stream_metadata = len(
[
metadata
for stream in streams.values()
for metadata in stream["metadata"]
if len(metadata["breadcrumb"]) == 0
]
)
assert stream_metadata == 1, "Extraneous stream metadata"
def test_visit(self, catalog):
executor = ListExecutor()
visit(catalog, executor)
assert dict(executor.properties) == {
"UniqueEntitiesName": {
"code",
"name",
"balance",
"created_at",
"id",
"active",
"payload",
"payload.content",
"payload.hash",
}
}
def test_visit(self, catalog, select_all_executor):
visit(catalog, select_all_executor)
self.assert_catalog_is_selected(catalog)
def test_visit(self, catalog):
executor = SchemaExecutor([
SchemaRule(
"UniqueEntitiesName",
["properties", "code"],
{"anyOf": [{
"type": "string"
}, {
"type": "null"
}]},
),
SchemaRule(
"UniqueEntitiesName",
["properties", "*_at"],
{
"type": "string",
"format": "date"
},
),
SchemaRule(
"UniqueEntitiesName",
["properties", "payload"],
{
"type": "object",
"properties": {
"content": {
"type": ["string", "null"]
},
"hash": {
"type": "string"
},
},
},
),
SchemaRule(
"UniqueEntitiesName",
["properties", "*load", "properties", "hash"],
{"type": ["string", "null"]},
),
])
visit(catalog, executor)
stream_node = next(s for s in catalog["streams"]
if s["tap_stream_id"] == "UniqueEntitiesName")
properties_node = stream_node["schema"]["properties"]
assert properties_node["code"] == {
"anyOf": [{
"type": "string"
}, {
"type": "null"
}]
}
if "created_at" in properties_node:
assert properties_node["created_at"] == {
"type": "string",
"format": "date"
}
else:
# If no matching properties were found for a glob-like pattern,
# no new property is created
assert "*_at" not in properties_node
assert properties_node["payload"] == {
"type": "object",
"properties": {
"content": {
"type": ["string", "null"]
},
"hash": {
"type": ["string", "null"]
},
},
}
