def test_visit(self, catalog): executor = MetadataExecutor( [ MetadataRule("UniqueEntitiesName", [], "replication-key", "created_at"), MetadataRule( "UniqueEntitiesName", ["properties", "created_at"], "is-replication-key", True, ), ] ) visit(catalog, executor) stream_node = next( s for s in catalog["streams"] if s["tap_stream_id"] == "UniqueEntitiesName" ) stream_metadata_node = next( m for m in stream_node["metadata"] if len(m["breadcrumb"]) == 0 ) property_metadata_node = next( m for m in stream_node["metadata"] if m["breadcrumb"] == ["properties", "created_at"] ) assert stream_node["replication_key"] == "created_at" assert stream_metadata_node["metadata"]["replication-key"] == "created_at" assert property_metadata_node["metadata"]["is-replication-key"] == True
def test_select_negated(self, catalog, attrs): selector = SelectExecutor( ["*.*", "!UniqueEntitiesName.code", "!UniqueEntitiesName.name"]) visit(catalog, selector) lister = ListSelectedExecutor() visit(catalog, lister) assert lister.selected_properties["UniqueEntitiesName"] == attrs
def assert_properties(properties): with properties_file.open() as catalog: schema = json.load(catalog) lister = ListSelectedExecutor() visit(schema, lister) assert lister.selected_properties[ "UniqueEntitiesName"] == properties
def test_select(self, catalog, attrs): selector = SelectExecutor( ["UniqueEntitiesName.name", "UniqueEntitiesName.code", "*.payload.content"] ) visit(catalog, selector) lister = ListSelectedExecutor() visit(catalog, lister) assert lister.selected_properties["UniqueEntitiesName"] == attrs
def test_select_all(self, catalog, select_all_executor): visit(catalog, select_all_executor) self.assert_catalog_is_selected(catalog) streams = {stream["tap_stream_id"]: stream for stream in catalog["streams"]} stream_metadata = len( [ metadata for stream in streams.values() for metadata in stream["metadata"] if len(metadata["breadcrumb"]) == 0 ] ) assert stream_metadata == 1, "Extraneous stream metadata"
def test_visit(self, catalog): executor = ListExecutor() visit(catalog, executor) assert dict(executor.properties) == { "UniqueEntitiesName": { "code", "name", "balance", "created_at", "id", "active", "payload", "payload.content", "payload.hash", } }
def test_visit(self, catalog, select_all_executor): visit(catalog, select_all_executor) self.assert_catalog_is_selected(catalog)
def test_visit(self, catalog): executor = SchemaExecutor([ SchemaRule( "UniqueEntitiesName", ["properties", "code"], {"anyOf": [{ "type": "string" }, { "type": "null" }]}, ), SchemaRule( "UniqueEntitiesName", ["properties", "*_at"], { "type": "string", "format": "date" }, ), SchemaRule( "UniqueEntitiesName", ["properties", "payload"], { "type": "object", "properties": { "content": { "type": ["string", "null"] }, "hash": { "type": "string" }, }, }, ), SchemaRule( "UniqueEntitiesName", ["properties", "*load", "properties", "hash"], {"type": ["string", "null"]}, ), ]) visit(catalog, executor) stream_node = next(s for s in catalog["streams"] if s["tap_stream_id"] == "UniqueEntitiesName") properties_node = stream_node["schema"]["properties"] assert properties_node["code"] == { "anyOf": [{ "type": "string" }, { "type": "null" }] } if "created_at" in properties_node: assert properties_node["created_at"] == { "type": "string", "format": "date" } else: # If no matching properties were found for a glob-like pattern, # no new property is created assert "*_at" not in properties_node assert properties_node["payload"] == { "type": "object", "properties": { "content": { "type": ["string", "null"] }, "hash": { "type": ["string", "null"] }, }, }