def test_parse_sheet_and_column_names_from_catalog(self):
sheet1 = "soccer_team"
sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"])
sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}}
sheet2 = "basketball_teams"
sheet2_columns = frozenset(["gsw", "lakers"])
sheet2_schema = {"properties": {c: {"type": "string"} for c in sheet2_columns}}
catalog = ConfiguredAirbyteCatalog(
streams=[
ConfiguredAirbyteStream(
stream=AirbyteStream(name=sheet1, json_schema=sheet1_schema, supported_sync_modes=["full_refresh"]),
sync_mode=SyncMode.full_refresh,
destination_sync_mode=DestinationSyncMode.overwrite,
),
ConfiguredAirbyteStream(
stream=AirbyteStream(name=sheet2, json_schema=sheet2_schema, supported_sync_modes=["full_refresh"]),
sync_mode=SyncMode.full_refresh,
destination_sync_mode=DestinationSyncMode.overwrite,
),
]
)
actual = Helpers.parse_sheet_and_column_names_from_catalog(catalog)
expected = {sheet1: sheet1_columns, sheet2: sheet2_columns}
self.assertEqual(actual, expected)
def configured_catalogue(test_table_name: str,
table_schema: str) -> ConfiguredAirbyteCatalog:
append_stream = ConfiguredAirbyteStream(
stream=AirbyteStream(name=test_table_name, json_schema=table_schema),
sync_mode=SyncMode.incremental,
destination_sync_mode=DestinationSyncMode.append,
)
return ConfiguredAirbyteCatalog(streams=[append_stream])
def test_configured_catalogue(test_table_name: str,
table_schema: str) -> ConfiguredAirbyteCatalog:
# Deleting one column to simulate manipulation in UI
del table_schema["properties"]["column1"]
append_stream = ConfiguredAirbyteStream(
stream=AirbyteStream(name=test_table_name, json_schema=table_schema),
sync_mode=SyncMode.incremental,
destination_sync_mode=DestinationSyncMode.append,
)
return ConfiguredAirbyteCatalog(streams=[append_stream])
def _configured_catalog() -> ConfiguredAirbyteCatalog:
stream_schema = {
"type": "object",
"properties": {
"name": {
"type": "string"
}
}
}
append_stream = ConfiguredAirbyteStream(
stream=AirbyteStream(name=TEST_STREAM, json_schema=stream_schema),
sync_mode=SyncMode.incremental,
destination_sync_mode=DestinationSyncMode.append,
)
return ConfiguredAirbyteCatalog(streams=[append_stream])