def test_parse_sheet_and_column_names_from_catalog(self): sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} sheet2 = "basketball_teams" sheet2_columns = frozenset(["gsw", "lakers"]) sheet2_schema = {"properties": {c: {"type": "string"} for c in sheet2_columns}} catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( stream=AirbyteStream(name=sheet1, json_schema=sheet1_schema, supported_sync_modes=["full_refresh"]), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ), ConfiguredAirbyteStream( stream=AirbyteStream(name=sheet2, json_schema=sheet2_schema, supported_sync_modes=["full_refresh"]), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ), ] ) actual = Helpers.parse_sheet_and_column_names_from_catalog(catalog) expected = {sheet1: sheet1_columns, sheet2: sheet2_columns} self.assertEqual(actual, expected)
def configured_catalogue(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: append_stream = ConfiguredAirbyteStream( stream=AirbyteStream(name=test_table_name, json_schema=table_schema), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, ) return ConfiguredAirbyteCatalog(streams=[append_stream])
def test_configured_catalogue(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: # Deleting one column to simulate manipulation in UI del table_schema["properties"]["column1"] append_stream = ConfiguredAirbyteStream( stream=AirbyteStream(name=test_table_name, json_schema=table_schema), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, ) return ConfiguredAirbyteCatalog(streams=[append_stream])
def _configured_catalog() -> ConfiguredAirbyteCatalog: stream_schema = { "type": "object", "properties": { "name": { "type": "string" } } } append_stream = ConfiguredAirbyteStream( stream=AirbyteStream(name=TEST_STREAM, json_schema=stream_schema), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, ) return ConfiguredAirbyteCatalog(streams=[append_stream])