def test_streams_outputs_records(self, catalog_path, config):
configured_catalog = ConfiguredAirbyteCatalog.parse_file(catalog_path)
records, states = self._read_records(config, configured_catalog)
assert records, "should have some records returned"
if configured_catalog.streams[0].sync_mode == SyncMode.incremental:
assert states, "should have some states returned"
def _run_write(
self, config: Mapping[str, Any], configured_catalog_path: str, input_stream: io.TextIOWrapper
) -> Iterable[AirbyteMessage]:
catalog = ConfiguredAirbyteCatalog.parse_file(configured_catalog_path)
input_messages = self._parse_input_stream(input_stream)
self.logger.info("Begin writing to the destination...")
yield from self.write(config=config, configured_catalog=catalog, input_messages=input_messages)
self.logger.info("Writing complete.")
def configured_catalog_fixture(
configured_catalog_path,
catalog_schemas) -> Optional[ConfiguredAirbyteCatalog]:
if configured_catalog_path:
catalog = ConfiguredAirbyteCatalog.parse_file(configured_catalog_path)
for configured_stream in catalog.streams:
configured_stream.stream.json_schema = catalog_schemas.get(
configured_stream.stream.name, {})
return catalog
return None
def configured_catalog_fixture(
configured_catalog_path,
discovered_catalog) -> Optional[ConfiguredAirbyteCatalog]:
if configured_catalog_path:
catalog = ConfiguredAirbyteCatalog.parse_file(configured_catalog_path)
for configured_stream in catalog.streams:
configured_stream.stream = discovered_catalog.get(
configured_stream.stream.name, configured_stream.stream)
return catalog
return None
def configured_catalog_fixture(configured_catalog_path,
discovered_catalog) -> ConfiguredAirbyteCatalog:
"""Take ConfiguredAirbyteCatalog from discover command by default"""
if configured_catalog_path:
catalog = ConfiguredAirbyteCatalog.parse_file(configured_catalog_path)
for configured_stream in catalog.streams:
configured_stream.stream = discovered_catalog.get(
configured_stream.stream.name, configured_stream.stream)
return catalog
streams = [
ConfiguredAirbyteStream(
stream=stream,
sync_mode=stream.supported_sync_modes[0],
destination_sync_mode=DestinationSyncMode.append,
cursor_field=stream.default_cursor_field,
primary_key=stream.source_defined_primary_key,
) for _, stream in discovered_catalog.items()
]
return ConfiguredAirbyteCatalog(streams=streams)
def configured_catalog_fixture():
return ConfiguredAirbyteCatalog.parse_file(
"integration_tests/configured_catalog.json")
def configured_catalog_fixture():
return ConfiguredAirbyteCatalog.parse_file("sample_files/configured_catalog.json")
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
import pytest
from airbyte_cdk.models import ConfiguredAirbyteCatalog
from destination_google_sheets.writer import GoogleSheetsWriter
from integration_tests.test_spreadsheet import TEST_SPREADSHEET
# ----- PREPARE ENV -----
# path to configured_catalog json file
TEST_CATALOG_PATH: str = "integration_tests/test_data/test_writer_catalog.json"
# reading prepared catalog with streams
TEST_CATALOG: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalog.parse_file(TEST_CATALOG_PATH)
# define test writer
TEST_WRITER: GoogleSheetsWriter = GoogleSheetsWriter(TEST_SPREADSHEET)
# set flush buffer interval
TEST_WRITER.flush_interval = 2
# test stream name
TEST_STREAM: str = "stream_1"
# ----- BEGIN TESTS -----
def _prepare_buffers():
for configured_stream in TEST_CATALOG.streams:
TEST_WRITER.init_buffer_stream(configured_stream)
def configured_catalog_fixture(
configured_catalog_path) -> Optional[ConfiguredAirbyteCatalog]:
if configured_catalog_path:
return ConfiguredAirbyteCatalog.parse_file(configured_catalog_path)
return None
