def test_configure_catalog():
stream = AirbyteStream(name="stream",
supported_sync_modes=[SyncMode.full_refresh],
json_schema={})
catalog = AirbyteCatalog(streams=[stream])
catalog_message = AirbyteMessage(type=Type.CATALOG, catalog=catalog)
sys.stdin = io.StringIO(catalog_message.json())
expected_configured_catalog = ConfiguredAirbyteCatalog(streams=[
ConfiguredAirbyteStream(
stream=stream,
sync_mode=SyncMode.full_refresh,
destination_sync_mode=DestinationSyncMode.append)
])
expected_configured_catalog_json = json.loads(
expected_configured_catalog.json())
with tempfile.TemporaryDirectory() as temp_dir:
os.chdir(temp_dir)
configure_catalog()
assert os.path.exists("integration_tests/configured_catalog.json")
with open("integration_tests/configured_catalog.json") as f:
configured_catalog_json = json.loads(f.read())
assert configured_catalog_json == expected_configured_catalog_json
def configure_catalog():
record = AirbyteMessage.parse_raw(input())
for stream in record.catalog.streams:
stream.json_schema = {}
streams = [
ConfiguredAirbyteStream(
stream=stream,
sync_mode=stream.supported_sync_modes[0],
destination_sync_mode=DestinationSyncMode.append)
for stream in record.catalog.streams
]
configured_catalog = ConfiguredAirbyteCatalog(streams=streams)
default_folder = os.path.join(os.getcwd(), "integration_tests")
if not os.path.exists(default_folder):
os.mkdir(default_folder)
output_file_name = os.path.join(default_folder, "configured_catalog.json")
with open(output_file_name, "w") as outfile:
json.dump(json.loads(configured_catalog.json()),
outfile,
indent=2,
sort_keys=True)
def test_run_write(self, mocker, destination: Destination, tmp_path,
monkeypatch):
config_path, dummy_config = tmp_path / "config.json", {
"user": "******"
}
write_file(config_path, dummy_config)
dummy_catalog = ConfiguredAirbyteCatalog(streams=[
ConfiguredAirbyteStream(
stream=AirbyteStream(name="mystream",
json_schema={"type": "object"}),
sync_mode=SyncMode.full_refresh,
destination_sync_mode=DestinationSyncMode.overwrite,
)
])
catalog_path = tmp_path / "catalog.json"
write_file(catalog_path, dummy_catalog.json(exclude_unset=True))
args = {
"command": "write",
"config": config_path,
"catalog": catalog_path
}
parsed_args = argparse.Namespace(**args)
expected_write_result = [
_wrapped(_state({"k1": "v1"})),
_wrapped(_state({"k2": "v2"}))
]
mocker.patch.object(
destination,
"write",
return_value=iter(expected_write_result),
autospec=True # convert to iterator to mimic real usage
)
spec_msg = ConnectorSpecification(connectionSpecification={})
mocker.patch.object(destination, "spec", return_value=spec_msg)
validate_mock = mocker.patch(
"airbyte_cdk.destinations.destination.check_config_against_spec_or_exit"
)
# mock input is a record followed by some state messages
mocked_input: List[AirbyteMessage] = [
_wrapped(_record("s1", {"k1": "v1"})), *expected_write_result
]
mocked_stdin_string = "\n".join(
[record.json(exclude_unset=True) for record in mocked_input])
mocked_stdin_string += "\n add this non-serializable string to verify the destination does not break on malformed input"
mocked_stdin = io.TextIOWrapper(
io.BytesIO(bytes(mocked_stdin_string, "utf-8")))
monkeypatch.setattr("sys.stdin", mocked_stdin)
returned_write_result = list(destination.run_cmd(parsed_args))
# verify method call with the correct params
# Affirm to Mypy that call_count is indeed a method on this mock
destination.write.assert_called_once() # type: ignore
# Affirm to Mypy that call_count is indeed a method on this mock
destination.write.assert_called_with( # type: ignore
config=dummy_config,
configured_catalog=dummy_catalog,
# Stdin is internally consumed as a generator so we use a custom matcher
# that iterates over two iterables to check equality
input_messages=OrderedIterableMatcher(mocked_input),
)
# Check if config validation has been called
validate_mock.assert_called_with(dummy_config, spec_msg,
destination.logger)
# verify output was correct
assert expected_write_result == returned_write_result
from airbyte_cdk.models import (
AirbyteCatalog,
AirbyteConnectionStatus,
AirbyteMessage,
AirbyteRecordMessage,
AirbyteStateMessage,
AirbyteStream,
ConfiguredAirbyteCatalog,
ConfiguredAirbyteStream,
ConnectorSpecification,
DestinationSyncMode,
Status,
SyncMode,
Type,
)
dummy_catalog = ConfiguredAirbyteCatalog(streams=[
ConfiguredAirbyteStream(
stream=AirbyteStream(name="mystream", json_schema={"type": "object"}),
sync_mode=SyncMode.full_refresh,
destination_sync_mode=DestinationSyncMode.overwrite,
)
])
print(dummy_catalog.json())
