def test_update(self, mocker, mock_api_client, connection_configuration):
mocker.patch.object(resources.Connection, "_create_or_update")
resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml")
resource.state = mocker.Mock(resource_id="foo")
update_result = resource.update()
assert update_result == resource._create_or_update.return_value
resource._create_or_update.assert_called_with(resource._update_fn, resource.update_payload)
def test__check_for_legacy_connection_configuration_keys(
self, mock_api_client, connection_configuration, legacy_connection_configurations
):
resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml")
assert resource._check_for_legacy_connection_configuration_keys(connection_configuration["configuration"]) is None
for legacy_configuration in legacy_connection_configurations:
with pytest.raises(resources.InvalidConfigurationError):
resource._check_for_legacy_connection_configuration_keys(legacy_configuration["configuration"])
def test_update_payload_with_normalization(self, mocker, mock_api_client, connection_configuration_with_normalization):
assert resources.Connection.__base__ == resources.BaseResource
mocker.patch.object(resources.Connection, "resource_id", "foo")
mocker.patch.object(resources.Connection, "source_id", "source_id")
mocker.patch.object(resources.Connection, "destination_id", "destination_id")
connection = resources.Connection(mock_api_client, "workspace_id", connection_configuration_with_normalization, "bar.yaml")
assert connection.update_payload == resources.WebBackendConnectionUpdate(
connection_id=connection.resource_id,
**connection.configuration,
)
assert isinstance(connection.update_payload["operations"][0], WebBackendOperationCreateOrUpdate)
def test_create_payload_no_normalization(self, mocker, mock_api_client, connection_configuration):
assert resources.Connection.__base__ == resources.BaseResource
mocker.patch.object(resources.Connection, "resource_id", "foo")
mocker.patch.object(resources.Connection, "source_id", "source_id")
mocker.patch.object(resources.Connection, "destination_id", "destination_id")
connection = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml")
assert connection.create_payload == resources.WebBackendConnectionCreate(
name=connection.resource_name,
source_id=connection.source_id,
destination_id=connection.destination_id,
**connection.configuration,
)
assert "operations" not in connection.create_payload
def test_get_remote_comparable_configuration(self, mocker, mock_api_client, connection_configuration, remote_resource):
mocker.patch.object(
resources.Connection,
"remote_resource",
mocker.Mock(to_dict=mocker.Mock(return_value=remote_resource)),
)
resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml")
comparable = resource._get_remote_comparable_configuration()
resource.remote_resource.to_dict.assert_called_once()
assert isinstance(comparable, dict)
assert all([k not in comparable for k in resource.remote_root_level_keys_to_filter_out_for_comparison])
if "operations" in remote_resource and "operations" in comparable:
assert all([k not in comparable["operations"][0] for k in resource.remote_operation_level_keys_to_filter_out])
if remote_resource["operations"][0]["operator_configuration"].get("normalization") is not None:
assert "dbt" not in remote_resource["operations"][0]["operator_configuration"]
if remote_resource["operations"][0]["operator_configuration"].get("dbt") is not None:
assert "normalization" not in remote_resource["operations"][0]["operator_configuration"]
if "operations" in remote_resource and len(remote_resource["operations"]) == 0:
assert "operations" not in comparable
def test__deserialize_raw_configuration(self, mock_api_client, connection_configuration):
resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml")
configuration = resource._deserialize_raw_configuration()
assert isinstance(configuration["sync_catalog"], AirbyteCatalog)
assert configuration["namespace_definition"] == NamespaceDefinitionType(
connection_configuration["configuration"]["namespace_definition"]
)
assert configuration["schedule"] == ConnectionSchedule(**connection_configuration["configuration"]["schedule"])
assert configuration["resource_requirements"] == ResourceRequirements(
**connection_configuration["configuration"]["resource_requirements"]
)
assert configuration["status"] == ConnectionStatus(connection_configuration["configuration"]["status"])
assert list(configuration.keys()) == [
"namespace_definition",
"namespace_format",
"prefix",
"sync_catalog",
"schedule",
"status",
"resource_requirements",
]
def test_init(self, mocker, mock_api_client, state, connection_configuration):
assert resources.Connection.__base__ == resources.BaseResource
mocker.patch.object(resources.Connection, "resource_id", "foo")
connection = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml")
mocker.patch.object(connection, "state", state)
assert connection.api == resources.web_backend_api.WebBackendApi
assert connection.create_function_name == "web_backend_create_connection"
assert connection.update_function_name == "web_backend_update_connection"
assert connection.resource_id_field == "connection_id"
assert connection.resource_type == "connection"
assert connection.APPLY_PRIORITY == 1
assert connection.update_payload == resources.WebBackendConnectionUpdate(
connection_id=connection.resource_id, **connection.configuration
)
if state is None:
assert connection.get_payload is None
else:
assert connection.get_payload == resources.WebBackendConnectionRequestBody(
connection_id=state.resource_id, with_refreshed_catalog=False
)
def test__deserialize_operations(self, mock_api_client, connection_configuration):
resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml")
operations = [
{
"operator_configuration": {"operator_type": "normalization", "normalization": {"option": "basic"}},
"name": "operation-with-normalization",
},
{
"operator_configuration": {
"operator_type": "dbt",
"dbt": {
"dbt_arguments": "run",
"docker_image": "fishtownanalytics/dbt:0.19.1",
"git_repo_branch": "my-branch-name",
"git_repo_url": "https://github.com/airbytehq/airbyte",
},
},
"name": "operation-with-custom_dbt",
},
]
deserialized_operations = resource._deserialize_operations(operations, OperationCreate)
assert len(deserialized_operations) == 2
assert all([isinstance(o, OperationCreate) for o in deserialized_operations])
assert "normalization" in deserialized_operations[0]["operator_configuration"] and deserialized_operations[0][
"operator_configuration"
]["operator_type"] == OperatorType("normalization")
assert "dbt" in deserialized_operations[1]["operator_configuration"]
assert deserialized_operations[1]["operator_configuration"]["operator_type"] == OperatorType("dbt")
with pytest.raises(ValueError):
resource._deserialize_operations(
[
{
"operator_configuration": {"operator_type": "not-supported", "normalization": {"option": "basic"}},
"name": "operation-not-supported",
},
],
OperationCreate,
)
def test__create_configured_catalog(self, mock_api_client, connection_configuration):
resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml")
created_catalog = resource._create_configured_catalog(connection_configuration["configuration"]["sync_catalog"])
stream, config = (
connection_configuration["configuration"]["sync_catalog"]["streams"][0]["stream"],
connection_configuration["configuration"]["sync_catalog"]["streams"][0]["config"],
)
assert len(created_catalog.streams) == len(connection_configuration["configuration"]["sync_catalog"]["streams"])
assert created_catalog.streams[0].stream.name == stream["name"]
assert created_catalog.streams[0].stream.json_schema == stream["json_schema"]
assert created_catalog.streams[0].stream.supported_sync_modes == stream["supported_sync_modes"]
assert created_catalog.streams[0].stream.source_defined_cursor == stream["source_defined_cursor"]
assert created_catalog.streams[0].stream.namespace == stream["namespace"]
assert created_catalog.streams[0].stream.source_defined_primary_key == stream["source_defined_primary_key"]
assert created_catalog.streams[0].stream.default_cursor_field == stream["default_cursor_field"]
assert created_catalog.streams[0].config.sync_mode == config["sync_mode"]
assert created_catalog.streams[0].config.cursor_field == config["cursor_field"]
assert created_catalog.streams[0].config.destination_sync_mode == config["destination_sync_mode"]
assert created_catalog.streams[0].config.primary_key == config["primary_key"]
assert created_catalog.streams[0].config.alias_name == config["alias_name"]
assert created_catalog.streams[0].config.selected == config["selected"]
def test_destination_id(self, mocker, mock_api_client, connection_configuration, file_not_found_error):
assert resources.Connection.__base__ == resources.BaseResource
mocker.patch.object(resources.Connection, "resource_id", "foo")
if file_not_found_error:
mocker.patch.object(
resources.ResourceState, "from_configuration_path_and_workspace", mocker.Mock(side_effect=FileNotFoundError())
)
else:
mocker.patch.object(
resources.ResourceState,
"from_configuration_path_and_workspace",
mocker.Mock(return_value=mocker.Mock(resource_id="expected_destination_id")),
)
connection = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml")
if file_not_found_error:
with pytest.raises(resources.MissingStateError):
connection.destination_id
else:
destination_id = connection.destination_id
assert destination_id == "expected_destination_id"
resources.ResourceState.from_configuration_path_and_workspace.assert_called_with(
connection_configuration["destination_configuration_path"], connection.workspace_id
)