def test_oneof_usage(connector_spec, should_fail): t = _TestSpec() if should_fail is True: with pytest.raises(AssertionError): t.test_oneof_usage(actual_connector_spec=ConnectorSpecification( connectionSpecification=connector_spec)) else: t.test_oneof_usage(actual_connector_spec=ConnectorSpecification( connectionSpecification=connector_spec))
def test_run_check(self, mocker, destination: Destination, tmp_path): file_path = tmp_path / "config.json" dummy_config = {"user": "******"} write_file(file_path, dummy_config) args = {"command": "check", "config": file_path} parsed_args = argparse.Namespace(**args) destination.run_cmd(parsed_args) spec_msg = ConnectorSpecification(connectionSpecification={}) mocker.patch.object(destination, "spec", return_value=spec_msg) validate_mock = mocker.patch( "airbyte_cdk.destinations.destination.check_config_against_spec_or_exit" ) expected_check_result = AirbyteConnectionStatus( status=Status.SUCCEEDED) mocker.patch.object(destination, "check", return_value=expected_check_result, autospec=True) returned_check_result = next(iter(destination.run_cmd(parsed_args))) # verify method call with the correct params # Affirm to Mypy that this is indeed a method on this mock destination.check.assert_called_once() # type: ignore # Affirm to Mypy that this is indeed a method on this mock destination.check.assert_called_with( logger=ANY, config=dummy_config) # type: ignore # Check if config validation has been called validate_mock.assert_called_with(dummy_config, spec_msg, destination.logger) # verify output was correct assert _wrapped(expected_check_result) == returned_check_result
def test_airbyte_secret_is_masked_on_logger_output(source_spec, mocker, config, caplog): caplog.set_level(logging.DEBUG, logger="airbyte.test") caplog.handler.setFormatter(AirbyteLogFormatter()) entrypoint = AirbyteEntrypoint(MockSource()) parsed_args = Namespace(command="read", config="", state="", catalog="") mocker.patch.object( MockSource, "spec", return_value=ConnectorSpecification( connectionSpecification=source_spec), ) mocker.patch.object(MockSource, "configure", return_value=config) mocker.patch.object(MockSource, "read_config", return_value=None) mocker.patch.object(MockSource, "read_state", return_value={}) mocker.patch.object(MockSource, "read_catalog", return_value={}) list(entrypoint.run(parsed_args)) log_result = caplog.text expected_secret_values = [ config[k] for k, v in source_spec["properties"].items() if v.get("airbyte_secret") ] expected_plain_text_values = [ config[k] for k, v in source_spec["properties"].items() if not v.get("airbyte_secret") ] assert all([str(v) not in log_result for v in expected_secret_values]) assert all([str(v) in log_result for v in expected_plain_text_values])
def spec(self, logger: AirbyteLogger) -> ConnectorSpecification: """Tries to find and remove a spec data about local storage settings""" parent_code_dir = os.path.dirname(source_file.source.__file__) parent_spec_file = os.path.join(parent_code_dir, "spec.json") with open(parent_spec_file, "r") as f: spec = ConnectorSpecification.parse_obj(json.load(f)) # correction of the "storage" property to const type for provider in spec.connectionSpecification["properties"]["provider"][ "oneOf"]: storage = provider["properties"]["storage"] if "enum" in storage: storage.pop("enum") storage["const"] = storage.pop("default") for i in range( len(spec.connectionSpecification["properties"]["provider"] ["oneOf"])): provider = spec.connectionSpecification["properties"]["provider"][ "oneOf"][i] if provider["properties"]["storage"][ "const"] == LOCAL_STORAGE_NAME: spec.connectionSpecification["properties"]["provider"][ "oneOf"].pop(i) return spec
def spec(self, *args, **kwargs) -> ConnectorSpecification: """Returns the spec for this integration.""" return ConnectorSpecification( documentationUrl=DOCUMENTATION_URL, changelogUrl=DOCUMENTATION_URL, supportsIncremental=True, supported_destination_sync_modes=[ DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup ], connectionSpecification=SourceTiktokMarketingSpec.schema(), additionalProperties=True, advanced_auth=AdvancedAuth( auth_flow_type=AuthFlowType.oauth2_0, predicate_key=["credentials", "auth_type"], predicate_value="oauth2.0", oauth_config_specification=OAuthConfigSpecification( complete_oauth_output_specification= CompleteOauthOutputSpecification.schema(), complete_oauth_server_input_specification= CompleteOauthServerInputSpecification.schema(), complete_oauth_server_output_specification= CompleteOauthServerOutputSpecification.schema(), ), ), )
def spec(self, logger: logging.Logger) -> ConnectorSpecification: """ Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password) required to run this integration. By default, this will be loaded from a "spec.yaml" or a "spec.json" in the package root. """ package = self.__class__.__module__.split(".")[0] yaml_spec = load_optional_package_file(package, "spec.yaml") json_spec = load_optional_package_file(package, "spec.json") if yaml_spec and json_spec: raise RuntimeError( "Found multiple spec files in the package. Only one of spec.yaml or spec.json should be provided." ) if yaml_spec: spec_obj = yaml.load(yaml_spec, Loader=yaml.SafeLoader) elif json_spec: spec_obj = json.loads(json_spec) else: raise FileNotFoundError( "Unable to find spec.yaml or spec.json in the package.") return ConnectorSpecification.parse_obj(spec_obj)
def spec(self, logger: AirbyteLogger) -> ConnectorSpecification: """ Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password) required to run this integration. """ raw_spec = pkgutil.get_data( self.__class__.__module__.split(".")[0], "spec.json") return ConnectorSpecification.parse_obj(json.loads(raw_spec))
def test_non_airbyte_secrets_are_not_masked_on_uncaught_exceptions( mocker, caplog, capsys): caplog.set_level(logging.DEBUG, logger="airbyte.test") caplog.handler.setFormatter(AirbyteLogFormatter()) class BrokenSource(MockSource): def read( self, logger: logging.Logger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, state: MutableMapping[str, Any] = None, ): raise Exception("Exception:" + NOT_A_SECRET_VALUE) entrypoint = AirbyteEntrypoint(BrokenSource()) parsed_args = Namespace(command="read", config="", state="", catalog="") source_spec = { "type": "object", "required": ["api_token"], "additionalProperties": False, "properties": { SECRET_PROPERTY: { "type": "string", "airbyte_secret": True }, NOT_SECRET_PROPERTY: { "type": "string", "airbyte_secret": False }, }, } simple_config = { SECRET_PROPERTY: I_AM_A_SECRET_VALUE, NOT_SECRET_PROPERTY: NOT_A_SECRET_VALUE, } mocker.patch.object( MockSource, "spec", return_value=ConnectorSpecification( connectionSpecification=source_spec), ) mocker.patch.object(MockSource, "configure", return_value=simple_config) mocker.patch.object(MockSource, "read_config", return_value=None) mocker.patch.object(MockSource, "read_state", return_value={}) mocker.patch.object(MockSource, "read_catalog", return_value={}) mocker.patch.object(MockSource, "read", side_effect=Exception("Exception:" + NOT_A_SECRET_VALUE)) try: list(entrypoint.run(parsed_args)) except Exception: sys.excepthook(*sys.exc_info()) assert NOT_A_SECRET_VALUE in capsys.readouterr( ).out, "Should not have filtered non-secret value from exception trace message" assert NOT_A_SECRET_VALUE in caplog.text, "Should not have filtered non-secret value from exception log message"
def spec(self, *args, **kwargs) -> ConnectorSpecification: """Returns the spec for this integration.""" return ConnectorSpecification( documentationUrl=DOCUMENTATION_URL, changelogUrl=DOCUMENTATION_URL, supportsIncremental=True, supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup], connectionSpecification=SourceTiktokMarketingSpec.schema(), )
def spec(self, logger: logging.Logger) -> ConnectorSpecification: """ Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password) required to run this integration. """ raw_spec: Optional[bytes] = pkgutil.get_data( self.__class__.__module__.split(".")[0], "spec.json") if not raw_spec: raise ValueError("Unable to find spec.json.") return ConnectorSpecification.parse_obj(json.loads(raw_spec))
def spec(self, *args, **kwargs) -> ConnectorSpecification: """ Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password) required to run this integration. """ return ConnectorSpecification( documentationUrl="https://docs.airbyte.io/integrations/sources/facebook-marketing", changelogUrl="https://docs.airbyte.io/integrations/sources/facebook-marketing", supportsIncremental=True, supported_destination_sync_modes=[DestinationSyncMode.append], connectionSpecification=ConnectorConfig.schema(), )
def spec(self, *args, **kwargs) -> ConnectorSpecification: return ConnectorSpecification( documentationUrl= "https://docs.airbyte.io/integrations/sources/lever-hiring", changelogUrl= "https://docs.airbyte.io/integrations/sources/lever-hiring#changelog", connectionSpecification=ConnectorConfig.schema(), authSpecification=AuthSpecification( auth_type="oauth2.0", oauth2Specification=OAuth2Specification( oauthFlowInitParameters=[["client_id"], ["client_secret"], ["refresh_token"]]), ), )
def spec(self, *args, **kwargs) -> ConnectorSpecification: """ Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password) required to run this integration. """ # FIXME: airbyte-cdk does not parse pydantic $ref correctly. This override won't be needed after the fix schema = ConnectorConfig.schema() schema["properties"]["aws_environment"] = schema["definitions"]["AWSEnvironment"] schema["properties"]["region"] = schema["definitions"]["AWSRegion"] return ConnectorSpecification( documentationUrl="https://docs.airbyte.io/integrations/sources/amazon-seller-partner", changelogUrl="https://docs.airbyte.io/integrations/sources/amazon-seller-partner", connectionSpecification=schema, )
def test_config_validate(entrypoint: AirbyteEntrypoint, mocker, config_mock, schema, config_valid): parsed_args = Namespace(command="check", config="config_path") check_value = AirbyteConnectionStatus(status=Status.SUCCEEDED) mocker.patch.object(MockSource, "check", return_value=check_value) mocker.patch.object( MockSource, "spec", return_value=ConnectorSpecification(connectionSpecification=schema)) if config_valid: messages = list(entrypoint.run(parsed_args)) assert [_wrap_message(check_value)] == messages else: with pytest.raises(Exception, match=r"(?i)Config Validation Error:.*"): list(entrypoint.run(parsed_args))
def spec(self, *args, **kwargs) -> ConnectorSpecification: """ Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password) required to run this integration. """ return ConnectorSpecification( documentationUrl="https://docs.airbyte.io/integrations/sources/facebook-marketing", changelogUrl="https://docs.airbyte.io/integrations/sources/facebook-marketing", supportsIncremental=True, supported_destination_sync_modes=[DestinationSyncMode.append], connectionSpecification=expand_local_ref(ConnectorConfig.schema()), authSpecification=AuthSpecification( auth_type="oauth2.0", oauth2Specification=OAuth2Specification( rootObject=[], oauthFlowInitParameters=[], oauthFlowOutputParameters=[["access_token"]] ), ), )
def test_run_spec(self, mocker, destination: Destination): args = {"command": "spec"} parsed_args = argparse.Namespace(**args) expected_spec = ConnectorSpecification( connectionSpecification={"json_schema": { "prop": "value" }}) mocker.patch.object(destination, "spec", return_value=expected_spec, autospec=True) spec_message = next(iter(destination.run_cmd(parsed_args))) # Mypy doesn't understand magicmock so it thinks spec doesn't have assert_called_once attr destination.spec.assert_called_once() # type: ignore # verify the output of spec was returned assert _wrapped(expected_spec) == spec_message
def spec(self, *args, **kwargs) -> ConnectorSpecification: """ Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password) required to run this integration. """ # make dummy instance of stream_class in order to get 'supports_incremental' property incremental = self.stream_class(dataset="", provider="", format="", path_pattern="").supports_incremental supported_dest_sync_modes = [DestinationSyncMode.overwrite] if incremental: supported_dest_sync_modes.extend( [DestinationSyncMode.append, DestinationSyncMode.append_dedup]) return ConnectorSpecification( documentationUrl=self.documentation_url, changelogUrl=self.documentation_url, supportsIncremental=incremental, supported_destination_sync_modes=supported_dest_sync_modes, connectionSpecification=self.spec_class.schema(), )
def connector_spec_fixture(connector_spec_path) -> ConnectorSpecification: spec_obj = load_yaml_or_json_path(connector_spec_path) return ConnectorSpecification.parse_obj(spec_obj)
def spec(self, *args) -> ConnectorSpecification: return ConnectorSpecification( documentationUrl= "https://docs.airbyte.io/integrations/sources/amazon-ads", connectionSpecification=AmazonAdsConfig.schema(), )
def spec_mock(mocker): expected = ConnectorSpecification(connectionSpecification={}) mock = MagicMock(return_value=expected) mocker.patch.object(MockSource, "spec", mock) return mock
def spec(self, logger: AirbyteLogger) -> ConnectorSpecification: return ConnectorSpecification( connectionSpecification=BraintreeConfig.schema(), documentationUrl= "https://docs.airbyte.io/integrations/sources/braintree")
def test_run_spec(entrypoint: AirbyteEntrypoint, mocker): parsed_args = Namespace(command="spec") expected = ConnectorSpecification(connectionSpecification={"hi": "hi"}) mocker.patch.object(MockSource, "spec", return_value=expected) assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args))
def connector_spec_fixture(connector_spec_path) -> ConnectorSpecification: return ConnectorSpecification.parse_file(connector_spec_path)
def _spec(schema: Dict[str, Any]) -> ConnectorSpecification: return ConnectorSpecification(connectionSpecification=schema)
with pytest.raises( AssertionError, match="stream should have some fields mentioned by json schema" ): t.test_read(None, catalog, input_config, [], docker_runner_mock, MagicMock()) else: t.test_read(None, catalog, input_config, [], docker_runner_mock, MagicMock()) @pytest.mark.parametrize( "connector_spec, expected_error", [ # SUCCESS: no authSpecification specified (ConnectorSpecification(connectionSpecification={}), ""), # FAIL: Field specified in root object does not exist ( ConnectorSpecification( connectionSpecification={"type": "object"}, authSpecification={ "auth_type": "oauth2.0", "oauth2Specification": { "rootObject": ["credentials", 0], "oauthFlowInitParameters": [["client_id"], ["client_secret"]], "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], }, }, ),
def test_run_write(self, mocker, destination: Destination, tmp_path, monkeypatch): config_path, dummy_config = tmp_path / "config.json", { "user": "******" } write_file(config_path, dummy_config) dummy_catalog = ConfiguredAirbyteCatalog(streams=[ ConfiguredAirbyteStream( stream=AirbyteStream(name="mystream", json_schema={"type": "object"}), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ) ]) catalog_path = tmp_path / "catalog.json" write_file(catalog_path, dummy_catalog.json(exclude_unset=True)) args = { "command": "write", "config": config_path, "catalog": catalog_path } parsed_args = argparse.Namespace(**args) expected_write_result = [ _wrapped(_state({"k1": "v1"})), _wrapped(_state({"k2": "v2"})) ] mocker.patch.object( destination, "write", return_value=iter(expected_write_result), autospec=True # convert to iterator to mimic real usage ) spec_msg = ConnectorSpecification(connectionSpecification={}) mocker.patch.object(destination, "spec", return_value=spec_msg) validate_mock = mocker.patch( "airbyte_cdk.destinations.destination.check_config_against_spec_or_exit" ) # mock input is a record followed by some state messages mocked_input: List[AirbyteMessage] = [ _wrapped(_record("s1", {"k1": "v1"})), *expected_write_result ] mocked_stdin_string = "\n".join( [record.json(exclude_unset=True) for record in mocked_input]) mocked_stdin_string += "\n add this non-serializable string to verify the destination does not break on malformed input" mocked_stdin = io.TextIOWrapper( io.BytesIO(bytes(mocked_stdin_string, "utf-8"))) monkeypatch.setattr("sys.stdin", mocked_stdin) returned_write_result = list(destination.run_cmd(parsed_args)) # verify method call with the correct params # Affirm to Mypy that call_count is indeed a method on this mock destination.write.assert_called_once() # type: ignore # Affirm to Mypy that call_count is indeed a method on this mock destination.write.assert_called_with( # type: ignore config=dummy_config, configured_catalog=dummy_catalog, # Stdin is internally consumed as a generator so we use a custom matcher # that iterates over two iterables to check equality input_messages=OrderedIterableMatcher(mocked_input), ) # Check if config validation has been called validate_mock.assert_called_with(dummy_config, spec_msg, destination.logger) # verify output was correct assert expected_write_result == returned_write_result