def test_airbyte_secret_is_masked_on_logger_output(source_spec, mocker, config,
caplog):
caplog.set_level(logging.DEBUG, logger="airbyte.test")
caplog.handler.setFormatter(AirbyteLogFormatter())
entrypoint = AirbyteEntrypoint(MockSource())
parsed_args = Namespace(command="read", config="", state="", catalog="")
mocker.patch.object(
MockSource,
"spec",
return_value=ConnectorSpecification(
connectionSpecification=source_spec),
)
mocker.patch.object(MockSource, "configure", return_value=config)
mocker.patch.object(MockSource, "read_config", return_value=None)
mocker.patch.object(MockSource, "read_state", return_value={})
mocker.patch.object(MockSource, "read_catalog", return_value={})
list(entrypoint.run(parsed_args))
log_result = caplog.text
expected_secret_values = [
config[k] for k, v in source_spec["properties"].items()
if v.get("airbyte_secret")
]
expected_plain_text_values = [
config[k] for k, v in source_spec["properties"].items()
if not v.get("airbyte_secret")
]
assert all([str(v) not in log_result for v in expected_secret_values])
assert all([str(v) in log_result for v in expected_plain_text_values])
def test_non_airbyte_secrets_are_not_masked_on_uncaught_exceptions(
mocker, caplog, capsys):
caplog.set_level(logging.DEBUG, logger="airbyte.test")
caplog.handler.setFormatter(AirbyteLogFormatter())
class BrokenSource(MockSource):
def read(
self,
logger: logging.Logger,
config: Mapping[str, Any],
catalog: ConfiguredAirbyteCatalog,
state: MutableMapping[str, Any] = None,
):
raise Exception("Exception:" + NOT_A_SECRET_VALUE)
entrypoint = AirbyteEntrypoint(BrokenSource())
parsed_args = Namespace(command="read", config="", state="", catalog="")
source_spec = {
"type": "object",
"required": ["api_token"],
"additionalProperties": False,
"properties": {
SECRET_PROPERTY: {
"type": "string",
"airbyte_secret": True
},
NOT_SECRET_PROPERTY: {
"type": "string",
"airbyte_secret": False
},
},
}
simple_config = {
SECRET_PROPERTY: I_AM_A_SECRET_VALUE,
NOT_SECRET_PROPERTY: NOT_A_SECRET_VALUE,
}
mocker.patch.object(
MockSource,
"spec",
return_value=ConnectorSpecification(
connectionSpecification=source_spec),
)
mocker.patch.object(MockSource, "configure", return_value=simple_config)
mocker.patch.object(MockSource, "read_config", return_value=None)
mocker.patch.object(MockSource, "read_state", return_value={})
mocker.patch.object(MockSource, "read_catalog", return_value={})
mocker.patch.object(MockSource,
"read",
side_effect=Exception("Exception:" +
NOT_A_SECRET_VALUE))
try:
list(entrypoint.run(parsed_args))
except Exception:
sys.excepthook(*sys.exc_info())
assert NOT_A_SECRET_VALUE in capsys.readouterr(
).out, "Should not have filtered non-secret value from exception trace message"
assert NOT_A_SECRET_VALUE in caplog.text, "Should not have filtered non-secret value from exception log message"
def test_parse_missing_required_args(cmd: str, args: MutableMapping[str, Any],
entrypoint: AirbyteEntrypoint):
required_args = {
"check": ["config"],
"discover": ["config"],
"read": ["config", "catalog"]
}
for required_arg in required_args[cmd]:
argcopy = deepcopy(args)
del argcopy[required_arg]
with pytest.raises(BaseException):
entrypoint.parse_args(_as_arglist(cmd, argcopy))
def test_config_validate(entrypoint: AirbyteEntrypoint, mocker, config_mock,
schema, config_valid):
parsed_args = Namespace(command="check", config="config_path")
check_value = AirbyteConnectionStatus(status=Status.SUCCEEDED)
mocker.patch.object(MockSource, "check", return_value=check_value)
mocker.patch.object(
MockSource,
"spec",
return_value=ConnectorSpecification(connectionSpecification=schema))
if config_valid:
messages = list(entrypoint.run(parsed_args))
assert [_wrap_message(check_value)] == messages
else:
with pytest.raises(Exception, match=r"(?i)Config Validation Error:.*"):
list(entrypoint.run(parsed_args))
def test_run_check(entrypoint: AirbyteEntrypoint, mocker, spec_mock,
config_mock):
parsed_args = Namespace(command="check", config="config_path")
check_value = AirbyteConnectionStatus(status=Status.SUCCEEDED)
mocker.patch.object(MockSource, "check", return_value=check_value)
assert [_wrap_message(check_value)] == list(entrypoint.run(parsed_args))
assert spec_mock.called
def test_run_check(entrypoint: AirbyteEntrypoint, mocker):
parsed_args = Namespace(command="check", config="config_path")
config = {"username": "******"}
check_value = AirbyteConnectionStatus(status=Status.SUCCEEDED)
mocker.patch.object(MockSource, "read_config", return_value=config)
mocker.patch.object(MockSource, "configure", return_value=config)
mocker.patch.object(MockSource, "check", return_value=check_value)
assert [_wrap_message(check_value)] == list(entrypoint.run(parsed_args))
def test_run_discover(entrypoint: AirbyteEntrypoint, mocker, spec_mock,
config_mock):
parsed_args = Namespace(command="discover", config="config_path")
expected = AirbyteCatalog(
streams=[AirbyteStream(name="stream", json_schema={"k": "v"})])
mocker.patch.object(MockSource, "discover", return_value=expected)
assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args))
assert spec_mock.called
def test_run_discover(entrypoint: AirbyteEntrypoint, mocker):
parsed_args = Namespace(command="discover", config="config_path")
config = {"username": "******"}
expected = AirbyteCatalog(
streams=[AirbyteStream(name="stream", json_schema={"k": "v"})])
mocker.patch.object(MockSource, "read_config", return_value=config)
mocker.patch.object(MockSource, "configure", return_value=config)
mocker.patch.object(MockSource, "discover", return_value=expected)
assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args))
def test_run_read(entrypoint: AirbyteEntrypoint, mocker, spec_mock,
config_mock):
parsed_args = Namespace(command="read",
config="config_path",
state="statepath",
catalog="catalogpath")
expected = AirbyteRecordMessage(stream="stream",
data={"data": "stuff"},
emitted_at=1)
mocker.patch.object(MockSource, "read_state", return_value={})
mocker.patch.object(MockSource, "read_catalog", return_value={})
mocker.patch.object(
MockSource,
"read",
return_value=[AirbyteMessage(record=expected, type=Type.RECORD)])
assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args))
assert spec_mock.called
def test_parse_valid_args(cmd: str, args: Mapping[str, Any],
entrypoint: AirbyteEntrypoint):
arglist = _as_arglist(cmd, args)
parsed_args = entrypoint.parse_args(arglist)
assert {"command": cmd, **args} == vars(parsed_args)
def entrypoint() -> AirbyteEntrypoint:
return AirbyteEntrypoint(MockSource())
def test_invalid_command(entrypoint: AirbyteEntrypoint, mocker):
with pytest.raises(Exception):
mocker.patch.object(MockSource, "read_config", return_value={})
mocker.patch.object(MockSource, "configure", return_value={})
list(entrypoint.run(Namespace(command="invalid", config="conf")))
def test_run_spec(entrypoint: AirbyteEntrypoint, mocker):
parsed_args = Namespace(command="spec")
expected = ConnectorSpecification(connectionSpecification={"hi": "hi"})
mocker.patch.object(MockSource, "spec", return_value=expected)
assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args))
def test_invalid_command(entrypoint: AirbyteEntrypoint, mocker, config_mock):
with pytest.raises(Exception):
list(entrypoint.run(Namespace(command="invalid", config="conf")))
def test_parse_valid_args(cmd: str, args: Mapping[str, Any], expected_args,
entrypoint: AirbyteEntrypoint):
arglist = _as_arglist(cmd, args)
parsed_args = entrypoint.parse_args(arglist)
assert vars(parsed_args) == expected_args