def test_emit_message(capsys):
traced_exc = AirbyteTracedException(internal_message="internal message",
message="user-friendly message",
exception=RuntimeError("oh no"))
expected_message = AirbyteMessage(
type="TRACE",
trace=AirbyteTraceMessage(
type="ERROR",
emitted_at=0.0,
error=AirbyteErrorTraceMessage(
failure_type="system_error",
message="user-friendly message",
internal_message="internal message",
stack_trace="RuntimeError: oh no\n",
),
),
)
traced_exc.emit_message()
stdout = capsys.readouterr().out
printed_message = AirbyteMessage.parse_obj(json.loads(stdout))
printed_message.trace.emitted_at = 0.0
assert printed_message == expected_message
def test_exception_as_airbyte_message():
traced_exc = AirbyteTracedException("an internal message")
airbyte_message = traced_exc.as_airbyte_message()
assert type(airbyte_message) == AirbyteMessage
assert airbyte_message.type == MessageType.TRACE
assert airbyte_message.trace.type == TraceType.ERROR
assert airbyte_message.trace.emitted_at > 0
assert airbyte_message.trace.error.failure_type == FailureType.system_error
assert airbyte_message.trace.error.message == "Something went wrong in the connector. See the logs for more details."
assert airbyte_message.trace.error.internal_message == "an internal message"
assert airbyte_message.trace.error.stack_trace == "airbyte_cdk.utils.traced_exception.AirbyteTracedException: an internal message\n"
def test_build_from_existing_exception(raised_exception):
traced_exc = AirbyteTracedException.from_exception(
raised_exception, message="my user-friendly message")
assert traced_exc.message == "my user-friendly message"
assert traced_exc.internal_message == "an error has occurred"
assert traced_exc.failure_type == FailureType.system_error
assert traced_exc._exception == raised_exception
def read(
self,
logger: logging.Logger,
config: Mapping[str, Any],
catalog: ConfiguredAirbyteCatalog,
state: MutableMapping[str,
Any] = None) -> Iterator[AirbyteMessage]:
"""
This method is overridden to check whether the stream `quotes` exists in the source, if not skip reading that stream.
"""
connector_state = copy.deepcopy(state or {})
logger.info(f"Starting syncing {self.name}")
config, internal_config = split_config(config)
# TODO assert all streams exist in the connector
# get the streams once in case the connector needs to make any queries to generate them
stream_instances = {s.name: s for s in self.streams(config)}
self._stream_to_instance_map = stream_instances
with create_timer(self.name) as timer:
for configured_stream in catalog.streams:
stream_instance = stream_instances.get(
configured_stream.stream.name)
if not stream_instance and configured_stream.stream.name == "quotes":
logger.warning(
"Stream `quotes` does not exist in the source. Skip reading `quotes` stream."
)
continue
if not stream_instance:
raise KeyError(
f"The requested stream {configured_stream.stream.name} was not found in the source. Available streams: {stream_instances.keys()}"
)
try:
yield from self._read_stream(
logger=logger,
stream_instance=stream_instance,
configured_stream=configured_stream,
connector_state=connector_state,
internal_config=internal_config,
)
except Exception as e:
logger.exception(
f"Encountered an exception while reading stream {configured_stream.stream.name}"
)
display_message = stream_instance.get_error_display_message(
e)
if display_message:
raise AirbyteTracedException.from_exception(
e, message=display_message) from e
raise e
finally:
logger.info(f"Finished syncing {self.name}")
logger.info(timer.report())
logger.info(f"Finished syncing {self.name}")
def read(
self,
logger: logging.Logger,
config: Mapping[str, Any],
catalog: ConfiguredAirbyteCatalog,
state: MutableMapping[str, Any] = None,
) -> Iterator[AirbyteMessage]:
"""Implements the Read operation from the Airbyte Specification. See https://docs.airbyte.io/architecture/airbyte-protocol."""
connector_state = copy.deepcopy(state or {})
logger.info(f"Starting syncing {self.name}")
config, internal_config = split_config(config)
# TODO assert all streams exist in the connector
# get the streams once in case the connector needs to make any queries to generate them
stream_instances = {s.name: s for s in self.streams(config)}
self._stream_to_instance_map = stream_instances
with create_timer(self.name) as timer:
for configured_stream in catalog.streams:
stream_instance = stream_instances.get(
configured_stream.stream.name)
if not stream_instance:
raise KeyError(
f"The requested stream {configured_stream.stream.name} was not found in the source."
f" Available streams: {stream_instances.keys()}")
try:
timer.start_event(
f"Syncing stream {configured_stream.stream.name}")
yield from self._read_stream(
logger=logger,
stream_instance=stream_instance,
configured_stream=configured_stream,
connector_state=connector_state,
internal_config=internal_config,
)
except AirbyteTracedException as e:
raise e
except Exception as e:
logger.exception(
f"Encountered an exception while reading stream {configured_stream.stream.name}"
)
display_message = stream_instance.get_error_display_message(
e)
if display_message:
raise AirbyteTracedException.from_exception(
e, message=display_message) from e
raise e
finally:
timer.finish_event()
logger.info(
f"Finished syncing {configured_stream.stream.name}")
logger.info(timer.report())
logger.info(f"Finished syncing {self.name}")
def hook_fn(exception_type, exception_value, traceback_):
# For developer ergonomics, we want to see the stack trace in the logs when we do a ctrl-c
if issubclass(exception_type, KeyboardInterrupt):
sys.__excepthook__(exception_type, exception_value, traceback_)
return
logger.fatal(exception_value, exc_info=exception_value)
# emit an AirbyteTraceMessage for any exception that gets to this spot
traced_exc = (exception_value if issubclass(
exception_type, AirbyteTracedException) else
AirbyteTracedException.from_exception(exception_value))
traced_exc.emit_message()
def test_existing_exception_as_airbyte_message(raised_exception):
traced_exc = AirbyteTracedException.from_exception(raised_exception)
airbyte_message = traced_exc.as_airbyte_message()
assert type(airbyte_message) == AirbyteMessage
assert airbyte_message.type == MessageType.TRACE
assert airbyte_message.trace.type == TraceType.ERROR
assert airbyte_message.trace.error.message == "Something went wrong in the connector. See the logs for more details."
assert airbyte_message.trace.error.internal_message == "an error has occurred"
assert airbyte_message.trace.error.stack_trace.startswith(
"Traceback (most recent call last):")
assert airbyte_message.trace.error.stack_trace.endswith(
'raise RuntimeError("an error has occurred")\n'
"RuntimeError: an error has occurred\n")