def read(
self, logger: AirbyteLogger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, state: MutableMapping[str, Any] = None
) -> Iterator[AirbyteMessage]:
"""Implements the Read operation from the Airbyte Specification. See https://docs.airbyte.io/architecture/airbyte-specification."""
connector_state = copy.deepcopy(state or {})
logger.info(f"Starting syncing {self.name}")
config, internal_config = split_config(config)
# TODO assert all streams exist in the connector
# get the streams once in case the connector needs to make any queries to generate them
stream_instances = {s.name: s for s in self.streams(config)}
self._stream_to_instance_map = stream_instances
with create_timer(self.name) as timer:
for configured_stream in catalog.streams:
stream_instance = stream_instances.get(configured_stream.stream.name)
if not stream_instance:
raise KeyError(
f"The requested stream {configured_stream.stream.name} was not found in the source. Available streams: {stream_instances.keys()}"
)
try:
yield from self._read_stream(
logger=logger,
stream_instance=stream_instance,
configured_stream=configured_stream,
connector_state=connector_state,
internal_config=internal_config,
)
except Exception as e:
logger.exception(f"Encountered an exception while reading stream {self.name}")
raise e
finally:
logger.info(f"Finished syncing {self.name}")
logger.info(timer.report())
logger.info(f"Finished syncing {self.name}")
def test_counter_finish_event():
with create_timer("Counter") as timer:
with mock.patch(
"airbyte_cdk.utils.event_timing.EventTimer.finish_event"
) as mock_finish_event:
timer.finish_event("test_event")
mock_finish_event.assert_called_with("test_event")
def test_report_is_ordered_by_name_by_default():
names = ["j", "b", "g", "d", "e", "f", "c", "h", "i", "a"]
with create_timer("Source Counter") as timer:
for name in names:
timer.start_event(name)
timer.finish_event()
report = timer.report().split("\n")[1:] # ignore the first line
report_names = [line.split(" ")[0] for line in report]
assert report_names == sorted(names)
def read(
self,
logger: logging.Logger,
config: Mapping[str, Any],
catalog: ConfiguredAirbyteCatalog,
state: MutableMapping[str,
Any] = None) -> Iterator[AirbyteMessage]:
"""
This method is overridden to check whether the stream `quotes` exists in the source, if not skip reading that stream.
"""
connector_state = copy.deepcopy(state or {})
logger.info(f"Starting syncing {self.name}")
config, internal_config = split_config(config)
# TODO assert all streams exist in the connector
# get the streams once in case the connector needs to make any queries to generate them
stream_instances = {s.name: s for s in self.streams(config)}
self._stream_to_instance_map = stream_instances
with create_timer(self.name) as timer:
for configured_stream in catalog.streams:
stream_instance = stream_instances.get(
configured_stream.stream.name)
if not stream_instance and configured_stream.stream.name == "quotes":
logger.warning(
"Stream `quotes` does not exist in the source. Skip reading `quotes` stream."
)
continue
if not stream_instance:
raise KeyError(
f"The requested stream {configured_stream.stream.name} was not found in the source. Available streams: {stream_instances.keys()}"
)
try:
yield from self._read_stream(
logger=logger,
stream_instance=stream_instance,
configured_stream=configured_stream,
connector_state=connector_state,
internal_config=internal_config,
)
except Exception as e:
logger.exception(
f"Encountered an exception while reading stream {configured_stream.stream.name}"
)
display_message = stream_instance.get_error_display_message(
e)
if display_message:
raise AirbyteTracedException.from_exception(
e, message=display_message) from e
raise e
finally:
logger.info(f"Finished syncing {self.name}")
logger.info(timer.report())
logger.info(f"Finished syncing {self.name}")
def test_counter_init():
with create_timer("Counter") as timer:
assert timer.name == "Counter"
def test_double_finish_is_safely_ignored():
with create_timer("Source Counter") as timer:
timer.start_event("test_event")
timer.finish_event()
timer.finish_event()
assert timer.count == 1
def test_timer_multiple_events():
with create_timer("Counter") as timer:
for i in range(10):
timer.start_event("test_event")
timer.finish_event()
assert timer.count == 10