def _read_incremental( self, logger: AirbyteLogger, stream_instance: Stream, configured_stream: ConfiguredAirbyteStream, connector_state: MutableMapping[str, Any], ) -> Iterator[AirbyteMessage]: stream_name = configured_stream.stream.name stream_state = connector_state.get(stream_name, {}) if stream_state: logger.info(f"Setting state of {stream_name} stream to {stream_state.get(stream_name)}") checkpoint_interval = stream_instance.state_checkpoint_interval slices = stream_instance.stream_slices( cursor_field=configured_stream.cursor_field, sync_mode=SyncMode.incremental, stream_state=stream_state ) for slice in slices: record_counter = 0 records = stream_instance.read_records( sync_mode=SyncMode.incremental, stream_slice=slice, stream_state=stream_state, cursor_field=configured_stream.cursor_field or None, ) for record_data in records: record_counter += 1 yield self._as_airbyte_record(stream_name, record_data) stream_state = stream_instance.get_updated_state(stream_state, record_data) if checkpoint_interval and record_counter % checkpoint_interval == 0: yield self._checkpoint_state(stream_name, stream_state, connector_state, logger) yield self._checkpoint_state(stream_name, stream_state, connector_state, logger)
def _read_full_refresh( self, stream_instance: Stream, configured_stream: ConfiguredAirbyteStream ) -> Iterator[AirbyteMessage]: args = { "sync_mode": SyncMode.full_refresh, "cursor_field": configured_stream.cursor_field } for slices in stream_instance.stream_slices(**args): for record in stream_instance.read_records(stream_slice=slices, **args): yield self._as_airbyte_record(configured_stream.stream.name, record)