def read( self, logger: AirbyteLogger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, state: MutableMapping[str, Any] = None ) -> Iterator[AirbyteMessage]: """ Overwritten to dynamically receive only those streams that are necessary for reading for significant speed gains (Salesforce has a strict API limit on requests). """ connector_state = copy.deepcopy(state or {}) config, internal_config = split_config(config) # get the streams once in case the connector needs to make any queries to generate them logger.info("Starting generating streams") stream_instances = {s.name: s for s in self.streams(config, catalog=catalog)} logger.info(f"Starting syncing {self.name}") self._stream_to_instance_map = stream_instances for configured_stream in catalog.streams: stream_instance = stream_instances.get(configured_stream.stream.name) if not stream_instance: raise KeyError( f"The requested stream {configured_stream.stream.name} was not found in the source. Available streams: {stream_instances.keys()}" ) try: yield from self._read_stream( logger=logger, stream_instance=stream_instance, configured_stream=configured_stream, connector_state=connector_state, internal_config=internal_config, ) except Exception as e: logger.exception(f"Encountered an exception while reading stream {self.name}") raise e logger.info(f"Finished syncing {self.name}")
def read( self, logger: AirbyteLogger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, state: Optional[MutableMapping[str, Any]] = None, ) -> Iterator[AirbyteMessage]: """ Overwritten to dynamically receive only those streams that are necessary for reading for significant speed gains (Salesforce has a strict API limit on requests). """ connector_state = copy.deepcopy(state or {}) config, internal_config = split_config(config) # get the streams once in case the connector needs to make any queries to generate them logger.info("Starting generating streams") stream_instances = { s.name: s for s in self.streams(config, catalog=catalog, state=state) } logger.info(f"Starting syncing {self.name}") self._stream_to_instance_map = stream_instances for configured_stream in catalog.streams: stream_instance = stream_instances.get( configured_stream.stream.name) if not stream_instance: raise KeyError( f"The requested stream {configured_stream.stream.name} was not found in the source. Available streams: {stream_instances.keys()}" ) try: yield from self._read_stream( logger=logger, stream_instance=stream_instance, configured_stream=configured_stream, connector_state=connector_state, internal_config=internal_config, ) except exceptions.HTTPError as error: error_data = error.response.json()[0] error_code = error_data.get("errorCode") if error.response.status_code == codes.FORBIDDEN and error_code == "REQUEST_LIMIT_EXCEEDED": logger.warn( f"API Call limit is exceeded. Error message: '{error_data.get('message')}'" ) break # if got 403 rate limit response, finish the sync with success. raise error except Exception as e: logger.exception( f"Encountered an exception while reading stream {self.name}" ) raise e logger.info(f"Finished syncing {self.name}")