class ConditionalPaginator:
"""
A paginator that performs pagination by incrementing a page number and stops based on a provided stop condition.
"""
def __init__(self, stop_condition: str, state: DictState, decoder: Decoder,
config):
self._stop_condition_interpolator = InterpolatedBoolean(stop_condition)
self._state: DictState = state
self._decoder = decoder
self._config = config
def next_page_token(
self, response: requests.Response,
last_records: List[Mapping[str,
Any]]) -> Optional[Mapping[str, Any]]:
decoded_response = self._decoder.decode(response)
headers = response.headers
should_stop = self._stop_condition_interpolator.eval(
self._config,
decoded_response=decoded_response,
headers=headers,
last_records=last_records)
if should_stop:
return None
next_page = self._get_page() + 1
self._update_page_state(next_page)
return {"page": next_page}
def _get_page(self):
return self._state.get_state("page")
def _update_page_state(self, page):
self._state.update_state(**{"page": page})
def __init__(self,
action: Union[ResponseAction, str],
*,
http_codes: Set[int] = None,
error_message_contain: str = None,
predicate: str = ""):
"""
:param action: action to execute if a request matches
:param http_codes: http code of matching requests
:param error_message_contain: error substring of matching requests
:param predicate: predicate to apply to determine if a request is matching
"""
if isinstance(action, str):
action = ResponseAction[action]
self._http_codes = http_codes or set()
self._predicate = InterpolatedBoolean(predicate)
self._error_message_contains = error_message_contain
self._action = action
class RecordFilter:
def __init__(self, config: Config, condition: str = None):
self._config = config
self._filter_interpolator = InterpolatedBoolean(condition)
def filter_records(
self,
records: List[Record],
stream_state: Mapping[str, Any],
stream_slice: Mapping[str, Any] = None,
next_page_token: Mapping[str, Any] = None,
) -> List[Record]:
kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token}
return [record for record in records if self._filter_interpolator.eval(self._config, record=record, **kwargs)]
class HttpResponseFilter:
TOO_MANY_REQUESTS_ERRORS = {429}
DEFAULT_RETRIABLE_ERRORS = set([x for x in range(500, 600)
]).union(TOO_MANY_REQUESTS_ERRORS)
def __init__(self,
action: Union[ResponseAction, str],
*,
http_codes: Set[int] = None,
error_message_contain: str = None,
predicate: str = ""):
"""
:param action: action to execute if a request matches
:param http_codes: http code of matching requests
:param error_message_contain: error substring of matching requests
:param predicate: predicate to apply to determine if a request is matching
"""
if isinstance(action, str):
action = ResponseAction[action]
self._http_codes = http_codes or set()
self._predicate = InterpolatedBoolean(predicate)
self._error_message_contains = error_message_contain
self._action = action
@property
def action(self):
return self._action
def matches(self, response: requests.Response) -> Optional[ResponseAction]:
if (response.status_code in self._http_codes
or (self._response_matches_predicate(response))
or (self._response_contains_error_message(response))):
return self._action
else:
return None
def _response_matches_predicate(self, response: requests.Response) -> bool:
return self._predicate and self._predicate.eval(
None, decoded_response=response.json())
def _response_contains_error_message(self,
response: requests.Response) -> bool:
if not self._error_message_contains:
return False
else:
return self._error_message_contains in HttpStream.parse_response_error_message(
response)
def __init__(self, config: Config, condition: str = None):
self._config = config
self._filter_interpolator = InterpolatedBoolean(condition)
def test_interpolated_boolean(test_name, template, expected_result):
interpolated_bool = InterpolatedBoolean(template)
assert interpolated_bool.eval(config) == expected_result
def __init__(self, stop_condition: str, state: DictState, decoder: Decoder,
config):
self._stop_condition_interpolator = InterpolatedBoolean(stop_condition)
self._state: DictState = state
self._decoder = decoder
self._config = config