class InterpolatedRequestInputProvider: """ Helper class that generically performs string interpolation on the provided dictionary or string input """ def __init__(self, *, config, request_inputs=None): self._config = config if request_inputs is None: request_inputs = {} if isinstance(request_inputs, str): self._interpolator = InterpolatedString(request_inputs, "") else: self._interpolator = InterpolatedMapping(request_inputs, JinjaInterpolation()) def request_inputs( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None) -> Union[Mapping, str]: kwargs = { "stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token } interpolated_value = self._interpolator.eval(self._config, **kwargs) if isinstance(interpolated_value, dict): non_null_tokens = { k: v for k, v in interpolated_value.items() if v } return non_null_tokens return interpolated_value
class InterpolatedPaginator(Paginator): def __init__(self, *, next_page_token_template: Mapping[str, str], config: Config, decoder: Optional[Decoder] = None): self._next_page_token_template = InterpolatedMapping( next_page_token_template, JinjaInterpolation()) self._decoder = decoder or JsonDecoder() self._config = config def next_page_token( self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]: decoded_response = self._decoder.decode(response) headers = response.headers interpolated_values = self._next_page_token_template.eval( self._config, decoded_response=decoded_response, headers=headers, last_records=last_records) non_null_tokens = { k: v for k, v in interpolated_values.items() if v is not None } return non_null_tokens if non_null_tokens else None
def test(): d = { "field": "value", "number": 100, "field_to_interpolate_from_config": "{{ config['c'] }}", "field_to_interpolate_from_kwargs": "{{ kwargs['a'] }}", } config = {"c": "VALUE_FROM_CONFIG"} kwargs = {"a": "VALUE_FROM_KWARGS"} mapping = InterpolatedMapping(d) interpolated = mapping.eval(config, **{"kwargs": kwargs}) assert interpolated["field"] == "value" assert interpolated["number"] == 100 assert interpolated["field_to_interpolate_from_config"] == "VALUE_FROM_CONFIG" assert interpolated["field_to_interpolate_from_kwargs"] == "VALUE_FROM_KWARGS"
def test(): d = { "field": "value", "field_to_interpolate_from_config": "{{ config['c'] }}", "field_to_interpolate_from_kwargs": "{{ kwargs['a'] }}", "a_field": "{{ value_passed_directly }}", } config = {"c": "VALUE_FROM_CONFIG"} kwargs = {"a": "VALUE_FROM_KWARGS"} mapping = InterpolatedMapping(d) value_passed_directly = "ABC" interpolated = mapping.eval(config, **{"kwargs": kwargs}, value_passed_directly=value_passed_directly) assert interpolated["field"] == "value" assert interpolated[ "field_to_interpolate_from_config"] == "VALUE_FROM_CONFIG" assert interpolated[ "field_to_interpolate_from_kwargs"] == "VALUE_FROM_KWARGS" assert interpolated["a_field"] == value_passed_directly
class SubstreamSlicer(StreamSlicer): """ Stream slicer that iterates over the parent's stream slices and records and emits slices by interpolating the slice_definition mapping Will populate the state with `parent_stream_slice` and `parent_record` so they can be accessed by other components """ def __init__(self, parent_streams: List[Stream], state: DictState, slice_definition: Mapping[str, Any]): self._parent_streams = parent_streams self._state = state self._interpolation = InterpolatedMapping(slice_definition, JinjaInterpolation()) def stream_slices( self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: """ Iterate over each parent stream. For each stream, iterate over its stream_slices. For each stream slice, iterate over each records. yield a stream slice for each such records. If a parent slice contains no record, emit a slice with parent_record=None. The template string can interpolate the following values: - parent_stream_slice: mapping representing the parent's stream slice - parent_record: mapping representing the parent record - parent_stream_name: string representing the parent stream name """ if not self._parent_streams: yield from [] else: for parent_stream in self._parent_streams: for parent_stream_slice in parent_stream.stream_slices( sync_mode=sync_mode, cursor_field=None, stream_state=stream_state): self._state.update_state( parent_stream_slice=parent_stream_slice) self._state.update_state(parent_record=None) empty_parent_slice = True for parent_record in parent_stream.read_records( sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=parent_stream_slice, stream_state=None): empty_parent_slice = False slice_definition = self._get_slice_definition( parent_stream_slice, parent_record, parent_stream.name) self._state.update_state(parent_record=parent_record) yield slice_definition # If the parent slice contains no records, # yield a slice definition with parent_record==None if empty_parent_slice: slice_definition = self._get_slice_definition( parent_stream_slice, None, parent_stream.name) yield slice_definition def _get_slice_definition(self, parent_stream_slice, parent_record, parent_stream_name): return self._interpolation.eval( None, parent_stream_slice=parent_stream_slice, parent_record=parent_record, parent_stream_name=parent_stream_name)