class InterpolatedRequestInputProvider:
"""
Helper class that generically performs string interpolation on the provided dictionary or string input
"""
def __init__(self, *, config, request_inputs=None):
self._config = config
if request_inputs is None:
request_inputs = {}
if isinstance(request_inputs, str):
self._interpolator = InterpolatedString(request_inputs, "")
else:
self._interpolator = InterpolatedMapping(request_inputs,
JinjaInterpolation())
def request_inputs(
self,
stream_state: Mapping[str, Any],
stream_slice: Mapping[str, Any] = None,
next_page_token: Mapping[str, Any] = None) -> Union[Mapping, str]:
kwargs = {
"stream_state": stream_state,
"stream_slice": stream_slice,
"next_page_token": next_page_token
}
interpolated_value = self._interpolator.eval(self._config, **kwargs)
if isinstance(interpolated_value, dict):
non_null_tokens = {
k: v
for k, v in interpolated_value.items() if v
}
return non_null_tokens
return interpolated_value
class InterpolatedPaginator(Paginator):
def __init__(self,
*,
next_page_token_template: Mapping[str, str],
config: Config,
decoder: Optional[Decoder] = None):
self._next_page_token_template = InterpolatedMapping(
next_page_token_template, JinjaInterpolation())
self._decoder = decoder or JsonDecoder()
self._config = config
def next_page_token(
self, response: requests.Response,
last_records: List[Mapping[str,
Any]]) -> Optional[Mapping[str, Any]]:
decoded_response = self._decoder.decode(response)
headers = response.headers
interpolated_values = self._next_page_token_template.eval(
self._config,
decoded_response=decoded_response,
headers=headers,
last_records=last_records)
non_null_tokens = {
k: v
for k, v in interpolated_values.items() if v is not None
}
return non_null_tokens if non_null_tokens else None
def test():
d = {
"field": "value",
"number": 100,
"field_to_interpolate_from_config": "{{ config['c'] }}",
"field_to_interpolate_from_kwargs": "{{ kwargs['a'] }}",
}
config = {"c": "VALUE_FROM_CONFIG"}
kwargs = {"a": "VALUE_FROM_KWARGS"}
mapping = InterpolatedMapping(d)
interpolated = mapping.eval(config, **{"kwargs": kwargs})
assert interpolated["field"] == "value"
assert interpolated["number"] == 100
assert interpolated["field_to_interpolate_from_config"] == "VALUE_FROM_CONFIG"
assert interpolated["field_to_interpolate_from_kwargs"] == "VALUE_FROM_KWARGS"
def test():
d = {
"field": "value",
"field_to_interpolate_from_config": "{{ config['c'] }}",
"field_to_interpolate_from_kwargs": "{{ kwargs['a'] }}",
"a_field": "{{ value_passed_directly }}",
}
config = {"c": "VALUE_FROM_CONFIG"}
kwargs = {"a": "VALUE_FROM_KWARGS"}
mapping = InterpolatedMapping(d)
value_passed_directly = "ABC"
interpolated = mapping.eval(config,
**{"kwargs": kwargs},
value_passed_directly=value_passed_directly)
assert interpolated["field"] == "value"
assert interpolated[
"field_to_interpolate_from_config"] == "VALUE_FROM_CONFIG"
assert interpolated[
"field_to_interpolate_from_kwargs"] == "VALUE_FROM_KWARGS"
assert interpolated["a_field"] == value_passed_directly
class SubstreamSlicer(StreamSlicer):
"""
Stream slicer that iterates over the parent's stream slices and records and emits slices by interpolating the slice_definition mapping
Will populate the state with `parent_stream_slice` and `parent_record` so they can be accessed by other components
"""
def __init__(self, parent_streams: List[Stream], state: DictState,
slice_definition: Mapping[str, Any]):
self._parent_streams = parent_streams
self._state = state
self._interpolation = InterpolatedMapping(slice_definition,
JinjaInterpolation())
def stream_slices(
self, sync_mode: SyncMode,
stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]:
"""
Iterate over each parent stream.
For each stream, iterate over its stream_slices.
For each stream slice, iterate over each records.
yield a stream slice for each such records.
If a parent slice contains no record, emit a slice with parent_record=None.
The template string can interpolate the following values:
- parent_stream_slice: mapping representing the parent's stream slice
- parent_record: mapping representing the parent record
- parent_stream_name: string representing the parent stream name
"""
if not self._parent_streams:
yield from []
else:
for parent_stream in self._parent_streams:
for parent_stream_slice in parent_stream.stream_slices(
sync_mode=sync_mode,
cursor_field=None,
stream_state=stream_state):
self._state.update_state(
parent_stream_slice=parent_stream_slice)
self._state.update_state(parent_record=None)
empty_parent_slice = True
for parent_record in parent_stream.read_records(
sync_mode=SyncMode.full_refresh,
cursor_field=None,
stream_slice=parent_stream_slice,
stream_state=None):
empty_parent_slice = False
slice_definition = self._get_slice_definition(
parent_stream_slice, parent_record,
parent_stream.name)
self._state.update_state(parent_record=parent_record)
yield slice_definition
# If the parent slice contains no records,
# yield a slice definition with parent_record==None
if empty_parent_slice:
slice_definition = self._get_slice_definition(
parent_stream_slice, None, parent_stream.name)
yield slice_definition
def _get_slice_definition(self, parent_stream_slice, parent_record,
parent_stream_name):
return self._interpolation.eval(
None,
parent_stream_slice=parent_stream_slice,
parent_record=parent_record,
parent_stream_name=parent_stream_name)