def __init__(self,
*,
next_page_token_template: Mapping[str, str],
config: Config,
decoder: Optional[Decoder] = None):
self._next_page_token_template = InterpolatedMapping(
next_page_token_template, JinjaInterpolation())
self._decoder = decoder or JsonDecoder()
self._config = config
def __init__(self, *, config, request_inputs=None):
self._config = config
if request_inputs is None:
request_inputs = {}
if isinstance(request_inputs, str):
self._interpolator = InterpolatedString(request_inputs, "")
else:
self._interpolator = InterpolatedMapping(request_inputs,
JinjaInterpolation())
def __init__(
self,
token_refresh_endpoint: str,
client_id: str,
client_secret: str,
refresh_token: str,
config: Mapping[str, Any],
scopes: List[str] = None,
token_expiry_date: str = None,
access_token_name: str = "access_token",
expires_in_name: str = "expires_in",
refresh_request_body: Mapping[str, Any] = None,
):
self.config = config
self.token_refresh_endpoint = InterpolatedString(token_refresh_endpoint)
self.client_secret = InterpolatedString(client_secret)
self.client_id = InterpolatedString(client_id)
self.refresh_token = InterpolatedString(refresh_token)
self.scopes = scopes
self.access_token_name = InterpolatedString(access_token_name)
self.expires_in_name = InterpolatedString(expires_in_name)
self.refresh_request_body = InterpolatedMapping(refresh_request_body)
self.token_expiry_date = (
pendulum.parse(InterpolatedString(token_expiry_date).eval(self.config))
if token_expiry_date
else pendulum.now().subtract(days=1)
)
self.access_token = None
class InterpolatedRequestInputProvider:
"""
Helper class that generically performs string interpolation on the provided dictionary or string input
"""
def __init__(self, *, config, request_inputs=None):
self._config = config
if request_inputs is None:
request_inputs = {}
if isinstance(request_inputs, str):
self._interpolator = InterpolatedString(request_inputs, "")
else:
self._interpolator = InterpolatedMapping(request_inputs,
JinjaInterpolation())
def request_inputs(
self,
stream_state: Mapping[str, Any],
stream_slice: Mapping[str, Any] = None,
next_page_token: Mapping[str, Any] = None) -> Union[Mapping, str]:
kwargs = {
"stream_state": stream_state,
"stream_slice": stream_slice,
"next_page_token": next_page_token
}
interpolated_value = self._interpolator.eval(self._config, **kwargs)
if isinstance(interpolated_value, dict):
non_null_tokens = {
k: v
for k, v in interpolated_value.items() if v
}
return non_null_tokens
return interpolated_value
class InterpolatedPaginator(Paginator):
def __init__(self,
*,
next_page_token_template: Mapping[str, str],
config: Config,
decoder: Optional[Decoder] = None):
self._next_page_token_template = InterpolatedMapping(
next_page_token_template, JinjaInterpolation())
self._decoder = decoder or JsonDecoder()
self._config = config
def next_page_token(
self, response: requests.Response,
last_records: List[Mapping[str,
Any]]) -> Optional[Mapping[str, Any]]:
decoded_response = self._decoder.decode(response)
headers = response.headers
interpolated_values = self._next_page_token_template.eval(
self._config,
decoded_response=decoded_response,
headers=headers,
last_records=last_records)
non_null_tokens = {
k: v
for k, v in interpolated_values.items() if v is not None
}
return non_null_tokens if non_null_tokens else None
def test():
d = {
"field": "value",
"number": 100,
"field_to_interpolate_from_config": "{{ config['c'] }}",
"field_to_interpolate_from_kwargs": "{{ kwargs['a'] }}",
}
config = {"c": "VALUE_FROM_CONFIG"}
kwargs = {"a": "VALUE_FROM_KWARGS"}
mapping = InterpolatedMapping(d)
interpolated = mapping.eval(config, **{"kwargs": kwargs})
assert interpolated["field"] == "value"
assert interpolated["number"] == 100
assert interpolated["field_to_interpolate_from_config"] == "VALUE_FROM_CONFIG"
assert interpolated["field_to_interpolate_from_kwargs"] == "VALUE_FROM_KWARGS"
def newfunc(*fargs, **fkeywords):
interpolation = JinjaInterpolation()
all_keywords = {**keywords}
all_keywords.update(fkeywords)
# config is a special keyword used for interpolation
config = all_keywords.pop("config", None)
# options is a special keyword used for interpolation and propagation
if "options" in all_keywords:
options = all_keywords.pop("options")
else:
options = dict()
# create object's partial parameters
fully_created = _create_inner_objects(all_keywords, options)
# interpolate the parameters
interpolated_keywords = InterpolatedMapping(
fully_created, interpolation).eval(config, **{"options": options})
interpolated_keywords = {
k: v
for k, v in interpolated_keywords.items() if v
}
all_keywords.update(interpolated_keywords)
# if config is not none, add it back to the keywords mapping
if config is not None:
all_keywords["config"] = config
kwargs_to_pass_down = _get_kwargs_to_pass_to_func(func, options)
all_keywords_to_pass_down = _get_kwargs_to_pass_to_func(
func, all_keywords)
try:
ret = func(*args, *fargs, **{
**all_keywords_to_pass_down,
**kwargs_to_pass_down
})
except TypeError as e:
raise Exception(
f"failed to create object of type {func} because {e}")
return ret
def test():
d = {
"field": "value",
"field_to_interpolate_from_config": "{{ config['c'] }}",
"field_to_interpolate_from_kwargs": "{{ kwargs['a'] }}",
"a_field": "{{ value_passed_directly }}",
}
config = {"c": "VALUE_FROM_CONFIG"}
kwargs = {"a": "VALUE_FROM_KWARGS"}
mapping = InterpolatedMapping(d)
value_passed_directly = "ABC"
interpolated = mapping.eval(config,
**{"kwargs": kwargs},
value_passed_directly=value_passed_directly)
assert interpolated["field"] == "value"
assert interpolated[
"field_to_interpolate_from_config"] == "VALUE_FROM_CONFIG"
assert interpolated[
"field_to_interpolate_from_kwargs"] == "VALUE_FROM_KWARGS"
assert interpolated["a_field"] == value_passed_directly
def __init__(self, parent_streams: List[Stream], state: DictState,
slice_definition: Mapping[str, Any]):
self._parent_streams = parent_streams
self._state = state
self._interpolation = InterpolatedMapping(slice_definition,
JinjaInterpolation())
class SubstreamSlicer(StreamSlicer):
"""
Stream slicer that iterates over the parent's stream slices and records and emits slices by interpolating the slice_definition mapping
Will populate the state with `parent_stream_slice` and `parent_record` so they can be accessed by other components
"""
def __init__(self, parent_streams: List[Stream], state: DictState,
slice_definition: Mapping[str, Any]):
self._parent_streams = parent_streams
self._state = state
self._interpolation = InterpolatedMapping(slice_definition,
JinjaInterpolation())
def stream_slices(
self, sync_mode: SyncMode,
stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]:
"""
Iterate over each parent stream.
For each stream, iterate over its stream_slices.
For each stream slice, iterate over each records.
yield a stream slice for each such records.
If a parent slice contains no record, emit a slice with parent_record=None.
The template string can interpolate the following values:
- parent_stream_slice: mapping representing the parent's stream slice
- parent_record: mapping representing the parent record
- parent_stream_name: string representing the parent stream name
"""
if not self._parent_streams:
yield from []
else:
for parent_stream in self._parent_streams:
for parent_stream_slice in parent_stream.stream_slices(
sync_mode=sync_mode,
cursor_field=None,
stream_state=stream_state):
self._state.update_state(
parent_stream_slice=parent_stream_slice)
self._state.update_state(parent_record=None)
empty_parent_slice = True
for parent_record in parent_stream.read_records(
sync_mode=SyncMode.full_refresh,
cursor_field=None,
stream_slice=parent_stream_slice,
stream_state=None):
empty_parent_slice = False
slice_definition = self._get_slice_definition(
parent_stream_slice, parent_record,
parent_stream.name)
self._state.update_state(parent_record=parent_record)
yield slice_definition
# If the parent slice contains no records,
# yield a slice definition with parent_record==None
if empty_parent_slice:
slice_definition = self._get_slice_definition(
parent_stream_slice, None, parent_stream.name)
yield slice_definition
def _get_slice_definition(self, parent_stream_slice, parent_record,
parent_stream_name):
return self._interpolation.eval(
None,
parent_stream_slice=parent_stream_slice,
parent_record=parent_record,
parent_stream_name=parent_stream_name)