def __init__(self, *, next_page_token_template: Mapping[str, str], config: Config, decoder: Optional[Decoder] = None): self._next_page_token_template = InterpolatedMapping( next_page_token_template, JinjaInterpolation()) self._decoder = decoder or JsonDecoder() self._config = config
def __init__(self, *, config, request_inputs=None): self._config = config if request_inputs is None: request_inputs = {} if isinstance(request_inputs, str): self._interpolator = InterpolatedString(request_inputs, "") else: self._interpolator = InterpolatedMapping(request_inputs, JinjaInterpolation())
def __init__( self, token_refresh_endpoint: str, client_id: str, client_secret: str, refresh_token: str, config: Mapping[str, Any], scopes: List[str] = None, token_expiry_date: str = None, access_token_name: str = "access_token", expires_in_name: str = "expires_in", refresh_request_body: Mapping[str, Any] = None, ): self.config = config self.token_refresh_endpoint = InterpolatedString(token_refresh_endpoint) self.client_secret = InterpolatedString(client_secret) self.client_id = InterpolatedString(client_id) self.refresh_token = InterpolatedString(refresh_token) self.scopes = scopes self.access_token_name = InterpolatedString(access_token_name) self.expires_in_name = InterpolatedString(expires_in_name) self.refresh_request_body = InterpolatedMapping(refresh_request_body) self.token_expiry_date = ( pendulum.parse(InterpolatedString(token_expiry_date).eval(self.config)) if token_expiry_date else pendulum.now().subtract(days=1) ) self.access_token = None
class InterpolatedRequestInputProvider: """ Helper class that generically performs string interpolation on the provided dictionary or string input """ def __init__(self, *, config, request_inputs=None): self._config = config if request_inputs is None: request_inputs = {} if isinstance(request_inputs, str): self._interpolator = InterpolatedString(request_inputs, "") else: self._interpolator = InterpolatedMapping(request_inputs, JinjaInterpolation()) def request_inputs( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None) -> Union[Mapping, str]: kwargs = { "stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token } interpolated_value = self._interpolator.eval(self._config, **kwargs) if isinstance(interpolated_value, dict): non_null_tokens = { k: v for k, v in interpolated_value.items() if v } return non_null_tokens return interpolated_value
class InterpolatedPaginator(Paginator): def __init__(self, *, next_page_token_template: Mapping[str, str], config: Config, decoder: Optional[Decoder] = None): self._next_page_token_template = InterpolatedMapping( next_page_token_template, JinjaInterpolation()) self._decoder = decoder or JsonDecoder() self._config = config def next_page_token( self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]: decoded_response = self._decoder.decode(response) headers = response.headers interpolated_values = self._next_page_token_template.eval( self._config, decoded_response=decoded_response, headers=headers, last_records=last_records) non_null_tokens = { k: v for k, v in interpolated_values.items() if v is not None } return non_null_tokens if non_null_tokens else None
def test(): d = { "field": "value", "number": 100, "field_to_interpolate_from_config": "{{ config['c'] }}", "field_to_interpolate_from_kwargs": "{{ kwargs['a'] }}", } config = {"c": "VALUE_FROM_CONFIG"} kwargs = {"a": "VALUE_FROM_KWARGS"} mapping = InterpolatedMapping(d) interpolated = mapping.eval(config, **{"kwargs": kwargs}) assert interpolated["field"] == "value" assert interpolated["number"] == 100 assert interpolated["field_to_interpolate_from_config"] == "VALUE_FROM_CONFIG" assert interpolated["field_to_interpolate_from_kwargs"] == "VALUE_FROM_KWARGS"
def newfunc(*fargs, **fkeywords): interpolation = JinjaInterpolation() all_keywords = {**keywords} all_keywords.update(fkeywords) # config is a special keyword used for interpolation config = all_keywords.pop("config", None) # options is a special keyword used for interpolation and propagation if "options" in all_keywords: options = all_keywords.pop("options") else: options = dict() # create object's partial parameters fully_created = _create_inner_objects(all_keywords, options) # interpolate the parameters interpolated_keywords = InterpolatedMapping( fully_created, interpolation).eval(config, **{"options": options}) interpolated_keywords = { k: v for k, v in interpolated_keywords.items() if v } all_keywords.update(interpolated_keywords) # if config is not none, add it back to the keywords mapping if config is not None: all_keywords["config"] = config kwargs_to_pass_down = _get_kwargs_to_pass_to_func(func, options) all_keywords_to_pass_down = _get_kwargs_to_pass_to_func( func, all_keywords) try: ret = func(*args, *fargs, **{ **all_keywords_to_pass_down, **kwargs_to_pass_down }) except TypeError as e: raise Exception( f"failed to create object of type {func} because {e}") return ret
def test(): d = { "field": "value", "field_to_interpolate_from_config": "{{ config['c'] }}", "field_to_interpolate_from_kwargs": "{{ kwargs['a'] }}", "a_field": "{{ value_passed_directly }}", } config = {"c": "VALUE_FROM_CONFIG"} kwargs = {"a": "VALUE_FROM_KWARGS"} mapping = InterpolatedMapping(d) value_passed_directly = "ABC" interpolated = mapping.eval(config, **{"kwargs": kwargs}, value_passed_directly=value_passed_directly) assert interpolated["field"] == "value" assert interpolated[ "field_to_interpolate_from_config"] == "VALUE_FROM_CONFIG" assert interpolated[ "field_to_interpolate_from_kwargs"] == "VALUE_FROM_KWARGS" assert interpolated["a_field"] == value_passed_directly
def __init__(self, parent_streams: List[Stream], state: DictState, slice_definition: Mapping[str, Any]): self._parent_streams = parent_streams self._state = state self._interpolation = InterpolatedMapping(slice_definition, JinjaInterpolation())
class SubstreamSlicer(StreamSlicer): """ Stream slicer that iterates over the parent's stream slices and records and emits slices by interpolating the slice_definition mapping Will populate the state with `parent_stream_slice` and `parent_record` so they can be accessed by other components """ def __init__(self, parent_streams: List[Stream], state: DictState, slice_definition: Mapping[str, Any]): self._parent_streams = parent_streams self._state = state self._interpolation = InterpolatedMapping(slice_definition, JinjaInterpolation()) def stream_slices( self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: """ Iterate over each parent stream. For each stream, iterate over its stream_slices. For each stream slice, iterate over each records. yield a stream slice for each such records. If a parent slice contains no record, emit a slice with parent_record=None. The template string can interpolate the following values: - parent_stream_slice: mapping representing the parent's stream slice - parent_record: mapping representing the parent record - parent_stream_name: string representing the parent stream name """ if not self._parent_streams: yield from [] else: for parent_stream in self._parent_streams: for parent_stream_slice in parent_stream.stream_slices( sync_mode=sync_mode, cursor_field=None, stream_state=stream_state): self._state.update_state( parent_stream_slice=parent_stream_slice) self._state.update_state(parent_record=None) empty_parent_slice = True for parent_record in parent_stream.read_records( sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=parent_stream_slice, stream_state=None): empty_parent_slice = False slice_definition = self._get_slice_definition( parent_stream_slice, parent_record, parent_stream.name) self._state.update_state(parent_record=parent_record) yield slice_definition # If the parent slice contains no records, # yield a slice definition with parent_record==None if empty_parent_slice: slice_definition = self._get_slice_definition( parent_stream_slice, None, parent_stream.name) yield slice_definition def _get_slice_definition(self, parent_stream_slice, parent_record, parent_stream_name): return self._interpolation.eval( None, parent_stream_slice=parent_stream_slice, parent_record=parent_record, parent_stream_name=parent_stream_name)