def test_negative_day_delta():
delta_template = "{{ day_delta(-25) }}"
interpolation = JinjaInterpolation()
val = interpolation.eval(delta_template, {})
assert val <= (
datetime.datetime.now(datetime.timezone.utc) -
datetime.timedelta(days=25)).strftime("%Y-%m-%dT%H:%M:%S.%f%z")
def test_positive_day_delta():
delta_template = "{{ day_delta(25) }}"
interpolation = JinjaInterpolation()
val = interpolation.eval(delta_template, {})
# We need to assert against an earlier delta since the interpolation function runs datetime.now() a few milliseconds earlier
assert val > (datetime.datetime.now(datetime.timezone.utc) +
datetime.timedelta(
days=24, hours=23)).strftime("%Y-%m-%dT%H:%M:%S.%f%z")
def __init__(self, initial_mapping: Mapping[str, str] = None, config=None):
if initial_mapping is None:
initial_mapping = dict()
if config is None:
config = dict()
self._templates_to_evaluate = initial_mapping
self._interpolator = JinjaInterpolation()
self._context = dict()
self._config = config
def __init__(self,
transform: str,
decoder: Optional[Decoder] = None,
config=None,
kwargs=None):
self._interpolator = JinjaInterpolation()
self._transform = transform
self._decoder = decoder or JsonDecoder()
self._config = config
self._kwargs = kwargs or dict()
def __init__(
self,
start_datetime: MinMaxDatetime,
end_datetime: MinMaxDatetime,
step: str,
cursor_value: InterpolatedString,
datetime_format: str,
config: Config,
lookback_window: Optional[InterpolatedString] = None,
):
self._timezone = datetime.timezone.utc
self._interpolation = JinjaInterpolation()
self._datetime_format = datetime_format
self._start_datetime = start_datetime
self._end_datetime = end_datetime
self._step = self._parse_timedelta(step)
self._config = config
self._cursor_value = cursor_value
self._lookback_window = lookback_window
# If datetime format is not specified then start/end datetime should inherit it from the stream slicer
if not self._start_datetime.datetime_format:
self._start_datetime.datetime_format = self._datetime_format
if not self._end_datetime.datetime_format:
self._end_datetime.datetime_format = self._datetime_format
class InterpolatedString:
def __init__(self, string: str, default: Optional[str] = None):
self._string = string
self._default = default or string
self._interpolation = JinjaInterpolation()
def eval(self, config, **kwargs):
return self._interpolation.eval(self._string, config, self._default, **kwargs)
def __init__(self,
*,
next_page_token_template: Mapping[str, str],
config: Config,
decoder: Optional[Decoder] = None):
self._next_page_token_template = InterpolatedMapping(
next_page_token_template, JinjaInterpolation())
self._decoder = decoder or JsonDecoder()
self._config = config
def __init__(self, *, config, request_inputs=None):
self._config = config
if request_inputs is None:
request_inputs = {}
if isinstance(request_inputs, str):
self._interpolator = InterpolatedString(request_inputs, "")
else:
self._interpolator = InterpolatedMapping(request_inputs,
JinjaInterpolation())
class DictState(State):
stream_state_field = "stream_state"
def __init__(self, initial_mapping: Mapping[str, str] = None, config=None):
if initial_mapping is None:
initial_mapping = dict()
if config is None:
config = dict()
self._templates_to_evaluate = initial_mapping
self._interpolator = JinjaInterpolation()
self._context = dict()
self._config = config
def set_state(self, state):
self._context[self.stream_state_field] = state
def update_state(self, **kwargs):
stream_state = kwargs.get(self.stream_state_field)
prev_stream_state = self.get_stream_state() or stream_state
self._context.update(**kwargs)
self._context[self.stream_state_field] = self._compute_state(prev_stream_state)
def get_state(self, state_field):
return self._context.get(state_field, {})
def get_stream_state(self):
return self.get_state(self.stream_state_field)
def _compute_state(self, prev_state):
updated_state = {
self._interpolator.eval(name, self._config): self._interpolator.eval(value, self._config, **self._context)
for name, value in self._templates_to_evaluate.items()
}
updated_state = {name: value for name, value in updated_state.items() if value}
if prev_state:
next_state = {name: _get_max(name=name, val=value, other_state=prev_state) for name, value in updated_state.items()}
else:
next_state = updated_state
self._context[self.stream_state_field] = next_state
return next_state
class InterpolatedBoolean:
def __init__(self, condition):
self._condition = condition
self._default = "False"
self._interpolation = JinjaInterpolation()
def eval(self, config, **kwargs):
if isinstance(self._condition, bool):
return self._condition
else:
evaluated = self._interpolation.eval(self._condition, config,
self._default, **kwargs)
if evaluated in false_values:
return False
# The presence of a value is generally regarded as truthy, so we treat it as such
return True
def newfunc(*fargs, **fkeywords):
interpolation = JinjaInterpolation()
all_keywords = {**keywords}
all_keywords.update(fkeywords)
# config is a special keyword used for interpolation
config = all_keywords.pop("config", None)
# options is a special keyword used for interpolation and propagation
if "options" in all_keywords:
options = all_keywords.pop("options")
else:
options = dict()
# create object's partial parameters
fully_created = _create_inner_objects(all_keywords, options)
# interpolate the parameters
interpolated_keywords = InterpolatedMapping(
fully_created, interpolation).eval(config, **{"options": options})
interpolated_keywords = {
k: v
for k, v in interpolated_keywords.items() if v
}
all_keywords.update(interpolated_keywords)
# if config is not none, add it back to the keywords mapping
if config is not None:
all_keywords["config"] = config
kwargs_to_pass_down = _get_kwargs_to_pass_to_func(func, options)
all_keywords_to_pass_down = _get_kwargs_to_pass_to_func(
func, all_keywords)
try:
ret = func(*args, *fargs, **{
**all_keywords_to_pass_down,
**kwargs_to_pass_down
})
except TypeError as e:
raise Exception(
f"failed to create object of type {func} because {e}")
return ret
class JelloExtractor:
default_transform = "."
def __init__(self,
transform: str,
decoder: Optional[Decoder] = None,
config=None,
kwargs=None):
self._interpolator = JinjaInterpolation()
self._transform = transform
self._decoder = decoder or JsonDecoder()
self._config = config
self._kwargs = kwargs or dict()
def extract_records(self, response: requests.Response) -> List[Record]:
response_body = self._decoder.decode(response)
script = self._interpolator.eval(self._transform,
self._config,
default=self.default_transform,
**{"kwargs": self._kwargs})
return jello_lib.pyquery(response_body, script)
def __init__(self):
self._interpolator = JinjaInterpolation()
def __init__(self, string: str, default: Optional[str] = None):
self._string = string
self._default = default or string
self._interpolation = JinjaInterpolation()
def __init__(self,
mapping: Mapping[str, Any],
interpolation: Interpolation = JinjaInterpolation()):
self._mapping = mapping
self._interpolation = interpolation
def test_macros(test_name, s, expected_value):
interpolation = JinjaInterpolation()
config = {}
val = interpolation.eval(s, config)
assert val == expected_value
def __init__(self, parent_streams: List[Stream], state: DictState,
slice_definition: Mapping[str, Any]):
self._parent_streams = parent_streams
self._state = state
self._interpolation = InterpolatedMapping(slice_definition,
JinjaInterpolation())
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
import datetime
import pytest
from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
interpolation = JinjaInterpolation()
def test_get_value_from_config():
s = "{{ config['date'] }}"
config = {"date": "2022-01-01"}
val = interpolation.eval(s, config)
assert val == "2022-01-01"
def test_get_value_from_stream_slice():
s = "{{ stream_slice['date'] }}"
config = {"date": "2022-01-01"}
stream_slice = {"date": "2020-09-09"}
val = interpolation.eval(s, config, **{"stream_slice": stream_slice})
assert val == "2020-09-09"
def test_get_value_from_a_list_of_mappings():
s = "{{ records[0]['date'] }}"
config = {"date": "2022-01-01"}
records = [{"date": "2020-09-09"}]
def __init__(self, condition):
self._condition = condition
self._default = "False"
self._interpolation = JinjaInterpolation()