def test_value_from_kwargs():
string = "{{ next_page_token['next_page_url'] }}"
interpolated_string = InterpolatedString(string)
evaluated_string = interpolated_string.eval(config, **kwargs)
assert evaluated_string == "https://airbyte.io"
def test_value_is_static():
static_value = "a_static_value"
interpolated_string = InterpolatedString(static_value)
evaluated_string = interpolated_string.eval(config, **kwargs)
assert evaluated_string == static_value
def test_value_from_config():
string = "{{ config['start'] }}"
interpolated_string = InterpolatedString(string)
evaluated_string = interpolated_string.eval(config, **kwargs)
assert evaluated_string == config["start"]
def test_interpolated_default_value():
static_value = "{{ config['end'] }}"
interpolated_string = InterpolatedString(static_value,
"{{ config['start'] }}")
evaluated_string = interpolated_string.eval(config, **kwargs)
assert evaluated_string == config["start"]
def test_default_value():
static_value = "{{ config['end'] }}"
default = 5678
interpolated_string = InterpolatedString(static_value, default)
evaluated_string = interpolated_string.eval(config, **kwargs)
assert evaluated_string == default
def test_http_requester():
http_method = "GET"
request_options_provider = MagicMock()
request_params = {"param": "value"}
request_body_data = "body_key_1=value_1&body_key_2=value2"
request_body_json = {"body_field": "body_value"}
request_options_provider.request_params.return_value = request_params
request_options_provider.request_body_data.return_value = request_body_data
request_options_provider.request_body_json.return_value = request_body_json
request_headers_provider = MagicMock()
request_headers = {"header": "value"}
request_headers_provider.request_headers.return_value = request_headers
authenticator = MagicMock()
error_handler = MagicMock()
max_retries = 10
should_retry = True
backoff_time = 1000
error_handler.max_retries = max_retries
error_handler.should_retry.return_value = should_retry
error_handler.backoff_time.return_value = backoff_time
config = {"url": "https://airbyte.io"}
stream_slice = {"id": "1234"}
name = "stream_name"
requester = HttpRequester(
name=name,
url_base=InterpolatedString("{{ config['url'] }}"),
path=InterpolatedString("v1/{{ stream_slice['id'] }}"),
http_method=http_method,
request_options_provider=request_options_provider,
authenticator=authenticator,
error_handler=error_handler,
config=config,
)
assert requester.get_url_base() == "https://airbyte.io"
assert requester.get_path(stream_state={},
stream_slice=stream_slice,
next_page_token={}) == "v1/1234"
assert requester.get_authenticator() == authenticator
assert requester.get_method() == HttpMethod.GET
assert requester.request_params(stream_state={},
stream_slice=None,
next_page_token=None) == request_params
assert requester.request_body_data(
stream_state={}, stream_slice=None,
next_page_token=None) == request_body_data
assert requester.request_body_json(
stream_state={}, stream_slice=None,
next_page_token=None) == request_body_json
assert requester.should_retry(requests.Response()) == should_retry
class MinMaxDatetime:
"""
Compares the provided date against optional minimum or maximum times. If date is earlier than
min_date, then min_date is returned. If date is greater than max_date, then max_date is returned.
If neither, the input date is returned.
"""
def __init__(
self,
datetime: str,
datetime_format: str = "",
min_datetime: str = "",
max_datetime: str = "",
):
self._datetime_interpolator = InterpolatedString(datetime)
self._datetime_format = datetime_format
self._timezone = dt.timezone.utc
self._min_datetime_interpolator = InterpolatedString(
min_datetime) if min_datetime else None
self._max_datetime_interpolator = InterpolatedString(
max_datetime) if max_datetime else None
def get_datetime(self, config, **kwargs) -> dt.datetime:
# We apply a default datetime format here instead of at instantiation, so it can be set by the parent first
datetime_format = self._datetime_format
if not datetime_format:
datetime_format = "%Y-%m-%dT%H:%M:%S.%f%z"
time = dt.datetime.strptime(
self._datetime_interpolator.eval(config, **kwargs),
datetime_format).replace(tzinfo=self._timezone)
if self._min_datetime_interpolator:
min_time = dt.datetime.strptime(
self._min_datetime_interpolator.eval(config, **kwargs),
datetime_format).replace(tzinfo=self._timezone)
time = max(time, min_time)
if self._max_datetime_interpolator:
max_time = dt.datetime.strptime(
self._max_datetime_interpolator.eval(config, **kwargs),
datetime_format).replace(tzinfo=self._timezone)
time = min(time, max_time)
return time
@property
def datetime_format(self):
return self._datetime_format
@datetime_format.setter
def datetime_format(self, value):
self._datetime_format = value
def __init__(
self,
token_refresh_endpoint: str,
client_id: str,
client_secret: str,
refresh_token: str,
config: Mapping[str, Any],
scopes: List[str] = None,
token_expiry_date: str = None,
access_token_name: str = "access_token",
expires_in_name: str = "expires_in",
refresh_request_body: Mapping[str, Any] = None,
):
self.config = config
self.token_refresh_endpoint = InterpolatedString(token_refresh_endpoint)
self.client_secret = InterpolatedString(client_secret)
self.client_id = InterpolatedString(client_id)
self.refresh_token = InterpolatedString(refresh_token)
self.scopes = scopes
self.access_token_name = InterpolatedString(access_token_name)
self.expires_in_name = InterpolatedString(expires_in_name)
self.refresh_request_body = InterpolatedMapping(refresh_request_body)
self.token_expiry_date = (
pendulum.parse(InterpolatedString(token_expiry_date).eval(self.config))
if token_expiry_date
else pendulum.now().subtract(days=1)
)
self.access_token = None
def __init__(self, *, config, request_inputs=None):
self._config = config
if request_inputs is None:
request_inputs = {}
if isinstance(request_inputs, str):
self._interpolator = InterpolatedString(request_inputs, "")
else:
self._interpolator = InterpolatedMapping(request_inputs,
JinjaInterpolation())
def test_stream_slices(mock_datetime_now, test_name, stream_state, start, end, cursor, step, lookback_window, expected_slices):
lookback_window = InterpolatedString(lookback_window) if lookback_window else None
slicer = DatetimeStreamSlicer(
start_datetime=start,
end_datetime=end,
step=step,
cursor_value=cursor,
datetime_format=datetime_format,
lookback_window=lookback_window,
config=config,
)
stream_slices = slicer.stream_slices(SyncMode.incremental, stream_state)
assert expected_slices == stream_slices
def __init__(
self,
datetime: str,
datetime_format: str = "",
min_datetime: str = "",
max_datetime: str = "",
):
self._datetime_interpolator = InterpolatedString(datetime)
self._datetime_format = datetime_format
self._timezone = dt.timezone.utc
self._min_datetime_interpolator = InterpolatedString(
min_datetime) if min_datetime else None
self._max_datetime_interpolator = InterpolatedString(
max_datetime) if max_datetime else None
def test_static_value():
static_value = "HELLO WORLD"
s = InterpolatedString(static_value)
assert s.eval(config) == "HELLO WORLD"
def test_eval_from_kwargs():
string = "{{ kwargs['c'] }}"
kwargs = {"c": "airbyte"}
s = InterpolatedString(string)
assert s.eval(config, **{"kwargs": kwargs}) == "airbyte"
def test_eval_from_config():
string = "{{ config['field'] }}"
s = InterpolatedString(string)
assert s.eval(config) == "value"
{"owner_resource": "customer", "letter": "B"},
{"owner_resource": "store", "letter": "A"},
{"owner_resource": "store", "letter": "B"},
{"owner_resource": "subscription", "letter": "A"},
{"owner_resource": "subscription", "letter": "B"},
],
),
(
"test_list_and_datetime",
[
ListStreamSlicer(["customer", "store", "subscription"], {"owner_resource": "{{ slice_value }}"}, None),
DatetimeStreamSlicer(
MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d"),
MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d"),
"1d",
InterpolatedString(""),
"%Y-%m-%d",
None,
),
],
[
{"owner_resource": "customer", "start_date": "2021-01-01", "end_date": "2021-01-01"},
{"owner_resource": "customer", "start_date": "2021-01-02", "end_date": "2021-01-02"},
{"owner_resource": "customer", "start_date": "2021-01-03", "end_date": "2021-01-03"},
{"owner_resource": "store", "start_date": "2021-01-01", "end_date": "2021-01-01"},
{"owner_resource": "store", "start_date": "2021-01-02", "end_date": "2021-01-02"},
{"owner_resource": "store", "start_date": "2021-01-03", "end_date": "2021-01-03"},
{"owner_resource": "subscription", "start_date": "2021-01-01", "end_date": "2021-01-01"},
{"owner_resource": "subscription", "start_date": "2021-01-02", "end_date": "2021-01-02"},
{"owner_resource": "subscription", "start_date": "2021-01-03", "end_date": "2021-01-03"},
],
import datetime
import unittest
import pytest
from airbyte_cdk.models import SyncMode
from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer
datetime_format = "%Y-%m-%d"
FAKE_NOW = datetime.datetime(2022, 1, 1, tzinfo=datetime.timezone.utc)
config = {"start_date": "2021-01-01"}
end_date_now = InterpolatedString(
"{{ today_utc() }}",
)
cursor_value = InterpolatedString("{{ stream_state['date'] }}")
timezone = datetime.timezone.utc
@pytest.fixture()
def mock_datetime_now(monkeypatch):
datetime_mock = unittest.mock.MagicMock(wraps=datetime.datetime)
datetime_mock.now.return_value = FAKE_NOW
monkeypatch.setattr(datetime, "datetime", datetime_mock)
@pytest.mark.parametrize(
"test_name, stream_state, start, end, step, cursor, lookback_window, expected_slices",
[