def test_http_requester():
http_method = "GET"
request_options_provider = MagicMock()
request_params = {"param": "value"}
request_body_data = "body_key_1=value_1&body_key_2=value2"
request_body_json = {"body_field": "body_value"}
request_options_provider.request_params.return_value = request_params
request_options_provider.request_body_data.return_value = request_body_data
request_options_provider.request_body_json.return_value = request_body_json
request_headers_provider = MagicMock()
request_headers = {"header": "value"}
request_headers_provider.request_headers.return_value = request_headers
authenticator = MagicMock()
error_handler = MagicMock()
max_retries = 10
should_retry = True
backoff_time = 1000
error_handler.max_retries = max_retries
error_handler.should_retry.return_value = should_retry
error_handler.backoff_time.return_value = backoff_time
config = {"url": "https://airbyte.io"}
stream_slice = {"id": "1234"}
name = "stream_name"
requester = HttpRequester(
name=name,
url_base=InterpolatedString("{{ config['url'] }}"),
path=InterpolatedString("v1/{{ stream_slice['id'] }}"),
http_method=http_method,
request_options_provider=request_options_provider,
authenticator=authenticator,
error_handler=error_handler,
config=config,
)
assert requester.get_url_base() == "https://airbyte.io"
assert requester.get_path(stream_state={},
stream_slice=stream_slice,
next_page_token={}) == "v1/1234"
assert requester.get_authenticator() == authenticator
assert requester.get_method() == HttpMethod.GET
assert requester.request_params(stream_state={},
stream_slice=None,
next_page_token=None) == request_params
assert requester.request_body_data(
stream_state={}, stream_slice=None,
next_page_token=None) == request_body_data
assert requester.request_body_json(
stream_state={}, stream_slice=None,
next_page_token=None) == request_body_json
assert requester.should_retry(requests.Response()) == should_retry
def __init__(
self,
datetime: str,
datetime_format: str = "",
min_datetime: str = "",
max_datetime: str = "",
):
self._datetime_interpolator = InterpolatedString(datetime)
self._datetime_format = datetime_format
self._timezone = dt.timezone.utc
self._min_datetime_interpolator = InterpolatedString(
min_datetime) if min_datetime else None
self._max_datetime_interpolator = InterpolatedString(
max_datetime) if max_datetime else None
def __init__(self, *, config, request_inputs=None):
self._config = config
if request_inputs is None:
request_inputs = {}
if isinstance(request_inputs, str):
self._interpolator = InterpolatedString(request_inputs, "")
else:
self._interpolator = InterpolatedMapping(request_inputs,
JinjaInterpolation())
def test_stream_slices(mock_datetime_now, test_name, stream_state, start, end, cursor, step, lookback_window, expected_slices):
lookback_window = InterpolatedString(lookback_window) if lookback_window else None
slicer = DatetimeStreamSlicer(
start_datetime=start,
end_datetime=end,
step=step,
cursor_value=cursor,
datetime_format=datetime_format,
lookback_window=lookback_window,
config=config,
)
stream_slices = slicer.stream_slices(SyncMode.incremental, stream_state)
assert expected_slices == stream_slices
def test_eval_from_kwargs():
string = "{{ kwargs['c'] }}"
kwargs = {"c": "airbyte"}
s = InterpolatedString(string)
assert s.eval(config, **{"kwargs": kwargs}) == "airbyte"
def test_eval_from_config():
string = "{{ config['field'] }}"
s = InterpolatedString(string)
assert s.eval(config) == "value"
{"owner_resource": "customer", "letter": "B"},
{"owner_resource": "store", "letter": "A"},
{"owner_resource": "store", "letter": "B"},
{"owner_resource": "subscription", "letter": "A"},
{"owner_resource": "subscription", "letter": "B"},
],
),
(
"test_list_and_datetime",
[
ListStreamSlicer(["customer", "store", "subscription"], {"owner_resource": "{{ slice_value }}"}, None),
DatetimeStreamSlicer(
MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d"),
MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d"),
"1d",
InterpolatedString(""),
"%Y-%m-%d",
None,
),
],
[
{"owner_resource": "customer", "start_date": "2021-01-01", "end_date": "2021-01-01"},
{"owner_resource": "customer", "start_date": "2021-01-02", "end_date": "2021-01-02"},
{"owner_resource": "customer", "start_date": "2021-01-03", "end_date": "2021-01-03"},
{"owner_resource": "store", "start_date": "2021-01-01", "end_date": "2021-01-01"},
{"owner_resource": "store", "start_date": "2021-01-02", "end_date": "2021-01-02"},
{"owner_resource": "store", "start_date": "2021-01-03", "end_date": "2021-01-03"},
{"owner_resource": "subscription", "start_date": "2021-01-01", "end_date": "2021-01-01"},
{"owner_resource": "subscription", "start_date": "2021-01-02", "end_date": "2021-01-02"},
{"owner_resource": "subscription", "start_date": "2021-01-03", "end_date": "2021-01-03"},
],
import datetime
import unittest
import pytest
from airbyte_cdk.models import SyncMode
from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer
datetime_format = "%Y-%m-%d"
FAKE_NOW = datetime.datetime(2022, 1, 1, tzinfo=datetime.timezone.utc)
config = {"start_date": "2021-01-01"}
end_date_now = InterpolatedString(
"{{ today_utc() }}",
)
cursor_value = InterpolatedString("{{ stream_state['date'] }}")
timezone = datetime.timezone.utc
@pytest.fixture()
def mock_datetime_now(monkeypatch):
datetime_mock = unittest.mock.MagicMock(wraps=datetime.datetime)
datetime_mock.now.return_value = FAKE_NOW
monkeypatch.setattr(datetime, "datetime", datetime_mock)
@pytest.mark.parametrize(
"test_name, stream_state, start, end, step, cursor, lookback_window, expected_slices",
[