def test_value_from_kwargs(): string = "{{ next_page_token['next_page_url'] }}" interpolated_string = InterpolatedString(string) evaluated_string = interpolated_string.eval(config, **kwargs) assert evaluated_string == "https://airbyte.io"
def test_value_is_static(): static_value = "a_static_value" interpolated_string = InterpolatedString(static_value) evaluated_string = interpolated_string.eval(config, **kwargs) assert evaluated_string == static_value
def test_value_from_config(): string = "{{ config['start'] }}" interpolated_string = InterpolatedString(string) evaluated_string = interpolated_string.eval(config, **kwargs) assert evaluated_string == config["start"]
def test_interpolated_default_value(): static_value = "{{ config['end'] }}" interpolated_string = InterpolatedString(static_value, "{{ config['start'] }}") evaluated_string = interpolated_string.eval(config, **kwargs) assert evaluated_string == config["start"]
def test_default_value(): static_value = "{{ config['end'] }}" default = 5678 interpolated_string = InterpolatedString(static_value, default) evaluated_string = interpolated_string.eval(config, **kwargs) assert evaluated_string == default
def test_http_requester(): http_method = "GET" request_options_provider = MagicMock() request_params = {"param": "value"} request_body_data = "body_key_1=value_1&body_key_2=value2" request_body_json = {"body_field": "body_value"} request_options_provider.request_params.return_value = request_params request_options_provider.request_body_data.return_value = request_body_data request_options_provider.request_body_json.return_value = request_body_json request_headers_provider = MagicMock() request_headers = {"header": "value"} request_headers_provider.request_headers.return_value = request_headers authenticator = MagicMock() error_handler = MagicMock() max_retries = 10 should_retry = True backoff_time = 1000 error_handler.max_retries = max_retries error_handler.should_retry.return_value = should_retry error_handler.backoff_time.return_value = backoff_time config = {"url": "https://airbyte.io"} stream_slice = {"id": "1234"} name = "stream_name" requester = HttpRequester( name=name, url_base=InterpolatedString("{{ config['url'] }}"), path=InterpolatedString("v1/{{ stream_slice['id'] }}"), http_method=http_method, request_options_provider=request_options_provider, authenticator=authenticator, error_handler=error_handler, config=config, ) assert requester.get_url_base() == "https://airbyte.io" assert requester.get_path(stream_state={}, stream_slice=stream_slice, next_page_token={}) == "v1/1234" assert requester.get_authenticator() == authenticator assert requester.get_method() == HttpMethod.GET assert requester.request_params(stream_state={}, stream_slice=None, next_page_token=None) == request_params assert requester.request_body_data( stream_state={}, stream_slice=None, next_page_token=None) == request_body_data assert requester.request_body_json( stream_state={}, stream_slice=None, next_page_token=None) == request_body_json assert requester.should_retry(requests.Response()) == should_retry
class MinMaxDatetime: """ Compares the provided date against optional minimum or maximum times. If date is earlier than min_date, then min_date is returned. If date is greater than max_date, then max_date is returned. If neither, the input date is returned. """ def __init__( self, datetime: str, datetime_format: str = "", min_datetime: str = "", max_datetime: str = "", ): self._datetime_interpolator = InterpolatedString(datetime) self._datetime_format = datetime_format self._timezone = dt.timezone.utc self._min_datetime_interpolator = InterpolatedString( min_datetime) if min_datetime else None self._max_datetime_interpolator = InterpolatedString( max_datetime) if max_datetime else None def get_datetime(self, config, **kwargs) -> dt.datetime: # We apply a default datetime format here instead of at instantiation, so it can be set by the parent first datetime_format = self._datetime_format if not datetime_format: datetime_format = "%Y-%m-%dT%H:%M:%S.%f%z" time = dt.datetime.strptime( self._datetime_interpolator.eval(config, **kwargs), datetime_format).replace(tzinfo=self._timezone) if self._min_datetime_interpolator: min_time = dt.datetime.strptime( self._min_datetime_interpolator.eval(config, **kwargs), datetime_format).replace(tzinfo=self._timezone) time = max(time, min_time) if self._max_datetime_interpolator: max_time = dt.datetime.strptime( self._max_datetime_interpolator.eval(config, **kwargs), datetime_format).replace(tzinfo=self._timezone) time = min(time, max_time) return time @property def datetime_format(self): return self._datetime_format @datetime_format.setter def datetime_format(self, value): self._datetime_format = value
def __init__( self, token_refresh_endpoint: str, client_id: str, client_secret: str, refresh_token: str, config: Mapping[str, Any], scopes: List[str] = None, token_expiry_date: str = None, access_token_name: str = "access_token", expires_in_name: str = "expires_in", refresh_request_body: Mapping[str, Any] = None, ): self.config = config self.token_refresh_endpoint = InterpolatedString(token_refresh_endpoint) self.client_secret = InterpolatedString(client_secret) self.client_id = InterpolatedString(client_id) self.refresh_token = InterpolatedString(refresh_token) self.scopes = scopes self.access_token_name = InterpolatedString(access_token_name) self.expires_in_name = InterpolatedString(expires_in_name) self.refresh_request_body = InterpolatedMapping(refresh_request_body) self.token_expiry_date = ( pendulum.parse(InterpolatedString(token_expiry_date).eval(self.config)) if token_expiry_date else pendulum.now().subtract(days=1) ) self.access_token = None
def __init__(self, *, config, request_inputs=None): self._config = config if request_inputs is None: request_inputs = {} if isinstance(request_inputs, str): self._interpolator = InterpolatedString(request_inputs, "") else: self._interpolator = InterpolatedMapping(request_inputs, JinjaInterpolation())
def test_stream_slices(mock_datetime_now, test_name, stream_state, start, end, cursor, step, lookback_window, expected_slices): lookback_window = InterpolatedString(lookback_window) if lookback_window else None slicer = DatetimeStreamSlicer( start_datetime=start, end_datetime=end, step=step, cursor_value=cursor, datetime_format=datetime_format, lookback_window=lookback_window, config=config, ) stream_slices = slicer.stream_slices(SyncMode.incremental, stream_state) assert expected_slices == stream_slices
def __init__( self, datetime: str, datetime_format: str = "", min_datetime: str = "", max_datetime: str = "", ): self._datetime_interpolator = InterpolatedString(datetime) self._datetime_format = datetime_format self._timezone = dt.timezone.utc self._min_datetime_interpolator = InterpolatedString( min_datetime) if min_datetime else None self._max_datetime_interpolator = InterpolatedString( max_datetime) if max_datetime else None
def test_static_value(): static_value = "HELLO WORLD" s = InterpolatedString(static_value) assert s.eval(config) == "HELLO WORLD"
def test_eval_from_kwargs(): string = "{{ kwargs['c'] }}" kwargs = {"c": "airbyte"} s = InterpolatedString(string) assert s.eval(config, **{"kwargs": kwargs}) == "airbyte"
def test_eval_from_config(): string = "{{ config['field'] }}" s = InterpolatedString(string) assert s.eval(config) == "value"
{"owner_resource": "customer", "letter": "B"}, {"owner_resource": "store", "letter": "A"}, {"owner_resource": "store", "letter": "B"}, {"owner_resource": "subscription", "letter": "A"}, {"owner_resource": "subscription", "letter": "B"}, ], ), ( "test_list_and_datetime", [ ListStreamSlicer(["customer", "store", "subscription"], {"owner_resource": "{{ slice_value }}"}, None), DatetimeStreamSlicer( MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d"), MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d"), "1d", InterpolatedString(""), "%Y-%m-%d", None, ), ], [ {"owner_resource": "customer", "start_date": "2021-01-01", "end_date": "2021-01-01"}, {"owner_resource": "customer", "start_date": "2021-01-02", "end_date": "2021-01-02"}, {"owner_resource": "customer", "start_date": "2021-01-03", "end_date": "2021-01-03"}, {"owner_resource": "store", "start_date": "2021-01-01", "end_date": "2021-01-01"}, {"owner_resource": "store", "start_date": "2021-01-02", "end_date": "2021-01-02"}, {"owner_resource": "store", "start_date": "2021-01-03", "end_date": "2021-01-03"}, {"owner_resource": "subscription", "start_date": "2021-01-01", "end_date": "2021-01-01"}, {"owner_resource": "subscription", "start_date": "2021-01-02", "end_date": "2021-01-02"}, {"owner_resource": "subscription", "start_date": "2021-01-03", "end_date": "2021-01-03"}, ],
import datetime import unittest import pytest from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer datetime_format = "%Y-%m-%d" FAKE_NOW = datetime.datetime(2022, 1, 1, tzinfo=datetime.timezone.utc) config = {"start_date": "2021-01-01"} end_date_now = InterpolatedString( "{{ today_utc() }}", ) cursor_value = InterpolatedString("{{ stream_state['date'] }}") timezone = datetime.timezone.utc @pytest.fixture() def mock_datetime_now(monkeypatch): datetime_mock = unittest.mock.MagicMock(wraps=datetime.datetime) datetime_mock.now.return_value = FAKE_NOW monkeypatch.setattr(datetime, "datetime", datetime_mock) @pytest.mark.parametrize( "test_name, stream_state, start, end, step, cursor, lookback_window, expected_slices", [