def test_interpolated_request_header(test_name, stop_condition_template,
expected_next_page_token, expected_page):
state = DictState()
state.update_state(page=1)
decoder = JsonDecoder()
config = {"response_override": "stop_if_you_see_me"}
response = requests.Response()
response.headers = {"has_more": True}
response_body = {
"_metadata": {
"content": "stop_if_you_see_me"
},
"accounts": [],
"end": 99,
"total": 200,
"characters": {}
}
response._content = json.dumps(response_body).encode("utf-8")
last_records = [{
"id": 0,
"more_records": True
}, {
"id": 1,
"more_records": True
}]
paginator = ConditionalPaginator(stop_condition_template, state, decoder,
config)
next_page_token = paginator.next_page_token(response, last_records)
assert next_page_token == expected_next_page_token
assert state.get_state("page") == expected_page
def test_record_filter(test_name, transform_template, filter_template, body,
expected_records):
config = {"response_override": "stop_if_you_see_me"}
stream_state = {"created_at": "06-06-21"}
stream_slice = {"last_seen": "06-10-21"}
next_page_token = {"last_seen_id": 14}
response = create_response(body)
decoder = JsonDecoder()
extractor = JelloExtractor(transform=transform_template,
decoder=decoder,
config=config,
kwargs={})
if filter_template is None:
record_filter = None
else:
record_filter = RecordFilter(config=config, condition=filter_template)
record_selector = RecordSelector(extractor=extractor,
record_filter=record_filter)
actual_records = record_selector.select_records(
response=response,
stream_state=stream_state,
stream_slice=stream_slice,
next_page_token=next_page_token)
assert actual_records == expected_records
def __init__(self,
*,
next_page_token_template: Mapping[str, str],
config: Config,
decoder: Optional[Decoder] = None):
self._next_page_token_template = InterpolatedMapping(
next_page_token_template, JinjaInterpolation())
self._decoder = decoder or JsonDecoder()
self._config = config
def __init__(self,
transform: str,
decoder: Optional[Decoder] = None,
config=None,
kwargs=None):
self._interpolator = JinjaInterpolation()
self._transform = transform
self._decoder = decoder or JsonDecoder()
self._config = config
self._kwargs = kwargs or dict()
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
import json
import pytest
import requests
from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor
config = {"field": "record_array"}
kwargs = {"data_field": "records"}
decoder = JsonDecoder()
@pytest.mark.parametrize(
"test_name, transform, body, expected_records",
[
("test_extract_from_array", "_.data", {
"data": [{
"id": 1
}, {
"id": 2
}]
}, [{
"id": 1
}, {
"id": 2
}]),