def streams(self, config: Mapping[str, Any]) -> List[Stream]:
auth = NoAuth()
return [
Users(authenticator=auth, api_key=config["api_key"]),
Leads(authenticator=auth, api_key=config["api_key"]),
Campaigns(authenticator=auth, api_key=config["api_key"]),
]
def test_slice(site_urls, sync_mode):
stream = SearchAnalyticsByDate(NoAuth(), site_urls, "2021-09-01",
"2021-09-07")
search_types = stream.search_types
stream_slice = stream.stream_slices(sync_mode=sync_mode)
for site_url in site_urls:
for search_type in search_types:
for range_ in [
{
"start_date": "2021-09-01",
"end_date": "2021-09-03"
},
{
"start_date": "2021-09-04",
"end_date": "2021-09-06"
},
{
"start_date": "2021-09-07",
"end_date": "2021-09-07"
},
]:
expected = {
"site_url": quote_plus(site_url),
"search_type": search_type,
"start_date": range_["start_date"],
"end_date": range_["end_date"],
}
assert expected == next(stream_slice)
def test_updated_state():
stream = SearchAnalyticsByDate(
NoAuth(), ["https://domain1.com", "https://domain2.com"], "start_date",
"end_date")
state = {}
record = {
"site_url": "https://domain1.com",
"search_type": "web",
"date": "2022-01-01"
}
state = stream.get_updated_state(state, record)
record = {
"site_url": "https://domain2.com",
"search_type": "web",
"date": "2022-01-01"
}
state = stream.get_updated_state(state, record)
assert state == {
"https://domain1.com": {
"web": {
"date": "2022-01-01"
}
},
"https://domain2.com": {
"web": {
"date": "2022-01-01"
}
},
"date": "2022-01-01",
}
def us_census_stream():
return UsCensusStream(
query_params={},
query_path="data/test",
api_key="MY_API_KEY",
authenticator=NoAuth(),
)
def test_config():
test_config = json.loads(
read_file("../integration_tests/sample_config.json"))
test_config["authenticator"] = NoAuth()
test_config["metrics"] = []
test_config["dimensions"] = []
return test_config
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
config["timezone"] = config.get("timezone", "UTC")
timezone = pendulum.timezone(config.get("timezone", "UTC"))
earliest_date = pendulum.today(timezone) - timedelta(days=90)
start_date = parse_date(
config.get("start_date") or pendulum.today(timezone), timezone)
config["start_date"] = self.is_start_date_before_earliest_date(
start_date, earliest_date)
config["end_date"] = pendulum.now(timezone)
AirbyteLogger().log(
"INFO",
f"Using start_date: {config['start_date']}, end_date: {config['end_date']}"
)
auth = NoAuth()
return [
InAppEvents(authenticator=auth, **config),
Installs(authenticator=auth, **config),
UninstallEvents(authenticator=auth, **config),
RetargetingInAppEvents(authenticator=auth, **config),
RetargetingConversions(authenticator=auth, **config),
PartnersReport(authenticator=auth, **config),
DailyReport(authenticator=auth, **config),
GeoReport(authenticator=auth, **config),
RetargetingPartnersReport(authenticator=auth, **config),
RetargetingDailyReport(authenticator=auth, **config),
RetargetingGeoReport(authenticator=auth, **config),
]
def test_switch_to_standard_endpoint_if_scroll_expired(requests_mock):
"""
Test shows that if scroll param expired we try sync with standard API.
"""
url = "https://api.intercom.io/companies/scroll"
requests_mock.get(
url,
json={"type": "company.list", "data": [{"type": "company", "id": "530370b477ad7120001d"}], "scroll_param": "expired_scroll_param"},
)
url = "https://api.intercom.io/companies/scroll?scroll_param=expired_scroll_param"
requests_mock.get(url, json={"errors": [{"code": "not_found", "message": "scroll parameter not found"}]}, status_code=404)
url = "https://api.intercom.io/companies"
requests_mock.get(url, json={"type": "company.list", "data": [{"type": "company", "id": "530370b477ad7120001d"}]})
stream1 = Companies(authenticator=NoAuth())
records = []
assert stream1._endpoint_type == Companies.EndpointType.scroll
for slice in stream1.stream_slices(sync_mode=SyncMode.full_refresh):
records += list(stream1.read_records(sync_mode=SyncMode, stream_slice=slice))
assert stream1._endpoint_type == Companies.EndpointType.standard
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
# NoAuth just means there is no authentication required for this API. It's only included for completeness
# of the example, but if you don't need authentication, you don't need to pass an authenticator at all.
# Other authenticators are available for API token-based auth and Oauth2.
auth = NoAuth()
# Parse the date from a string into a datetime object
start_date = datetime.strptime(config["start_date"], "%Y-%m-%d")
return [ExchangeRates(authenticator=auth, base=config["base"], start_date=start_date)]
def __init__(self, advertiser_id: int, app_id: int, secret: str, access_token: str):
super().__init__(authenticator=NoAuth())
self._advertiser_ids = []
# for Sandbox env
self._advertiser_id = advertiser_id
if not self._advertiser_id:
# for Production env
self._secret = secret
self._app_id = app_id
self._access_token = access_token
else:
self._advertiser_ids.append(self._advertiser_id)
def test_no_auth():
"""
Should always return empty body, no matter how many times token is retrieved.
"""
no_auth = NoAuth()
assert {} == no_auth.get_auth_header()
no_auth = NoAuth()
assert {} == no_auth.get_auth_header()
def __init__(
self,
url_base: str,
aws_signature: AWSSignature,
replication_start_date: str,
marketplace_ids: List[str],
authenticator: HttpAuthenticator = NoAuth(),
):
self._authenticator = authenticator
self._session = requests.Session()
self._url_base = url_base
self._session.auth = aws_signature
self._replication_start_date = replication_start_date
self.marketplace_ids = marketplace_ids
def test_lookup_metrics_dimensions_data_type(
metrics_dimensions_mapping, mock_metrics_dimensions_type_list_link):
test_config = json.loads(
read_file("../integration_tests/sample_config.json"))
test_config["authenticator"] = NoAuth()
test_config["metrics"] = []
test_config["dimensions"] = []
field_type, attribute, expected = metrics_dimensions_mapping
g = GoogleAnalyticsV4Stream(config=test_config)
test = g.lookup_data_type(field_type, attribute)
assert test == expected
def test_slice(site_urls, sync_mode):
stream = SearchAnalyticsByDate(NoAuth(), site_urls, "start_date",
"end_date")
search_types = stream.search_types
stream_slice = stream.stream_slices(sync_mode=sync_mode)
for site_url in site_urls:
for search_type in search_types:
expected = {
"site_url": quote_plus(site_url),
"search_type": search_type
}
assert expected == next(stream_slice)
def test_conversation_part_has_conversation_id(requests_mock, single_conversation_response):
"""
Test shows that conversation_part records include the `conversation_id` field.
"""
conversation_id = single_conversation_response["id"]
url = f"https://api.intercom.io/conversations/{conversation_id}"
requests_mock.get(url, json=single_conversation_response)
conversation_parts = ConversationParts(authenticator=NoAuth())
record_count = 0
for record in conversation_parts.read_records(sync_mode=SyncMode.incremental, stream_slice={"id": conversation_id}):
assert record["conversation_id"] == "151272900024304"
record_count += 1
assert record_count == 2
def __init__(
self,
url_base: str,
aws_signature: AWSSignature,
replication_start_date: str,
marketplace_ids: List[str],
period_in_days: Optional[int],
authenticator: HttpAuthenticator = NoAuth(),
):
self._authenticator = authenticator
self._session = requests.Session()
self._url_base = url_base.rstrip("/") + "/"
self._session.auth = aws_signature
self._replication_start_date = replication_start_date
self.marketplace_ids = marketplace_ids
self.period_in_days = period_in_days
def reports_stream():
aws_signature = AWSSignature(
service="execute-api",
aws_access_key_id="AccessKeyId",
aws_secret_access_key="SecretAccessKey",
aws_session_token="SessionToken",
region="US",
)
stream = MerchantListingsReports(
url_base="https://test.url",
aws_signature=aws_signature,
replication_start_date="2017-01-25T00:00:00Z",
marketplace_ids=["id"],
authenticator=NoAuth(),
)
return stream
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
"""
The US Census website hosts many APIs: https://www.census.gov/data/developers/data-sets.html
We provide one generic stream of all the US Census APIs rather than one stream per API.
:param config: A Mapping of the user input configuration as defined in the connector spec.
"""
return [
UsCensusStream(
query_params=config.get("query_params"),
query_path=config.get("query_path"),
api_key=config.get("api_key"),
authenticator=NoAuth(),
)
]
def test_config():
test_conf = {
"view_id": "1234567",
"window_in_days": 1,
"authenticator": NoAuth(),
"metrics": [],
"start_date":
pendulum.now().subtract(days=2).date().strftime("%Y-%m-%d"),
"dimensions": [],
"credentials": {
"auth_type": "Client",
"client_id": "client_id_val",
"client_secret": "client_secret_val",
"refresh_token": "refresh_token_val",
},
}
return copy.deepcopy(test_conf)
def test_read(stream, http_method, endpoint, response, expected, requests_mock):
def get_mock(http_method, endpoint, response):
if http_method == "POST":
requests_mock.post(endpoint, json=response)
elif http_method == "GET":
requests_mock.get(endpoint, json=response)
requests_mock.post("https://api.intercom.io/conversations/search", json=response)
requests_mock.get("/companies/scroll", json=response)
get_mock(http_method, endpoint, response)
stream = stream(authenticator=NoAuth(), start_date=0)
records = []
for slice in stream.stream_slices(sync_mode=SyncMode.full_refresh):
records += list(stream.read_records(sync_mode=SyncMode, stream_slice=slice))
assert records == expected
def test_conversation_part_filtering_based_on_conversation(requests_mock, conversation_parts_responses):
"""
Test shows that conversation_parts filters conversations (from parent stream) correctly
"""
cursor_value = 1650988200
state = {"updated_at": cursor_value}
expected_record_ids = set()
for response_tuple in conversation_parts_responses:
http_method = response_tuple[0]
url = response_tuple[1]
response = response_tuple[2]
requests_mock.register_uri(http_method, url, json=response)
if "conversation_parts" in response:
expected_record_ids.update([cp["id"] for cp in response["conversation_parts"]["conversation_parts"]])
records = []
conversation_parts = ConversationParts(authenticator=NoAuth(), start_date=0)
for slice in conversation_parts.stream_slices(sync_mode=SyncMode.incremental, stream_state=state):
records.extend(list(conversation_parts.read_records(sync_mode=SyncMode.incremental, stream_slice=slice, stream_state=state)))
assert expected_record_ids == {r["id"] for r in records}
def test_balances_stream_slices():
"""Test slices for Balance stream.
Note that <end_date> is not used by this stream.
"""
now = datetime.now().replace(microsecond=0).astimezone()
# Test without end_date (it equal <now> by default)
balance = Balances(authenticator=NoAuth(), start_date=now)
balance.get_last_refreshed_datetime = lambda x: None
stream_slices = balance.stream_slices(sync_mode="any")
assert 1 == len(stream_slices)
balance = Balances(authenticator=NoAuth(), start_date=now - timedelta(minutes=1))
balance.get_last_refreshed_datetime = lambda x: None
stream_slices = balance.stream_slices(sync_mode="any")
assert 1 == len(stream_slices)
balance = Balances(
authenticator=NoAuth(),
start_date=now - timedelta(hours=23),
)
balance.get_last_refreshed_datetime = lambda x: None
stream_slices = balance.stream_slices(sync_mode="any")
assert 1 == len(stream_slices)
balance = Balances(
authenticator=NoAuth(),
start_date=now - timedelta(days=1),
)
balance.get_last_refreshed_datetime = lambda x: None
stream_slices = balance.stream_slices(sync_mode="any")
assert 2 == len(stream_slices)
balance = Balances(
authenticator=NoAuth(),
start_date=now - timedelta(days=1, minutes=1),
)
balance.get_last_refreshed_datetime = lambda x: None
stream_slices = balance.stream_slices(sync_mode="any")
assert 2 == len(stream_slices)
# test with custom end_date
balance = Balances(
authenticator=NoAuth(),
start_date=isoparse("2021-06-01T10:00:00+00:00"),
end_date=isoparse("2021-06-03T12:00:00+00:00"),
)
balance.get_last_refreshed_datetime = lambda x: None
stream_slices = balance.stream_slices(sync_mode="any")
assert [
{"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"},
{"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"},
{"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"},
] == stream_slices
# Test with stream state
balance = Balances(
authenticator=NoAuth(),
start_date=isoparse("2021-06-01T10:00:00+00:00"),
end_date=isoparse("2021-06-03T12:00:00+00:00"),
)
balance.get_last_refreshed_datetime = lambda x: None
stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"})
assert [
{"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"},
{"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"},
] == stream_slices
balance = Balances(
authenticator=NoAuth(),
start_date=isoparse("2021-06-01T10:00:00+00:00"),
end_date=isoparse("2021-06-03T12:00:00+00:00"),
)
balance.get_last_refreshed_datetime = lambda x: None
stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T11:00:00+00:00"})
assert [{"start_date": "2021-06-03T11:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices
balance = Balances(
authenticator=NoAuth(),
start_date=isoparse("2021-06-01T10:00:00+00:00"),
end_date=isoparse("2021-06-03T12:00:00+00:00"),
)
balance.get_last_refreshed_datetime = lambda x: None
stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T12:00:00+00:00"})
assert [{"start_date": "2021-06-03T12:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices
def test_pagination(count, expected):
stream = SearchAnalyticsByDate(NoAuth(), ["https://example.com"],
"start_date", "end_date")
response = MockResponse(stream.data_field, count)
stream.next_page_token(response)
assert stream.start_row == expected
def test_transactions_stream_slices():
start_date_max = {"hours": 0}
# if start_date > now - **start_date_max then no slices
transactions = Transactions(
authenticator=NoAuth(),
start_date=now() - timedelta(**start_date_max) - timedelta(minutes=2),
)
transactions.get_last_refreshed_datetime = lambda x: None
stream_slices = transactions.stream_slices(sync_mode="any")
assert 1 == len(stream_slices)
# start_date <= now - **start_date_max
transactions = Transactions(
authenticator=NoAuth(),
start_date=now() - timedelta(**start_date_max),
)
transactions.get_last_refreshed_datetime = lambda x: None
stream_slices = transactions.stream_slices(sync_mode="any")
assert 1 == len(stream_slices)
transactions = Transactions(
authenticator=NoAuth(),
start_date=now() - timedelta(**start_date_max) + timedelta(minutes=2),
)
transactions.get_last_refreshed_datetime = lambda x: None
stream_slices = transactions.stream_slices(sync_mode="any")
assert 1 == len(stream_slices)
transactions = Transactions(
authenticator=NoAuth(),
start_date=now() - timedelta(**start_date_max) - timedelta(hours=2),
)
transactions.get_last_refreshed_datetime = lambda x: None
stream_slices = transactions.stream_slices(sync_mode="any")
assert 1 == len(stream_slices)
transactions = Transactions(
authenticator=NoAuth(),
start_date=now() - timedelta(**start_date_max) - timedelta(days=1),
)
transactions.get_last_refreshed_datetime = lambda x: None
stream_slices = transactions.stream_slices(sync_mode="any")
assert 2 == len(stream_slices)
transactions = Transactions(
authenticator=NoAuth(),
start_date=now() - timedelta(**start_date_max) - timedelta(days=1, hours=2),
)
transactions.get_last_refreshed_datetime = lambda x: None
stream_slices = transactions.stream_slices(sync_mode="any")
assert 2 == len(stream_slices)
transactions = Transactions(
authenticator=NoAuth(),
start_date=now() - timedelta(**start_date_max) - timedelta(days=30, minutes=1),
)
transactions.get_last_refreshed_datetime = lambda x: None
stream_slices = transactions.stream_slices(sync_mode="any")
assert 31 == len(stream_slices)
# tests with specified end_date
transactions = Transactions(
authenticator=NoAuth(),
start_date=isoparse("2021-06-01T10:00:00+00:00"),
end_date=isoparse("2021-06-04T12:00:00+00:00"),
)
transactions.get_last_refreshed_datetime = lambda x: None
stream_slices = transactions.stream_slices(sync_mode="any")
assert [
{"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"},
{"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"},
{"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"},
{"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"},
] == stream_slices
# tests with specified end_date and stream_state
transactions = Transactions(
authenticator=NoAuth(),
start_date=isoparse("2021-06-01T10:00:00+00:00"),
end_date=isoparse("2021-06-04T12:00:00+00:00"),
)
transactions.get_last_refreshed_datetime = lambda x: None
stream_slices = transactions.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"})
assert [
{"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"},
{"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"},
{"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"},
] == stream_slices
transactions = Transactions(
authenticator=NoAuth(),
start_date=isoparse("2021-06-01T10:00:00+00:00"),
end_date=isoparse("2021-06-04T12:00:00+00:00"),
)
transactions.get_last_refreshed_datetime = lambda x: None
stream_slices = transactions.stream_slices(sync_mode="any", stream_state={"date": "2021-06-04T10:00:00+00:00"})
assert [{"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}] == stream_slices
def test_state(current_stream_state, latest_record, expected):
stream = SearchAnalyticsByDate(NoAuth(), ["https://example.com"],
"start_date", "end_date")
value = stream.get_updated_state(current_stream_state, latest_record)
assert value == expected
def test_date_slices():
now = date.today()
# Test with start_date now range
stream_slices = Annotations(authenticator=NoAuth(),
start_date=now,
end_date=now,
date_window_size=1,
region="EU").stream_slices(sync_mode="any")
assert 1 == len(stream_slices)
stream_slices = Annotations(authenticator=NoAuth(),
start_date=now - timedelta(days=1),
end_date=now,
date_window_size=1,
region="US").stream_slices(sync_mode="any")
assert 2 == len(stream_slices)
stream_slices = Annotations(
authenticator=NoAuth(),
start_date=now - timedelta(days=2),
end_date=now,
date_window_size=1).stream_slices(sync_mode="any")
assert 3 == len(stream_slices)
stream_slices = Annotations(
authenticator=NoAuth(),
start_date=now - timedelta(days=2),
end_date=now,
date_window_size=10).stream_slices(sync_mode="any")
assert 1 == len(stream_slices)
# test with attribution_window
stream_slices = Annotations(
authenticator=NoAuth(),
start_date=now - timedelta(days=2),
end_date=now,
date_window_size=1,
attribution_window=5,
region="US",
).stream_slices(sync_mode="any")
assert 8 == len(stream_slices)
# Test with start_date end_date range
stream_slices = Annotations(
authenticator=NoAuth(),
start_date=date.fromisoformat("2021-07-01"),
end_date=date.fromisoformat("2021-07-01"),
date_window_size=1,
region="US",
).stream_slices(sync_mode="any")
assert [{
"start_date": "2021-07-01",
"end_date": "2021-07-01"
}] == stream_slices
stream_slices = Annotations(
authenticator=NoAuth(),
start_date=date.fromisoformat("2021-07-01"),
end_date=date.fromisoformat("2021-07-02"),
date_window_size=1,
region="EU",
).stream_slices(sync_mode="any")
assert [{
"start_date": "2021-07-01",
"end_date": "2021-07-01"
}, {
"start_date": "2021-07-02",
"end_date": "2021-07-02"
}] == stream_slices
stream_slices = Annotations(
authenticator=NoAuth(),
start_date=date.fromisoformat("2021-07-01"),
end_date=date.fromisoformat("2021-07-03"),
date_window_size=1,
region="US",
).stream_slices(sync_mode="any")
assert [
{
"start_date": "2021-07-01",
"end_date": "2021-07-01"
},
{
"start_date": "2021-07-02",
"end_date": "2021-07-02"
},
{
"start_date": "2021-07-03",
"end_date": "2021-07-03"
},
] == stream_slices
stream_slices = Annotations(
authenticator=NoAuth(),
start_date=date.fromisoformat("2021-07-01"),
end_date=date.fromisoformat("2021-07-03"),
date_window_size=2,
region="US",
).stream_slices(sync_mode="any")
assert [{
"start_date": "2021-07-01",
"end_date": "2021-07-02"
}, {
"start_date": "2021-07-03",
"end_date": "2021-07-03"
}] == stream_slices
# test with stream_state
stream_slices = Annotations(
authenticator=NoAuth(),
start_date=date.fromisoformat("2021-07-01"),
end_date=date.fromisoformat("2021-07-03"),
date_window_size=1,
).stream_slices(sync_mode="any", stream_state={"date": "2021-07-02"})
assert [{
"start_date": "2021-07-02",
"end_date": "2021-07-02"
}, {
"start_date": "2021-07-03",
"end_date": "2021-07-03"
}] == stream_slices
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
import pytest
from airbyte_cdk.sources.streams.http.auth import NoAuth
from source_intercom.source import ConversationParts, Conversations
from source_intercom.utils import EagerlyCachedStreamState as stream_state_cache
# Define the Stream instances for the tests
INTERCOM_STREAM = Conversations(authenticator=NoAuth(), start_date=0)
INTERCOM_SUB_STREAM = ConversationParts(authenticator=NoAuth(), start_date=0)
@pytest.mark.parametrize(
"stream, cur_stream_state, state_object, expected_output",
[
# When Full-Refresh: state_object: empty.
(INTERCOM_STREAM, {
INTERCOM_STREAM.cursor_field: ""
}, {}, {
INTERCOM_STREAM.name: {
INTERCOM_STREAM.cursor_field: ""
}
}),
(
INTERCOM_SUB_STREAM,
{
INTERCOM_SUB_STREAM.cursor_field: ""
},
{},
