async def test__get_data_w_incorrect_filter(base_connector, ds, mocker):
"""It should return an array with None if the jq filter does not match the data"""
fake_fetch = mocker.patch(fetch_fn_name,
return_value=helpers.build_future(FAKE_DATA))
with pytest.raises(ScriptRuntimeError):
await base_connector._get_data(f'/{ds.endpoint}', '.putzes')
assert fake_fetch.call_count == 1
async def test__get_data_w_bad_jq_filter(base_connector, ds, mocker):
"""It should throw a ValueError if jq filter is not valid"""
fake_fetch = mocker.patch(fetch_fn_name,
return_value=helpers.build_future(FAKE_DATA))
with pytest.raises(ValueError):
await base_connector._get_data(f'/{ds.endpoint}', 'putzes')
assert fake_fetch.call_count == 1
async def test_fetch_page_with_no_meta(mocker):
"""Tests that no meta object in response is not an issue"""
dataset = 'calls'
fake_data = {'data': {'stuff': 'stuff'}}
fake_data = helpers.build_future(fake_data)
fake_fetch = mocker.patch(fetch_fn_name, return_value=fake_data)
# despite there being a limit of 10 runs, only one run should occur
res = await fetch_page(dataset, [], {}, 10, 0)
assert len(res) == 1
assert_called_with(fake_fetch, expected_count=1)
async def test__get_data_tags_case(con, mocker):
"""Tests with tags happy case"""
dataset = 'tags'
fake_res = fake_tags
fake_res = helpers.build_future(fake_res)
fake_fetch_page = mocker.patch(fetch_fn_name, return_value=fake_res)
ds = build_ds(dataset)
res = await con._get_tags(ds.dataset, 10)
assert fake_fetch_page.call_count == 1
assert res == filtered_tags
def test__run_fetch(base_connector, ds, mocker):
"""It should call _run_fetch and produce a dict"""
fake__get_data = mocker.patch.object(
RevinateConnector,
'_get_data',
return_value=helpers.build_future(JQ_FILTERED_DATA))
result = base_connector._run_fetch(f'/{ds.endpoint}', '.')
assert result == JQ_FILTERED_DATA
assert fake__get_data.call_count == 1
def test__retrieve_users_from_fetch(con, mocker):
"""Tests _retrieve_data for users from fetch_page function"""
fake_res = [fake_teams, fake_users]
fake_res = [helpers.build_future(item) for item in fake_res]
mocker.patch(fetch_fn_name, side_effect=fake_res)
ds = build_ds('users')
df = con._retrieve_data(ds)
assert df.shape == (11, 4)
mocker.patch(fetch_fn_name, side_effect=Exception('Youch!'))
with pytest.raises(Exception):
con._retrieve_data(ds)
async def test_fetch_page_with_error(mocker):
"""
Tests error sent from API goes through a retry policy and then throws an exception
"""
dataset = 'tags'
fake_data = {'error': 'Oops!', 'troubleshoot': 'Blah blah blah'}
fake_data = helpers.build_future(fake_data)
fake_fetch = mocker.patch(fetch_fn_name, return_value=fake_data)
with pytest.raises(Exception) as e:
await fetch_page(dataset, [], {}, 1, 0)
fake_fetch.call_count == 4
assert str(e.value) == 'Aborting Aircall requests due to Oops!'
async def test__get_data_happy_case(base_connector, ds, mocker):
"""It should return valid data if everything is valid"""
fake_fetch = mocker.patch(
fetch_fn_name,
return_value=helpers.build_future(FAKE_DATA),
)
fake_endpoint = f'/{ds.endpoint}?page=2&size=50'
res = await base_connector._get_data(fake_endpoint, jq_filter='.')
assert res == JQ_FILTERED_DATA
assert fake_fetch.call_count == 1
def test__retrieve_tags_from_fetch(con, mocker):
"""Tests _retrieve_tags from the fetch_page function"""
fake_res = fake_tags
fake_res = helpers.build_future(fake_tags)
mocker.patch(fetch_fn_name, return_value=fake_res)
ds = build_ds('tags')
df = con._retrieve_data(ds)
assert df.shape == (3, 4)
mocker.patch(fetch_fn_name, side_effect=Exception('Oh noez !!!'))
with pytest.raises(Exception):
con._retrieve_data(ds)
async def test_fetch_page_with_no_next(mocker):
"""Bearer version: tests that no next page returns an array of one response"""
dataset = 'tags'
fake_data = {'data': {'stuff': 'stuff'}, 'meta': {'next_page_link': None, 'current_page': 1}}
fake_data = helpers.build_future(fake_data)
fake_fetch = mocker.patch(fetch_fn_name, return_value=fake_data)
result = await fetch_page(dataset, [], {}, 10, 0)
assert len(result) == 1
first_dict = result[0]
assert 'data' in first_dict
assert 'meta' in first_dict
assert_called_with(
fake_fetch, ['https://proxy.bearer.sh/aircall_oauth/tags?per_page=50&page=1', {}]
)
async def test__get_data_users_case(con, mocker):
"""Tests users call happy case"""
dataset = 'users'
fake_res = [fake_teams, fake_users]
fake_res = [helpers.build_future(item) for item in fake_res]
fake_fetch_page = mocker.patch(fetch_fn_name, side_effect=fake_res)
ds = build_ds(dataset)
res = await con._get_data(ds.dataset, 10)
assert fake_fetch_page.call_count == 2
assert len(res) == 2
teams, users = res
assert teams == more_filtered_teams
assert list(teams[0].keys()) == columns_for_teams
assert users == more_filtered_users
assert list(users[0].keys()) == columns_for_users
async def test_fetch_page_with_low_limit(mocker):
"""Test fetch_page to see limit and current pass stopping recursion"""
dataset = 'users'
data_list = [
{
'data': {'stuff': 'stuff'},
'meta': {'next_page_link': '/users?page=2', 'current_page': 1},
},
{
'data': {'stuff': 'stuff'},
'meta': {'next_page_link': '/users?page=3', 'current_page': 2},
},
{'data': {'stuff': 'stuff'}, 'meta': {'next_page_link': None, 'current_page': 3}},
]
data_list = [helpers.build_future(item) for item in data_list]
fake_fetch = mocker.patch(fetch_fn_name, side_effect=data_list)
# limit is only 1 and run is 0 i.e. this is the first run
res = await fetch_page(dataset, [], {}, 1, 0)
assert len(res) == 1
assert_called_with(
fake_fetch, ['https://proxy.bearer.sh/aircall_oauth/users?per_page=50&page=1', {}]
)
async def test_fetch_page_with_next_page(mocker):
"""Test fetch_page to see multiple pages"""
dataset = 'calls'
data_list = [
{
'data': {'stuff': 'stuff'},
'meta': {'next_page_link': '/calls?page=2', 'current_page': 1},
},
{'data': {'stuff': 'stuff'}, 'meta': {'next_page_link': None, 'current_page': 2}},
]
data_list = [helpers.build_future(item) for item in data_list]
fake_fetch = mocker.patch(fetch_fn_name, side_effect=data_list)
# limit is 10 and run is 0 i.e. this is the first run
# want to see that, despite the limit being set to 10,
# only two objects are sent in the array because no next_page_link
# on second object means that there is no more to be fetched
fake_res = await fetch_page(dataset, [], {}, 10, 0)
assert len(fake_res) == 2
assert_called_with(
fake_fetch, ['https://proxy.bearer.sh/aircall_oauth/calls?per_page=50&page=2', {}], 2
)
async def test__get_data_w_no_content(base_connector, ds, mocker):
"""It should still succeed even with an empty content array"""
mocker.patch(
fetch_fn_name,
return_value=helpers.build_future({
'links': [{
'rel': '',
'href': '',
'templated': False
}],
'content': [],
'page': {
'size': 50,
'totalElements': 498,
'totalPages': 10,
'number': 2
},
}),
)
result = await base_connector._get_data(f'/{ds.endpoint}', '.')
assert len(result) == 0