def test_should_output_records(self, mock_stdout, requests_mock):
requests_mock.get(
"https://api.nikabot.com/api/v1/roles?limit=1000&page=0",
json=json.loads(ROLES_RESPONSE))
requests_mock.get(
"https://api.nikabot.com/api/v1/roles?limit=1000&page=1",
json=json.loads(EMPTY_RESPONSE))
config = {"access_token": "my-access-token", "page_size": 1000}
state = {}
catalog = Catalog(streams=[
CatalogEntry(
tap_stream_id="roles",
stream="roles",
schema=Schema.from_dict({}),
key_properties=["id"],
metadata=[{
"breadcrumb": [],
"metadata": {
"selected": True
}
}],
)
])
sync(config, state, catalog)
assert mock_stdout.mock_calls == [
call(
'{"type": "SCHEMA", "stream": "roles", "schema": {}, "key_properties": ["id"]}\n'
),
call(
'{"type": "RECORD", "stream": "roles", "record": {"id": "d893ebf32d49c35c1d754774", "team_id": "T034F9NPW", "name": "0.5"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
call(
'{"type": "RECORD", "stream": "roles", "record": {"id": "cfabd9aa6f3e6381a716da58", "team_id": "T034F9NPW", "name": "0.1"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
]
def test_should_output_records(self, mock_stdout, requests_mock):
requests_mock.get("https://api.nikabot.com/api/v1/teams",
json=json.loads(TEAMS_RESPONSE))
config = {"access_token": "my-access-token", "page_size": 1000}
state = {}
catalog = Catalog(streams=[
CatalogEntry(
tap_stream_id="teams",
stream="teams",
schema=Schema.from_dict({}),
key_properties=["id"],
metadata=[{
"breadcrumb": [],
"metadata": {
"selected": True
}
}],
)
])
sync(config, state, catalog)
assert mock_stdout.mock_calls == [
call(
'{"type": "SCHEMA", "stream": "teams", "schema": {}, "key_properties": ["id"]}\n'
),
call(
'{"type": "RECORD", "stream": "teams", "record": {"id": "5d6ca50762a07c00045125fb", "domain": "pageup", "bot_token": "e31d3b7ae51ff1feec8be578f23eb017e8143f66a7a085342c664544b81618ec41b87810d61a9c1f6133fe0c7d88aa3976232bb2a2665c4f89c38058b51cd20c", "activated_by": "U6K26HMGV", "status": "ACTIVE", "platform_id": "T034F9NPW", "created_at": "2019-09-02T05:13:43.151", "subscription": {"active_until": "2020-07-08T23:59:59", "status": "active", "number_of_users": 69, "subscriber_id": "U93KT77T6"}, "icon": {"image_34": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_34.png", "image_44": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_44.png", "image_68": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_68.png", "image_88": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_88.png", "image_102": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_102.png", "image_132": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_132.png", "image_230": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_230.png", "image_original": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_original.png"}}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
]
def test_should_output_records(self, mock_stdout, requests_mock):
requests_mock.get("https://api.nikabot.com/api/v1/groups?limit=1000&page=0", json=json.loads(GROUPS_RESPONSE))
requests_mock.get("https://api.nikabot.com/api/v1/groups?limit=1000&page=1", json=json.loads(EMPTY_RESPONSE))
config = {"access_token": "my-access-token", "page_size": 1000}
state = {}
catalog = Catalog(
streams=[
CatalogEntry(
tap_stream_id="groups",
stream="groups",
schema=Schema.from_dict({}),
key_properties=["id"],
metadata=[{"breadcrumb": [], "metadata": {"selected": True}}],
)
]
)
sync(config, state, catalog)
assert mock_stdout.mock_calls == [
call('{"type": "SCHEMA", "stream": "groups", "schema": {}, "key_properties": ["id"]}\n'),
call(
'{"type": "RECORD", "stream": "groups", "record": {"id": "f1b4b37cc2658672770b789f", "team_id": "T034F9NPW", "name": "TA Squad 5"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
call(
'{"type": "RECORD", "stream": "groups", "record": {"id": "3176700ac4f2203b825fae6c", "team_id": "T034F9NPW", "name": "Platform Toolkit"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
]
def test_should_use_bookmark_when_bookmark_has_timezone_info(
self, mock_stdout, requests_mock, mock_catalog):
requests_page0 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=20200610&dateEnd=20200610",
json=json.loads(RECORDS_RESPONSE),
)
requests_page1 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=20200610&dateEnd=20200610",
json=json.loads(EMPTY_RESPONSE),
)
config = {
"access_token": "my-access-token",
"page_size": 1000,
"cutoff_days": 10,
"start_date": "2020-01-01",
"end_date": "2020-06-10",
}
state = {"records": "2020-06-09T00:00:00.000+00:00"}
sync(config, state, mock_catalog)
assert requests_page0.call_count == 1
assert requests_page1.call_count == 1
mock_stdout.assert_has_calls([
call(
'{"type": "STATE", "value": {"records": "2020-06-10T00:00:00"}}\n'
),
])
def test_should_output_records(self, mock_stdout, requests_mock):
requests_mock.get(
"https://api.nikabot.com/api/v1/projects?limit=1000&page=0",
json=json.loads(PROJECTS_RESPONSE))
requests_mock.get(
"https://api.nikabot.com/api/v1/projects?limit=1000&page=1",
json=json.loads(EMPTY_RESPONSE))
config = {"access_token": "my-access-token", "page_size": 1000}
state = {}
catalog = Catalog(streams=[
CatalogEntry(
tap_stream_id="projects",
stream="projects",
schema=Schema.from_dict({}),
key_properties=["id"],
metadata=[{
"breadcrumb": [],
"metadata": {
"selected": True
}
}],
)
])
sync(config, state, catalog)
assert mock_stdout.mock_calls == [
call(
'{"type": "SCHEMA", "stream": "projects", "schema": {}, "key_properties": ["id"]}\n'
),
call(
'{"type": "RECORD", "stream": "projects", "record": {"id": "5d6ca95e62a07c00045126e7", "project_name": "CAP - Analytics", "team_id": "T034F9NPW", "author": "U6K26HMGV", "pto": {"status": false}, "custom_ref": "", "create_date": "2019-09-02T05:32:14.23", "client": "", "type": "Capability Custodian", "created_at": "2019-09-02T05:32:14.23", "assigned_groups": ["Analytics"]}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
call(
'{"type": "RECORD", "stream": "projects", "record": {"id": "5d6ca97c62a07c00045126e8", "project_name": "CAP - Authentication", "team_id": "T034F9NPW", "author": "U6K26HMGV", "pto": {"status": false}, "custom_ref": "", "create_date": "2019-09-02T05:32:44.172", "client": "", "type": "Capability Custodian", "created_at": "2019-09-02T05:32:44.172", "assigned_groups": ["Authentication"]}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
]
def test_should_output_multiple_pages(self, mock_stdout, requests_mock,
mock_catalog):
requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=00010101&dateEnd=99991231",
json=json.loads(RECORDS_RESPONSE),
)
requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=00010101&dateEnd=99991231",
json=json.loads(RECORDS_PAGE2_RESPONSE),
)
requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=2&dateStart=00010101&dateEnd=99991231",
json=json.loads(EMPTY_RESPONSE),
)
config = {"access_token": "my-access-token", "page_size": 1000}
state = {}
sync(config, state, mock_catalog)
assert mock_stdout.mock_calls == [
call('{"type": "SCHEMA", "stream": "records", "schema": ' +
SCHEMA + ', "key_properties": ["id"]}\n'),
call(
'{"type": "RECORD", "stream": "records", "record": {"id": "5ee2ca823e056d00141896a0", "team_id": "T034F9NPW", "user_id": "UBM1DQ9RB", "project_name": "CAP - Data Lifecycle", "project_id": "5d6ca9e462a07c00045126ed", "hours": 2.0, "date": "2000-01-01T00:00:00.000000Z", "created_at": "2020-01-01T00:21:22.779000Z"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
call(
'{"type": "RECORD", "stream": "records", "record": {"id": "5ee1d52e5cff9100146de745", "team_id": "T034F9NPW", "user_id": "U107SJ4N6", "project_name": "DUX", "project_id": "5d6df702956de30004dc0198", "hours": 7.5, "date": "2020-06-10T00:00:00.000000Z", "created_at": "2020-06-11T06:54:38.138000Z"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
call(
'{"type": "RECORD", "stream": "records", "record": {"id": "5d9d7a035da6700004970476", "team_id": "T034F9NPW", "user_id": "UBM1DQ9RB", "project_name": "Leave (All Kinds)", "project_id": "5d6ca50762a07c00045125fc", "hours": 7.5, "date": "2019-08-20T00:00:00.000000Z", "created_at": "2019-10-09T06:11:15.976000Z"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
call(
'{"type": "RECORD", "stream": "records", "record": {"id": "5d9d79c45da6700004970475", "team_id": "T034F9NPW", "user_id": "UBM1DQ9RB", "project_name": "TX - M1 Assign due dates", "project_id": "5d6e06c9956de30004dc01b0", "hours": 7.5, "date": "2019-08-21T00:00:00.000000Z", "created_at": "2019-10-09T06:10:12.673000Z"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
]
def test_should_output_records_given_default_config(
self, mock_stdout, requests_mock, mock_catalog):
requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=00010101&dateEnd=20191222",
json=json.loads(RECORDS_RESPONSE),
)
requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=00010101&dateEnd=20191222",
json=json.loads(EMPTY_RESPONSE),
)
config = {
"access_token": "my-access-token",
"page_size": 1000,
"cutoff_days": 10
}
state = {}
sync(config, state, mock_catalog)
assert mock_stdout.mock_calls == [
call(
'{"type": "SCHEMA", "stream": "records", "schema": ' + SCHEMA +
', "key_properties": ["id"], "bookmark_properties": ["date"]}\n'
),
call(
'{"type": "RECORD", "stream": "records", "record": {"id": "5ee2ca823e056d00141896a0", "team_id": "T034F9NPW", "user_id": "UBM1DQ9RB", "project_name": "CAP - Data Lifecycle", "project_id": "5d6ca9e462a07c00045126ed", "hours": 2.0, "date": "2000-01-01T00:00:00+00:00", "created_at": "2020-01-01T00:21:22.779000+00:00"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
call(
'{"type": "RECORD", "stream": "records", "record": {"id": "5ee1d52e5cff9100146de745", "team_id": "T034F9NPW", "user_id": "U107SJ4N6", "project_name": "DUX", "project_id": "5d6df702956de30004dc0198", "hours": 7.5, "date": "2020-06-10T00:00:00+00:00", "created_at": "2020-06-11T06:54:38.138000+00:00"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
call(
'{"type": "STATE", "value": {"records": "2020-06-10T00:00:00"}}\n'
),
]
def test_should_output_records(self, mock_stdout, requests_mock):
requests_mock.get(
"https://api.nikabot.com/api/v1/users?limit=1000&page=0",
json=json.loads(USERS_RESPONSE))
requests_mock.get(
"https://api.nikabot.com/api/v1/users?limit=1000&page=1",
json=json.loads(EMPTY_RESPONSE))
config = {"access_token": "my-access-token", "page_size": 1000}
state = {}
catalog = Catalog(streams=[
CatalogEntry(
tap_stream_id="users",
stream="users",
schema=Schema.from_dict({}),
key_properties=["id"],
metadata=[{
"breadcrumb": [],
"metadata": {
"selected": True
}
}],
)
])
sync(config, state, catalog)
assert mock_stdout.mock_calls == [
call(
'{"type": "SCHEMA", "stream": "users", "schema": {}, "key_properties": ["id"]}\n'
),
call(
'{"type": "RECORD", "stream": "users", "record": {"id": "5de459977292020014fb601c", "name": "Billy", "deleted": true, "presence": "away", "user_id": "UR5B0QABX", "team_id": "T034F9NPW", "is_restricted": false, "is_ultra_restricted": false, "is_admin": false, "is_nikabot_admin": false, "tz": "Australia/Canberra", "tz_label": "Australian Eastern Standard Time", "tz_offset": 36000, "is_checkin_excluded": true, "created_at": "2019-12-02T00:23:51.087", "groups": [], "updated_at": "2020-06-14T22:47:29.617"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
call(
'{"type": "RECORD", "stream": "users", "record": {"id": "68QMxnnt8YcpPdfmM", "name": "paul.heasley", "deleted": false, "presence": "active", "user_id": "U04AX35QP", "team_id": "T034F9NPW", "is_restricted": false, "is_ultra_restricted": false, "is_admin": false, "is_nikabot_admin": true, "tz": "Australia/Canberra", "tz_label": "Australian Eastern Standard Time", "tz_offset": 36000, "is_checkin_excluded": false, "create_date": "2019-09-02T05:13:47.88", "created_at": "2019-09-02T05:13:47.882", "role": "0.1", "groups": ["TA Stream", "TA Squad 1", "TA Squad 2", "TA Squad 3", "TA Squad 4", "Learning Applications", "Notification Capability"], "updated_at": "2020-06-15T06:07:58.272"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n'
),
]
assert LOGGER.info.mock_calls == [
call("Syncing stream: %s", "users"),
call(
"Making %s request to %s with params %s",
"GET",
"https://api.nikabot.com/api/v1/users",
{
"limit": "1000",
"page": "0"
},
),
call(
"Making %s request to %s with params %s",
"GET",
"https://api.nikabot.com/api/v1/users",
{
"limit": "1000",
"page": "1"
},
),
]
def test_should_raise_error_when_start_date_greater_than_end_date(self, mock_catalog):
config = {
"access_token": "my-access-token",
"page_size": 1000,
"cutoff_days": 10,
"start_date": "2020-06-11",
"end_date": "2020-06-10",
}
state = {}
with pytest.raises(StartDateAfterEndDateError):
sync(config, state, mock_catalog)
def test_should_raise_error_when_log_based_replication_requested(
self, mock_catalog):
config = {"access_token": "my-access-token", "page_size": 1000}
state = {}
mock_catalog.streams[0].replication_method = "LOG_BASED"
with pytest.raises(InvalidReplicationMethodError) as excinfo:
sync(config, state, mock_catalog)
assert (
str(excinfo.value) ==
"Invalid replication method selected 'LOG_BASED', valid options are 'FULL_TABLE'"
)
def test_should_raise_error_when_incremental_replication_requested(
self, mock_catalog):
config = {"access_token": "my-access-token", "page_size": 1000}
state = {}
mock_catalog.streams[0].replication_key = ("date", )
mock_catalog.streams[0].replication_method = "INCREMENTAL"
with pytest.raises(InvalidReplicationMethodError) as excinfo:
sync(config, state, mock_catalog)
assert (
str(excinfo.value) ==
"Invalid replication method selected 'INCREMENTAL', valid options are 'FULL_TABLE'"
)
def test_should_raise_error_when_start_date_greater_than_end_date(
self, mock_catalog):
config = {
"access_token": "my-access-token",
"page_size": 1000,
"start_date": "2020-06-11",
"end_date": "2020-06-10",
}
state = {}
with pytest.raises(StartDateAfterEndDateError) as excinfo:
sync(config, state, mock_catalog)
assert str(
excinfo.value
) == "Start date '2020-06-11' cannot be later than end date '2020-06-10'"
def test_should_sync_future_dates_given_cutoff_days_not_set(self, requests_mock, mock_catalog):
requests_page0 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=00010101&dateEnd=99991231",
json=json.loads(RECORDS_RESPONSE),
)
requests_page1 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=00010101&dateEnd=99991231",
json=json.loads(EMPTY_RESPONSE),
)
config = {"access_token": "my-access-token", "page_size": 1000, "cutoff_days": None}
state = {}
sync(config, state, mock_catalog)
assert requests_page0.call_count == 1
assert requests_page1.call_count == 1
def test_should_return_no_records_when_bookmark_greater_than_cutoff_date(
self, mock_stdout, mock_catalog):
config = {
"access_token": "my-access-token",
"page_size": 1000,
"cutoff_days": 10,
}
state = {"records": "2020-01-01T00:00:00.000"}
sync(config, state, mock_catalog)
assert mock_stdout.mock_calls == [
call(
'{"type": "SCHEMA", "stream": "records", "schema": ' + SCHEMA +
', "key_properties": ["id"], "bookmark_properties": ["date"]}\n'
),
]
def test_should_not_use_cutoff_days_when_full_replication(self, mock_stdout, requests_mock, mock_catalog):
requests_page0 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=00010101&dateEnd=99991231",
json=json.loads(RECORDS_RESPONSE),
)
requests_page1 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=00010101&dateEnd=99991231",
json=json.loads(EMPTY_RESPONSE),
)
config = {"access_token": "my-access-token", "page_size": 1000, "cutoff_days": 10}
state = {}
mock_catalog.streams[0].replication_method = "FULL_TABLE"
sync(config, state, mock_catalog)
assert requests_page0.call_count == 1
assert requests_page1.call_count == 1
def test_should_output_nothing_given_no_streams_selected(
self, mock_stdout):
config = {"access_token": "my-access-token", "page_size": 1000}
state = {}
catalog = Catalog(streams=[
CatalogEntry(
tap_stream_id="users",
stream="users",
schema=Schema.from_dict({}),
key_properties=["id"],
metadata=[],
)
])
sync(config, state, catalog)
mock_stdout.assert_not_called()
assert LOGGER.info.mock_calls == [call("Skipping stream: %s", "users")]
def test_should_start_from_start_date_given_bookmark_less_than_start_date(self, requests_mock, mock_catalog):
requests_page0 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=20200101&dateEnd=20200610",
json=json.loads(RECORDS_RESPONSE),
)
requests_page1 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=20200101&dateEnd=20200610",
json=json.loads(EMPTY_RESPONSE),
)
config = {
"access_token": "my-access-token",
"page_size": 1000,
"cutoff_days": 10,
"start_date": "2020-01-01",
"end_date": "2020-06-10",
}
state = {"records": "2010-12-31T00:00:00.000"}
sync(config, state, mock_catalog)
assert requests_page0.call_count == 1
assert requests_page1.call_count == 1
def test_should_use_end_date_when_cutoff_date_greater_than_end_date(self, requests_mock, mock_catalog):
requests_page0 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=20191219&dateEnd=20191220",
json=json.loads(RECORDS_RESPONSE),
)
requests_page1 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=20191219&dateEnd=20191220",
json=json.loads(EMPTY_RESPONSE),
)
config = {
"access_token": "my-access-token",
"page_size": 1000,
"cutoff_days": 10,
"start_date": "2019-12-19",
"end_date": "2019-12-20",
}
state = {}
sync(config, state, mock_catalog)
assert requests_page0.call_count == 1
assert requests_page1.call_count == 1
def test_should_not_use_bookmark_when_full_replication_given_start_and_end_dates(self, requests_mock, mock_catalog):
requests_page0 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=20200101&dateEnd=20200610",
json=json.loads(RECORDS_RESPONSE),
)
requests_page1 = requests_mock.get(
"https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=20200101&dateEnd=20200610",
json=json.loads(EMPTY_RESPONSE),
)
config = {
"access_token": "my-access-token",
"page_size": 1000,
"cutoff_days": 10,
"start_date": "2020-01-01",
"end_date": "2020-06-10",
}
state = {"records": "2020-06-09T00:00:00.000"}
mock_catalog.streams[0].replication_method = "FULL_TABLE"
sync(config, state, mock_catalog)
assert requests_page0.call_count == 1
assert requests_page1.call_count == 1
def test_should_output_no_records_given_no_records_available(
self, mock_stdout, requests_mock):
requests_mock.get(
"https://api.nikabot.com/api/v1/users?limit=1000&page=0",
json=json.loads(EMPTY_RESPONSE))
config = {"access_token": "my-access-token", "page_size": 1000}
state = {}
catalog = Catalog(streams=[
CatalogEntry(
tap_stream_id="users",
stream="users",
schema=Schema.from_dict({}),
key_properties=["id"],
metadata=[{
"breadcrumb": [],
"metadata": {
"selected": True
}
}],
)
])
sync(config, state, catalog)
assert mock_stdout.mock_calls == [
call(
'{"type": "SCHEMA", "stream": "users", "schema": {}, "key_properties": ["id"]}\n'
)
]
assert LOGGER.info.mock_calls == [
call("Syncing stream: %s", "users"),
call(
"Making %s request to %s with params %s",
"GET",
"https://api.nikabot.com/api/v1/users",
{
"limit": "1000",
"page": "0"
},
),
]
