def test_should_output_records(self, mock_stdout, requests_mock): requests_mock.get( "https://api.nikabot.com/api/v1/roles?limit=1000&page=0", json=json.loads(ROLES_RESPONSE)) requests_mock.get( "https://api.nikabot.com/api/v1/roles?limit=1000&page=1", json=json.loads(EMPTY_RESPONSE)) config = {"access_token": "my-access-token", "page_size": 1000} state = {} catalog = Catalog(streams=[ CatalogEntry( tap_stream_id="roles", stream="roles", schema=Schema.from_dict({}), key_properties=["id"], metadata=[{ "breadcrumb": [], "metadata": { "selected": True } }], ) ]) sync(config, state, catalog) assert mock_stdout.mock_calls == [ call( '{"type": "SCHEMA", "stream": "roles", "schema": {}, "key_properties": ["id"]}\n' ), call( '{"type": "RECORD", "stream": "roles", "record": {"id": "d893ebf32d49c35c1d754774", "team_id": "T034F9NPW", "name": "0.5"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), call( '{"type": "RECORD", "stream": "roles", "record": {"id": "cfabd9aa6f3e6381a716da58", "team_id": "T034F9NPW", "name": "0.1"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), ]
def test_should_output_records(self, mock_stdout, requests_mock): requests_mock.get("https://api.nikabot.com/api/v1/teams", json=json.loads(TEAMS_RESPONSE)) config = {"access_token": "my-access-token", "page_size": 1000} state = {} catalog = Catalog(streams=[ CatalogEntry( tap_stream_id="teams", stream="teams", schema=Schema.from_dict({}), key_properties=["id"], metadata=[{ "breadcrumb": [], "metadata": { "selected": True } }], ) ]) sync(config, state, catalog) assert mock_stdout.mock_calls == [ call( '{"type": "SCHEMA", "stream": "teams", "schema": {}, "key_properties": ["id"]}\n' ), call( '{"type": "RECORD", "stream": "teams", "record": {"id": "5d6ca50762a07c00045125fb", "domain": "pageup", "bot_token": "e31d3b7ae51ff1feec8be578f23eb017e8143f66a7a085342c664544b81618ec41b87810d61a9c1f6133fe0c7d88aa3976232bb2a2665c4f89c38058b51cd20c", "activated_by": "U6K26HMGV", "status": "ACTIVE", "platform_id": "T034F9NPW", "created_at": "2019-09-02T05:13:43.151", "subscription": {"active_until": "2020-07-08T23:59:59", "status": "active", "number_of_users": 69, "subscriber_id": "U93KT77T6"}, "icon": {"image_34": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_34.png", "image_44": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_44.png", "image_68": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_68.png", "image_88": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_88.png", "image_102": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_102.png", "image_132": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_132.png", "image_230": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_230.png", "image_original": "https://avatars.slack-edge.com/2017-09-15/241678543093_b2ad80be9268cdbd89c3_original.png"}}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), ]
def test_should_output_records(self, mock_stdout, requests_mock): requests_mock.get("https://api.nikabot.com/api/v1/groups?limit=1000&page=0", json=json.loads(GROUPS_RESPONSE)) requests_mock.get("https://api.nikabot.com/api/v1/groups?limit=1000&page=1", json=json.loads(EMPTY_RESPONSE)) config = {"access_token": "my-access-token", "page_size": 1000} state = {} catalog = Catalog( streams=[ CatalogEntry( tap_stream_id="groups", stream="groups", schema=Schema.from_dict({}), key_properties=["id"], metadata=[{"breadcrumb": [], "metadata": {"selected": True}}], ) ] ) sync(config, state, catalog) assert mock_stdout.mock_calls == [ call('{"type": "SCHEMA", "stream": "groups", "schema": {}, "key_properties": ["id"]}\n'), call( '{"type": "RECORD", "stream": "groups", "record": {"id": "f1b4b37cc2658672770b789f", "team_id": "T034F9NPW", "name": "TA Squad 5"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), call( '{"type": "RECORD", "stream": "groups", "record": {"id": "3176700ac4f2203b825fae6c", "team_id": "T034F9NPW", "name": "Platform Toolkit"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), ]
def test_should_use_bookmark_when_bookmark_has_timezone_info( self, mock_stdout, requests_mock, mock_catalog): requests_page0 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=20200610&dateEnd=20200610", json=json.loads(RECORDS_RESPONSE), ) requests_page1 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=20200610&dateEnd=20200610", json=json.loads(EMPTY_RESPONSE), ) config = { "access_token": "my-access-token", "page_size": 1000, "cutoff_days": 10, "start_date": "2020-01-01", "end_date": "2020-06-10", } state = {"records": "2020-06-09T00:00:00.000+00:00"} sync(config, state, mock_catalog) assert requests_page0.call_count == 1 assert requests_page1.call_count == 1 mock_stdout.assert_has_calls([ call( '{"type": "STATE", "value": {"records": "2020-06-10T00:00:00"}}\n' ), ])
def test_should_output_records(self, mock_stdout, requests_mock): requests_mock.get( "https://api.nikabot.com/api/v1/projects?limit=1000&page=0", json=json.loads(PROJECTS_RESPONSE)) requests_mock.get( "https://api.nikabot.com/api/v1/projects?limit=1000&page=1", json=json.loads(EMPTY_RESPONSE)) config = {"access_token": "my-access-token", "page_size": 1000} state = {} catalog = Catalog(streams=[ CatalogEntry( tap_stream_id="projects", stream="projects", schema=Schema.from_dict({}), key_properties=["id"], metadata=[{ "breadcrumb": [], "metadata": { "selected": True } }], ) ]) sync(config, state, catalog) assert mock_stdout.mock_calls == [ call( '{"type": "SCHEMA", "stream": "projects", "schema": {}, "key_properties": ["id"]}\n' ), call( '{"type": "RECORD", "stream": "projects", "record": {"id": "5d6ca95e62a07c00045126e7", "project_name": "CAP - Analytics", "team_id": "T034F9NPW", "author": "U6K26HMGV", "pto": {"status": false}, "custom_ref": "", "create_date": "2019-09-02T05:32:14.23", "client": "", "type": "Capability Custodian", "created_at": "2019-09-02T05:32:14.23", "assigned_groups": ["Analytics"]}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), call( '{"type": "RECORD", "stream": "projects", "record": {"id": "5d6ca97c62a07c00045126e8", "project_name": "CAP - Authentication", "team_id": "T034F9NPW", "author": "U6K26HMGV", "pto": {"status": false}, "custom_ref": "", "create_date": "2019-09-02T05:32:44.172", "client": "", "type": "Capability Custodian", "created_at": "2019-09-02T05:32:44.172", "assigned_groups": ["Authentication"]}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), ]
def test_should_output_multiple_pages(self, mock_stdout, requests_mock, mock_catalog): requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=00010101&dateEnd=99991231", json=json.loads(RECORDS_RESPONSE), ) requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=00010101&dateEnd=99991231", json=json.loads(RECORDS_PAGE2_RESPONSE), ) requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=2&dateStart=00010101&dateEnd=99991231", json=json.loads(EMPTY_RESPONSE), ) config = {"access_token": "my-access-token", "page_size": 1000} state = {} sync(config, state, mock_catalog) assert mock_stdout.mock_calls == [ call('{"type": "SCHEMA", "stream": "records", "schema": ' + SCHEMA + ', "key_properties": ["id"]}\n'), call( '{"type": "RECORD", "stream": "records", "record": {"id": "5ee2ca823e056d00141896a0", "team_id": "T034F9NPW", "user_id": "UBM1DQ9RB", "project_name": "CAP - Data Lifecycle", "project_id": "5d6ca9e462a07c00045126ed", "hours": 2.0, "date": "2000-01-01T00:00:00.000000Z", "created_at": "2020-01-01T00:21:22.779000Z"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), call( '{"type": "RECORD", "stream": "records", "record": {"id": "5ee1d52e5cff9100146de745", "team_id": "T034F9NPW", "user_id": "U107SJ4N6", "project_name": "DUX", "project_id": "5d6df702956de30004dc0198", "hours": 7.5, "date": "2020-06-10T00:00:00.000000Z", "created_at": "2020-06-11T06:54:38.138000Z"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), call( '{"type": "RECORD", "stream": "records", "record": {"id": "5d9d7a035da6700004970476", "team_id": "T034F9NPW", "user_id": "UBM1DQ9RB", "project_name": "Leave (All Kinds)", "project_id": "5d6ca50762a07c00045125fc", "hours": 7.5, "date": "2019-08-20T00:00:00.000000Z", "created_at": "2019-10-09T06:11:15.976000Z"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), call( '{"type": "RECORD", "stream": "records", "record": {"id": "5d9d79c45da6700004970475", "team_id": "T034F9NPW", "user_id": "UBM1DQ9RB", "project_name": "TX - M1 Assign due dates", "project_id": "5d6e06c9956de30004dc01b0", "hours": 7.5, "date": "2019-08-21T00:00:00.000000Z", "created_at": "2019-10-09T06:10:12.673000Z"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), ]
def test_should_output_records_given_default_config( self, mock_stdout, requests_mock, mock_catalog): requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=00010101&dateEnd=20191222", json=json.loads(RECORDS_RESPONSE), ) requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=00010101&dateEnd=20191222", json=json.loads(EMPTY_RESPONSE), ) config = { "access_token": "my-access-token", "page_size": 1000, "cutoff_days": 10 } state = {} sync(config, state, mock_catalog) assert mock_stdout.mock_calls == [ call( '{"type": "SCHEMA", "stream": "records", "schema": ' + SCHEMA + ', "key_properties": ["id"], "bookmark_properties": ["date"]}\n' ), call( '{"type": "RECORD", "stream": "records", "record": {"id": "5ee2ca823e056d00141896a0", "team_id": "T034F9NPW", "user_id": "UBM1DQ9RB", "project_name": "CAP - Data Lifecycle", "project_id": "5d6ca9e462a07c00045126ed", "hours": 2.0, "date": "2000-01-01T00:00:00+00:00", "created_at": "2020-01-01T00:21:22.779000+00:00"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), call( '{"type": "RECORD", "stream": "records", "record": {"id": "5ee1d52e5cff9100146de745", "team_id": "T034F9NPW", "user_id": "U107SJ4N6", "project_name": "DUX", "project_id": "5d6df702956de30004dc0198", "hours": 7.5, "date": "2020-06-10T00:00:00+00:00", "created_at": "2020-06-11T06:54:38.138000+00:00"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), call( '{"type": "STATE", "value": {"records": "2020-06-10T00:00:00"}}\n' ), ]
def test_should_output_records(self, mock_stdout, requests_mock): requests_mock.get( "https://api.nikabot.com/api/v1/users?limit=1000&page=0", json=json.loads(USERS_RESPONSE)) requests_mock.get( "https://api.nikabot.com/api/v1/users?limit=1000&page=1", json=json.loads(EMPTY_RESPONSE)) config = {"access_token": "my-access-token", "page_size": 1000} state = {} catalog = Catalog(streams=[ CatalogEntry( tap_stream_id="users", stream="users", schema=Schema.from_dict({}), key_properties=["id"], metadata=[{ "breadcrumb": [], "metadata": { "selected": True } }], ) ]) sync(config, state, catalog) assert mock_stdout.mock_calls == [ call( '{"type": "SCHEMA", "stream": "users", "schema": {}, "key_properties": ["id"]}\n' ), call( '{"type": "RECORD", "stream": "users", "record": {"id": "5de459977292020014fb601c", "name": "Billy", "deleted": true, "presence": "away", "user_id": "UR5B0QABX", "team_id": "T034F9NPW", "is_restricted": false, "is_ultra_restricted": false, "is_admin": false, "is_nikabot_admin": false, "tz": "Australia/Canberra", "tz_label": "Australian Eastern Standard Time", "tz_offset": 36000, "is_checkin_excluded": true, "created_at": "2019-12-02T00:23:51.087", "groups": [], "updated_at": "2020-06-14T22:47:29.617"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), call( '{"type": "RECORD", "stream": "users", "record": {"id": "68QMxnnt8YcpPdfmM", "name": "paul.heasley", "deleted": false, "presence": "active", "user_id": "U04AX35QP", "team_id": "T034F9NPW", "is_restricted": false, "is_ultra_restricted": false, "is_admin": false, "is_nikabot_admin": true, "tz": "Australia/Canberra", "tz_label": "Australian Eastern Standard Time", "tz_offset": 36000, "is_checkin_excluded": false, "create_date": "2019-09-02T05:13:47.88", "created_at": "2019-09-02T05:13:47.882", "role": "0.1", "groups": ["TA Stream", "TA Squad 1", "TA Squad 2", "TA Squad 3", "TA Squad 4", "Learning Applications", "Notification Capability"], "updated_at": "2020-06-15T06:07:58.272"}, "time_extracted": "2020-01-01T00:00:00.000000Z"}\n' ), ] assert LOGGER.info.mock_calls == [ call("Syncing stream: %s", "users"), call( "Making %s request to %s with params %s", "GET", "https://api.nikabot.com/api/v1/users", { "limit": "1000", "page": "0" }, ), call( "Making %s request to %s with params %s", "GET", "https://api.nikabot.com/api/v1/users", { "limit": "1000", "page": "1" }, ), ]
def test_should_raise_error_when_start_date_greater_than_end_date(self, mock_catalog): config = { "access_token": "my-access-token", "page_size": 1000, "cutoff_days": 10, "start_date": "2020-06-11", "end_date": "2020-06-10", } state = {} with pytest.raises(StartDateAfterEndDateError): sync(config, state, mock_catalog)
def test_should_raise_error_when_log_based_replication_requested( self, mock_catalog): config = {"access_token": "my-access-token", "page_size": 1000} state = {} mock_catalog.streams[0].replication_method = "LOG_BASED" with pytest.raises(InvalidReplicationMethodError) as excinfo: sync(config, state, mock_catalog) assert ( str(excinfo.value) == "Invalid replication method selected 'LOG_BASED', valid options are 'FULL_TABLE'" )
def test_should_raise_error_when_incremental_replication_requested( self, mock_catalog): config = {"access_token": "my-access-token", "page_size": 1000} state = {} mock_catalog.streams[0].replication_key = ("date", ) mock_catalog.streams[0].replication_method = "INCREMENTAL" with pytest.raises(InvalidReplicationMethodError) as excinfo: sync(config, state, mock_catalog) assert ( str(excinfo.value) == "Invalid replication method selected 'INCREMENTAL', valid options are 'FULL_TABLE'" )
def test_should_raise_error_when_start_date_greater_than_end_date( self, mock_catalog): config = { "access_token": "my-access-token", "page_size": 1000, "start_date": "2020-06-11", "end_date": "2020-06-10", } state = {} with pytest.raises(StartDateAfterEndDateError) as excinfo: sync(config, state, mock_catalog) assert str( excinfo.value ) == "Start date '2020-06-11' cannot be later than end date '2020-06-10'"
def test_should_sync_future_dates_given_cutoff_days_not_set(self, requests_mock, mock_catalog): requests_page0 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=00010101&dateEnd=99991231", json=json.loads(RECORDS_RESPONSE), ) requests_page1 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=00010101&dateEnd=99991231", json=json.loads(EMPTY_RESPONSE), ) config = {"access_token": "my-access-token", "page_size": 1000, "cutoff_days": None} state = {} sync(config, state, mock_catalog) assert requests_page0.call_count == 1 assert requests_page1.call_count == 1
def test_should_return_no_records_when_bookmark_greater_than_cutoff_date( self, mock_stdout, mock_catalog): config = { "access_token": "my-access-token", "page_size": 1000, "cutoff_days": 10, } state = {"records": "2020-01-01T00:00:00.000"} sync(config, state, mock_catalog) assert mock_stdout.mock_calls == [ call( '{"type": "SCHEMA", "stream": "records", "schema": ' + SCHEMA + ', "key_properties": ["id"], "bookmark_properties": ["date"]}\n' ), ]
def test_should_not_use_cutoff_days_when_full_replication(self, mock_stdout, requests_mock, mock_catalog): requests_page0 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=00010101&dateEnd=99991231", json=json.loads(RECORDS_RESPONSE), ) requests_page1 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=00010101&dateEnd=99991231", json=json.loads(EMPTY_RESPONSE), ) config = {"access_token": "my-access-token", "page_size": 1000, "cutoff_days": 10} state = {} mock_catalog.streams[0].replication_method = "FULL_TABLE" sync(config, state, mock_catalog) assert requests_page0.call_count == 1 assert requests_page1.call_count == 1
def test_should_output_nothing_given_no_streams_selected( self, mock_stdout): config = {"access_token": "my-access-token", "page_size": 1000} state = {} catalog = Catalog(streams=[ CatalogEntry( tap_stream_id="users", stream="users", schema=Schema.from_dict({}), key_properties=["id"], metadata=[], ) ]) sync(config, state, catalog) mock_stdout.assert_not_called() assert LOGGER.info.mock_calls == [call("Skipping stream: %s", "users")]
def test_should_start_from_start_date_given_bookmark_less_than_start_date(self, requests_mock, mock_catalog): requests_page0 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=20200101&dateEnd=20200610", json=json.loads(RECORDS_RESPONSE), ) requests_page1 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=20200101&dateEnd=20200610", json=json.loads(EMPTY_RESPONSE), ) config = { "access_token": "my-access-token", "page_size": 1000, "cutoff_days": 10, "start_date": "2020-01-01", "end_date": "2020-06-10", } state = {"records": "2010-12-31T00:00:00.000"} sync(config, state, mock_catalog) assert requests_page0.call_count == 1 assert requests_page1.call_count == 1
def test_should_use_end_date_when_cutoff_date_greater_than_end_date(self, requests_mock, mock_catalog): requests_page0 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=20191219&dateEnd=20191220", json=json.loads(RECORDS_RESPONSE), ) requests_page1 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=20191219&dateEnd=20191220", json=json.loads(EMPTY_RESPONSE), ) config = { "access_token": "my-access-token", "page_size": 1000, "cutoff_days": 10, "start_date": "2019-12-19", "end_date": "2019-12-20", } state = {} sync(config, state, mock_catalog) assert requests_page0.call_count == 1 assert requests_page1.call_count == 1
def test_should_not_use_bookmark_when_full_replication_given_start_and_end_dates(self, requests_mock, mock_catalog): requests_page0 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=0&dateStart=20200101&dateEnd=20200610", json=json.loads(RECORDS_RESPONSE), ) requests_page1 = requests_mock.get( "https://api.nikabot.com/api/v1/records?limit=1000&page=1&dateStart=20200101&dateEnd=20200610", json=json.loads(EMPTY_RESPONSE), ) config = { "access_token": "my-access-token", "page_size": 1000, "cutoff_days": 10, "start_date": "2020-01-01", "end_date": "2020-06-10", } state = {"records": "2020-06-09T00:00:00.000"} mock_catalog.streams[0].replication_method = "FULL_TABLE" sync(config, state, mock_catalog) assert requests_page0.call_count == 1 assert requests_page1.call_count == 1
def test_should_output_no_records_given_no_records_available( self, mock_stdout, requests_mock): requests_mock.get( "https://api.nikabot.com/api/v1/users?limit=1000&page=0", json=json.loads(EMPTY_RESPONSE)) config = {"access_token": "my-access-token", "page_size": 1000} state = {} catalog = Catalog(streams=[ CatalogEntry( tap_stream_id="users", stream="users", schema=Schema.from_dict({}), key_properties=["id"], metadata=[{ "breadcrumb": [], "metadata": { "selected": True } }], ) ]) sync(config, state, catalog) assert mock_stdout.mock_calls == [ call( '{"type": "SCHEMA", "stream": "users", "schema": {}, "key_properties": ["id"]}\n' ) ] assert LOGGER.info.mock_calls == [ call("Syncing stream: %s", "users"), call( "Making %s request to %s with params %s", "GET", "https://api.nikabot.com/api/v1/users", { "limit": "1000", "page": "0" }, ), ]