def test_datetime_date_method(): now = datetime.datetime.now() assert now.date() == FakeDate(2012, 1, 14)
def test_add_mailchimp_resource_latest_yesterday(self, m): '''Latest data was yesterday, so get today and yesterday only. Yesterday's values are now final and will overwrite what's there.''' # Mock API responses m.get('https://dc1.api.mailchimp.com/3.0/lists/123456', json=GENERAL_LIST_STATS_RESPONSE) activity_date = dateutil.parser.parse("2014-10-18").date() list_activity_response = _make_list_activity_response(activity_date, 2) m.get('https://dc1.api.mailchimp.com/3.0/lists/123456/activity?count=2', # noqa json=list_activity_response) m.get('https://dc1.api.mailchimp.com/3.0/campaigns/?list_id=123456&since_send_time=2014-10-17', # noqa json=CAMPAIGNS_RESPONSE) # input arguments used by our mock `ingest` datapackage = { 'name': 'my-datapackage', 'project': 'my-project', 'resources': [{ 'name': 'latest-project-entries', 'schema': { 'fields': [] } }] } params = { 'list_id': '123456' } def latest_entries_res(): yield { 'campaigns_sent': 0, 'list_id': '123456', 'subs': 1, 'subscribers': 29000, 'unsubs': 0, 'date': FakeDate(2014, 10, 17), 'source': 'mailchimp' } # Path to the processor we want to test processor_dir = \ os.path.dirname(datapackage_pipelines_measure.processors.__file__) processor_path = os.path.join(processor_dir, 'add_mailchimp_resource.py') # Trigger the processor with our mock `ingest` and capture what it will # returned to `spew`. spew_args, _ = mock_processor_test(processor_path, (params, datapackage, iter([latest_entries_res()]))) spew_dp = spew_args[0] spew_res_iter = spew_args[1] # Asserts for the datapackage dp_resources = spew_dp['resources'] assert len(dp_resources) == 2 assert dp_resources[0]['name'] == 'latest-project-entries' assert dp_resources[1]['name'] == '123456' # Asserts for the res_iter # two resources in res_iter resources = list(spew_res_iter) assert len(resources) == 2 # first resource will look like latest_entries_res assert list(resources[0])[0] == next(latest_entries_res()) # second resource is from the mailchimp processor res_rows = resources[1] assert len(res_rows) == 2 assert res_rows[0] == { 'campaigns_sent': 0, 'list_id': '123456', 'subs': 2, 'subscribers': 30000, 'unsubs': 4, 'date': FakeDate(2014, 10, 18), 'source': 'mailchimp' } assert res_rows[1] == { 'campaigns_sent': 0, 'list_id': '123456', 'subs': 4, # subs and unsubs have been updated 'unsubs': 6, 'date': FakeDate(2014, 10, 17), 'source': 'mailchimp', 'subscribers': 29000 # subscribers number is retained from db }
def test_add_mailchimp_resource_no_latest(self, m): '''No latest data in email table, so populate with historical data.''' # Mock API responses m.get('https://dc1.api.mailchimp.com/3.0/lists/123456', json=GENERAL_LIST_STATS_RESPONSE) activity_date = dateutil.parser.parse("2014-10-18").date() list_activity_response = _make_list_activity_response(activity_date, 30) m.get('https://dc1.api.mailchimp.com/3.0/lists/123456/activity?count=29', # noqa json=list_activity_response) m.get('https://dc1.api.mailchimp.com/3.0/campaigns/?list_id=123456&since_send_time=2014-09-20', # noqa json=CAMPAIGNS_RESPONSE) m.get('https://dc1.api.mailchimp.com/3.0/lists/123456/growth-history/2014-09', # noqa json=GROWTH_HISTORY_RESPONSE) # input arguments used by our mock `ingest` datapackage = { 'name': 'my-datapackage', 'project': 'my-project', 'resources': [] # nothing here } params = { 'list_id': '123456' } # Path to the processor we want to test processor_dir = \ os.path.dirname(datapackage_pipelines_measure.processors.__file__) processor_path = os.path.join(processor_dir, 'add_mailchimp_resource.py') # Trigger the processor with our mock `ingest` and capture what it will # returned to `spew`. spew_args, _ = mock_processor_test(processor_path, (params, datapackage, iter([]))) spew_dp = spew_args[0] spew_res_iter = spew_args[1] # Asserts for the datapackage dp_resources = spew_dp['resources'] assert len(dp_resources) == 1 assert dp_resources[0]['name'] == '123456' field_names = \ [field['name'] for field in dp_resources[0]['schema']['fields']] assert field_names == ['source', 'date', 'list_id', 'subs', 'unsubs', 'subscribers', 'campaigns_sent'] # Asserts for the res_iter spew_res_iter_contents = list(spew_res_iter) assert len(list(spew_res_iter_contents)) == 1 rows = list(spew_res_iter_contents)[0] assert len(rows) == 30 assert rows[0] == { 'campaigns_sent': 0, 'list_id': '123456', 'subs': 2, 'subscribers': 30000, 'unsubs': 4, 'date': FakeDate(2014, 10, 18), 'source': 'mailchimp' } # third one has a campaign sent assert rows[2] == { 'campaigns_sent': 1, 'list_id': '123456', 'subs': 6, 'unsubs': 8, 'date': FakeDate(2014, 10, 16), 'source': 'mailchimp' }
def test_add_github_resource_processor(self, mock_request): # mock the github response mock_repo_response = { 'name': 'my-repository', 'subscribers_count': 4, 'stargazers_count': 1, 'forks_count': 10, 'full_name': 'org/my_github_repo' } mock_search_response = { 'total_count': 5 } mock_request.get('https://api.github.com/repos/org/my_github_repo', json=mock_repo_response) mock_request.get('https://api.github.com/search/issues', json=mock_search_response) # input arguments used by our mock `ingest` datapackage = { 'name': 'my-datapackage', 'project': 'my-project', 'resources': [] } params = { 'name': 'hello', 'repo': 'org/my_github_repo' } # Path to the processor we want to test processor_dir = \ os.path.dirname(datapackage_pipelines_measure.processors.__file__) processor_path = os.path.join(processor_dir, 'add_github_resource.py') # Trigger the processor with our mock `ingest` and capture what it will # returned to `spew`. spew_args, _ = mock_processor_test(processor_path, (params, datapackage, [])) spew_dp = spew_args[0] spew_res_iter = spew_args[1] # Asserts for the datapackage dp_resources = spew_dp['resources'] assert len(dp_resources) == 1 assert dp_resources[0]['name'] == 'hello' field_names = \ [field['name'] for field in dp_resources[0]['schema']['fields']] assert field_names == ['repository', 'watchers', 'stars', 'forks', 'source', 'date', 'open_prs', 'closed_prs', 'open_issues', 'closed_issues'] # Asserts for the res_iter spew_res_iter_contents = list(spew_res_iter) assert len(spew_res_iter_contents) == 1 assert list(spew_res_iter_contents[0]) == \ [{ 'repository': 'my-repository', 'watchers': 4, 'stars': 1, 'forks': 10, 'source': 'github', 'date': FakeDate(2017, 10, 12), 'open_prs': 5, 'closed_prs': 5, 'open_issues': 5, 'closed_issues': 5 }]