def test_datetime_date_method():
now = datetime.datetime.now()
assert now.date() == FakeDate(2012, 1, 14)
def test_add_mailchimp_resource_latest_yesterday(self, m):
'''Latest data was yesterday, so get today and yesterday only.
Yesterday's values are now final and will overwrite what's there.'''
# Mock API responses
m.get('https://dc1.api.mailchimp.com/3.0/lists/123456',
json=GENERAL_LIST_STATS_RESPONSE)
activity_date = dateutil.parser.parse("2014-10-18").date()
list_activity_response = _make_list_activity_response(activity_date, 2)
m.get('https://dc1.api.mailchimp.com/3.0/lists/123456/activity?count=2', # noqa
json=list_activity_response)
m.get('https://dc1.api.mailchimp.com/3.0/campaigns/?list_id=123456&since_send_time=2014-10-17', # noqa
json=CAMPAIGNS_RESPONSE)
# input arguments used by our mock `ingest`
datapackage = {
'name': 'my-datapackage',
'project': 'my-project',
'resources': [{
'name': 'latest-project-entries',
'schema': {
'fields': []
}
}]
}
params = {
'list_id': '123456'
}
def latest_entries_res():
yield {
'campaigns_sent': 0,
'list_id': '123456',
'subs': 1,
'subscribers': 29000,
'unsubs': 0,
'date': FakeDate(2014, 10, 17),
'source': 'mailchimp'
}
# Path to the processor we want to test
processor_dir = \
os.path.dirname(datapackage_pipelines_measure.processors.__file__)
processor_path = os.path.join(processor_dir,
'add_mailchimp_resource.py')
# Trigger the processor with our mock `ingest` and capture what it will
# returned to `spew`.
spew_args, _ = mock_processor_test(processor_path,
(params, datapackage,
iter([latest_entries_res()])))
spew_dp = spew_args[0]
spew_res_iter = spew_args[1]
# Asserts for the datapackage
dp_resources = spew_dp['resources']
assert len(dp_resources) == 2
assert dp_resources[0]['name'] == 'latest-project-entries'
assert dp_resources[1]['name'] == '123456'
# Asserts for the res_iter
# two resources in res_iter
resources = list(spew_res_iter)
assert len(resources) == 2
# first resource will look like latest_entries_res
assert list(resources[0])[0] == next(latest_entries_res())
# second resource is from the mailchimp processor
res_rows = resources[1]
assert len(res_rows) == 2
assert res_rows[0] == {
'campaigns_sent': 0,
'list_id': '123456',
'subs': 2,
'subscribers': 30000,
'unsubs': 4,
'date': FakeDate(2014, 10, 18),
'source': 'mailchimp'
}
assert res_rows[1] == {
'campaigns_sent': 0,
'list_id': '123456',
'subs': 4, # subs and unsubs have been updated
'unsubs': 6,
'date': FakeDate(2014, 10, 17),
'source': 'mailchimp',
'subscribers': 29000 # subscribers number is retained from db
}
def test_add_mailchimp_resource_no_latest(self, m):
'''No latest data in email table, so populate with historical data.'''
# Mock API responses
m.get('https://dc1.api.mailchimp.com/3.0/lists/123456',
json=GENERAL_LIST_STATS_RESPONSE)
activity_date = dateutil.parser.parse("2014-10-18").date()
list_activity_response = _make_list_activity_response(activity_date,
30)
m.get('https://dc1.api.mailchimp.com/3.0/lists/123456/activity?count=29', # noqa
json=list_activity_response)
m.get('https://dc1.api.mailchimp.com/3.0/campaigns/?list_id=123456&since_send_time=2014-09-20', # noqa
json=CAMPAIGNS_RESPONSE)
m.get('https://dc1.api.mailchimp.com/3.0/lists/123456/growth-history/2014-09', # noqa
json=GROWTH_HISTORY_RESPONSE)
# input arguments used by our mock `ingest`
datapackage = {
'name': 'my-datapackage',
'project': 'my-project',
'resources': [] # nothing here
}
params = {
'list_id': '123456'
}
# Path to the processor we want to test
processor_dir = \
os.path.dirname(datapackage_pipelines_measure.processors.__file__)
processor_path = os.path.join(processor_dir,
'add_mailchimp_resource.py')
# Trigger the processor with our mock `ingest` and capture what it will
# returned to `spew`.
spew_args, _ = mock_processor_test(processor_path,
(params, datapackage, iter([])))
spew_dp = spew_args[0]
spew_res_iter = spew_args[1]
# Asserts for the datapackage
dp_resources = spew_dp['resources']
assert len(dp_resources) == 1
assert dp_resources[0]['name'] == '123456'
field_names = \
[field['name'] for field in dp_resources[0]['schema']['fields']]
assert field_names == ['source', 'date', 'list_id', 'subs', 'unsubs',
'subscribers', 'campaigns_sent']
# Asserts for the res_iter
spew_res_iter_contents = list(spew_res_iter)
assert len(list(spew_res_iter_contents)) == 1
rows = list(spew_res_iter_contents)[0]
assert len(rows) == 30
assert rows[0] == {
'campaigns_sent': 0,
'list_id': '123456',
'subs': 2,
'subscribers': 30000,
'unsubs': 4,
'date': FakeDate(2014, 10, 18),
'source': 'mailchimp'
}
# third one has a campaign sent
assert rows[2] == {
'campaigns_sent': 1,
'list_id': '123456',
'subs': 6,
'unsubs': 8,
'date': FakeDate(2014, 10, 16),
'source': 'mailchimp'
}
def test_add_github_resource_processor(self, mock_request):
# mock the github response
mock_repo_response = {
'name': 'my-repository',
'subscribers_count': 4,
'stargazers_count': 1,
'forks_count': 10,
'full_name': 'org/my_github_repo'
}
mock_search_response = {
'total_count': 5
}
mock_request.get('https://api.github.com/repos/org/my_github_repo',
json=mock_repo_response)
mock_request.get('https://api.github.com/search/issues',
json=mock_search_response)
# input arguments used by our mock `ingest`
datapackage = {
'name': 'my-datapackage',
'project': 'my-project',
'resources': []
}
params = {
'name': 'hello',
'repo': 'org/my_github_repo'
}
# Path to the processor we want to test
processor_dir = \
os.path.dirname(datapackage_pipelines_measure.processors.__file__)
processor_path = os.path.join(processor_dir, 'add_github_resource.py')
# Trigger the processor with our mock `ingest` and capture what it will
# returned to `spew`.
spew_args, _ = mock_processor_test(processor_path,
(params, datapackage, []))
spew_dp = spew_args[0]
spew_res_iter = spew_args[1]
# Asserts for the datapackage
dp_resources = spew_dp['resources']
assert len(dp_resources) == 1
assert dp_resources[0]['name'] == 'hello'
field_names = \
[field['name'] for field in dp_resources[0]['schema']['fields']]
assert field_names == ['repository', 'watchers', 'stars', 'forks',
'source', 'date', 'open_prs', 'closed_prs',
'open_issues', 'closed_issues']
# Asserts for the res_iter
spew_res_iter_contents = list(spew_res_iter)
assert len(spew_res_iter_contents) == 1
assert list(spew_res_iter_contents[0]) == \
[{
'repository': 'my-repository',
'watchers': 4,
'stars': 1,
'forks': 10,
'source': 'github',
'date': FakeDate(2017, 10, 12),
'open_prs': 5,
'closed_prs': 5,
'open_issues': 5,
'closed_issues': 5
}]