def test_resuming(self):
"""Test if it resumes when a failure is reached"""
http_requests = setup_mock_redmine_server(max_failures=1)
args = {
'url': REDMINE_URL,
'api_token': 'AAAA',
'max_issues': 3
}
job = PercevalJob('arthur-job-1234567890', 'mytask', 'redmine', 'issue',
self.conn, 'items')
with self.assertRaises(requests.exceptions.HTTPError):
job.run(args)
result = job.result
self.assertEqual(result.job_id, 'arthur-job-1234567890')
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'redmine')
self.assertEqual(result.category, 'issue')
self.assertEqual(result.last_uuid, '3c3d67925b108a37f88cc6663f7f7dd493fa818c')
self.assertEqual(result.max_date, 1323367117.0)
self.assertEqual(result.nitems, 3)
self.assertEqual(result.offset, None)
self.assertEqual(result.nresumed, 0)
issues = self.conn.lrange('items', 0, -1)
issues = [pickle.loads(i) for i in issues]
issues = [i['uuid'] for i in issues]
self.conn.ltrim('items', 1, 0)
expected = ['91a8349c2f6ebffcccc49409529c61cfd3825563',
'c4aeb9e77fec8e4679caa23d4012e7cc36ae8b98',
'3c3d67925b108a37f88cc6663f7f7dd493fa818c']
self.assertEqual(issues, expected)
job.run(args, resume=True)
result = job.result
self.assertEqual(result.job_id, 'arthur-job-1234567890')
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'redmine')
self.assertEqual(result.last_uuid, '4ab289ab60aee93a66e5490529799cf4a2b4d94c')
self.assertEqual(result.max_date, 1469607427.0)
self.assertEqual(result.nitems, 4)
self.assertEqual(result.offset, None)
self.assertEqual(result.nresumed, 1)
issues = self.conn.lrange('items', 0, -1)
issues = [pickle.loads(i) for i in issues]
issues = [i['uuid'] for i in issues]
self.conn.ltrim('items', 1, 0)
expected = ['4ab289ab60aee93a66e5490529799cf4a2b4d94c']
self.assertEqual(issues, expected)
def test_resuming(self):
"""Test if it resumes when a failure is reached"""
http_requests = setup_mock_redmine_server(max_failures=1)
args = {'url': REDMINE_URL, 'api_token': 'AAAA', 'max_issues': 3}
job = PercevalJob('arthur-job-1234567890', 'mytask', 'redmine',
'issue', self.conn, 'items')
with self.assertRaises(requests.exceptions.HTTPError):
job.run(args)
result = job.result
self.assertEqual(result.job_id, 'arthur-job-1234567890')
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'redmine')
self.assertEqual(result.category, 'issue')
self.assertEqual(result.last_uuid,
'3c3d67925b108a37f88cc6663f7f7dd493fa818c')
self.assertEqual(result.max_date, 1323367117.0)
self.assertEqual(result.nitems, 3)
self.assertEqual(result.offset, None)
self.assertEqual(result.nresumed, 0)
issues = self.conn.lrange('items', 0, -1)
issues = [pickle.loads(i) for i in issues]
issues = [i['uuid'] for i in issues]
self.conn.ltrim('items', 1, 0)
expected = [
'91a8349c2f6ebffcccc49409529c61cfd3825563',
'c4aeb9e77fec8e4679caa23d4012e7cc36ae8b98',
'3c3d67925b108a37f88cc6663f7f7dd493fa818c'
]
self.assertEqual(issues, expected)
job.run(args, resume=True)
result = job.result
self.assertEqual(result.job_id, 'arthur-job-1234567890')
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'redmine')
self.assertEqual(result.last_uuid,
'4ab289ab60aee93a66e5490529799cf4a2b4d94c')
self.assertEqual(result.max_date, 1469607427.0)
self.assertEqual(result.nitems, 4)
self.assertEqual(result.offset, None)
self.assertEqual(result.nresumed, 1)
issues = self.conn.lrange('items', 0, -1)
issues = [pickle.loads(i) for i in issues]
issues = [i['uuid'] for i in issues]
self.conn.ltrim('items', 1, 0)
expected = ['4ab289ab60aee93a66e5490529799cf4a2b4d94c']
self.assertEqual(issues, expected)
def test_run_not_found_parameters(self):
"""Check if it fails when a required backend parameter is not found"""
job = PercevalJob('arthur-job-1234567890', 'mytask', 'git', self.conn,
'items')
args = {'uri': 'http://example.com/'}
with self.assertRaises(AttributeError) as e:
job.run(args, fetch_from_cache=False)
self.assertEqual(e.exception.args[1], 'gitlog')
def test_run(self):
"""Test run method using the Git backend"""
job = PercevalJob('1234567890', 8, 'mytask', 'git', 'commit',
self.conn, 'items')
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
archive_args = {
'archive_path': self.tmp_path,
'fetch_from_archive': False
}
job.run(args, archive_args)
self.assertIsInstance(job.archive_manager, ArchiveManager)
result = job.result
self.assertIsInstance(job.result, JobResult)
self.assertEqual(result.job_id, '1234567890')
self.assertEqual(result.job_number, 8)
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'git')
self.assertEqual(result.category, 'commit')
self.assertEqual(result.summary.last_uuid,
'1375b60d3c23ac9b81da92523e4144abc4489d4c')
self.assertEqual(result.summary.max_updated_on,
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(
result.summary.last_updated_on,
datetime.datetime(2012, 8, 14, 17, 30, 13, tzinfo=UTC))
self.assertEqual(result.summary.fetched, 9)
self.assertEqual(result.summary.last_offset, None)
commits = self.conn.lrange('items', 0, -1)
commits = [pickle.loads(c) for c in commits]
commits = [commit['data']['commit'] for commit in commits]
expected = [
'456a68ee1407a77f3e804a30dff245bb6c6b872f',
'51a3b654f252210572297f47597b31527c475fb8',
'ce8e0b86a1e9877f42fe9453ede418519115f367',
'589bb080f059834829a2a5955bebfd7c2baa110a',
'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb',
'c0d66f92a95e31c77be08dc9d0f11a16715d1885',
'7debcf8a2f57f86663809c58b5c07a398be7674c',
'87783129c3f00d2c81a3a8e585eb86a47e39891a',
'bc57a9209f096a130dcc5ba7089a8663f758a703'
]
self.assertEqual(commits, expected)
def test_run_not_found_parameters(self):
"""Check if it fails when a required backend parameter is not found"""
job = PercevalJob('1234567890', 8, 'mytask', 'git', 'commit',
self.conn, 'items')
args = {'uri': 'http://example.com/'}
archive_args = {
'archive_path': self.tmp_path,
'fetch_from_archive': False
}
with self.assertRaises(AttributeError) as e:
job.run(args, archive_args)
self.assertEqual(e.exception.args[1], 'gitlog')
def test_run_not_found_parameters(self):
"""Check if it fails when a required backend parameter is not found"""
job = PercevalJob('arthur-job-1234567890', 'mytask', 'git', 'commit',
self.conn, 'items')
args = {
'uri': 'http://example.com/'
}
archive_args = {
'archive_path': self.tmp_path,
'fetch_from_archive': False
}
with self.assertRaises(AttributeError) as e:
job.run(args, archive_args)
self.assertEqual(e.exception.args[1], 'gitlog')
def test_run(self):
"""Test run method using the Git backend"""
job = PercevalJob('arthur-job-1234567890', 'mytask', 'git', 'commit',
self.conn, 'items')
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
archive_args = {
'archive_path': self.tmp_path,
'fetch_from_archive': False
}
job.run(args, archive_args)
self.assertIsInstance(job.archive_manager, ArchiveManager)
result = job.result
self.assertIsInstance(job.result, JobResult)
self.assertEqual(result.job_id, 'arthur-job-1234567890')
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'git')
self.assertEqual(result.category, 'commit')
self.assertEqual(result.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c')
self.assertEqual(result.max_date, 1392185439.0)
self.assertEqual(result.nitems, 9)
self.assertEqual(result.offset, None)
self.assertEqual(result.nresumed, 0)
commits = self.conn.lrange('items', 0, -1)
commits = [pickle.loads(c) for c in commits]
commits = [commit['data']['commit'] for commit in commits]
expected = ['456a68ee1407a77f3e804a30dff245bb6c6b872f',
'51a3b654f252210572297f47597b31527c475fb8',
'ce8e0b86a1e9877f42fe9453ede418519115f367',
'589bb080f059834829a2a5955bebfd7c2baa110a',
'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb',
'c0d66f92a95e31c77be08dc9d0f11a16715d1885',
'7debcf8a2f57f86663809c58b5c07a398be7674c',
'87783129c3f00d2c81a3a8e585eb86a47e39891a',
'bc57a9209f096a130dcc5ba7089a8663f758a703']
self.assertEqual(commits, expected)
def test_run(self):
"""Test run method using the Git backend"""
job = PercevalJob('arthur-job-1234567890', 'mytask', 'git', self.conn,
'items')
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
job.run(args, fetch_from_cache=False)
result = job.result
self.assertIsInstance(job.result, JobResult)
self.assertEqual(result.job_id, 'arthur-job-1234567890')
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'git')
self.assertEqual(result.last_uuid,
'1375b60d3c23ac9b81da92523e4144abc4489d4c')
self.assertEqual(result.max_date, 1392185439.0)
self.assertEqual(result.nitems, 9)
self.assertEqual(result.offset, None)
self.assertEqual(result.nresumed, 0)
commits = self.conn.lrange('items', 0, -1)
commits = [pickle.loads(c) for c in commits]
commits = [commit['data']['commit'] for commit in commits]
expected = [
'456a68ee1407a77f3e804a30dff245bb6c6b872f',
'51a3b654f252210572297f47597b31527c475fb8',
'ce8e0b86a1e9877f42fe9453ede418519115f367',
'589bb080f059834829a2a5955bebfd7c2baa110a',
'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb',
'c0d66f92a95e31c77be08dc9d0f11a16715d1885',
'7debcf8a2f57f86663809c58b5c07a398be7674c',
'87783129c3f00d2c81a3a8e585eb86a47e39891a',
'bc57a9209f096a130dcc5ba7089a8663f758a703'
]
self.assertEqual(commits, expected)
def test_metadata(self):
"""Check if metadata parameters are correctly set"""
job = PercevalJob('1234567890', 8, 'mytask', 'git', 'commit',
self.conn, 'items')
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
archive_args = {
'archive_path': self.tmp_path,
'fetch_from_archive': False
}
job.run(args, archive_args)
items = self.conn.lrange('items', 0, -1)
items = [pickle.loads(item) for item in items]
for item in items:
self.assertEqual(item['arthur_version'], __version__)
self.assertEqual(item['job_id'], '1234567890')
def _test_fetch_from_archive(self):
"""Test run method fetching from the archive"""
http_requests = setup_mock_bugzilla_server()
expected = ['5a8a1e25dfda86b961b4146050883cbfc928f8ec',
'1fd4514e56f25a39ffd78eab19e77cfe4dfb7769',
'6a4cb244067c3cfa38f9f563e2ab3cd8ac21762f',
'7e033ed0110032ead6b918be43c1f3f88cd98fd7',
'f90d12b380ffdb47f2b6e96b321f08000181a9d6',
'4b166308f205121bc57704032acdc81b6c9bb8b1',
'b4009442d38f4241a4e22e3e61b7cd8ef5ced35c']
# First, we fetch the bugs from the server, storing them
# in a archive
args = {
'url': BUGZILLA_SERVER_URL,
'max_bugs': 5
}
archive_args = {
'archive_path': self.tmp_path,
'fetch_from_archive': False
}
job = PercevalJob('arthur-job-1234567890', 'mytask', 'bugzilla', 'issue',
self.conn, 'items')
job.run(args, archive_args=archive_args)
bugs = self.conn.lrange('items', 0, -1)
bugs = [pickle.loads(b) for b in bugs]
bugs = [bug['uuid'] for bug in bugs]
self.conn.ltrim('items', 1, 0)
result = job.result
self.assertIsInstance(job.archive_manager, ArchiveManager)
self.assertEqual(result.job_id, job.job_id)
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'bugzilla')
self.assertEqual(result.category, 'issue')
self.assertEqual(result.last_uuid, 'b4009442d38f4241a4e22e3e61b7cd8ef5ced35c')
self.assertEqual(result.max_date, 1439404330.0)
self.assertEqual(result.nitems, 7)
self.assertEqual(result.offset, None)
self.assertEqual(result.nresumed, 0)
self.assertEqual(len(http_requests), 13)
self.assertListEqual(bugs, expected)
# Now, we get the bugs from the archive.
# The contents should be the same and there won't be
# any new request to the server
job_archive = PercevalJob('arthur-job-1234567890-bis', 'mytask', 'bugzilla', 'issue',
self.conn, 'items')
archive_args['fetch_from_archive'] = True
job_archive.run(args, archive_args=archive_args)
archived_bugs = self.conn.lrange('items', 0, -1)
archived_bugs = [pickle.loads(b) for b in archived_bugs]
archived_bugs = [bug['uuid'] for bug in archived_bugs]
self.conn.ltrim('items', 1, 0)
result = job_archive.result
self.assertIsInstance(job.archive_manager, ArchiveManager)
self.assertEqual(result.job_id, job_archive.job_id)
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'bugzilla')
self.assertEqual(result.category, 'issue')
self.assertEqual(result.last_uuid, 'b4009442d38f4241a4e22e3e61b7cd8ef5ced35c')
self.assertEqual(result.max_date, 1439404330.0)
self.assertEqual(result.nitems, 7)
self.assertEqual(result.offset, None)
self.assertEqual(result.nresumed, 0)
self.assertEqual(len(http_requests), 13)
self.assertListEqual(archived_bugs, bugs)
def test_fetch_from_cache(self):
"""Test run method fetching from the cache"""
http_requests = setup_mock_bugzilla_server()
expected = ['5a8a1e25dfda86b961b4146050883cbfc928f8ec',
'1fd4514e56f25a39ffd78eab19e77cfe4dfb7769',
'6a4cb244067c3cfa38f9f563e2ab3cd8ac21762f',
'7e033ed0110032ead6b918be43c1f3f88cd98fd7',
'f90d12b380ffdb47f2b6e96b321f08000181a9d6',
'4b166308f205121bc57704032acdc81b6c9bb8b1',
'b4009442d38f4241a4e22e3e61b7cd8ef5ced35c']
# First, we fetch the bugs from the server, storing them
# in a cache
args = {
'url': BUGZILLA_SERVER_URL,
'max_bugs': 5
}
job = PercevalJob('arthur-job-1234567890', 'mytask', 'bugzilla',
self.conn, 'items')
job.initialize_cache(self.tmp_path)
job.run(args, fetch_from_cache=False)
bugs = self.conn.lrange('items', 0, -1)
bugs = [pickle.loads(b) for b in bugs]
bugs = [bug['uuid'] for bug in bugs]
self.conn.ltrim('items', 1, 0)
result = job.result
self.assertEqual(result.job_id, job.job_id)
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'bugzilla')
self.assertEqual(result.last_uuid, 'b4009442d38f4241a4e22e3e61b7cd8ef5ced35c')
self.assertEqual(result.max_date, 1439404330.0)
self.assertEqual(result.nitems, 7)
self.assertEqual(result.offset, None)
self.assertEqual(result.nresumed, 0)
self.assertEqual(len(http_requests), 13)
self.assertListEqual(bugs, expected)
# Now, we get the bugs from the cache.
# The contents should be the same and there won't be
# any new request to the server
job_cache = PercevalJob('arthur-job-1234567890-bis', 'mytask', 'bugzilla',
self.conn, 'items')
job_cache.initialize_cache(self.tmp_path)
job_cache.run(args, fetch_from_cache=True)
cached_bugs = self.conn.lrange('items', 0, -1)
cached_bugs = [pickle.loads(b) for b in cached_bugs]
cached_bugs = [bug['uuid'] for bug in cached_bugs]
self.conn.ltrim('items', 1, 0)
result = job_cache.result
self.assertEqual(result.job_id, job_cache.job_id)
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'bugzilla')
self.assertEqual(result.last_uuid, 'b4009442d38f4241a4e22e3e61b7cd8ef5ced35c')
self.assertEqual(result.max_date, 1439404330.0)
self.assertEqual(result.nitems, 7)
self.assertEqual(result.offset, None)
self.assertEqual(result.nresumed, 0)
self.assertEqual(len(http_requests), 13)
self.assertListEqual(cached_bugs, bugs)
def _test_fetch_from_archive(self):
"""Test run method fetching from the archive"""
http_requests = setup_mock_bugzilla_server()
expected = [
'5a8a1e25dfda86b961b4146050883cbfc928f8ec',
'1fd4514e56f25a39ffd78eab19e77cfe4dfb7769',
'6a4cb244067c3cfa38f9f563e2ab3cd8ac21762f',
'7e033ed0110032ead6b918be43c1f3f88cd98fd7',
'f90d12b380ffdb47f2b6e96b321f08000181a9d6',
'4b166308f205121bc57704032acdc81b6c9bb8b1',
'b4009442d38f4241a4e22e3e61b7cd8ef5ced35c'
]
# First, we fetch the bugs from the server, storing them
# in a archive
args = {'url': BUGZILLA_SERVER_URL, 'max_bugs': 5}
archive_args = {
'archive_path': self.tmp_path,
'fetch_from_archive': False
}
job = PercevalJob('1234567890', 8, 'mytask', 'bugzilla', 'issue',
self.conn, 'items')
job.run(args, archive_args=archive_args)
bugs = self.conn.lrange('items', 0, -1)
bugs = [pickle.loads(b) for b in bugs]
bugs = [bug['uuid'] for bug in bugs]
self.conn.ltrim('items', 1, 0)
result = job.result
self.assertIsInstance(job.archive_manager, ArchiveManager)
self.assertEqual(result.job_id, job.job_id)
self.assertEqual(result.job_number, job.job_number)
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'bugzilla')
self.assertEqual(result.category, 'issue')
self.assertEqual(result.summary.last_uuid,
'b4009442d38f4241a4e22e3e61b7cd8ef5ced35c')
self.assertEqual(
result.summary.max_updated_on,
datetime.datetime(2011, 12, 8, 17, 58, 37, tzinfo=UTC))
self.assertEqual(
result.summary.last_updated_on,
datetime.datetime(2011, 12, 8, 17, 58, 37, tzinfo=UTC))
self.assertEqual(result.summary.total, 7)
self.assertEqual(result.summary.offset, None)
self.assertEqual(len(http_requests), 13)
self.assertListEqual(bugs, expected)
# Now, we get the bugs from the archive.
# The contents should be the same and there won't be
# any new request to the server
job_archive = PercevalJob('1234567890-bis', 'mytask', 'bugzilla',
'issue', self.conn, 'items')
archive_args['fetch_from_archive'] = True
job_archive.run(args, archive_args=archive_args)
archived_bugs = self.conn.lrange('items', 0, -1)
archived_bugs = [pickle.loads(b) for b in archived_bugs]
archived_bugs = [bug['uuid'] for bug in archived_bugs]
self.conn.ltrim('items', 1, 0)
result = job_archive.result
self.assertIsInstance(job.archive_manager, ArchiveManager)
self.assertEqual(result.job_id, job_archive.job_id)
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'bugzilla')
self.assertEqual(result.category, 'issue')
self.assertEqual(result.summary.last_uuid,
'b4009442d38f4241a4e22e3e61b7cd8ef5ced35c')
self.assertEqual(
result.summary.max_updated_on,
datetime.datetime(2011, 12, 8, 17, 58, 37, tzinfo=UTC))
self.assertEqual(
result.summary.last_updated_on,
datetime.datetime(2011, 12, 8, 17, 58, 37, tzinfo=UTC))
self.assertEqual(result.summary.total, 7)
self.assertEqual(result.summary.offset, None)
self.assertEqual(len(http_requests), 13)
self.assertListEqual(archived_bugs, bugs)