def test_task_not_rescheduled_not_resume(self):
"""Check if tasks unable to resume are not rescheduled"""
handler = FailedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3)
task = self.registry.add('mytask', 'gerrit', 'review', {},
scheduling_cfg=scheduler_opts)
result = JobResult(0, 1, 'mytask', 'gerrit', 'review')
summary = Summary()
summary.fetched = 2
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
result.summary = summary
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.FAILED)
self.assertEqual(task.num_failures, 1)
def test_task_rescheduled_with_next_offset(self):
"""Check if tasks are rescheduled updating next_offset"""
handler = CompletedJobHandler(self.task_scheduler)
task = self.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)
summary.last_offset = 800
summary.max_offset = 1000
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# Both fields are updated to the max value received
# within the result
self.assertEqual(task.backend_args['next_from_date'],
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(task.backend_args['next_offset'], 1000)
def test_failed_task_rescheduled_with_next_offset(self):
"""Check if failed tasks are rescheduled updating next_offset"""
handler = FailedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 2
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)
summary.last_offset = 800
summary.max_offset = 1000
result.summary = summary
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# Both fields are updated to the max value received
# within the result
self.assertEqual(task.backend_args['next_from_date'],
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(task.backend_args['next_offset'], 1000)
self.assertEqual(task.num_failures, 1)
def test_failed_task_not_rescheduled_max_retries(self):
"""Check if the task is not re-scheduled when num failures reaches its limit"""
handler = FailedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
# Force to a pre-defined number of failures
task.num_failures = 2
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 2
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
result.summary = summary
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.FAILED)
self.assertEqual(task.num_failures, 3)
def test_task_rescheduling(self):
"""Check if the task related to the event is re-scheduled"""
handler = CompletedJobHandler(self.task_scheduler)
_ = self.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_task_rescheduled_no_new_items(self):
"""Check if tasks are rescheduled when no items where generated before"""
handler = CompletedJobHandler(self.task_scheduler)
_ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 0
summary.max_updated_on = None
summary.max_offset = None
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.task_scheduler.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# Both fields are not updated
self.assertNotIn('next_from_date', task.backend_args)
self.assertNotIn('next_offset', task.backend_args)
def test_failed_task_rescheduled_no_new_items(self):
"""Check if tasks are rescheduled when no items where generated before"""
handler = FailedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 0
summary.max_updated_on = None
summary.max_offset = None
result.summary = summary
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# Both fields are not updated
self.assertNotIn('next_from_date', task.backend_args)
self.assertNotIn('next_offset', task.backend_args)
def test_job_result_init(self):
result = JobResult('arthur-job-1234567890', 'mytask', 'mock_backend',
'category', 'ABCDEFGHIJK', 1344965413.0, 58)
self.assertEqual(result.job_id, 'arthur-job-1234567890')
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'mock_backend')
self.assertEqual(result.category, 'category')
self.assertEqual(result.last_uuid, 'ABCDEFGHIJK')
self.assertEqual(result.max_date, 1344965413.0)
self.assertEqual(result.nitems, 58)
self.assertEqual(result.offset, None)
self.assertEqual(result.nresumed, 0)
result = JobResult('arthur-job-1234567890',
'mytask',
'mock_backend',
'category',
'ABCDEFGHIJK',
1344965413.0,
58,
offset=128,
nresumed=10)
self.assertEqual(result.offset, 128)
self.assertEqual(result.nresumed, 10)
def test_handle_event(self):
"""Check if the event is handled correctly"""
handler = FailedJobHandler(self.task_scheduler)
task = self.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 2
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
result.summary = summary
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
# It won't be scheduled because max_retries is not set
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.FAILED)
def test_to_dict(self):
"""Test whether a JobResult object is converted to a dict"""
result = JobResult('arthur-job-1234567890', 'mytask',
'mock_backend', 'category')
expected = {
'job_id': 'arthur-job-1234567890',
'task_id': 'mytask'
}
d = result.to_dict()
self.assertEqual(d, expected)
def test_task_rescheduled_with_next_offset(self):
"""Check if tasks are rescheduled updating next_offset"""
handler = CompletedJobHandler(self.task_scheduler)
task = self.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0,
'mytask',
'git',
'commit',
'FFFFFFFF',
1392185439.0,
9,
offset=1000)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# Both fields are updated to the last value received
# within the result
self.assertEqual(task.backend_args['next_from_date'],
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(task.backend_args['next_offset'], 1000)
def test_to_dict(self):
"""Test whether a JobResult object is converted to a dict"""
result = JobResult('arthur-job-1234567890', 'mytask', 'mock_backend',
'category', 'ABCDEFGHIJK', 1344965413.0, 58)
expected = {
'job_id': 'arthur-job-1234567890',
'task_id': 'mytask',
'last_uuid': 'ABCDEFGHIJK',
'max_date': 1344965413.0,
'nitems': 58,
'offset': None,
}
d = result.to_dict()
self.assertEqual(d, expected)
def test_job_result_init(self):
result = JobResult('arthur-job-1234567890', 'mytask',
'mock_backend', 'category')
self.assertEqual(result.job_id, 'arthur-job-1234567890')
self.assertEqual(result.task_id, 'mytask')
self.assertEqual(result.backend, 'mock_backend')
self.assertEqual(result.category, 'category')
self.assertEqual(result.summary, None)
def test_task_not_rescheduled_age_limit(self):
"""Check if the task is not re-scheduled when age reaches its limit"""
handler = CompletedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0,
max_retries=0,
max_age=3)
task = self.task_scheduler.registry.add('mytask',
'git',
'commit', {},
scheduling_cfg=scheduler_opts)
# Force the age to its pre-defined limit
task.age = 3
self.task_scheduler.registry.update('mytask', task)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.task_scheduler.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_ignore_orphan_event(self):
"""Check if an orphan event is ignored"""
handler = CompletedJobHandler(self.task_scheduler)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, False)
def test_job_result_init(self):
result = JobResult('http://example.com/', 'mock_backend',
'ABCDEFGHIJK', 1344965413.0, 58)
self.assertEqual(result.origin, 'http://example.com/')
self.assertEqual(result.backend, 'mock_backend')
self.assertEqual(result.last_uuid, 'ABCDEFGHIJK')
self.assertEqual(result.max_date, 1344965413.0)
self.assertEqual(result.nitems, 58)
self.assertEqual(result.offset, None)
result = JobResult('http://example.com/',
'mock_backend',
'ABCDEFGHIJK',
1344965413.0,
58,
offset=128)
self.assertEqual(result.offset, 128)
def test_task_rescheduling_unlimited_age(self):
"""Check if the task is re-scheduled when unlimited age is set"""
handler = CompletedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0,
max_retries=0,
max_age=None)
task = self.registry.add('mytask',
'git',
'commit', {},
scheduling_cfg=scheduler_opts)
# Force the age to be large value
task.age = 1000000
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_ignore_orphan_event(self):
"""Check if an orphan event is ignored"""
handler = FailedJobHandler(self.task_scheduler)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
payload = {'error': "Error", 'result': result}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, False)
def test_task_rescheduling(self):
"""Check if the task related to the event is re-scheduled"""
handler = CompletedJobHandler(self.task_scheduler)
task = self.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_ignore_event_on_task_registry_error(self, mock_redis_get):
"""Check if an event is ignored when a TaskRegistryError is thrown"""
mock_redis_get.side_effect = RedisError
self.task_scheduler.registry.add('mytask', 'git', 'commit', {})
handler = CompletedJobHandler(self.task_scheduler)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, False)
def test_task_not_rescheduled_archive_task(self):
"""Check if archive tasks are not rescheduled"""
handler = CompletedJobHandler(self.task_scheduler)
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True)
task = self.registry.add('mytask', 'git', 'commit', {},
archiving_cfg=archiving_cfg)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_reset_num_failures(self):
"""Check if the number of failures is reset if the task is successful"""
handler = CompletedJobHandler(self.task_scheduler)
task = self.task_scheduler.registry.add('mytask', 'git', 'commit', {})
# Force to a pre-defined number of failures
task.num_failures = 2
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.task_scheduler.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.num_failures, 0)
def test_ignore_event_on_update_task_registry_error(
self, mock_redis_exists):
"""Check if an event is ignored when a TaskRegistryError is thrown"""
mock_redis_exists.side_effect = [False, True, RedisError]
self.task_scheduler.registry.add('mytask', 'git', 'commit', {})
handler = StartedJobHandler(self.task_scheduler)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
event = JobEvent(JobEventType.STARTED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, False)
def test_task_not_rescheduled_archive_task(self):
"""Check if archive tasks are not rescheduled"""
handler = CompletedJobHandler(self.task_scheduler)
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True)
task = self.registry.add('mytask', 'git', 'commit', {},
archiving_cfg=archiving_cfg)
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_not_rescheduled_age_limit(self):
"""Check if the task is not re-scheduled when age reaches its limit"""
handler = CompletedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
# Force the age to its pre-defined limit
task.age = 3
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_rescheduling_unlimited_age(self):
"""Check if the task is re-scheduled when unlimited age is set"""
handler = CompletedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
# Force the age to be large value
task.age = 1000000
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
