def test_task_not_rescheduled_not_resume(self): """Check if tasks unable to resume are not rescheduled""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'gerrit', 'review', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'gerrit', 'review') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) self.assertEqual(task.num_failures, 1)
def test_task_rescheduled_with_next_offset(self): """Check if tasks are rescheduled updating next_offset""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) summary.last_offset = 800 summary.max_offset = 1000 result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the max value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000)
def test_failed_task_rescheduled_with_next_offset(self): """Check if failed tasks are rescheduled updating next_offset""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) summary.last_offset = 800 summary.max_offset = 1000 result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the max value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000) self.assertEqual(task.num_failures, 1)
def test_failed_task_not_rescheduled_max_retries(self): """Check if the task is not re-scheduled when num failures reaches its limit""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force to a pre-defined number of failures task.num_failures = 2 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) self.assertEqual(task.num_failures, 3)
def test_task_rescheduling(self): """Check if the task related to the event is re-scheduled""" handler = CompletedJobHandler(self.task_scheduler) _ = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_task_rescheduled_no_new_items(self): """Check if tasks are rescheduled when no items where generated before""" handler = CompletedJobHandler(self.task_scheduler) _ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 0 summary.max_updated_on = None summary.max_offset = None result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are not updated self.assertNotIn('next_from_date', task.backend_args) self.assertNotIn('next_offset', task.backend_args)
def test_failed_task_rescheduled_no_new_items(self): """Check if tasks are rescheduled when no items where generated before""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 0 summary.max_updated_on = None summary.max_offset = None result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are not updated self.assertNotIn('next_from_date', task.backend_args) self.assertNotIn('next_offset', task.backend_args)
def test_job_result_init(self): result = JobResult('arthur-job-1234567890', 'mytask', 'mock_backend', 'category', 'ABCDEFGHIJK', 1344965413.0, 58) self.assertEqual(result.job_id, 'arthur-job-1234567890') self.assertEqual(result.task_id, 'mytask') self.assertEqual(result.backend, 'mock_backend') self.assertEqual(result.category, 'category') self.assertEqual(result.last_uuid, 'ABCDEFGHIJK') self.assertEqual(result.max_date, 1344965413.0) self.assertEqual(result.nitems, 58) self.assertEqual(result.offset, None) self.assertEqual(result.nresumed, 0) result = JobResult('arthur-job-1234567890', 'mytask', 'mock_backend', 'category', 'ABCDEFGHIJK', 1344965413.0, 58, offset=128, nresumed=10) self.assertEqual(result.offset, 128) self.assertEqual(result.nresumed, 10)
def test_handle_event(self): """Check if the event is handled correctly""" handler = FailedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) # It won't be scheduled because max_retries is not set handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED)
def test_to_dict(self): """Test whether a JobResult object is converted to a dict""" result = JobResult('arthur-job-1234567890', 'mytask', 'mock_backend', 'category') expected = { 'job_id': 'arthur-job-1234567890', 'task_id': 'mytask' } d = result.to_dict() self.assertEqual(d, expected)
def test_task_rescheduled_with_next_offset(self): """Check if tasks are rescheduled updating next_offset""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9, offset=1000) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the last value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000)
def test_to_dict(self): """Test whether a JobResult object is converted to a dict""" result = JobResult('arthur-job-1234567890', 'mytask', 'mock_backend', 'category', 'ABCDEFGHIJK', 1344965413.0, 58) expected = { 'job_id': 'arthur-job-1234567890', 'task_id': 'mytask', 'last_uuid': 'ABCDEFGHIJK', 'max_date': 1344965413.0, 'nitems': 58, 'offset': None, } d = result.to_dict() self.assertEqual(d, expected)
def test_job_result_init(self): result = JobResult('arthur-job-1234567890', 'mytask', 'mock_backend', 'category') self.assertEqual(result.job_id, 'arthur-job-1234567890') self.assertEqual(result.task_id, 'mytask') self.assertEqual(result.backend, 'mock_backend') self.assertEqual(result.category, 'category') self.assertEqual(result.summary, None)
def test_task_not_rescheduled_age_limit(self): """Check if the task is not re-scheduled when age reaches its limit""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to its pre-defined limit task.age = 3 self.task_scheduler.registry.update('mytask', task) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = CompletedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False)
def test_job_result_init(self): result = JobResult('http://example.com/', 'mock_backend', 'ABCDEFGHIJK', 1344965413.0, 58) self.assertEqual(result.origin, 'http://example.com/') self.assertEqual(result.backend, 'mock_backend') self.assertEqual(result.last_uuid, 'ABCDEFGHIJK') self.assertEqual(result.max_date, 1344965413.0) self.assertEqual(result.nitems, 58) self.assertEqual(result.offset, None) result = JobResult('http://example.com/', 'mock_backend', 'ABCDEFGHIJK', 1344965413.0, 58, offset=128) self.assertEqual(result.offset, 128)
def test_task_rescheduling_unlimited_age(self): """Check if the task is re-scheduled when unlimited age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be large value task.age = 1000000 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = FailedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') payload = {'error': "Error", 'result': result} event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, False)
def test_task_rescheduling(self): """Check if the task related to the event is re-scheduled""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_ignore_event_on_task_registry_error(self, mock_redis_get): """Check if an event is ignored when a TaskRegistryError is thrown""" mock_redis_get.side_effect = RedisError self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) handler = CompletedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False)
def test_task_not_rescheduled_archive_task(self): """Check if archive tasks are not rescheduled""" handler = CompletedJobHandler(self.task_scheduler) archiving_cfg = ArchivingTaskConfig('/tmp/archive', True) task = self.registry.add('mytask', 'git', 'commit', {}, archiving_cfg=archiving_cfg) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_reset_num_failures(self): """Check if the number of failures is reset if the task is successful""" handler = CompletedJobHandler(self.task_scheduler) task = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) # Force to a pre-defined number of failures task.num_failures = 2 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.num_failures, 0)
def test_ignore_event_on_update_task_registry_error( self, mock_redis_exists): """Check if an event is ignored when a TaskRegistryError is thrown""" mock_redis_exists.side_effect = [False, True, RedisError] self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) handler = StartedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.STARTED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False)
def test_task_not_rescheduled_archive_task(self): """Check if archive tasks are not rescheduled""" handler = CompletedJobHandler(self.task_scheduler) archiving_cfg = ArchivingTaskConfig('/tmp/archive', True) task = self.registry.add('mytask', 'git', 'commit', {}, archiving_cfg=archiving_cfg) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_not_rescheduled_age_limit(self): """Check if the task is not re-scheduled when age reaches its limit""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to its pre-defined limit task.age = 3 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_rescheduling_unlimited_age(self): """Check if the task is re-scheduled when unlimited age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be large value task.age = 1000000 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED)