def test_add_task(self): """Test to add tasks to the registry""" args = { 'from_date': '1970-01-01', 'component': 'test' } archive = { 'archive_path': '/tmp/archive', 'fetch_from_archive': False, 'archived_after': None, } sched = { 'stime': 10 } registry = TaskRegistry() before = datetime.now().timestamp() new_task = registry.add('mytask', 'mock_backend', 'category', args) tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertIsInstance(task, Task) self.assertEqual(task, new_task) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.category, 'category') self.assertEqual(task.backend, 'mock_backend') self.assertDictEqual(task.backend_args, args) self.assertDictEqual(task.archive_args, {}) self.assertDictEqual(task.sched_args, {}) self.assertGreater(task.created_on, before) before = datetime.now().timestamp() _ = registry.add('atask', 'mock_backend', 'category', args, archive_args=archive, sched_args=sched) tasks = registry.tasks self.assertEqual(len(tasks), 2) task0 = tasks[0] self.assertIsInstance(task0, Task) self.assertEqual(task0.task_id, 'atask') self.assertEqual(task0.backend, 'mock_backend') self.assertEqual(task0.category, 'category') self.assertDictEqual(task0.backend_args, args) self.assertDictEqual(task0.archive_args, archive) self.assertDictEqual(task0.sched_args, sched) self.assertGreater(task0.created_on, before) task1 = tasks[1] self.assertEqual(task1.task_id, 'mytask') self.assertGreater(task0.created_on, task1.created_on)
def test_get_task_not_found(self): """Check whether it raises an exception when a task is not found""" registry = TaskRegistry() with self.assertRaises(NotFoundError): registry.get('mytask') registry.add('newtask', 'mock_backend', 'mocked_category', {}) with self.assertRaises(NotFoundError): registry.get('mytask')
def test_remove_not_found(self): """Check whether it raises an exception when a task is not found""" registry = TaskRegistry() with self.assertRaises(NotFoundError): registry.remove('mytask') registry.add('task', 'mock_backend', None, '/tmp/example') with self.assertRaises(NotFoundError): registry.remove('mytask') self.assertEqual(len(registry.tasks), 1)
def test_schedule_task(self): """Jobs should be added and executed""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archive_args = {} sched_args = { 'delay': 0, 'max_retries_job': 0 } registry = TaskRegistry() task = registry.add('mytask', 'git', category, args, archive_args=archive_args, sched_args=sched_args) schlr = Scheduler(self.conn, registry, async_mode=False) job_id = schlr.schedule_task(task.task_id) schlr.schedule() job = schlr._scheduler._queues[Q_CREATION_JOBS].fetch_job(job_id) result = job.return_value self.assertEqual(result.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c') self.assertEqual(result.max_date, 1392185439.0) self.assertEqual(result.nitems, 9)
def test_add_existing_task(self): """Check if it raises an exception when an exisiting task is added again""" registry = TaskRegistry() registry.add('mytask', 'mock_backend', 'category', {}) with self.assertRaises(AlreadyExistsError): registry.add('mytask', 'new_backend', 'category', {}) # Only one tasks is on the registry tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.backend, 'mock_backend') self.assertEqual(task.category, 'category') self.assertDictEqual(task.backend_args, {})
def test_remove_task(self): """Test to remove a task from the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} registry = TaskRegistry() registry.add('mytask', 'mock_backend', 'mocked_category', args) registry.add('newtask', 'to_remove', 'mocked_category', None) registry.add('atask', 'test_backend', 'mocked_category', None) tasks = registry.tasks self.assertEqual(len(tasks), 3) registry.remove('newtask') tasks = registry.tasks self.assertEqual(len(tasks), 2) self.assertEqual(tasks[0].task_id, 'atask') self.assertEqual(tasks[1].task_id, 'mytask')
def test_remove_task(self): """Test to remove a task from the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} registry = TaskRegistry() registry.add('mytask', 'git', 'commit', args) registry.add('newtask', 'bugzilla', 'issue', None) registry.add('atask', 'git', 'commit', None) tasks = registry.tasks self.assertEqual(len(tasks), 3) registry.remove('newtask') tasks = registry.tasks self.assertEqual(len(tasks), 2) self.assertEqual(tasks[0].task_id, 'atask') self.assertEqual(tasks[1].task_id, 'mytask')
def test_get_task(self): """Test to get a task from the registry""" args = { 'from_date': '1970-01-01', 'component': 'test' } registry = TaskRegistry() registry.add('mytask', 'mock_backend', 'category', args) registry.add('newtask', 'to_remove', 'category', None) registry.add('atask', 'test_backend', 'category', None) task = registry.get('atask') self.assertIsInstance(task, Task) self.assertEqual(task.task_id, 'atask') self.assertEqual(task.category, 'category') self.assertEqual(task.backend, 'test_backend') self.assertEqual(task.backend_args, None)
def test_get_task(self): """Test to get a task from the registry""" args = { 'from_date': '1970-01-01', 'component': 'test' } registry = TaskRegistry() registry.add('mytask', 'mock_backend', 'category', args) registry.add('newtask', 'to_remove', 'category', None) registry.add('atask', 'test_backend', 'category', None) task = registry.get('atask') self.assertIsInstance(task, Task) self.assertEqual(task.task_id, 'atask') self.assertEqual(task.status, TaskStatus.NEW) self.assertEqual(task.category, 'category') self.assertEqual(task.backend, 'test_backend') self.assertEqual(task.backend_args, None)
def test_remove_task(self): """Test to remove a task from the registry""" args = { 'from_date': '1970-01-01', 'component': 'test' } registry = TaskRegistry() registry.add('mytask', 'mock_backend', 'mocked_category', args) registry.add('newtask', 'to_remove', 'mocked_category', None) registry.add('atask', 'test_backend', 'mocked_category', None) tasks = registry.tasks self.assertEqual(len(tasks), 3) registry.remove('newtask') tasks = registry.tasks self.assertEqual(len(tasks), 2) self.assertEqual(tasks[0].task_id, 'atask') self.assertEqual(tasks[1].task_id, 'mytask')
class TestCompletedJobHandler(TestBaseRQ): """Unit tests for CompletedJobHandler""" def setUp(self): super().setUp() self.registry = TaskRegistry() self.task_scheduler = _TaskScheduler(self.registry, self.conn, []) def test_initialization(self): """Check if the handler is correctly initialized""" handler = CompletedJobHandler(self.task_scheduler) self.assertEqual(handler.task_scheduler, self.task_scheduler) def test_task_rescheduling(self): """Check if the task related to the event is re-scheduled""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_rescheduling_age(self): """Check if the task is re-scheduled when maximum age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be lower than its limit task.age = 2 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_rescheduling_unlimited_age(self): """Check if the task is re-scheduled when unlimited age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be large value task.age = 1000000 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_not_rescheduled_age_limit(self): """Check if the task is not re-scheduled when age reaches its limit""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to its pre-defined limit task.age = 3 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.COMPLETED) def test_task_not_rescheduled_archive_task(self): """Check if archive tasks are not rescheduled""" handler = CompletedJobHandler(self.task_scheduler) archiving_cfg = ArchivingTaskConfig('/tmp/archive', True) task = self.registry.add('mytask', 'git', 'commit', {}, archiving_cfg=archiving_cfg) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.COMPLETED) def test_task_rescheduled_with_next_from_date(self): """Check if tasks are rescheduled updating next_from_date""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # The field is updated to the last date received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) def test_task_rescheduled_with_next_offset(self): """Check if tasks are rescheduled updating next_offset""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9, offset=1000) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the last value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000) def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = CompletedJobHandler(self.task_scheduler) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False)
def test_add_task(self): """Test to add tasks to the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} archive = ArchivingTaskConfig('/tmp/archive', False, archived_after=None) sched = SchedulingTaskConfig(delay=10, max_retries=2) registry = TaskRegistry(self.conn) before = datetime.datetime.now().timestamp() _ = registry.add('mytask', 'git', 'commit', args) tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertIsInstance(task, Task) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.status, TaskStatus.NEW) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) self.assertEqual(task.category, 'commit') self.assertEqual(task.backend, 'git') self.assertDictEqual(task.backend_args, args) self.assertEqual(task.archiving_cfg, None) self.assertEqual(task.scheduling_cfg, None) self.assertGreater(task.created_on, before) before = datetime.datetime.now().timestamp() _ = registry.add('atask', 'git', 'commit', args, archiving_cfg=archive, scheduling_cfg=sched) tasks = registry.tasks self.assertEqual(len(tasks), 2) task0 = tasks[0] self.assertIsInstance(task0, Task) self.assertEqual(task0.task_id, 'atask') self.assertEqual(task0.status, TaskStatus.NEW) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) self.assertEqual(task0.backend, 'git') self.assertEqual(task0.category, 'commit') self.assertDictEqual(task0.backend_args, args) self.assertEqual(task0.archiving_cfg.archive_path, archive.archive_path) self.assertEqual(task0.archiving_cfg.archived_after, archive.archived_after) self.assertEqual(task0.archiving_cfg.fetch_from_archive, archive.fetch_from_archive) self.assertEqual(task0.scheduling_cfg.delay, sched.delay) self.assertEqual(task0.scheduling_cfg.max_retries, sched.max_retries) self.assertGreater(task0.created_on, before) task1 = tasks[1] self.assertEqual(task1.task_id, 'mytask') self.assertGreater(task0.created_on, task1.created_on)
class TestFailedJobHandler(TestBaseRQ): """Unit tests for CompletedJobHandler""" def setUp(self): super().setUp() self.registry = TaskRegistry() self.task_scheduler = _TaskScheduler(self.registry, self.conn, []) def test_initialization(self): """Check if the handler is correctly initialized""" handler = FailedJobHandler(self.task_scheduler) self.assertEqual(handler.task_scheduler, self.task_scheduler) def test_handle_event(self): """Check if the event is handled correctly""" handler = FailedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) # It won't be scheduled because max_retries is not set handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) def test_task_not_rescheduled_not_resume(self): """Check if tasks unable to resume are not rescheduled""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'gerrit', 'review', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'gerrit', 'review') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) self.assertEqual(task.num_failures, 1) def test_failed_task_not_rescheduled_max_retries(self): """Check if the task is not re-scheduled when num failures reaches its limit""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force to a pre-defined number of failures task.num_failures = 2 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) self.assertEqual(task.num_failures, 3) def test_failed_task_rescheduled_with_next_from_date(self): """Check if failed tasks are rescheduled updating next_from_date""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # The field is updated to the max date received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.num_failures, 1) def test_failed_task_rescheduled_with_next_offset(self): """Check if failed tasks are rescheduled updating next_offset""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) summary.last_offset = 800 summary.max_offset = 1000 result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the max value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000) self.assertEqual(task.num_failures, 1) def test_failed_task_rescheduled_no_new_items(self): """Check if tasks are rescheduled when no items where generated before""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 0 summary.max_updated_on = None summary.max_offset = None result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are not updated self.assertNotIn('next_from_date', task.backend_args) self.assertNotIn('next_offset', task.backend_args) def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = FailedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, False)
class TestCompletedJobHandler(TestBaseRQ): """Unit tests for CompletedJobHandler""" def setUp(self): super().setUp() self.registry = TaskRegistry(self.conn) self.task_scheduler = _TaskScheduler(self.registry, self.conn, []) def test_initialization(self): """Check if the handler is correctly initialized""" handler = CompletedJobHandler(self.task_scheduler) self.assertEqual(handler.task_scheduler, self.task_scheduler) def test_task_rescheduling(self): """Check if the task related to the event is re-scheduled""" handler = CompletedJobHandler(self.task_scheduler) _ = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_rescheduling_age(self): """Check if the task is re-scheduled when maximum age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be lower than its limit task.age = 2 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_rescheduling_unlimited_age(self): """Check if the task is re-scheduled when unlimited age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be large value task.age = 1000000 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_not_rescheduled_age_limit(self): """Check if the task is not re-scheduled when age reaches its limit""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to its pre-defined limit task.age = 3 self.task_scheduler.registry.update('mytask', task) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.COMPLETED) def test_task_not_rescheduled_archive_task(self): """Check if archive tasks are not rescheduled""" handler = CompletedJobHandler(self.task_scheduler) archiving_cfg = ArchivingTaskConfig('/tmp/archive', True) _ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}, archiving_cfg=archiving_cfg) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.COMPLETED) def test_task_rescheduled_with_next_from_date(self): """Check if tasks are rescheduled updating next_from_date""" handler = CompletedJobHandler(self.task_scheduler) _ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) # The field is updated to the max date received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) def test_task_rescheduled_with_next_offset(self): """Check if tasks are rescheduled updating next_offset""" handler = CompletedJobHandler(self.task_scheduler) _ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) summary.last_offset = 800 summary.max_offset = 1000 result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the max value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000) def test_task_rescheduled_no_new_items(self): """Check if tasks are rescheduled when no items where generated before""" handler = CompletedJobHandler(self.task_scheduler) _ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 0 summary.max_updated_on = None summary.max_offset = None result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are not updated self.assertNotIn('next_from_date', task.backend_args) self.assertNotIn('next_offset', task.backend_args) def test_task_reset_num_failures(self): """Check if the number of failures is reset if the task is successful""" handler = CompletedJobHandler(self.task_scheduler) task = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) # Force to a pre-defined number of failures task.num_failures = 2 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.num_failures, 0) def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = CompletedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False) @unittest.mock.patch('redis.StrictRedis.get') def test_ignore_event_on_task_registry_error(self, mock_redis_get): """Check if an event is ignored when a TaskRegistryError is thrown""" mock_redis_get.side_effect = RedisError self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) handler = CompletedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False)
class TestStartedJobHandler(TestBaseRQ): """Unit tests for StartedJobHandler""" def setUp(self): super().setUp() self.registry = TaskRegistry(self.conn) self.task_scheduler = _TaskScheduler(self.registry, self.conn, []) def test_initialization(self): """Check if the handler is correctly initialized""" handler = StartedJobHandler(self.task_scheduler) self.assertEqual(handler.task_scheduler, self.task_scheduler) def test_task_status(self): """Check if the handler changes the task status to running""" handler = StartedJobHandler(self.task_scheduler) _ = self.registry.add('mytask', 'git', 'commit', {}) event = JobEvent(JobEventType.STARTED, 0, 'mytask', None) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.RUNNING) def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = StartedJobHandler(self.task_scheduler) event = JobEvent(JobEventType.STARTED, 0, 'mytask', None) handled = handler(event) self.assertEqual(handled, False) @unittest.mock.patch('redis.StrictRedis.get') def test_ignore_event_on_get_task_registry_error(self, mock_redis_get): """Check if an event is ignored when a TaskRegistryError is thrown""" mock_redis_get.side_effect = RedisError self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) handler = StartedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.STARTED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False) @unittest.mock.patch('redis.StrictRedis.exists') def test_ignore_event_on_update_task_registry_error( self, mock_redis_exists): """Check if an event is ignored when a TaskRegistryError is thrown""" mock_redis_exists.side_effect = [False, True, RedisError] self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) handler = StartedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.STARTED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False)