def _publish_job_events(self, pubsub_channel, pubsub_override=False): """Generic job events tests""" pubsub = self.conn.pubsub() pubsub.subscribe(pubsub_channel) q = rq.Queue('foo') w = MockArthurWorker([q]) if pubsub_override: w.pubsub_channel = pubsub_channel job_a = q.enqueue(mock_sum, task_id=0, a=2, b=3) job_b = q.enqueue(mock_failure, task_id=1) status = w.work(burst=True) self.assertEqual(status, True) # Ignore the first messages because it is a # subscription notification _ = pubsub.get_message() # STARTED event for job 'a' msg_a = pubsub.get_message() event = JobEvent.deserialize(msg_a['data']) self.assertEqual(event.job_id, job_a.id) self.assertEqual(event.task_id, 0) self.assertEqual(event.type, JobEventType.STARTED) self.assertEqual(event.payload, None) # COMPLETED event for job 'a' msg_a = pubsub.get_message() event = JobEvent.deserialize(msg_a['data']) self.assertEqual(job_a.result, 5) self.assertEqual(event.job_id, job_a.id) self.assertEqual(event.task_id, 0) self.assertEqual(event.type, JobEventType.COMPLETED) self.assertEqual(event.payload, 5) # STARTED event for job 'b' msg_b = pubsub.get_message() event = JobEvent.deserialize(msg_b['data']) self.assertEqual(event.job_id, job_b.id) self.assertEqual(event.task_id, 1) self.assertEqual(event.type, JobEventType.STARTED) self.assertEqual(event.payload, None) # FAILURE event for job 'b' msg_b = pubsub.get_message() event = JobEvent.deserialize(msg_b['data']) self.assertEqual(job_b.result, None) self.assertEqual(event.job_id, job_b.id) self.assertEqual(event.task_id, 1) self.assertEqual(event.type, JobEventType.FAILURE) self.assertRegex(event.payload['error'], "Traceback") self.assertEqual(event.payload['result'], None)
def test_unique_identifier(self): """Test if different identifiers create unique identifiers""" event_a = JobEvent(JobEventType.COMPLETED, '1', None) event_b = JobEvent(JobEventType.COMPLETED, '2', None) event_c = JobEvent(JobEventType.FAILURE, '3', None) self.assertNotEqual(event_a.uuid, None) self.assertNotEqual(event_a.uuid, event_b.uuid) self.assertNotEqual(event_b.uuid, event_c.uuid) self.assertNotEqual(event_c.uuid, event_a.uuid)
def test_handle_event(self): """Check if the event is handled correctly""" handler = FailedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) # It won't be scheduled because max_retries is not set handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED)
def test_task_rescheduled_with_next_offset(self): """Check if tasks are rescheduled updating next_offset""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) summary.last_offset = 800 summary.max_offset = 1000 result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the max value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000)
def test_failed_task_rescheduled_with_next_offset(self): """Check if failed tasks are rescheduled updating next_offset""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) summary.last_offset = 800 summary.max_offset = 1000 result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the max value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000) self.assertEqual(task.num_failures, 1)
def test_failed_task_rescheduled_no_new_items(self): """Check if tasks are rescheduled when no items where generated before""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 0 summary.max_updated_on = None summary.max_offset = None result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are not updated self.assertNotIn('next_from_date', task.backend_args) self.assertNotIn('next_offset', task.backend_args)
def test_task_rescheduled_no_new_items(self): """Check if tasks are rescheduled when no items where generated before""" handler = CompletedJobHandler(self.task_scheduler) _ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 0 summary.max_updated_on = None summary.max_offset = None result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are not updated self.assertNotIn('next_from_date', task.backend_args) self.assertNotIn('next_offset', task.backend_args)
def test_task_rescheduling(self): """Check if the task related to the event is re-scheduled""" handler = CompletedJobHandler(self.task_scheduler) _ = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_task_rescheduled_with_next_offset(self): """Check if tasks are rescheduled updating next_offset""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9, offset=1000) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the last value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000)
def test_failed_task_not_rescheduled_max_retries(self): """Check if the task is not re-scheduled when num failures reaches its limit""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force to a pre-defined number of failures task.num_failures = 2 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) self.assertEqual(task.num_failures, 3)
def test_task_not_rescheduled_not_resume(self): """Check if tasks unable to resume are not rescheduled""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'gerrit', 'review', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'gerrit', 'review') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) self.assertEqual(task.num_failures, 1)
def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = StartedJobHandler(self.task_scheduler) event = JobEvent(JobEventType.STARTED, 0, 'mytask', None) handled = handler(event) self.assertEqual(handled, False)
def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = CompletedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False)
def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = FailedJobHandler(self.task_scheduler) payload = {'error': "Error"} event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, False)
def test_serializer(self): """Test if an event is properly serialized and deserialized""" result = MockJobResult(10, 'mockbackend') event_a = JobEvent(JobEventType.COMPLETED, '1', result) data = event_a.serialize() event = JobEvent.deserialize(data) self.assertIsInstance(event, JobEvent) self.assertEqual(event.uuid, event_a.uuid) self.assertEqual(event.timestamp, event_a.timestamp) self.assertEqual(event.type, event_a.type) self.assertEqual(event.job_id, event_a.job_id) payload = event.payload self.assertIsInstance(payload, MockJobResult) self.assertEqual(payload.result, result.result) self.assertEqual(payload.category, result.category) self.assertEqual(payload.timestamp, result.timestamp)
def test_task_status(self): """Check if the handler changes the task status to running""" handler = StartedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) event = JobEvent(JobEventType.STARTED, 0, 'mytask', None) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.RUNNING)
def test_handle_event(self): """Check if the event is handled correctly""" handler = FailedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) payload = {'error': "Error"} event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED)
def test_task_rescheduling(self): """Check if the task related to the event is re-scheduled""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_ignore_event_on_task_registry_error(self, mock_redis_get): """Check if an event is ignored when a TaskRegistryError is thrown""" mock_redis_get.side_effect = RedisError self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) handler = CompletedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False)
def test_initialization(self): """Test if the instance is correctly initialized""" dt_before = datetime_utcnow() event = JobEvent(JobEventType.COMPLETED, '1', None) dt_after = datetime_utcnow() self.assertEqual(event.type, JobEventType.COMPLETED) self.assertEqual(event.job_id, '1') self.assertEqual(event.payload, None) self.assertGreater(event.timestamp, dt_before) self.assertLess(event.timestamp, dt_after) dt_before = datetime_utcnow() event = JobEvent(JobEventType.FAILURE, '2', "Error") dt_after = datetime_utcnow() self.assertEqual(event.type, JobEventType.FAILURE) self.assertEqual(event.job_id, '2') self.assertEqual(event.payload, "Error") self.assertGreater(event.timestamp, dt_before) self.assertLess(event.timestamp, dt_after)
def test_ignore_event_on_update_task_registry_error( self, mock_redis_exists): """Check if an event is ignored when a TaskRegistryError is thrown""" mock_redis_exists.side_effect = [False, True, RedisError] self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) handler = StartedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.STARTED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False)
def _publish_finished_job_status(self, pubsub_channel, skip_pubsub_override=False): """Test whether the worker publishes the status of a finished job""" pubsub = self.conn.pubsub() pubsub.subscribe(pubsub_channel) q = rq.Queue('foo') w = MockArthurWorker([q]) if not skip_pubsub_override: w.pubsub_channel = pubsub_channel job_a = q.enqueue(mock_sum, a=2, b=3) job_b = q.enqueue(mock_failure) status = w.work(burst=True) self.assertEqual(status, True) # Ignore the first message because it is a # subscription notification _ = pubsub.get_message() msg_a = pubsub.get_message() msg_b = pubsub.get_message() event = JobEvent.deserialize(msg_a['data']) self.assertEqual(job_a.result, 5) self.assertEqual(event.job_id, job_a.id) self.assertEqual(event.type, JobEventType.COMPLETED) self.assertEqual(event.payload, 5) event = JobEvent.deserialize(msg_b['data']) self.assertEqual(job_b.result, None) self.assertEqual(event.job_id, job_b.id) self.assertEqual(event.type, JobEventType.FAILURE) self.assertRegex(event.payload, "Traceback")
def test_task_not_rescheduled_archive_task(self): """Check if archive tasks are not rescheduled""" handler = CompletedJobHandler(self.task_scheduler) archiving_cfg = ArchivingTaskConfig('/tmp/archive', True) task = self.registry.add('mytask', 'git', 'commit', {}, archiving_cfg=archiving_cfg) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_listen_after_unsubscribing(self): """Check if only handles subscribed messages after unsubscribing. Due to listening on a channel is a blocking call (because Redis lib implementation), this test mocks the connection with it to simulate events reception. """ class TrackedEvents: def __init__(self): self.handled = 0 self.ok = 0 self.failures = 0 global tracked_events tracked_events = TrackedEvents() def handle_successful_job(job): tracked_events.handled += 1 tracked_events.ok += 1 def handle_failed_job(job): tracked_events.handled += 1 tracked_events.failures += 1 events = [ JobEvent(JobEventType.COMPLETED, 1, 'A', MockJobResult(20, 'A')), JobEvent(JobEventType.FAILURE, 2, 'B', 'ERROR'), JobEvent(JobEventType.FAILURE, 3, 'C', 'ERROR'), JobEvent(JobEventType.FAILURE, 4, 'D', 'ERROR'), JobEvent(JobEventType.COMPLETED, 5, 'E', MockJobResult(22, 'B')), JobEvent(JobEventType.FAILURE, 6, 'F', 'ERROR'), JobEvent(JobEventType.UNDEFINED, 7, 'G', 'OK'), ] conn = MockRedisPubSubConnection(events) listener = JobEventsListener(conn) # COMPLETED and FAILURE events are handled on the first call # to run(); on the next call only FAILURE events are handled, # tracked_events keeps track of both calls listener.subscribe(JobEventType.COMPLETED, handle_successful_job) listener.subscribe(JobEventType.FAILURE, handle_failed_job) listener.run() listener.unsubscribe(JobEventType.COMPLETED) listener.run() self.assertEqual(tracked_events.handled, 10) self.assertEqual(tracked_events.ok, 2) self.assertEqual(tracked_events.failures, 8)
def test_task_not_rescheduled_age_limit(self): """Check if the task is not re-scheduled when age reaches its limit""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to its pre-defined limit task.age = 3 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_rescheduling_unlimited_age(self): """Check if the task is re-scheduled when unlimited age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be large value task.age = 1000000 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_listen(self): """Check if if listens and handles event messages. Due to listening on a channel is a blocking call (because Redis lib implementation), this test mocks the connection with it to simulate events reception. """ class TrackedEvents: def __init__(self): self.handled = 0 self.ok = 0 self.failures = 0 global tracked_events tracked_events = TrackedEvents() def handle_successful_job(job): tracked_events.handled += 1 tracked_events.ok += 1 def handle_failed_job(job): tracked_events.handled += 1 tracked_events.failures += 1 events = [ JobEvent(JobEventType.COMPLETED, 1, 'A', MockJobResult(20, 'A')), JobEvent(JobEventType.FAILURE, 2, 'B', 'ERROR'), JobEvent(JobEventType.FAILURE, 3, 'C', 'ERROR'), JobEvent(JobEventType.FAILURE, 4, 'D', 'ERROR'), JobEvent(JobEventType.COMPLETED, 5, 'E', MockJobResult(22, 'B')), JobEvent(JobEventType.FAILURE, 6, 'F', 'ERROR'), JobEvent(JobEventType.UNDEFINED, 7, 'G', 'OK'), ] conn = MockRedisPubSubConnection(events) listener = JobEventsListener(conn) listener.subscribe(JobEventType.COMPLETED, handle_successful_job) listener.subscribe(JobEventType.FAILURE, handle_failed_job) listener.run() # UNDEFINED event was not handled self.assertEqual(tracked_events.handled, 6) self.assertEqual(tracked_events.ok, 2) self.assertEqual(tracked_events.failures, 4)
def test_task_not_rescheduled_age_limit(self): """Check if the task is not re-scheduled when age reaches its limit""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to its pre-defined limit task.age = 3 self.task_scheduler.registry.update('mytask', task) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_rescheduling_unlimited_age(self): """Check if the task is re-scheduled when unlimited age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be large value task.age = 1000000 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_task_not_rescheduled_archive_task(self): """Check if archive tasks are not rescheduled""" handler = CompletedJobHandler(self.task_scheduler) archiving_cfg = ArchivingTaskConfig('/tmp/archive', True) task = self.registry.add('mytask', 'git', 'commit', {}, archiving_cfg=archiving_cfg) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.COMPLETED)