def test_task_jobs_is_updated(self): """Test if the list of job ids for a task is updated""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0) registry = TaskRegistry() task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) schlr.schedule() self.assertEqual(task.age, 1) self.assertEqual(len(task.jobs), 1)
def test_schedule_task(self): """Jobs should be added and executed""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archive_args = {} sched_args = { 'delay': 0, 'max_retries_job': 0 } registry = TaskRegistry() task = registry.add('mytask', 'git', category, args, archive_args=archive_args, sched_args=sched_args) schlr = Scheduler(self.conn, registry, async_mode=False) job_id = schlr.schedule_task(task.task_id) schlr.schedule() job = schlr._scheduler._queues[Q_CREATION_JOBS].fetch_job(job_id) result = job.return_value self.assertEqual(result.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c') self.assertEqual(result.max_date, 1392185439.0) self.assertEqual(result.nitems, 9)
def test_add_task(self): """Test to add tasks to the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} archive = ArchivingTaskConfig('/tmp/archive', False, archived_after=None) sched = SchedulingTaskConfig(delay=10, max_retries=2) registry = TaskRegistry() before = datetime.datetime.now().timestamp() new_task = registry.add('mytask', 'git', 'commit', args) tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertIsInstance(task, Task) self.assertEqual(task, new_task) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.status, TaskStatus.NEW) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) self.assertEqual(task.category, 'commit') self.assertEqual(task.backend, 'git') self.assertDictEqual(task.backend_args, args) self.assertEqual(task.archiving_cfg, None) self.assertEqual(task.scheduling_cfg, None) self.assertGreater(task.created_on, before) before = datetime.datetime.now().timestamp() _ = registry.add('atask', 'git', 'commit', args, archiving_cfg=archive, scheduling_cfg=sched) tasks = registry.tasks self.assertEqual(len(tasks), 2) task0 = tasks[0] self.assertIsInstance(task0, Task) self.assertEqual(task0.task_id, 'atask') self.assertEqual(task0.status, TaskStatus.NEW) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) self.assertEqual(task0.backend, 'git') self.assertEqual(task0.category, 'commit') self.assertDictEqual(task0.backend_args, args) self.assertEqual(task0.archiving_cfg, archive) self.assertEqual(task0.scheduling_cfg, sched) self.assertGreater(task0.created_on, before) task1 = tasks[1] self.assertEqual(task1.task_id, 'mytask') self.assertGreater(task0.created_on, task1.created_on)
def test_update_task_not_found(self): """Check whether it raises an exception when a task is not found""" registry = TaskRegistry(self.conn) with self.assertLogs(logger, level='WARNING') as cm: registry.update('mytask', None) self.assertEqual( cm.output[0], 'WARNING:arthur.tasks:Task mytask not found, adding it')
def test_update_task_error(self, mock_redis_cmd): """Test whether the lock is released in case of a Redis error and a TaskRegistryError is thrown""" mock_redis_cmd.side_effect = RedisError registry = TaskRegistry(self.conn) self.assertGreater(registry._rwlock._access_mutex._value, 0) with self.assertRaises(TaskRegistryError): registry.update('mytask', None) self.assertGreater(registry._rwlock._access_mutex._value, 0)
def test_tasks_error(self, mock_redis_set): """Test whether the lock is released in case of a Redis error and a TaskRegistryError is thrown""" mock_redis_set.side_effect = RedisError registry = TaskRegistry(self.conn) registry.add('mytask', 'git', 'commit', {}) self.assertGreater(registry._rwlock._readers_mutex._value, 0) with self.assertRaises(TaskRegistryError): _ = registry.tasks self.assertGreater(registry._rwlock._readers_mutex._value, 0)
def test_add_task(self): """Test to add tasks to the registry""" args = { 'from_date': '1970-01-01', 'component': 'test' } archive = { 'archive_path': '/tmp/archive', 'fetch_from_archive': False, 'archived_after': None, } sched = { 'stime': 10 } registry = TaskRegistry() before = datetime.now().timestamp() new_task = registry.add('mytask', 'mock_backend', 'category', args) tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertIsInstance(task, Task) self.assertEqual(task, new_task) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.category, 'category') self.assertEqual(task.backend, 'mock_backend') self.assertDictEqual(task.backend_args, args) self.assertDictEqual(task.archive_args, {}) self.assertDictEqual(task.sched_args, {}) self.assertGreater(task.created_on, before) before = datetime.now().timestamp() _ = registry.add('atask', 'mock_backend', 'category', args, archive_args=archive, sched_args=sched) tasks = registry.tasks self.assertEqual(len(tasks), 2) task0 = tasks[0] self.assertIsInstance(task0, Task) self.assertEqual(task0.task_id, 'atask') self.assertEqual(task0.backend, 'mock_backend') self.assertEqual(task0.category, 'category') self.assertDictEqual(task0.backend_args, args) self.assertDictEqual(task0.archive_args, archive) self.assertDictEqual(task0.sched_args, sched) self.assertGreater(task0.created_on, before) task1 = tasks[1] self.assertEqual(task1.task_id, 'mytask') self.assertGreater(task0.created_on, task1.created_on)
def test_add_task(self): """Test to add tasks to the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} archive = { 'archive_path': '/tmp/archive', 'fetch_from_archive': False, 'archived_after': None, } sched = {'stime': 10} registry = TaskRegistry() before = datetime.now().timestamp() new_task = registry.add('mytask', 'mock_backend', 'category', args) tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertIsInstance(task, Task) self.assertEqual(task, new_task) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.category, 'category') self.assertEqual(task.backend, 'mock_backend') self.assertDictEqual(task.backend_args, args) self.assertDictEqual(task.archive_args, {}) self.assertDictEqual(task.sched_args, {}) self.assertGreater(task.created_on, before) before = datetime.now().timestamp() _ = registry.add('atask', 'mock_backend', 'category', args, archive_args=archive, sched_args=sched) tasks = registry.tasks self.assertEqual(len(tasks), 2) task0 = tasks[0] self.assertIsInstance(task0, Task) self.assertEqual(task0.task_id, 'atask') self.assertEqual(task0.backend, 'mock_backend') self.assertEqual(task0.category, 'category') self.assertDictEqual(task0.backend_args, args) self.assertDictEqual(task0.archive_args, archive) self.assertDictEqual(task0.sched_args, sched) self.assertGreater(task0.created_on, before) task1 = tasks[1] self.assertEqual(task1.task_id, 'mytask') self.assertGreater(task0.created_on, task1.created_on)
def test_tasks(self): """Test to list tasks in the registry""" registry = TaskRegistry(self.conn) expected = [] for i in range(0, 20): task_id = 'mytask{}'.format(i) expected.append(task_id) registry.add(task_id, 'git', 'commit', {}) tasks = registry.tasks self.assertEqual(len(tasks), 20) for t in tasks: self.assertIn(t.task_id, expected)
class TestStartedJobHandler(TestBaseRQ): """Unit tests for StartedJobHandler""" def setUp(self): super().setUp() self.registry = TaskRegistry() self.task_scheduler = _TaskScheduler(self.registry, self.conn, []) def test_initialization(self): """Check if the handler is correctly initialized""" handler = StartedJobHandler(self.task_scheduler) self.assertEqual(handler.task_scheduler, self.task_scheduler) def test_task_status(self): """Check if the handler changes the task status to running""" handler = StartedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) event = JobEvent(JobEventType.STARTED, 0, 'mytask', None) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.RUNNING) def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = StartedJobHandler(self.task_scheduler) event = JobEvent(JobEventType.STARTED, 0, 'mytask', None) handled = handler(event) self.assertEqual(handled, False)
def test_not_found_task(self): """Raises an error when the task to schedule does not exist""" registry = TaskRegistry() schlr = Scheduler(self.conn, registry, async_mode=False) self.assertRaises(NotFoundError, schlr.schedule_task, 'mytask')
def test_empty_registry(self): """Check empty registry on init""" registry = TaskRegistry() tasks = registry.tasks self.assertListEqual(tasks, [])
def test_add_existing_task(self): """Check if it raises an exception when an exisiting task is added again""" registry = TaskRegistry() registry.add('mytask', 'mock_backend', 'category', {}) with self.assertRaises(AlreadyExistsError): registry.add('mytask', 'new_backend', 'category', {}) # Only one tasks is on the registry tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.backend, 'mock_backend') self.assertEqual(task.category, 'category') self.assertDictEqual(task.backend_args, {})
def test_update_task(self): """Test to update tasks in the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} registry = TaskRegistry(self.conn) before = datetime.datetime.now().timestamp() _ = registry.add('mytask', 'git', 'commit', args) tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertIsInstance(task, Task) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.status, TaskStatus.NEW) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) self.assertEqual(task.category, 'commit') self.assertEqual(task.backend, 'git') self.assertDictEqual(task.backend_args, args) self.assertEqual(task.archiving_cfg, None) self.assertEqual(task.scheduling_cfg, None) self.assertGreater(task.created_on, before) task.status = TaskStatus.COMPLETED task.age = 1 registry.update('mytask', task) tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertIsInstance(task, Task) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.status, TaskStatus.COMPLETED) self.assertEqual(task.age, 1) self.assertListEqual(task.jobs, []) self.assertEqual(task.category, 'commit') self.assertEqual(task.backend, 'git') self.assertDictEqual(task.backend_args, args) self.assertEqual(task.archiving_cfg, None) self.assertEqual(task.scheduling_cfg, None) self.assertGreater(task.created_on, before)
def test_get_task(self): """Test to get a task from the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} registry = TaskRegistry() registry.add('mytask', 'mock_backend', args) registry.add('newtask', 'to_remove', None) registry.add('atask', 'test_backend', None) task = registry.get('atask') self.assertIsInstance(task, Task) self.assertEqual(task.task_id, 'atask') self.assertEqual(task.backend, 'test_backend') self.assertEqual(task.backend_args, None)
def test_schedule_task_user_queue(self): """Task should be added and executed in the user's queue""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, queue='myqueue') registry = TaskRegistry(self.conn) task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) task = registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) schlr.schedule() task = registry.get('mytask') self.assertEqual(task.age, 1) job = schlr._scheduler._queues['myqueue'].fetch_job(task.jobs[0].id) result = job.return_value self.assertEqual(result.task_id, task.task_id) self.assertEqual(result.job_number, 1) self.assertEqual(result.summary.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c') self.assertEqual(result.summary.max_updated_on, datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(result.summary.total, 9)
def test_get_task_not_found(self): """Check whether it raises an exception when a task is not found""" registry = TaskRegistry() with self.assertRaises(NotFoundError): registry.get('mytask') registry.add('newtask', 'mock_backend', 'mocked_category', {}) with self.assertRaises(NotFoundError): registry.get('mytask')
def test_remove_task(self): """Test to remove a task from the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} registry = TaskRegistry() registry.add('mytask', 'git', 'commit', args) registry.add('newtask', 'bugzilla', 'issue', None) registry.add('atask', 'git', 'commit', None) tasks = registry.tasks self.assertEqual(len(tasks), 3) registry.remove('newtask') tasks = registry.tasks self.assertEqual(len(tasks), 2) self.assertEqual(tasks[0].task_id, 'atask') self.assertEqual(tasks[1].task_id, 'mytask')
def test_remove_task(self): """Test to remove a task from the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} registry = TaskRegistry() registry.add('mytask', 'mock_backend', 'mocked_category', args) registry.add('newtask', 'to_remove', 'mocked_category', None) registry.add('atask', 'test_backend', 'mocked_category', None) tasks = registry.tasks self.assertEqual(len(tasks), 3) registry.remove('newtask') tasks = registry.tasks self.assertEqual(len(tasks), 2) self.assertEqual(tasks[0].task_id, 'atask') self.assertEqual(tasks[1].task_id, 'mytask')
def test_set_job_number(self): """Check if the job number is set correctly when a new job is enqueued""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, queue='myqueue') registry = TaskRegistry() task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) # Modify the list of jobs to pretend this is not the first # time running this task. Job number will be calculated # adding one to the length of jobs. task.jobs = ['A', 'B', 'C'] schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) schlr.schedule() self.assertEqual(task.age, 1) # Get the last job run and check the result job = schlr._scheduler._queues['myqueue'].fetch_job(task.jobs[3]) result = job.return_value self.assertEqual(result.task_id, task.task_id) self.assertEqual(result.job_number, 4)
def test_matched_tasks(self): """Test to list tasks in the registry when other data is stored in Redis""" registry = TaskRegistry(self.conn) self.conn.set("key1", 'value') self.conn.set("key2", 1) self.conn.set("key3", b'x') expected = [] for i in range(0, 20): task_id = 'mytask{}'.format(i) expected.append(task_id) registry.add(task_id, 'git', 'commit', {}) tasks = registry.tasks self.assertEqual(len(tasks), 20) for t in tasks: self.assertIn(t.task_id, expected)
def test_get_task(self): """Test to get a task from the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} registry = TaskRegistry() registry.add('mytask', 'git', 'commit', args) registry.add('newtask', 'bugzilla', 'issue', None) registry.add('atask', 'git', 'commit', None) task = registry.get('atask') self.assertIsInstance(task, Task) self.assertEqual(task.task_id, 'atask') self.assertEqual(task.status, TaskStatus.NEW) self.assertEqual(task.category, 'commit') self.assertEqual(task.backend, 'git') self.assertEqual(task.backend_args, None)
def test_remove_not_found(self): """Check whether it raises an exception when a task is not found""" registry = TaskRegistry() with self.assertRaises(NotFoundError): registry.remove('mytask') registry.add('task', 'git', "mock_category", {}) with self.assertRaises(NotFoundError): registry.remove('mytask') self.assertEqual(len(registry.tasks), 1)
def test_remove_not_found(self): """Check whether it raises an exception when a task is not found""" registry = TaskRegistry() with self.assertRaises(NotFoundError): registry.remove('mytask') registry.add('task', 'mock_backend', None, '/tmp/example') with self.assertRaises(NotFoundError): registry.remove('mytask') self.assertEqual(len(registry.tasks), 1)
def test_schedule_task(self): """Jobs should be added and executed""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0) registry = TaskRegistry() task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) schlr.schedule() self.assertEqual(task.age, 1) job = schlr._scheduler._queues[Q_CREATION_JOBS].fetch_job( task.last_job) result = job.return_value self.assertEqual(result.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c') self.assertEqual(result.max_date, 1392185439.0) self.assertEqual(result.nitems, 9)
def test_schedule_task_with_reset(self): """Task should be added and executing reseting some of its counters""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0) registry = TaskRegistry(self.conn) task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) # Force values to some of the counters task.age = 100 task.num_failures = 5 registry.update('mytask', task) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id, reset=True) # Counters were reset task = registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) self.assertEqual(task.num_failures, 0) schlr.schedule() task = registry.get('mytask') self.assertEqual(task.age, 1) job = schlr._scheduler._queues[Q_CREATION_JOBS].fetch_job( task.jobs[0].id) result = job.return_value self.assertEqual(result.task_id, task.task_id) self.assertEqual(result.job_number, 1) self.assertEqual(result.summary.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c') self.assertEqual(result.summary.max_updated_on, datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(result.summary.total, 9)
def test_remove_task_error(self, mock_redis_cmd): """Test whether the lock is released in case of a Redis error and an error TaskRegistryError is thrown""" mock_redis_cmd.side_effect = RedisError registry = TaskRegistry(self.conn) registry.add('mytask', 'git', 'commit', {}) self.assertGreater(registry._rwlock._access_mutex._value, 0) with self.assertRaises(TaskRegistryError): registry.remove('mytask') self.assertGreater(registry._rwlock._access_mutex._value, 0)
def test_remove_task(self): """Test to remove a task from the registry""" args = { 'from_date': '1970-01-01', 'component': 'test' } registry = TaskRegistry() registry.add('mytask', 'mock_backend', 'mocked_category', args) registry.add('newtask', 'to_remove', 'mocked_category', None) registry.add('atask', 'test_backend', 'mocked_category', None) tasks = registry.tasks self.assertEqual(len(tasks), 3) registry.remove('newtask') tasks = registry.tasks self.assertEqual(len(tasks), 2) self.assertEqual(tasks[0].task_id, 'atask') self.assertEqual(tasks[1].task_id, 'mytask')
def test_get_task(self): """Test to get a task from the registry""" args = { 'from_date': '1970-01-01', 'component': 'test' } registry = TaskRegistry() registry.add('mytask', 'mock_backend', 'category', args) registry.add('newtask', 'to_remove', 'category', None) registry.add('atask', 'test_backend', 'category', None) task = registry.get('atask') self.assertIsInstance(task, Task) self.assertEqual(task.task_id, 'atask') self.assertEqual(task.category, 'category') self.assertEqual(task.backend, 'test_backend') self.assertEqual(task.backend_args, None)
class TestFailedJobHandler(TestBaseRQ): """Unit tests for CompletedJobHandler""" def setUp(self): super().setUp() self.registry = TaskRegistry() self.task_scheduler = _TaskScheduler(self.registry, self.conn, []) def test_initialization(self): """Check if the handler is correctly initialized""" handler = FailedJobHandler(self.task_scheduler) self.assertEqual(handler.task_scheduler, self.task_scheduler) def test_handle_event(self): """Check if the event is handled correctly""" handler = FailedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) payload = { 'error': "Error" } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = FailedJobHandler(self.task_scheduler) payload = { 'error': "Error" } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, False)
def test_add_existing_task(self): """Check if it raises an exception when an exisiting task is added again""" registry = TaskRegistry() registry.add('mytask', 'mock_backend', {}) with self.assertRaises(AlreadyExistsError): registry.add('mytask', 'new_backend', {}) # Only one tasks is on the registry tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.backend, 'mock_backend') self.assertDictEqual(task.backend_args, {})
class TestFailedJobHandler(TestBaseRQ): """Unit tests for CompletedJobHandler""" def setUp(self): super().setUp() self.registry = TaskRegistry() self.task_scheduler = _TaskScheduler(self.registry, self.conn, []) def test_initialization(self): """Check if the handler is correctly initialized""" handler = FailedJobHandler(self.task_scheduler) self.assertEqual(handler.task_scheduler, self.task_scheduler) def test_handle_event(self): """Check if the event is handled correctly""" handler = FailedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) # It won't be scheduled because max_retries is not set handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) def test_task_not_rescheduled_not_resume(self): """Check if tasks unable to resume are not rescheduled""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'gerrit', 'review', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'gerrit', 'review') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) self.assertEqual(task.num_failures, 1) def test_failed_task_not_rescheduled_max_retries(self): """Check if the task is not re-scheduled when num failures reaches its limit""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force to a pre-defined number of failures task.num_failures = 2 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) self.assertEqual(task.num_failures, 3) def test_failed_task_rescheduled_with_next_from_date(self): """Check if failed tasks are rescheduled updating next_from_date""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # The field is updated to the max date received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.num_failures, 1) def test_failed_task_rescheduled_with_next_offset(self): """Check if failed tasks are rescheduled updating next_offset""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) summary.last_offset = 800 summary.max_offset = 1000 result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the max value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000) self.assertEqual(task.num_failures, 1) def test_failed_task_rescheduled_no_new_items(self): """Check if tasks are rescheduled when no items where generated before""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 0 summary.max_updated_on = None summary.max_offset = None result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are not updated self.assertNotIn('next_from_date', task.backend_args) self.assertNotIn('next_offset', task.backend_args) def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = FailedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, False)
class TestCompletedJobHandler(TestBaseRQ): """Unit tests for CompletedJobHandler""" def setUp(self): super().setUp() self.registry = TaskRegistry() self.task_scheduler = _TaskScheduler(self.registry, self.conn, []) def test_initialization(self): """Check if the handler is correctly initialized""" handler = CompletedJobHandler(self.task_scheduler) self.assertEqual(handler.task_scheduler, self.task_scheduler) def test_task_rescheduling(self): """Check if the task related to the event is re-scheduled""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_rescheduling_age(self): """Check if the task is re-scheduled when maximum age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be lower than its limit task.age = 2 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_rescheduling_unlimited_age(self): """Check if the task is re-scheduled when unlimited age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be large value task.age = 1000000 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_not_rescheduled_age_limit(self): """Check if the task is not re-scheduled when age reaches its limit""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to its pre-defined limit task.age = 3 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.COMPLETED) def test_task_not_rescheduled_archive_task(self): """Check if archive tasks are not rescheduled""" handler = CompletedJobHandler(self.task_scheduler) archiving_cfg = ArchivingTaskConfig('/tmp/archive', True) task = self.registry.add('mytask', 'git', 'commit', {}, archiving_cfg=archiving_cfg) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.COMPLETED) def test_task_rescheduled_with_next_from_date(self): """Check if tasks are rescheduled updating next_from_date""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # The field is updated to the last date received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) def test_task_rescheduled_with_next_offset(self): """Check if tasks are rescheduled updating next_offset""" handler = CompletedJobHandler(self.task_scheduler) task = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9, offset=1000) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the last value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000) def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = CompletedJobHandler(self.task_scheduler) result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False)
def setUp(self): super().setUp() self.registry = TaskRegistry() self.task_scheduler = _TaskScheduler(self.registry, self.conn, [])
class TestCompletedJobHandler(TestBaseRQ): """Unit tests for CompletedJobHandler""" def setUp(self): super().setUp() self.registry = TaskRegistry(self.conn) self.task_scheduler = _TaskScheduler(self.registry, self.conn, []) def test_initialization(self): """Check if the handler is correctly initialized""" handler = CompletedJobHandler(self.task_scheduler) self.assertEqual(handler.task_scheduler, self.task_scheduler) def test_task_rescheduling(self): """Check if the task related to the event is re-scheduled""" handler = CompletedJobHandler(self.task_scheduler) _ = self.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_rescheduling_age(self): """Check if the task is re-scheduled when maximum age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be lower than its limit task.age = 2 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_rescheduling_unlimited_age(self): """Check if the task is re-scheduled when unlimited age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be large value task.age = 1000000 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) def test_task_not_rescheduled_age_limit(self): """Check if the task is not re-scheduled when age reaches its limit""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to its pre-defined limit task.age = 3 self.task_scheduler.registry.update('mytask', task) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.COMPLETED) def test_task_not_rescheduled_archive_task(self): """Check if archive tasks are not rescheduled""" handler = CompletedJobHandler(self.task_scheduler) archiving_cfg = ArchivingTaskConfig('/tmp/archive', True) _ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}, archiving_cfg=archiving_cfg) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.COMPLETED) def test_task_rescheduled_with_next_from_date(self): """Check if tasks are rescheduled updating next_from_date""" handler = CompletedJobHandler(self.task_scheduler) _ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) # The field is updated to the max date received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) def test_task_rescheduled_with_next_offset(self): """Check if tasks are rescheduled updating next_offset""" handler = CompletedJobHandler(self.task_scheduler) _ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) summary.last_offset = 800 summary.max_offset = 1000 result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the max value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000) def test_task_rescheduled_no_new_items(self): """Check if tasks are rescheduled when no items where generated before""" handler = CompletedJobHandler(self.task_scheduler) _ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 0 summary.max_updated_on = None summary.max_offset = None result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are not updated self.assertNotIn('next_from_date', task.backend_args) self.assertNotIn('next_offset', task.backend_args) def test_task_reset_num_failures(self): """Check if the number of failures is reset if the task is successful""" handler = CompletedJobHandler(self.task_scheduler) task = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) # Force to a pre-defined number of failures task.num_failures = 2 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.num_failures, 0) def test_ignore_orphan_event(self): """Check if an orphan event is ignored""" handler = CompletedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False) @unittest.mock.patch('redis.StrictRedis.get') def test_ignore_event_on_task_registry_error(self, mock_redis_get): """Check if an event is ignored when a TaskRegistryError is thrown""" mock_redis_get.side_effect = RedisError self.task_scheduler.registry.add('mytask', 'git', 'commit', {}) handler = CompletedJobHandler(self.task_scheduler) result = JobResult(0, 1, 'mytask', 'git', 'commit') event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, False)