def test_update_task_error(self, mock_redis_cmd): """Test whether the lock is released in case of a Redis error and a TaskRegistryError is thrown""" mock_redis_cmd.side_effect = RedisError registry = TaskRegistry(self.conn) self.assertGreater(registry._rwlock._access_mutex._value, 0) with self.assertRaises(TaskRegistryError): registry.update('mytask', None) self.assertGreater(registry._rwlock._access_mutex._value, 0)
def test_update_task_not_found(self): """Check whether it raises an exception when a task is not found""" registry = TaskRegistry(self.conn) with self.assertLogs(logger, level='WARNING') as cm: registry.update('mytask', None) self.assertEqual( cm.output[0], 'WARNING:arthur.tasks:Task mytask not found, adding it')
def test_set_job_number(self): """Check if the job number is set correctly when a new job is enqueued""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, queue='myqueue') registry = TaskRegistry(self.conn) task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) task = registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) # Modify the list of jobs to pretend this is not the first # time running this task. Job number will be calculated # adding one to the length of jobs. task.jobs = ['A', 'B', 'C'] registry.update('mytask', task) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) task = registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) schlr.schedule() task = registry.get('mytask') self.assertEqual(task.age, 1) # Get the last job run and check the result job = schlr._scheduler._queues['myqueue'].fetch_job(task.jobs[3].id) result = job.return_value self.assertEqual(result.task_id, task.task_id) self.assertEqual(result.job_number, 4)
def test_schedule_task_with_reset(self): """Task should be added and executing reseting some of its counters""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0) registry = TaskRegistry(self.conn) task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) # Force values to some of the counters task.age = 100 task.num_failures = 5 registry.update('mytask', task) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id, reset=True) # Counters were reset task = registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) self.assertEqual(task.num_failures, 0) schlr.schedule() task = registry.get('mytask') self.assertEqual(task.age, 1) job = schlr._scheduler._queues[Q_CREATION_JOBS].fetch_job( task.jobs[0].id) result = job.return_value self.assertEqual(result.task_id, task.task_id) self.assertEqual(result.job_number, 1) self.assertEqual(result.summary.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c') self.assertEqual(result.summary.max_updated_on, datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(result.summary.total, 9)
def test_schudule_task_retries_queue(self): """Failed task should be rescheduled to the retries queue""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0) registry = TaskRegistry(self.conn) task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) task.num_failures = 2 # Force number of failures before scheduling the task registry.update(task.task_id, task) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) task = registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) schlr.schedule() task = registry.get('mytask') self.assertEqual(task.age, 1) job = schlr._scheduler._queues[Q_RETRYING_JOBS].fetch_job( task.jobs[0].id) result = job.return_value self.assertEqual(result.task_id, task.task_id) self.assertEqual(result.job_number, 1) self.assertEqual(result.summary.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c') self.assertEqual(result.summary.max_updated_on, datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(result.summary.total, 9)
def test_update_task(self): """Test to update tasks in the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} registry = TaskRegistry(self.conn) before = datetime.datetime.now().timestamp() _ = registry.add('mytask', 'git', 'commit', args) tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertIsInstance(task, Task) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.status, TaskStatus.NEW) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) self.assertEqual(task.category, 'commit') self.assertEqual(task.backend, 'git') self.assertDictEqual(task.backend_args, args) self.assertEqual(task.archiving_cfg, None) self.assertEqual(task.scheduling_cfg, None) self.assertGreater(task.created_on, before) task.status = TaskStatus.COMPLETED task.age = 1 registry.update('mytask', task) tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertIsInstance(task, Task) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.status, TaskStatus.COMPLETED) self.assertEqual(task.age, 1) self.assertListEqual(task.jobs, []) self.assertEqual(task.category, 'commit') self.assertEqual(task.backend, 'git') self.assertDictEqual(task.backend_args, args) self.assertEqual(task.archiving_cfg, None) self.assertEqual(task.scheduling_cfg, None) self.assertGreater(task.created_on, before)