def test_update_task_error(self, mock_redis_cmd):
"""Test whether the lock is released in case of a Redis error and a TaskRegistryError is thrown"""
mock_redis_cmd.side_effect = RedisError
registry = TaskRegistry(self.conn)
self.assertGreater(registry._rwlock._access_mutex._value, 0)
with self.assertRaises(TaskRegistryError):
registry.update('mytask', None)
self.assertGreater(registry._rwlock._access_mutex._value, 0)
def test_update_task_not_found(self):
"""Check whether it raises an exception when a task is not found"""
registry = TaskRegistry(self.conn)
with self.assertLogs(logger, level='WARNING') as cm:
registry.update('mytask', None)
self.assertEqual(
cm.output[0],
'WARNING:arthur.tasks:Task mytask not found, adding it')
def test_set_job_number(self):
"""Check if the job number is set correctly when a new job is enqueued"""
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
category = 'commit'
archiving_opts = None
scheduler_opts = SchedulingTaskConfig(delay=0,
max_retries=0,
queue='myqueue')
registry = TaskRegistry(self.conn)
task = registry.add('mytask',
'git',
category,
args,
archiving_cfg=archiving_opts,
scheduling_cfg=scheduler_opts)
schlr = Scheduler(self.conn, registry, async_mode=False)
schlr.schedule_task(task.task_id)
task = registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.age, 0)
# Modify the list of jobs to pretend this is not the first
# time running this task. Job number will be calculated
# adding one to the length of jobs.
task.jobs = ['A', 'B', 'C']
registry.update('mytask', task)
schlr = Scheduler(self.conn, registry, async_mode=False)
schlr.schedule_task(task.task_id)
task = registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.age, 0)
schlr.schedule()
task = registry.get('mytask')
self.assertEqual(task.age, 1)
# Get the last job run and check the result
job = schlr._scheduler._queues['myqueue'].fetch_job(task.jobs[3].id)
result = job.return_value
self.assertEqual(result.task_id, task.task_id)
self.assertEqual(result.job_number, 4)
def test_schedule_task_with_reset(self):
"""Task should be added and executing reseting some of its counters"""
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
category = 'commit'
archiving_opts = None
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0)
registry = TaskRegistry(self.conn)
task = registry.add('mytask',
'git',
category,
args,
archiving_cfg=archiving_opts,
scheduling_cfg=scheduler_opts)
# Force values to some of the counters
task.age = 100
task.num_failures = 5
registry.update('mytask', task)
schlr = Scheduler(self.conn, registry, async_mode=False)
schlr.schedule_task(task.task_id, reset=True)
# Counters were reset
task = registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.age, 0)
self.assertEqual(task.num_failures, 0)
schlr.schedule()
task = registry.get('mytask')
self.assertEqual(task.age, 1)
job = schlr._scheduler._queues[Q_CREATION_JOBS].fetch_job(
task.jobs[0].id)
result = job.return_value
self.assertEqual(result.task_id, task.task_id)
self.assertEqual(result.job_number, 1)
self.assertEqual(result.summary.last_uuid,
'1375b60d3c23ac9b81da92523e4144abc4489d4c')
self.assertEqual(result.summary.max_updated_on,
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(result.summary.total, 9)
def test_schudule_task_retries_queue(self):
"""Failed task should be rescheduled to the retries queue"""
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
category = 'commit'
archiving_opts = None
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0)
registry = TaskRegistry(self.conn)
task = registry.add('mytask',
'git',
category,
args,
archiving_cfg=archiving_opts,
scheduling_cfg=scheduler_opts)
task.num_failures = 2 # Force number of failures before scheduling the task
registry.update(task.task_id, task)
schlr = Scheduler(self.conn, registry, async_mode=False)
schlr.schedule_task(task.task_id)
task = registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.age, 0)
schlr.schedule()
task = registry.get('mytask')
self.assertEqual(task.age, 1)
job = schlr._scheduler._queues[Q_RETRYING_JOBS].fetch_job(
task.jobs[0].id)
result = job.return_value
self.assertEqual(result.task_id, task.task_id)
self.assertEqual(result.job_number, 1)
self.assertEqual(result.summary.last_uuid,
'1375b60d3c23ac9b81da92523e4144abc4489d4c')
self.assertEqual(result.summary.max_updated_on,
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(result.summary.total, 9)
def test_update_task(self):
"""Test to update tasks in the registry"""
args = {'from_date': '1970-01-01', 'component': 'test'}
registry = TaskRegistry(self.conn)
before = datetime.datetime.now().timestamp()
_ = registry.add('mytask', 'git', 'commit', args)
tasks = registry.tasks
self.assertEqual(len(tasks), 1)
task = tasks[0]
self.assertIsInstance(task, Task)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.status, TaskStatus.NEW)
self.assertEqual(task.age, 0)
self.assertListEqual(task.jobs, [])
self.assertEqual(task.category, 'commit')
self.assertEqual(task.backend, 'git')
self.assertDictEqual(task.backend_args, args)
self.assertEqual(task.archiving_cfg, None)
self.assertEqual(task.scheduling_cfg, None)
self.assertGreater(task.created_on, before)
task.status = TaskStatus.COMPLETED
task.age = 1
registry.update('mytask', task)
tasks = registry.tasks
self.assertEqual(len(tasks), 1)
task = tasks[0]
self.assertIsInstance(task, Task)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.status, TaskStatus.COMPLETED)
self.assertEqual(task.age, 1)
self.assertListEqual(task.jobs, [])
self.assertEqual(task.category, 'commit')
self.assertEqual(task.backend, 'git')
self.assertDictEqual(task.backend_args, args)
self.assertEqual(task.archiving_cfg, None)
self.assertEqual(task.scheduling_cfg, None)
self.assertGreater(task.created_on, before)