def test_task_jobs_is_updated(self):
"""Test if the list of job ids for a task is updated"""
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
category = 'commit'
archiving_opts = None
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0)
registry = TaskRegistry()
task = registry.add('mytask', 'git', category, args,
archiving_cfg=archiving_opts,
scheduling_cfg=scheduler_opts)
schlr = Scheduler(self.conn, registry, async_mode=False)
schlr.schedule_task(task.task_id)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.age, 0)
self.assertListEqual(task.jobs, [])
schlr.schedule()
self.assertEqual(task.age, 1)
self.assertEqual(len(task.jobs), 1)
def test_schedule_task(self):
"""Jobs should be added and executed"""
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
category = 'commit'
archive_args = {}
sched_args = {
'delay': 0,
'max_retries_job': 0
}
registry = TaskRegistry()
task = registry.add('mytask', 'git', category, args,
archive_args=archive_args,
sched_args=sched_args)
schlr = Scheduler(self.conn, registry, async_mode=False)
job_id = schlr.schedule_task(task.task_id)
schlr.schedule()
job = schlr._scheduler._queues[Q_CREATION_JOBS].fetch_job(job_id)
result = job.return_value
self.assertEqual(result.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c')
self.assertEqual(result.max_date, 1392185439.0)
self.assertEqual(result.nitems, 9)
def test_add_task(self):
"""Test to add tasks to the registry"""
args = {'from_date': '1970-01-01', 'component': 'test'}
archive = ArchivingTaskConfig('/tmp/archive',
False,
archived_after=None)
sched = SchedulingTaskConfig(delay=10, max_retries=2)
registry = TaskRegistry()
before = datetime.datetime.now().timestamp()
new_task = registry.add('mytask', 'git', 'commit', args)
tasks = registry.tasks
self.assertEqual(len(tasks), 1)
task = tasks[0]
self.assertIsInstance(task, Task)
self.assertEqual(task, new_task)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.status, TaskStatus.NEW)
self.assertEqual(task.age, 0)
self.assertListEqual(task.jobs, [])
self.assertEqual(task.category, 'commit')
self.assertEqual(task.backend, 'git')
self.assertDictEqual(task.backend_args, args)
self.assertEqual(task.archiving_cfg, None)
self.assertEqual(task.scheduling_cfg, None)
self.assertGreater(task.created_on, before)
before = datetime.datetime.now().timestamp()
_ = registry.add('atask',
'git',
'commit',
args,
archiving_cfg=archive,
scheduling_cfg=sched)
tasks = registry.tasks
self.assertEqual(len(tasks), 2)
task0 = tasks[0]
self.assertIsInstance(task0, Task)
self.assertEqual(task0.task_id, 'atask')
self.assertEqual(task0.status, TaskStatus.NEW)
self.assertEqual(task.age, 0)
self.assertListEqual(task.jobs, [])
self.assertEqual(task0.backend, 'git')
self.assertEqual(task0.category, 'commit')
self.assertDictEqual(task0.backend_args, args)
self.assertEqual(task0.archiving_cfg, archive)
self.assertEqual(task0.scheduling_cfg, sched)
self.assertGreater(task0.created_on, before)
task1 = tasks[1]
self.assertEqual(task1.task_id, 'mytask')
self.assertGreater(task0.created_on, task1.created_on)
def test_update_task_not_found(self):
"""Check whether it raises an exception when a task is not found"""
registry = TaskRegistry(self.conn)
with self.assertLogs(logger, level='WARNING') as cm:
registry.update('mytask', None)
self.assertEqual(
cm.output[0],
'WARNING:arthur.tasks:Task mytask not found, adding it')
def test_update_task_error(self, mock_redis_cmd):
"""Test whether the lock is released in case of a Redis error and a TaskRegistryError is thrown"""
mock_redis_cmd.side_effect = RedisError
registry = TaskRegistry(self.conn)
self.assertGreater(registry._rwlock._access_mutex._value, 0)
with self.assertRaises(TaskRegistryError):
registry.update('mytask', None)
self.assertGreater(registry._rwlock._access_mutex._value, 0)
def test_tasks_error(self, mock_redis_set):
"""Test whether the lock is released in case of a Redis error and a TaskRegistryError is thrown"""
mock_redis_set.side_effect = RedisError
registry = TaskRegistry(self.conn)
registry.add('mytask', 'git', 'commit', {})
self.assertGreater(registry._rwlock._readers_mutex._value, 0)
with self.assertRaises(TaskRegistryError):
_ = registry.tasks
self.assertGreater(registry._rwlock._readers_mutex._value, 0)
def test_add_task(self):
"""Test to add tasks to the registry"""
args = {
'from_date': '1970-01-01',
'component': 'test'
}
archive = {
'archive_path': '/tmp/archive',
'fetch_from_archive': False,
'archived_after': None,
}
sched = {
'stime': 10
}
registry = TaskRegistry()
before = datetime.now().timestamp()
new_task = registry.add('mytask', 'mock_backend', 'category', args)
tasks = registry.tasks
self.assertEqual(len(tasks), 1)
task = tasks[0]
self.assertIsInstance(task, Task)
self.assertEqual(task, new_task)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.category, 'category')
self.assertEqual(task.backend, 'mock_backend')
self.assertDictEqual(task.backend_args, args)
self.assertDictEqual(task.archive_args, {})
self.assertDictEqual(task.sched_args, {})
self.assertGreater(task.created_on, before)
before = datetime.now().timestamp()
_ = registry.add('atask', 'mock_backend', 'category', args,
archive_args=archive, sched_args=sched)
tasks = registry.tasks
self.assertEqual(len(tasks), 2)
task0 = tasks[0]
self.assertIsInstance(task0, Task)
self.assertEqual(task0.task_id, 'atask')
self.assertEqual(task0.backend, 'mock_backend')
self.assertEqual(task0.category, 'category')
self.assertDictEqual(task0.backend_args, args)
self.assertDictEqual(task0.archive_args, archive)
self.assertDictEqual(task0.sched_args, sched)
self.assertGreater(task0.created_on, before)
task1 = tasks[1]
self.assertEqual(task1.task_id, 'mytask')
self.assertGreater(task0.created_on, task1.created_on)
def test_add_task(self):
"""Test to add tasks to the registry"""
args = {'from_date': '1970-01-01', 'component': 'test'}
archive = {
'archive_path': '/tmp/archive',
'fetch_from_archive': False,
'archived_after': None,
}
sched = {'stime': 10}
registry = TaskRegistry()
before = datetime.now().timestamp()
new_task = registry.add('mytask', 'mock_backend', 'category', args)
tasks = registry.tasks
self.assertEqual(len(tasks), 1)
task = tasks[0]
self.assertIsInstance(task, Task)
self.assertEqual(task, new_task)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.category, 'category')
self.assertEqual(task.backend, 'mock_backend')
self.assertDictEqual(task.backend_args, args)
self.assertDictEqual(task.archive_args, {})
self.assertDictEqual(task.sched_args, {})
self.assertGreater(task.created_on, before)
before = datetime.now().timestamp()
_ = registry.add('atask',
'mock_backend',
'category',
args,
archive_args=archive,
sched_args=sched)
tasks = registry.tasks
self.assertEqual(len(tasks), 2)
task0 = tasks[0]
self.assertIsInstance(task0, Task)
self.assertEqual(task0.task_id, 'atask')
self.assertEqual(task0.backend, 'mock_backend')
self.assertEqual(task0.category, 'category')
self.assertDictEqual(task0.backend_args, args)
self.assertDictEqual(task0.archive_args, archive)
self.assertDictEqual(task0.sched_args, sched)
self.assertGreater(task0.created_on, before)
task1 = tasks[1]
self.assertEqual(task1.task_id, 'mytask')
self.assertGreater(task0.created_on, task1.created_on)
def test_tasks(self):
"""Test to list tasks in the registry"""
registry = TaskRegistry(self.conn)
expected = []
for i in range(0, 20):
task_id = 'mytask{}'.format(i)
expected.append(task_id)
registry.add(task_id, 'git', 'commit', {})
tasks = registry.tasks
self.assertEqual(len(tasks), 20)
for t in tasks:
self.assertIn(t.task_id, expected)
class TestStartedJobHandler(TestBaseRQ):
"""Unit tests for StartedJobHandler"""
def setUp(self):
super().setUp()
self.registry = TaskRegistry()
self.task_scheduler = _TaskScheduler(self.registry, self.conn, [])
def test_initialization(self):
"""Check if the handler is correctly initialized"""
handler = StartedJobHandler(self.task_scheduler)
self.assertEqual(handler.task_scheduler, self.task_scheduler)
def test_task_status(self):
"""Check if the handler changes the task status to running"""
handler = StartedJobHandler(self.task_scheduler)
task = self.registry.add('mytask', 'git', 'commit', {})
event = JobEvent(JobEventType.STARTED, 0, 'mytask', None)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.RUNNING)
def test_ignore_orphan_event(self):
"""Check if an orphan event is ignored"""
handler = StartedJobHandler(self.task_scheduler)
event = JobEvent(JobEventType.STARTED, 0, 'mytask', None)
handled = handler(event)
self.assertEqual(handled, False)
def test_not_found_task(self):
"""Raises an error when the task to schedule does not exist"""
registry = TaskRegistry()
schlr = Scheduler(self.conn, registry, async_mode=False)
self.assertRaises(NotFoundError, schlr.schedule_task, 'mytask')
def test_empty_registry(self):
"""Check empty registry on init"""
registry = TaskRegistry()
tasks = registry.tasks
self.assertListEqual(tasks, [])
def test_add_existing_task(self):
"""Check if it raises an exception when an exisiting task is added again"""
registry = TaskRegistry()
registry.add('mytask', 'mock_backend', 'category', {})
with self.assertRaises(AlreadyExistsError):
registry.add('mytask', 'new_backend', 'category', {})
# Only one tasks is on the registry
tasks = registry.tasks
self.assertEqual(len(tasks), 1)
task = tasks[0]
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.backend, 'mock_backend')
self.assertEqual(task.category, 'category')
self.assertDictEqual(task.backend_args, {})
def test_update_task(self):
"""Test to update tasks in the registry"""
args = {'from_date': '1970-01-01', 'component': 'test'}
registry = TaskRegistry(self.conn)
before = datetime.datetime.now().timestamp()
_ = registry.add('mytask', 'git', 'commit', args)
tasks = registry.tasks
self.assertEqual(len(tasks), 1)
task = tasks[0]
self.assertIsInstance(task, Task)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.status, TaskStatus.NEW)
self.assertEqual(task.age, 0)
self.assertListEqual(task.jobs, [])
self.assertEqual(task.category, 'commit')
self.assertEqual(task.backend, 'git')
self.assertDictEqual(task.backend_args, args)
self.assertEqual(task.archiving_cfg, None)
self.assertEqual(task.scheduling_cfg, None)
self.assertGreater(task.created_on, before)
task.status = TaskStatus.COMPLETED
task.age = 1
registry.update('mytask', task)
tasks = registry.tasks
self.assertEqual(len(tasks), 1)
task = tasks[0]
self.assertIsInstance(task, Task)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.status, TaskStatus.COMPLETED)
self.assertEqual(task.age, 1)
self.assertListEqual(task.jobs, [])
self.assertEqual(task.category, 'commit')
self.assertEqual(task.backend, 'git')
self.assertDictEqual(task.backend_args, args)
self.assertEqual(task.archiving_cfg, None)
self.assertEqual(task.scheduling_cfg, None)
self.assertGreater(task.created_on, before)
def test_get_task(self):
"""Test to get a task from the registry"""
args = {'from_date': '1970-01-01', 'component': 'test'}
registry = TaskRegistry()
registry.add('mytask', 'mock_backend', args)
registry.add('newtask', 'to_remove', None)
registry.add('atask', 'test_backend', None)
task = registry.get('atask')
self.assertIsInstance(task, Task)
self.assertEqual(task.task_id, 'atask')
self.assertEqual(task.backend, 'test_backend')
self.assertEqual(task.backend_args, None)
def test_schedule_task_user_queue(self):
"""Task should be added and executed in the user's queue"""
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
category = 'commit'
archiving_opts = None
scheduler_opts = SchedulingTaskConfig(delay=0,
max_retries=0,
queue='myqueue')
registry = TaskRegistry(self.conn)
task = registry.add('mytask',
'git',
category,
args,
archiving_cfg=archiving_opts,
scheduling_cfg=scheduler_opts)
schlr = Scheduler(self.conn, registry, async_mode=False)
schlr.schedule_task(task.task_id)
task = registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.age, 0)
schlr.schedule()
task = registry.get('mytask')
self.assertEqual(task.age, 1)
job = schlr._scheduler._queues['myqueue'].fetch_job(task.jobs[0].id)
result = job.return_value
self.assertEqual(result.task_id, task.task_id)
self.assertEqual(result.job_number, 1)
self.assertEqual(result.summary.last_uuid,
'1375b60d3c23ac9b81da92523e4144abc4489d4c')
self.assertEqual(result.summary.max_updated_on,
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(result.summary.total, 9)
def test_get_task_not_found(self):
"""Check whether it raises an exception when a task is not found"""
registry = TaskRegistry()
with self.assertRaises(NotFoundError):
registry.get('mytask')
registry.add('newtask', 'mock_backend', 'mocked_category', {})
with self.assertRaises(NotFoundError):
registry.get('mytask')
def test_remove_task(self):
"""Test to remove a task from the registry"""
args = {'from_date': '1970-01-01', 'component': 'test'}
registry = TaskRegistry()
registry.add('mytask', 'git', 'commit', args)
registry.add('newtask', 'bugzilla', 'issue', None)
registry.add('atask', 'git', 'commit', None)
tasks = registry.tasks
self.assertEqual(len(tasks), 3)
registry.remove('newtask')
tasks = registry.tasks
self.assertEqual(len(tasks), 2)
self.assertEqual(tasks[0].task_id, 'atask')
self.assertEqual(tasks[1].task_id, 'mytask')
def test_remove_task(self):
"""Test to remove a task from the registry"""
args = {'from_date': '1970-01-01', 'component': 'test'}
registry = TaskRegistry()
registry.add('mytask', 'mock_backend', 'mocked_category', args)
registry.add('newtask', 'to_remove', 'mocked_category', None)
registry.add('atask', 'test_backend', 'mocked_category', None)
tasks = registry.tasks
self.assertEqual(len(tasks), 3)
registry.remove('newtask')
tasks = registry.tasks
self.assertEqual(len(tasks), 2)
self.assertEqual(tasks[0].task_id, 'atask')
self.assertEqual(tasks[1].task_id, 'mytask')
def test_set_job_number(self):
"""Check if the job number is set correctly when a new job is enqueued"""
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
category = 'commit'
archiving_opts = None
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0,
queue='myqueue')
registry = TaskRegistry()
task = registry.add('mytask', 'git', category, args,
archiving_cfg=archiving_opts,
scheduling_cfg=scheduler_opts)
schlr = Scheduler(self.conn, registry, async_mode=False)
schlr.schedule_task(task.task_id)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.age, 0)
# Modify the list of jobs to pretend this is not the first
# time running this task. Job number will be calculated
# adding one to the length of jobs.
task.jobs = ['A', 'B', 'C']
schlr = Scheduler(self.conn, registry, async_mode=False)
schlr.schedule_task(task.task_id)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.age, 0)
schlr.schedule()
self.assertEqual(task.age, 1)
# Get the last job run and check the result
job = schlr._scheduler._queues['myqueue'].fetch_job(task.jobs[3])
result = job.return_value
self.assertEqual(result.task_id, task.task_id)
self.assertEqual(result.job_number, 4)
def test_matched_tasks(self):
"""Test to list tasks in the registry when other data is stored in Redis"""
registry = TaskRegistry(self.conn)
self.conn.set("key1", 'value')
self.conn.set("key2", 1)
self.conn.set("key3", b'x')
expected = []
for i in range(0, 20):
task_id = 'mytask{}'.format(i)
expected.append(task_id)
registry.add(task_id, 'git', 'commit', {})
tasks = registry.tasks
self.assertEqual(len(tasks), 20)
for t in tasks:
self.assertIn(t.task_id, expected)
def test_get_task(self):
"""Test to get a task from the registry"""
args = {'from_date': '1970-01-01', 'component': 'test'}
registry = TaskRegistry()
registry.add('mytask', 'git', 'commit', args)
registry.add('newtask', 'bugzilla', 'issue', None)
registry.add('atask', 'git', 'commit', None)
task = registry.get('atask')
self.assertIsInstance(task, Task)
self.assertEqual(task.task_id, 'atask')
self.assertEqual(task.status, TaskStatus.NEW)
self.assertEqual(task.category, 'commit')
self.assertEqual(task.backend, 'git')
self.assertEqual(task.backend_args, None)
def test_remove_not_found(self):
"""Check whether it raises an exception when a task is not found"""
registry = TaskRegistry()
with self.assertRaises(NotFoundError):
registry.remove('mytask')
registry.add('task', 'git', "mock_category", {})
with self.assertRaises(NotFoundError):
registry.remove('mytask')
self.assertEqual(len(registry.tasks), 1)
def test_remove_not_found(self):
"""Check whether it raises an exception when a task is not found"""
registry = TaskRegistry()
with self.assertRaises(NotFoundError):
registry.remove('mytask')
registry.add('task', 'mock_backend', None, '/tmp/example')
with self.assertRaises(NotFoundError):
registry.remove('mytask')
self.assertEqual(len(registry.tasks), 1)
def test_get_task_not_found(self):
"""Check whether it raises an exception when a task is not found"""
registry = TaskRegistry()
with self.assertRaises(NotFoundError):
registry.get('mytask')
registry.add('newtask', 'mock_backend', 'mocked_category', {})
with self.assertRaises(NotFoundError):
registry.get('mytask')
def test_schedule_task(self):
"""Jobs should be added and executed"""
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
category = 'commit'
archiving_opts = None
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0)
registry = TaskRegistry()
task = registry.add('mytask',
'git',
category,
args,
archiving_cfg=archiving_opts,
scheduling_cfg=scheduler_opts)
schlr = Scheduler(self.conn, registry, async_mode=False)
schlr.schedule_task(task.task_id)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.age, 0)
schlr.schedule()
self.assertEqual(task.age, 1)
job = schlr._scheduler._queues[Q_CREATION_JOBS].fetch_job(
task.last_job)
result = job.return_value
self.assertEqual(result.last_uuid,
'1375b60d3c23ac9b81da92523e4144abc4489d4c')
self.assertEqual(result.max_date, 1392185439.0)
self.assertEqual(result.nitems, 9)
def test_remove_not_found(self):
"""Check whether it raises an exception when a task is not found"""
registry = TaskRegistry()
with self.assertRaises(NotFoundError):
registry.remove('mytask')
registry.add('task', 'mock_backend', None, '/tmp/example')
with self.assertRaises(NotFoundError):
registry.remove('mytask')
self.assertEqual(len(registry.tasks), 1)
def test_schedule_task_with_reset(self):
"""Task should be added and executing reseting some of its counters"""
args = {
'uri': 'http://example.com/',
'gitpath': os.path.join(self.dir, 'data/git_log.txt')
}
category = 'commit'
archiving_opts = None
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0)
registry = TaskRegistry(self.conn)
task = registry.add('mytask',
'git',
category,
args,
archiving_cfg=archiving_opts,
scheduling_cfg=scheduler_opts)
# Force values to some of the counters
task.age = 100
task.num_failures = 5
registry.update('mytask', task)
schlr = Scheduler(self.conn, registry, async_mode=False)
schlr.schedule_task(task.task_id, reset=True)
# Counters were reset
task = registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.age, 0)
self.assertEqual(task.num_failures, 0)
schlr.schedule()
task = registry.get('mytask')
self.assertEqual(task.age, 1)
job = schlr._scheduler._queues[Q_CREATION_JOBS].fetch_job(
task.jobs[0].id)
result = job.return_value
self.assertEqual(result.task_id, task.task_id)
self.assertEqual(result.job_number, 1)
self.assertEqual(result.summary.last_uuid,
'1375b60d3c23ac9b81da92523e4144abc4489d4c')
self.assertEqual(result.summary.max_updated_on,
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(result.summary.total, 9)
def test_remove_task_error(self, mock_redis_cmd):
"""Test whether the lock is released in case of a Redis error and an error TaskRegistryError is thrown"""
mock_redis_cmd.side_effect = RedisError
registry = TaskRegistry(self.conn)
registry.add('mytask', 'git', 'commit', {})
self.assertGreater(registry._rwlock._access_mutex._value, 0)
with self.assertRaises(TaskRegistryError):
registry.remove('mytask')
self.assertGreater(registry._rwlock._access_mutex._value, 0)
def test_remove_task(self):
"""Test to remove a task from the registry"""
args = {
'from_date': '1970-01-01',
'component': 'test'
}
registry = TaskRegistry()
registry.add('mytask', 'mock_backend', 'mocked_category', args)
registry.add('newtask', 'to_remove', 'mocked_category', None)
registry.add('atask', 'test_backend', 'mocked_category', None)
tasks = registry.tasks
self.assertEqual(len(tasks), 3)
registry.remove('newtask')
tasks = registry.tasks
self.assertEqual(len(tasks), 2)
self.assertEqual(tasks[0].task_id, 'atask')
self.assertEqual(tasks[1].task_id, 'mytask')
def test_get_task(self):
"""Test to get a task from the registry"""
args = {
'from_date': '1970-01-01',
'component': 'test'
}
registry = TaskRegistry()
registry.add('mytask', 'mock_backend', 'category', args)
registry.add('newtask', 'to_remove', 'category', None)
registry.add('atask', 'test_backend', 'category', None)
task = registry.get('atask')
self.assertIsInstance(task, Task)
self.assertEqual(task.task_id, 'atask')
self.assertEqual(task.category, 'category')
self.assertEqual(task.backend, 'test_backend')
self.assertEqual(task.backend_args, None)
class TestFailedJobHandler(TestBaseRQ):
"""Unit tests for CompletedJobHandler"""
def setUp(self):
super().setUp()
self.registry = TaskRegistry()
self.task_scheduler = _TaskScheduler(self.registry, self.conn, [])
def test_initialization(self):
"""Check if the handler is correctly initialized"""
handler = FailedJobHandler(self.task_scheduler)
self.assertEqual(handler.task_scheduler, self.task_scheduler)
def test_handle_event(self):
"""Check if the event is handled correctly"""
handler = FailedJobHandler(self.task_scheduler)
task = self.registry.add('mytask', 'git', 'commit', {})
payload = {
'error': "Error"
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.FAILED)
def test_ignore_orphan_event(self):
"""Check if an orphan event is ignored"""
handler = FailedJobHandler(self.task_scheduler)
payload = {
'error': "Error"
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, False)
def test_add_existing_task(self):
"""Check if it raises an exception when an exisiting task is added again"""
registry = TaskRegistry()
registry.add('mytask', 'mock_backend', {})
with self.assertRaises(AlreadyExistsError):
registry.add('mytask', 'new_backend', {})
# Only one tasks is on the registry
tasks = registry.tasks
self.assertEqual(len(tasks), 1)
task = tasks[0]
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.backend, 'mock_backend')
self.assertDictEqual(task.backend_args, {})
class TestFailedJobHandler(TestBaseRQ):
"""Unit tests for CompletedJobHandler"""
def setUp(self):
super().setUp()
self.registry = TaskRegistry()
self.task_scheduler = _TaskScheduler(self.registry, self.conn, [])
def test_initialization(self):
"""Check if the handler is correctly initialized"""
handler = FailedJobHandler(self.task_scheduler)
self.assertEqual(handler.task_scheduler, self.task_scheduler)
def test_handle_event(self):
"""Check if the event is handled correctly"""
handler = FailedJobHandler(self.task_scheduler)
task = self.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 2
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
result.summary = summary
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
# It won't be scheduled because max_retries is not set
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.FAILED)
def test_task_not_rescheduled_not_resume(self):
"""Check if tasks unable to resume are not rescheduled"""
handler = FailedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3)
task = self.registry.add('mytask', 'gerrit', 'review', {},
scheduling_cfg=scheduler_opts)
result = JobResult(0, 1, 'mytask', 'gerrit', 'review')
summary = Summary()
summary.fetched = 2
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
result.summary = summary
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.FAILED)
self.assertEqual(task.num_failures, 1)
def test_failed_task_not_rescheduled_max_retries(self):
"""Check if the task is not re-scheduled when num failures reaches its limit"""
handler = FailedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
# Force to a pre-defined number of failures
task.num_failures = 2
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 2
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
result.summary = summary
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.FAILED)
self.assertEqual(task.num_failures, 3)
def test_failed_task_rescheduled_with_next_from_date(self):
"""Check if failed tasks are rescheduled updating next_from_date"""
handler = FailedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 2
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)
result.summary = summary
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# The field is updated to the max date received
# within the result
self.assertEqual(task.backend_args['next_from_date'],
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(task.num_failures, 1)
def test_failed_task_rescheduled_with_next_offset(self):
"""Check if failed tasks are rescheduled updating next_offset"""
handler = FailedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 2
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)
summary.last_offset = 800
summary.max_offset = 1000
result.summary = summary
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# Both fields are updated to the max value received
# within the result
self.assertEqual(task.backend_args['next_from_date'],
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(task.backend_args['next_offset'], 1000)
self.assertEqual(task.num_failures, 1)
def test_failed_task_rescheduled_no_new_items(self):
"""Check if tasks are rescheduled when no items where generated before"""
handler = FailedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 0
summary.max_updated_on = None
summary.max_offset = None
result.summary = summary
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# Both fields are not updated
self.assertNotIn('next_from_date', task.backend_args)
self.assertNotIn('next_offset', task.backend_args)
def test_ignore_orphan_event(self):
"""Check if an orphan event is ignored"""
handler = FailedJobHandler(self.task_scheduler)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
payload = {
'error': "Error",
'result': result
}
event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload)
handled = handler(event)
self.assertEqual(handled, False)
class TestCompletedJobHandler(TestBaseRQ):
"""Unit tests for CompletedJobHandler"""
def setUp(self):
super().setUp()
self.registry = TaskRegistry()
self.task_scheduler = _TaskScheduler(self.registry, self.conn, [])
def test_initialization(self):
"""Check if the handler is correctly initialized"""
handler = CompletedJobHandler(self.task_scheduler)
self.assertEqual(handler.task_scheduler, self.task_scheduler)
def test_task_rescheduling(self):
"""Check if the task related to the event is re-scheduled"""
handler = CompletedJobHandler(self.task_scheduler)
task = self.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_task_rescheduling_age(self):
"""Check if the task is re-scheduled when maximum age is set"""
handler = CompletedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
# Force the age to be lower than its limit
task.age = 2
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_task_rescheduling_unlimited_age(self):
"""Check if the task is re-scheduled when unlimited age is set"""
handler = CompletedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
# Force the age to be large value
task.age = 1000000
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_task_not_rescheduled_age_limit(self):
"""Check if the task is not re-scheduled when age reaches its limit"""
handler = CompletedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3)
task = self.registry.add('mytask', 'git', 'commit', {},
scheduling_cfg=scheduler_opts)
# Force the age to its pre-defined limit
task.age = 3
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_not_rescheduled_archive_task(self):
"""Check if archive tasks are not rescheduled"""
handler = CompletedJobHandler(self.task_scheduler)
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True)
task = self.registry.add('mytask', 'git', 'commit', {},
archiving_cfg=archiving_cfg)
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_rescheduled_with_next_from_date(self):
"""Check if tasks are rescheduled updating next_from_date"""
handler = CompletedJobHandler(self.task_scheduler)
task = self.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# The field is updated to the last date received
# within the result
self.assertEqual(task.backend_args['next_from_date'],
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
def test_task_rescheduled_with_next_offset(self):
"""Check if tasks are rescheduled updating next_offset"""
handler = CompletedJobHandler(self.task_scheduler)
task = self.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9,
offset=1000)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# Both fields are updated to the last value received
# within the result
self.assertEqual(task.backend_args['next_from_date'],
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(task.backend_args['next_offset'], 1000)
def test_ignore_orphan_event(self):
"""Check if an orphan event is ignored"""
handler = CompletedJobHandler(self.task_scheduler)
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, False)
def setUp(self):
super().setUp()
self.registry = TaskRegistry()
self.task_scheduler = _TaskScheduler(self.registry, self.conn, [])
class TestCompletedJobHandler(TestBaseRQ):
"""Unit tests for CompletedJobHandler"""
def setUp(self):
super().setUp()
self.registry = TaskRegistry(self.conn)
self.task_scheduler = _TaskScheduler(self.registry, self.conn, [])
def test_initialization(self):
"""Check if the handler is correctly initialized"""
handler = CompletedJobHandler(self.task_scheduler)
self.assertEqual(handler.task_scheduler, self.task_scheduler)
def test_task_rescheduling(self):
"""Check if the task related to the event is re-scheduled"""
handler = CompletedJobHandler(self.task_scheduler)
_ = self.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_task_rescheduling_age(self):
"""Check if the task is re-scheduled when maximum age is set"""
handler = CompletedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0,
max_retries=0,
max_age=3)
task = self.registry.add('mytask',
'git',
'commit', {},
scheduling_cfg=scheduler_opts)
# Force the age to be lower than its limit
task.age = 2
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_task_rescheduling_unlimited_age(self):
"""Check if the task is re-scheduled when unlimited age is set"""
handler = CompletedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0,
max_retries=0,
max_age=None)
task = self.registry.add('mytask',
'git',
'commit', {},
scheduling_cfg=scheduler_opts)
# Force the age to be large value
task.age = 1000000
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_task_not_rescheduled_age_limit(self):
"""Check if the task is not re-scheduled when age reaches its limit"""
handler = CompletedJobHandler(self.task_scheduler)
scheduler_opts = SchedulingTaskConfig(delay=0,
max_retries=0,
max_age=3)
task = self.task_scheduler.registry.add('mytask',
'git',
'commit', {},
scheduling_cfg=scheduler_opts)
# Force the age to its pre-defined limit
task.age = 3
self.task_scheduler.registry.update('mytask', task)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.task_scheduler.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_not_rescheduled_archive_task(self):
"""Check if archive tasks are not rescheduled"""
handler = CompletedJobHandler(self.task_scheduler)
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True)
_ = self.task_scheduler.registry.add('mytask',
'git',
'commit', {},
archiving_cfg=archiving_cfg)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.task_scheduler.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_rescheduled_with_next_from_date(self):
"""Check if tasks are rescheduled updating next_from_date"""
handler = CompletedJobHandler(self.task_scheduler)
_ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.task_scheduler.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# The field is updated to the max date received
# within the result
self.assertEqual(task.backend_args['next_from_date'],
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
def test_task_rescheduled_with_next_offset(self):
"""Check if tasks are rescheduled updating next_offset"""
handler = CompletedJobHandler(self.task_scheduler)
_ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
summary.last_offset = 800
summary.max_offset = 1000
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.task_scheduler.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# Both fields are updated to the max value received
# within the result
self.assertEqual(task.backend_args['next_from_date'],
datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC))
self.assertEqual(task.backend_args['next_offset'], 1000)
def test_task_rescheduled_no_new_items(self):
"""Check if tasks are rescheduled when no items where generated before"""
handler = CompletedJobHandler(self.task_scheduler)
_ = self.task_scheduler.registry.add('mytask', 'git', 'commit', {})
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 0
summary.max_updated_on = None
summary.max_offset = None
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.task_scheduler.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
# Both fields are not updated
self.assertNotIn('next_from_date', task.backend_args)
self.assertNotIn('next_offset', task.backend_args)
def test_task_reset_num_failures(self):
"""Check if the number of failures is reset if the task is successful"""
handler = CompletedJobHandler(self.task_scheduler)
task = self.task_scheduler.registry.add('mytask', 'git', 'commit', {})
# Force to a pre-defined number of failures
task.num_failures = 2
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010,
1,
1,
1,
0,
0,
tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014,
2,
12,
6,
10,
39,
tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
task = self.task_scheduler.registry.get('mytask')
self.assertEqual(task.status, TaskStatus.SCHEDULED)
self.assertEqual(task.num_failures, 0)
def test_ignore_orphan_event(self):
"""Check if an orphan event is ignored"""
handler = CompletedJobHandler(self.task_scheduler)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, False)
@unittest.mock.patch('redis.StrictRedis.get')
def test_ignore_event_on_task_registry_error(self, mock_redis_get):
"""Check if an event is ignored when a TaskRegistryError is thrown"""
mock_redis_get.side_effect = RedisError
self.task_scheduler.registry.add('mytask', 'git', 'commit', {})
handler = CompletedJobHandler(self.task_scheduler)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, False)
