def test_set_max_retries(self): """Test if max_retries property can be set""" scheduling_cfg = SchedulingTaskConfig(max_retries=3) self.assertEqual(scheduling_cfg.max_retries, 3) scheduling_cfg.max_retries = 1 self.assertEqual(scheduling_cfg.max_retries, 1)
def test_set_queue(self): """Test if queue property can be set""" scheduling_cfg = SchedulingTaskConfig(queue='myqueue') self.assertEqual(scheduling_cfg.queue, 'myqueue') scheduling_cfg.queue = None self.assertEqual(scheduling_cfg.queue, None)
def test_set_delay(self): """Test if delay property can be set""" scheduling_cfg = SchedulingTaskConfig(delay=5) self.assertEqual(scheduling_cfg.delay, 5) scheduling_cfg.delay = 1 self.assertEqual(scheduling_cfg.delay, 1)
def test_set_max_age(self): """Test if max_age property can be set""" scheduling_cfg = SchedulingTaskConfig(max_age=3) self.assertEqual(scheduling_cfg.max_age, 3) scheduling_cfg.max_age = None self.assertEqual(scheduling_cfg.max_age, None)
def test_to_dict(self): """Check whether the object is converted into a dictionary""" scheduling_cfg = SchedulingTaskConfig(delay=5, max_retries=1) d = scheduling_cfg.to_dict() expected = {'delay': 5, 'max_retries': 1} self.assertDictEqual(d, expected)
def test_set_invalid_queue(self): """Check if an exception is raised for invalid queue values""" with self.assertRaises(ValueError): _ = SchedulingTaskConfig(queue=1.0) scheduling_cfg = SchedulingTaskConfig(queue='myqueue') with self.assertRaises(ValueError): scheduling_cfg.queue = 0 with self.assertRaises(ValueError): scheduling_cfg.queue = -1 self.assertEqual(scheduling_cfg.queue, 'myqueue')
def test_task_jobs_is_updated(self): """Test if the list of job ids for a task is updated""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0) registry = TaskRegistry() task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) schlr.schedule() self.assertEqual(task.age, 1) self.assertEqual(len(task.jobs), 1)
def test_schedule_task(self): """Jobs should be added and executed""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0) registry = TaskRegistry() task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) schlr.schedule() self.assertEqual(task.age, 1) job = schlr._scheduler._queues[Q_CREATION_JOBS].fetch_job(task.jobs[0]) result = job.return_value self.assertEqual(result.task_id, task.task_id) self.assertEqual(result.job_number, 1) self.assertEqual(result.summary.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c') self.assertEqual(result.summary.max_updated_on, datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(result.summary.total, 9)
def test_schedule_task_user_queue(self): """Task should be added and executed in the user's queue""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, queue='myqueue') registry = TaskRegistry() task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) schlr.schedule() self.assertEqual(task.age, 1) job = schlr._scheduler._queues['myqueue'].fetch_job(task.jobs[0]) result = job.return_value self.assertEqual(result.task_id, task.task_id) self.assertEqual(result.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c') self.assertEqual(result.max_date, 1392185439.0) self.assertEqual(result.nitems, 9)
def test_task_not_rescheduled_not_resume(self): """Check if tasks unable to resume are not rescheduled""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'gerrit', 'review', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'gerrit', 'review') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) self.assertEqual(task.num_failures, 1)
def test_schedule_task(self): """Jobs should be added and executed""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0) registry = TaskRegistry() task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) schlr = Scheduler(self.conn, registry, async_mode=False) job_id = schlr.schedule_task(task.task_id) schlr.schedule() job = schlr._scheduler._queues[Q_CREATION_JOBS].fetch_job(job_id) result = job.return_value self.assertEqual(result.last_uuid, '1375b60d3c23ac9b81da92523e4144abc4489d4c') self.assertEqual(result.max_date, 1392185439.0) self.assertEqual(result.nitems, 9)
def test_failed_task_rescheduled_no_new_items(self): """Check if tasks are rescheduled when no items where generated before""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 0 summary.max_updated_on = None summary.max_offset = None result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are not updated self.assertNotIn('next_from_date', task.backend_args) self.assertNotIn('next_offset', task.backend_args)
def test_failed_task_rescheduled_with_next_offset(self): """Check if failed tasks are rescheduled updating next_offset""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) summary.last_offset = 800 summary.max_offset = 1000 result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED) # Both fields are updated to the max value received # within the result self.assertEqual(task.backend_args['next_from_date'], datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)) self.assertEqual(task.backend_args['next_offset'], 1000) self.assertEqual(task.num_failures, 1)
def test_failed_task_not_rescheduled_max_retries(self): """Check if the task is not re-scheduled when num failures reaches its limit""" handler = FailedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force to a pre-defined number of failures task.num_failures = 2 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 2 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) result.summary = summary payload = { 'error': "Error", 'result': result } event = JobEvent(JobEventType.FAILURE, 0, 'mytask', payload) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.FAILED) self.assertEqual(task.num_failures, 3)
def test_init(self): """Test whether object properties are initialized""" scheduling_cfg = SchedulingTaskConfig() self.assertEqual(scheduling_cfg.delay, WAIT_FOR_QUEUING) self.assertEqual(scheduling_cfg.max_retries, MAX_JOB_RETRIES) self.assertEqual(scheduling_cfg.max_age, None) self.assertEqual(scheduling_cfg.queue, None) scheduling_cfg = SchedulingTaskConfig(delay=5, max_retries=1, max_age=10, queue='myqueue') self.assertEqual(scheduling_cfg.delay, 5) self.assertEqual(scheduling_cfg.max_retries, 1) self.assertEqual(scheduling_cfg.max_age, 10) self.assertEqual(scheduling_cfg.queue, 'myqueue')
def test_from_dict_invalid_option(self): """Check if an exception is raised when an invalid option is given""" opts = {'delay': 1, 'max_retries': 3, 'custom_param': 'myparam'} with self.assertRaisesRegex( ValueError, "unknown 'custom_param' task config parameter"): _ = SchedulingTaskConfig.from_dict(opts)
def test_add_task(self): """Test to add tasks to the registry""" args = {'from_date': '1970-01-01', 'component': 'test'} archive = ArchivingTaskConfig('/tmp/archive', False, archived_after=None) sched = SchedulingTaskConfig(delay=10, max_retries=2) registry = TaskRegistry() before = datetime.datetime.now().timestamp() new_task = registry.add('mytask', 'git', 'commit', args) tasks = registry.tasks self.assertEqual(len(tasks), 1) task = tasks[0] self.assertIsInstance(task, Task) self.assertEqual(task, new_task) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.status, TaskStatus.NEW) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) self.assertEqual(task.category, 'commit') self.assertEqual(task.backend, 'git') self.assertDictEqual(task.backend_args, args) self.assertEqual(task.archiving_cfg, None) self.assertEqual(task.scheduling_cfg, None) self.assertGreater(task.created_on, before) before = datetime.datetime.now().timestamp() _ = registry.add('atask', 'git', 'commit', args, archiving_cfg=archive, scheduling_cfg=sched) tasks = registry.tasks self.assertEqual(len(tasks), 2) task0 = tasks[0] self.assertIsInstance(task0, Task) self.assertEqual(task0.task_id, 'atask') self.assertEqual(task0.status, TaskStatus.NEW) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) self.assertEqual(task0.backend, 'git') self.assertEqual(task0.category, 'commit') self.assertDictEqual(task0.backend_args, args) self.assertEqual(task0.archiving_cfg, archive) self.assertEqual(task0.scheduling_cfg, sched) self.assertGreater(task0.created_on, before) task1 = tasks[1] self.assertEqual(task1.task_id, 'mytask') self.assertGreater(task0.created_on, task1.created_on)
def test_from_dict_queue_none(self): """Check if an object is created from a dict when queue is None""" # Test None options opts = {'queue': None} scheduling_cfg = SchedulingTaskConfig.from_dict(opts) self.assertIsInstance(scheduling_cfg, SchedulingTaskConfig) self.assertEqual(scheduling_cfg.queue, None)
def test_from_dict(self): """Check if an object is created when its properties are given from a dict""" opts = {'delay': 1, 'max_retries': 3} scheduling_cfg = SchedulingTaskConfig.from_dict(opts) self.assertIsInstance(scheduling_cfg, SchedulingTaskConfig) self.assertEqual(scheduling_cfg.delay, 1) self.assertEqual(scheduling_cfg.max_retries, 3)
def test_to_dict(self): """Check whether the object is converted into a dict""" args = {'from_date': '1970-01-01', 'component': 'test'} category = 'commit' archive = ArchivingTaskConfig('/tmp/archive', False, archived_after=None) sched = SchedulingTaskConfig(delay=10, max_retries=2, max_age=5, queue='myqueue') before = datetime.datetime.now().timestamp() task = Task('mytask', 'git', category, args, archiving_cfg=archive, scheduling_cfg=sched) task.set_job('job-0', 0) task.set_job('job-1', 1) d = task.to_dict() expected_job_0 = {'job_id': 'job-0', 'job_number': 0} expected_job_1 = {'job_id': 'job-1', 'job_number': 1} expected = { 'task_id': 'mytask', 'status': 'NEW', 'age': 0, 'num_failures': 0, 'jobs': [expected_job_0, expected_job_1], 'backend': 'git', 'backend_args': args, 'category': category, 'archiving_cfg': { 'archive_path': '/tmp/archive', 'archived_after': None, 'fetch_from_archive': False }, 'scheduling_cfg': { 'delay': 10, 'max_retries': 2, 'max_age': 5, 'queue': 'myqueue' } } created_on = d.pop('created_on') self.assertGreater(created_on, before) self.assertDictEqual(d, expected)
def test_set_job_number(self): """Check if the job number is set correctly when a new job is enqueued""" args = { 'uri': 'http://example.com/', 'gitpath': os.path.join(self.dir, 'data/git_log.txt') } category = 'commit' archiving_opts = None scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, queue='myqueue') registry = TaskRegistry(self.conn) task = registry.add('mytask', 'git', category, args, archiving_cfg=archiving_opts, scheduling_cfg=scheduler_opts) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) task = registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) # Modify the list of jobs to pretend this is not the first # time running this task. Job number will be calculated # adding one to the length of jobs. task.jobs = ['A', 'B', 'C'] registry.update('mytask', task) schlr = Scheduler(self.conn, registry, async_mode=False) schlr.schedule_task(task.task_id) task = registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED) self.assertEqual(task.age, 0) schlr.schedule() task = registry.get('mytask') self.assertEqual(task.age, 1) # Get the last job run and check the result job = schlr._scheduler._queues['myqueue'].fetch_job(task.jobs[3].id) result = job.return_value self.assertEqual(result.task_id, task.task_id) self.assertEqual(result.job_number, 4)
def test_backend_not_found(self): """Test if it raises an exception when a backend is not found""" archive = ArchivingTaskConfig('/tmp/archive', False, archived_after=None) sched = SchedulingTaskConfig(delay=10, max_retries=2, max_age=5) with self.assertRaises(NotFoundError) as e: _ = Task('mytask', 'mock_backend', 'mock_item', {}, archiving_cfg=archive, scheduling_cfg=sched) self.assertEqual(e.exception.element, 'mock_backend')
def test_task_not_rescheduled_age_limit(self): """Check if the task is not re-scheduled when age reaches its limit""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to its pre-defined limit task.age = 3 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_rescheduling_unlimited_age(self): """Check if the task is re-scheduled when unlimited age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be large value task.age = 1000000 result = JobResult(0, 'mytask', 'git', 'commit', 'FFFFFFFF', 1392185439.0, 9) event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_to_dict(self): """Check whether the object is converted into a dict""" args = {'from_date': '1970-01-01', 'component': 'test'} category = 'mocked_category' archive = ArchivingTaskConfig('/tmp/archive', False, archived_after=None) sched = SchedulingTaskConfig(delay=10, max_retries=2) before = datetime.datetime.now().timestamp() task = Task('mytask', 'mock_backend', category, args, archiving_cfg=archive, scheduling_cfg=sched) task.jobs.append('job-0') task.jobs.append('job-1') d = task.to_dict() expected = { 'task_id': 'mytask', 'status': 'NEW', 'age': 0, 'jobs': ['job-0', 'job-1'], 'backend': 'mock_backend', 'backend_args': args, 'category': category, 'archiving_cfg': { 'archive_path': '/tmp/archive', 'archived_after': None, 'fetch_from_archive': False }, 'scheduling_cfg': { 'delay': 10, 'max_retries': 2 } } created_on = d.pop('created_on') self.assertGreater(created_on, before) self.assertDictEqual(d, expected)
def test_init(self, mock_utcnow): """Check arguments initialization""" mock_utcnow.return_value = datetime.datetime( 2017, 1, 1, tzinfo=dateutil.tz.tzutc()) args = {'from_date': '1970-01-01', 'component': 'test'} task = Task('mytask', 'git', 'commit', args) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.status, TaskStatus.NEW) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) self.assertEqual(task.backend, 'git') self.assertEqual(task.category, 'commit') self.assertDictEqual(task.backend_args, args) self.assertEqual(task.archiving_cfg, None) self.assertEqual(task.scheduling_cfg, None) self.assertEqual(task.created_on, 1483228800.0) # Test when archive and scheduler arguments are given archive = ArchivingTaskConfig('/tmp/archive', False, archived_after=None) sched = SchedulingTaskConfig(delay=10, max_retries=2, max_age=5) task = Task('mytask', 'git', 'commit', args, archiving_cfg=archive, scheduling_cfg=sched) self.assertEqual(task.task_id, 'mytask') self.assertEqual(task.status, TaskStatus.NEW) self.assertEqual(task.age, 0) self.assertListEqual(task.jobs, []) self.assertEqual(task.backend, 'git') self.assertEqual(task.category, 'commit') self.assertDictEqual(task.backend_args, args) self.assertEqual(task.archiving_cfg, archive) self.assertEqual(task.scheduling_cfg, sched) self.assertEqual(task.created_on, 1483228800.0)
def test_task_not_rescheduled_age_limit(self): """Check if the task is not re-scheduled when age reaches its limit""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=3) task = self.task_scheduler.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to its pre-defined limit task.age = 3 self.task_scheduler.registry.update('mytask', task) result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.task_scheduler.registry.get('mytask') self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_task_rescheduling_unlimited_age(self): """Check if the task is re-scheduled when unlimited age is set""" handler = CompletedJobHandler(self.task_scheduler) scheduler_opts = SchedulingTaskConfig(delay=0, max_retries=0, max_age=None) task = self.registry.add('mytask', 'git', 'commit', {}, scheduling_cfg=scheduler_opts) # Force the age to be large value task.age = 1000000 result = JobResult(0, 1, 'mytask', 'git', 'commit') summary = Summary() summary.fetched = 10 summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC) summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC) result.summary = summary event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result) handled = handler(event) self.assertEqual(handled, True) task = self.registry.get('mytask') self.assertEqual(task.status, TaskStatus.SCHEDULED)
def test_set_invalid_max_retries(self): """Check if an exception is raised for invalid max_retries values""" with self.assertRaises(ValueError): scheduling_cfg = SchedulingTaskConfig(max_retries=2.0) scheduling_cfg = SchedulingTaskConfig(max_retries=3) with self.assertRaises(ValueError): scheduling_cfg.max_retries = 5.0 with self.assertRaises(ValueError): scheduling_cfg.max_retries = '5' self.assertEqual(scheduling_cfg.max_retries, 3)
def test_set_invalid_delay(self): """Check if an exception is raised for invalid delay values""" with self.assertRaises(ValueError): scheduling_cfg = SchedulingTaskConfig(delay=5.0) scheduling_cfg = SchedulingTaskConfig(delay=5) with self.assertRaises(ValueError): scheduling_cfg.delay = 1.0 with self.assertRaises(ValueError): scheduling_cfg.delay = '1' self.assertEqual(scheduling_cfg.delay, 5)