def test_set_archived_after_to_utc(self):
"""Check whether dates are converted to UTC"""
dt = datetime.datetime(2001,
12,
1,
23,
15,
32,
tzinfo=dateutil.tz.tzoffset(None, -21600))
expected = datetime.datetime(2001,
12,
2,
5,
15,
32,
tzinfo=dateutil.tz.tzutc())
archiving_cfg = ArchivingTaskConfig('/tmp/archive',
True,
archived_after=dt)
# Date should have been converted to UTC
self.assertEqual(archiving_cfg.archived_after, expected)
archiving_cfg.archived_after = datetime.datetime(
2001, 12, 2, 5, 15, 32)
self.assertEqual(archiving_cfg.archived_after, expected)
def test_set_fetch_from_archive(self):
"""Test if fetch_from_archive property can be set"""
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True)
self.assertEqual(archiving_cfg.fetch_from_archive, True)
archiving_cfg.fetch_from_archive = False
self.assertEqual(archiving_cfg.fetch_from_archive, False)
def test_set_archive_path(self):
"""Test if archive_path property can be set"""
archiving_cfg = ArchivingTaskConfig('/tmp/archive', False)
self.assertEqual(archiving_cfg.archive_path, '/tmp/archive')
archiving_cfg.archive_path = '/tmp/oldarchive'
self.assertEqual(archiving_cfg.archive_path, '/tmp/oldarchive')
def test_add_task(self):
"""Test to add tasks to the registry"""
args = {'from_date': '1970-01-01', 'component': 'test'}
archive = ArchivingTaskConfig('/tmp/archive',
False,
archived_after=None)
sched = SchedulingTaskConfig(delay=10, max_retries=2)
registry = TaskRegistry()
before = datetime.datetime.now().timestamp()
new_task = registry.add('mytask', 'git', 'commit', args)
tasks = registry.tasks
self.assertEqual(len(tasks), 1)
task = tasks[0]
self.assertIsInstance(task, Task)
self.assertEqual(task, new_task)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.status, TaskStatus.NEW)
self.assertEqual(task.age, 0)
self.assertListEqual(task.jobs, [])
self.assertEqual(task.category, 'commit')
self.assertEqual(task.backend, 'git')
self.assertDictEqual(task.backend_args, args)
self.assertEqual(task.archiving_cfg, None)
self.assertEqual(task.scheduling_cfg, None)
self.assertGreater(task.created_on, before)
before = datetime.datetime.now().timestamp()
_ = registry.add('atask',
'git',
'commit',
args,
archiving_cfg=archive,
scheduling_cfg=sched)
tasks = registry.tasks
self.assertEqual(len(tasks), 2)
task0 = tasks[0]
self.assertIsInstance(task0, Task)
self.assertEqual(task0.task_id, 'atask')
self.assertEqual(task0.status, TaskStatus.NEW)
self.assertEqual(task.age, 0)
self.assertListEqual(task.jobs, [])
self.assertEqual(task0.backend, 'git')
self.assertEqual(task0.category, 'commit')
self.assertDictEqual(task0.backend_args, args)
self.assertEqual(task0.archiving_cfg, archive)
self.assertEqual(task0.scheduling_cfg, sched)
self.assertGreater(task0.created_on, before)
task1 = tasks[1]
self.assertEqual(task1.task_id, 'mytask')
self.assertGreater(task0.created_on, task1.created_on)
def test_set_archived_after(self):
"""Test if archived_after property can be set"""
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True)
self.assertEqual(archiving_cfg.archived_after, None)
dt = datetime.datetime(2001, 12, 1, 23, 15, 32,
tzinfo=dateutil.tz.tzutc())
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True,
archived_after=dt)
self.assertEqual(archiving_cfg.archived_after, dt)
dt = datetime.datetime(2018, 1, 1,
tzinfo=dateutil.tz.tzutc())
archiving_cfg.archived_after = dt
self.assertEqual(archiving_cfg.archived_after, dt)
def test_set_archived_after_from_date_string(self):
"""Test if archived_after property can be set from a date string"""
date_str = '2001-12-01 23:15:32 -0600'
expected = datetime.datetime(2001, 12, 2, 5, 15, 32,
tzinfo=dateutil.tz.tzutc())
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True,
archived_after=date_str)
self.assertEqual(archiving_cfg.archived_after, expected)
def test_init(self):
"""Test whether object properties are initialized"""
dt = datetime.datetime(2001, 12, 1, 23, 15, 32,
tzinfo=dateutil.tz.tzutc())
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True,
archived_after=dt)
self.assertEqual(archiving_cfg.archive_path, '/tmp/archive')
self.assertEqual(archiving_cfg.fetch_from_archive, True)
self.assertEqual(archiving_cfg.archived_after, dt)
def test_to_dict(self):
"""Check whether the object is converted into a dict"""
args = {'from_date': '1970-01-01', 'component': 'test'}
category = 'commit'
archive = ArchivingTaskConfig('/tmp/archive',
False,
archived_after=None)
sched = SchedulingTaskConfig(delay=10,
max_retries=2,
max_age=5,
queue='myqueue')
before = datetime.datetime.now().timestamp()
task = Task('mytask',
'git',
category,
args,
archiving_cfg=archive,
scheduling_cfg=sched)
task.set_job('job-0', 0)
task.set_job('job-1', 1)
d = task.to_dict()
expected_job_0 = {'job_id': 'job-0', 'job_number': 0}
expected_job_1 = {'job_id': 'job-1', 'job_number': 1}
expected = {
'task_id': 'mytask',
'status': 'NEW',
'age': 0,
'num_failures': 0,
'jobs': [expected_job_0, expected_job_1],
'backend': 'git',
'backend_args': args,
'category': category,
'archiving_cfg': {
'archive_path': '/tmp/archive',
'archived_after': None,
'fetch_from_archive': False
},
'scheduling_cfg': {
'delay': 10,
'max_retries': 2,
'max_age': 5,
'queue': 'myqueue'
}
}
created_on = d.pop('created_on')
self.assertGreater(created_on, before)
self.assertDictEqual(d, expected)
def test_backend_not_found(self):
"""Test if it raises an exception when a backend is not found"""
archive = ArchivingTaskConfig('/tmp/archive',
False,
archived_after=None)
sched = SchedulingTaskConfig(delay=10, max_retries=2, max_age=5)
with self.assertRaises(NotFoundError) as e:
_ = Task('mytask',
'mock_backend',
'mock_item', {},
archiving_cfg=archive,
scheduling_cfg=sched)
self.assertEqual(e.exception.element, 'mock_backend')
def test_task_not_rescheduled_archive_task(self):
"""Check if archive tasks are not rescheduled"""
handler = CompletedJobHandler(self.task_scheduler)
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True)
task = self.registry.add('mytask', 'git', 'commit', {},
archiving_cfg=archiving_cfg)
result = JobResult(0, 'mytask', 'git', 'commit',
'FFFFFFFF', 1392185439.0, 9)
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_set_invalid_archived_after(self):
"""Check if an exception is raised for invalid archived_after values"""
with self.assertRaisesRegex(ValueError, INVALID_ARCHIVED_AFTER_ERROR):
archiving_cfg = ArchivingTaskConfig('/tmp/archive', False,
archived_after=1.0)
with self.assertRaisesRegex(ValueError, INVALID_ARCHIVED_AFTER_INVALID_DATE_ERROR):
archiving_cfg = ArchivingTaskConfig('/tmp/archive', False,
archived_after='this date')
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True)
with self.assertRaisesRegex(ValueError, INVALID_ARCHIVED_AFTER_ERROR):
archiving_cfg.archived_after = 1.0
with self.assertRaisesRegex(ValueError, INVALID_ARCHIVED_AFTER_INVALID_DATE_ERROR):
archiving_cfg.archived_after = ''
self.assertEqual(archiving_cfg.archived_after, None)
def test_init(self, mock_utcnow):
"""Check arguments initialization"""
mock_utcnow.return_value = datetime.datetime(
2017, 1, 1, tzinfo=dateutil.tz.tzutc())
args = {'from_date': '1970-01-01', 'component': 'test'}
task = Task('mytask', 'git', 'commit', args)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.status, TaskStatus.NEW)
self.assertEqual(task.age, 0)
self.assertListEqual(task.jobs, [])
self.assertEqual(task.backend, 'git')
self.assertEqual(task.category, 'commit')
self.assertDictEqual(task.backend_args, args)
self.assertEqual(task.archiving_cfg, None)
self.assertEqual(task.scheduling_cfg, None)
self.assertEqual(task.created_on, 1483228800.0)
# Test when archive and scheduler arguments are given
archive = ArchivingTaskConfig('/tmp/archive',
False,
archived_after=None)
sched = SchedulingTaskConfig(delay=10, max_retries=2, max_age=5)
task = Task('mytask',
'git',
'commit',
args,
archiving_cfg=archive,
scheduling_cfg=sched)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.status, TaskStatus.NEW)
self.assertEqual(task.age, 0)
self.assertListEqual(task.jobs, [])
self.assertEqual(task.backend, 'git')
self.assertEqual(task.category, 'commit')
self.assertDictEqual(task.backend_args, args)
self.assertEqual(task.archiving_cfg, archive)
self.assertEqual(task.scheduling_cfg, sched)
self.assertEqual(task.created_on, 1483228800.0)
def test_to_dict(self):
"""Check whether the object is converted into a dict"""
args = {'from_date': '1970-01-01', 'component': 'test'}
category = 'mocked_category'
archive = ArchivingTaskConfig('/tmp/archive',
False,
archived_after=None)
sched = SchedulingTaskConfig(delay=10, max_retries=2)
before = datetime.datetime.now().timestamp()
task = Task('mytask',
'mock_backend',
category,
args,
archiving_cfg=archive,
scheduling_cfg=sched)
task.jobs.append('job-0')
task.jobs.append('job-1')
d = task.to_dict()
expected = {
'task_id': 'mytask',
'status': 'NEW',
'age': 0,
'jobs': ['job-0', 'job-1'],
'backend': 'mock_backend',
'backend_args': args,
'category': category,
'archiving_cfg': {
'archive_path': '/tmp/archive',
'archived_after': None,
'fetch_from_archive': False
},
'scheduling_cfg': {
'delay': 10,
'max_retries': 2
}
}
created_on = d.pop('created_on')
self.assertGreater(created_on, before)
self.assertDictEqual(d, expected)
def test_set_invalid_fetch_from_archive(self):
"""Check if an exception is raised for invalid fetch_from_archive values"""
with self.assertRaisesRegex(ValueError, INVALID_FETCH_FROM_ARCHIVE_ERROR):
archiving_cfg = ArchivingTaskConfig('/tmp/archive', 'False')
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True)
with self.assertRaisesRegex(ValueError, INVALID_FETCH_FROM_ARCHIVE_ERROR):
archiving_cfg.fetch_from_archive = 1.0
with self.assertRaisesRegex(ValueError, INVALID_FETCH_FROM_ARCHIVE_ERROR):
archiving_cfg.fetch_from_archive = 'False'
self.assertEqual(archiving_cfg.fetch_from_archive, True)
def test_set_invalid_archive_path(self):
"""Check if an exception is raised for invalid archive_path values"""
with self.assertRaisesRegex(ValueError, INVALID_ARCHIVE_PATH_ERROR):
archiving_cfg = ArchivingTaskConfig(5.0, False)
archiving_cfg = ArchivingTaskConfig('/tmp/archive', False)
with self.assertRaisesRegex(ValueError, INVALID_ARCHIVE_PATH_ERROR):
archiving_cfg.archive_path = 1.0
with self.assertRaisesRegex(ValueError, INVALID_ARCHIVE_PATH_ERROR):
archiving_cfg.archive_path = True
self.assertEqual(archiving_cfg.archive_path, '/tmp/archive')
def test_task_not_rescheduled_archive_task(self):
"""Check if archive tasks are not rescheduled"""
handler = CompletedJobHandler(self.task_scheduler)
archiving_cfg = ArchivingTaskConfig('/tmp/archive', True)
task = self.registry.add('mytask', 'git', 'commit', {},
archiving_cfg=archiving_cfg)
result = JobResult(0, 1, 'mytask', 'git', 'commit')
summary = Summary()
summary.fetched = 10
summary.last_updated_on = datetime.datetime(2010, 1, 1, 1, 0, 0, tzinfo=UTC)
summary.max_updated_on = datetime.datetime(2014, 2, 12, 6, 10, 39, tzinfo=UTC)
result.summary = summary
event = JobEvent(JobEventType.COMPLETED, 0, 'mytask', result)
handled = handler(event)
self.assertEqual(handled, True)
self.assertEqual(task.status, TaskStatus.COMPLETED)
def test_init(self):
"""Check arguments initialization"""
args = {'from_date': '1970-01-01', 'component': 'test'}
before = datetime.datetime.now().timestamp()
task = Task('mytask', 'mock_backend', 'category', args)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.backend, 'mock_backend')
self.assertEqual(task.category, 'category')
self.assertDictEqual(task.backend_args, args)
self.assertEqual(task.archiving_cfg, None)
self.assertEqual(task.scheduling_cfg, None)
self.assertGreater(task.created_on, before)
# Test when archive and scheduler arguments are given
archive = ArchivingTaskConfig('/tmp/archive',
False,
archived_after=None)
sched = SchedulingTaskConfig(delay=10, max_retries=2)
before = datetime.datetime.now().timestamp()
task = Task('mytask',
'mock_backend',
'category',
args,
archiving_cfg=archive,
scheduling_cfg=sched)
self.assertEqual(task.task_id, 'mytask')
self.assertEqual(task.backend, 'mock_backend')
self.assertEqual(task.category, 'category')
self.assertDictEqual(task.backend_args, args)
self.assertEqual(task.archiving_cfg, archive)
self.assertEqual(task.scheduling_cfg, sched)
self.assertGreater(task.created_on, before)
