def test_task_wrong_archive_after(self): """Check whether an exception is thrown when archived_after parameter is not properly set""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} archive_params = {"fetch_from_archive": True, "archived_after": "X"} app = Arthur(self.conn, async_mode=False) with self.assertRaises(ValueError) as ex: app.add_task(task_id, backend, category, backend_params, archive_params) self.assertEqual( ex.exception.args[0], "archive_args.archived_after datetime format not valid") archive_params = {"fetch_from_archive": True} with self.assertRaises(ValueError) as ex: app.add_task(task_id, backend, category, backend_params, archive_params) self.assertEqual(ex.exception.args[0], "archive_args.archived_after not defined")
def test_task_wrong_fetch_from_archive(self): """Check whether an exception is thrown when fetch_from_archive parameter is not properly set""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} archive_params = { "fetch_from_archive": 100, "archived_after": "2010-10-10" } app = Arthur(self.conn, async_mode=False) with self.assertRaises(ValueError) as ex: app.add_task(task_id, backend, category, backend_params, archive_params) self.assertEqual(ex.exception.args[0], "archive_args.fetch_from_archive not boolean") archive_params = {"archived_after": "2010-10-10"} with self.assertRaises(ValueError) as ex: app.add_task(task_id, backend, category, backend_params, archive_params) self.assertEqual(ex.exception.args[0], "archive_args.fetch_from_archive not defined")
def test_task_default_archive_path(self): """Check whether a default archive path is added when not defined in the archive params""" task_id = "arthur.task" backend = "backend" category = "mock_category" backend_params = {"a": "a", "b": "b"} archive_params = { 'fetch_from_archive': True, 'archived_after': "2010-10-10" } app = Arthur(self.conn, async_mode=False) app.add_task(task_id, backend, category, backend_params, archive_params) t = app._tasks.tasks[0] archiving_opts = t.archiving_cfg expected_dt = datetime.datetime(2010, 10, 10, 0, 0, 0, tzinfo=dateutil.tz.tzutc()) self.assertIsInstance(archiving_opts, ArchivingTaskConfig) self.assertEqual(archiving_opts.archive_path, os.path.expanduser(ARCHIVES_DEFAULT_PATH)) self.assertEqual(archiving_opts.fetch_from_archive, True) self.assertEqual(archiving_opts.archived_after, expected_dt)
def test_add_task_archive(self): """Check whether tasks are added""" task_id = "arthur.task" backend = "backend" category = 'acme-product' backend_params = {"a": "a", "b": "b"} archive_params = { 'fetch_from_archive': True, 'archived_after': '2100-01-01', 'archive_path': './acme-plan' } sched_params = {"delay": 10, "max_retries_job": 10} app = Arthur(self.conn, base_archive_path=self.tmp_path, async_mode=False) initial_tasks = len(app._tasks.tasks) app.add_task(task_id, backend, category, backend_params, archive_params, sched_params) after_tasks = len(app._tasks.tasks) t = app._tasks.tasks[0] self.assertEqual(t.task_id, task_id) self.assertEqual(t.backend, backend) self.assertEqual(t.category, category) self.assertDictEqual(t.backend_args, backend_params) self.assertDictEqual(t.archive_args, archive_params) self.assertDictEqual(t.sched_args, sched_params) self.assertEqual(initial_tasks, 0) self.assertEqual(after_tasks, 1)
def test_remove_task(self): """Check whether the removal of tasks is properly handled""" task_1 = "arthur.task-1" task_2 = "arthur.task-2" task_params = { "backend": "backend", "category": "category", "backend_params": { "a": "a", "b": "b" } } app = Arthur(self.conn, async_mode=False) app.add_task(task_1, task_params['backend'], task_params['category'], task_params['backend_params']) app.add_task(task_2, task_params['backend'], task_params['category'], task_params['backend_params']) tasks = len(app._tasks.tasks) self.assertEqual(tasks, 2) self.assertTrue(app.remove_task(task_1)) self.assertTrue(app.remove_task(task_2)) tasks = len(app._tasks.tasks) self.assertEqual(tasks, 0)
def test_add_task_empty_archive(self): """Check when a task has an empty archive""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} archive_params = {} app = Arthur(self.conn, async_mode=False) initial_tasks = len(app._tasks.tasks) app.add_task(task_id, backend, category, backend_params, archive_params) after_tasks = len(app._tasks.tasks) t = app._tasks.tasks[0] self.assertEqual(t.task_id, task_id) self.assertEqual(t.backend, backend) self.assertDictEqual(t.backend_args, backend_params) self.assertDictEqual(t.archive_args, archive_params) self.assertDictEqual(t.sched_args, {}) self.assertEqual(initial_tasks, 0) self.assertEqual(after_tasks, 1)
def test_reschedule(self): """Check whether it reschedules a failed task""" task_id = "arthur.task" backend = "git" category = "commit" backend_params = {"a": "a", "b": "b"} sched_params = {"max_retries": 5} app = Arthur(self.conn, async_mode=False) app.add_task(task_id, backend, category, backend_params, sched_args=sched_params) # Set task as failed t = app._tasks.tasks[0] t.status = TaskStatus.FAILED t.age = 100 t.num_failures = 5 app._tasks.update(task_id, t) # Task will be scheduled again after calling the method result = app.reschedule_task(task_id) # Update task values t = app._tasks.get(task_id) self.assertEqual(result, True) self.assertEqual(t.status, TaskStatus.SCHEDULED) self.assertEqual(t.age, 0) self.assertEqual(t.num_failures, 0)
def test_items(self): new_path = os.path.join(self.tmp_path, 'newgit') app = Arthur(self.conn, async_mode=False) app.add_task('test', 'git', {'uri': self.git_path, 'gitpath': new_path}) app.start() commits = [item['data']['commit'] for item in app.items()] expected = ['bc57a9209f096a130dcc5ba7089a8663f758a703', '87783129c3f00d2c81a3a8e585eb86a47e39891a', '7debcf8a2f57f86663809c58b5c07a398be7674c', 'c0d66f92a95e31c77be08dc9d0f11a16715d1885', 'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb', '589bb080f059834829a2a5955bebfd7c2baa110a', 'ce8e0b86a1e9877f42fe9453ede418519115f367', '51a3b654f252210572297f47597b31527c475fb8', '456a68ee1407a77f3e804a30dff245bb6c6b872f'] self.assertListEqual(commits, expected) commits = [item for item in app.items()] self.assertListEqual(commits, []) shutil.rmtree(new_path)
def test_add_task_registry_error(self, mock_redis_set): """Check whether a TaskRegistryError exception is thrown when a task cannot be added""" mock_redis_set.side_effect = RedisError app = Arthur(self.conn, async_mode=False) with self.assertRaises(TaskRegistryError): app.add_task("arthur.task", "git", "commit", {"a": "a", "b": "b"})
def test_add_duplicated_task(self): """Check whether an exception is thrown when a duplicated task is added""" app = Arthur(self.conn, async_mode=False) app.add_task("arthur.task", "git", "commit", {"a": "a", "b": "b"}) with self.assertRaises(AlreadyExistsError): app.add_task("arthur.task", "backend", "category", { "a": "a", "b": "b" })
def test_add_task_invalid_format_backend_args(self): """Check whether an exception is thrown when the backend args is not a dict""" task_id = "arthur.task" backend = "backend" category = "mock_item" backend_params = "wrong_params" app = Arthur(self.conn, async_mode=False) with self.assertRaisesRegex(ValueError, re.compile(INVALID_BACKEND_ARGS_ERROR)): app.add_task(task_id, backend, category, backend_params)
def test_add_task_archive(self): """Check whether tasks are added""" task_id = "arthur.task" backend = "backend" category = 'acme-product' backend_params = {"a": "a", "b": "b"} archive_args = { 'fetch_from_archive': True, 'archived_after': '2100-01-01' } sched_args = {'delay': 10, 'max_retries': 5} app = Arthur(self.conn, base_archive_path=self.tmp_path, async_mode=False) initial_tasks = len(app._tasks.tasks) app.add_task(task_id, backend, category, backend_params, archive_args, sched_args) after_tasks = len(app._tasks.tasks) t = app._tasks.tasks[0] self.assertEqual(t.task_id, task_id) self.assertEqual(t.backend, backend) self.assertEqual(t.category, category) self.assertDictEqual(t.backend_args, backend_params) archiving_opts = t.archiving_cfg expected_dt = datetime.datetime(2100, 1, 1, 0, 0, 0, tzinfo=dateutil.tz.tzutc()) self.assertIsInstance(archiving_opts, ArchivingTaskConfig) self.assertEqual(archiving_opts.archive_path, self.tmp_path) self.assertEqual(archiving_opts.fetch_from_archive, True) self.assertEqual(archiving_opts.archived_after, expected_dt) sched_opts = t.scheduling_cfg self.assertIsInstance(sched_opts, SchedulingTaskConfig) self.assertEqual(sched_opts.delay, 10) self.assertEqual(sched_opts.max_retries, 5) self.assertEqual(initial_tasks, 0) self.assertEqual(after_tasks, 1)
def test_task_wrong_archive_after(self): """Check whether an exception is thrown when archived_after parameter is not properly set""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} archive_params = {"fetch_from_archive": True, "archived_after": "X"} app = Arthur(self.conn, async_mode=False) with self.assertRaisesRegex(ValueError, INVALID_ARCHIVED_AFTER_INVALID_DATE_ERROR): app.add_task(task_id, backend, category, backend_params, archive_params)
def test_task_ignore_archive_after(self): """Check whether the archived_after parameter is not set when fetch_from_archive is false""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} archive_params = {"fetch_from_archive": False, "archived_after": "X"} app = Arthur(self.conn, async_mode=False) app.add_task(task_id, backend, category, backend_params, archive_params) t = app._tasks.tasks[0] self.assertEqual(t.archive_args['fetch_from_archive'], False) self.assertIsNone(t.archive_args['archived_after'])
def test_add_task_no_task_id(self): """Check whether an exception is thrown when the task id is missing""" task_id = None backend = "backend" category = "mock_category" backend_params = {"a": "a", "b": "b"} app = Arthur(self.conn, async_mode=False) with self.assertRaisesRegex(ValueError, MISSING_TASK_ID_ERROR): app.add_task(task_id, backend, category, backend_params) task_id = " " with self.assertRaisesRegex(ValueError, MISSING_TASK_ID_ERROR): app.add_task(task_id, backend, category, backend_params)
def test_task_ignore_archive_after(self): """Check whether the archived_after parameter is not set when fetch_from_archive is false""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} archive_params = {"fetch_from_archive": False, "archived_after": "X"} app = Arthur(self.conn, async_mode=False) with self.assertRaisesRegex(ValueError, INVALID_ARCHIVED_AFTER_INVALID_DATE_ERROR): app.add_task(task_id, backend, category, backend_params, archive_params) self.assertEqual(len(app._tasks.tasks), 0)
def test_task_wrong_delay(self): """Check whether an exception is thrown when delay parameter is not properly set""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} sched_params = {"delay": "1", "max_retries": 10} app = Arthur(self.conn, async_mode=False) with self.assertRaisesRegex(ValueError, INVALID_SCHEDULER_DELAY) as ex: app.add_task(task_id, backend, category, backend_params, sched_args=sched_params)
def test_add_task_no_category(self): """Check whether an exception is thrown when the backend category is missing""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} app = Arthur(self.conn, async_mode=False) with self.assertRaisesRegex(ValueError, re.compile(MISSING_CATEGORY_ERROR)): app.add_task(task_id, backend, category, backend_params) category = " " with self.assertRaisesRegex(ValueError, re.compile(MISSING_CATEGORY_ERROR)): app.add_task(task_id, backend, category, backend_params)
def test_task_unknown_scheduler_parameter(self): """Check whether an exception is thrown when an unknown parameter is in the sched params""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} sched_params = {"not_valid_param": 1, "delay": 10, "max_retries": 10} app = Arthur(self.conn, async_mode=False) with self.assertRaisesRegex(ValueError, re.compile(INVALID_SCHEDULER_PARAM)): app.add_task(task_id, backend, category, backend_params, sched_args=sched_params)
def test_task_wrong_delay(self): """Check whether an exception is thrown when delay parameter is not properly set""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} sched_params = {"delay": "1", "max_retries_job": 10} app = Arthur(self.conn, async_mode=False) with self.assertRaises(ValueError) as ex: app.add_task(task_id, backend, category, backend_params, sched_args=sched_params) self.assertEqual(ex.exception.args[0], "sched_args.delay not int")
def test_task_unknown_archive_parameter(self): """Check whether an exception is thrown when an unknown parameter is in the archive params""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} archive_params = { "unknown": 1, "fetch_from_archive": True, "archived_after": "2010-10-10" } app = Arthur(self.conn, async_mode=False) with self.assertRaisesRegex(ValueError, re.compile(UNKNOWN_ARCHIVE_PARAMETER)): app.add_task(task_id, backend, category, backend_params, archive_params)
def test_task_unknown_scheduler_parameter(self): """Check whether an exception is thrown when an unknown parameter is in the sched params""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} sched_params = {"unknown": 1, "delay": 10, "max_retries_job": 10} app = Arthur(self.conn, async_mode=False) with self.assertRaises(ValueError) as ex: app.add_task(task_id, backend, category, backend_params, sched_args=sched_params) self.assertEqual(ex.exception.args[0], "unknown not accepted in schedule_args")
def test_task_unknown_archive_parameter(self): """Check whether an exception is thrown when an unknown parameter is in the archive params""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} archive_params = { "unknown": 1, "fetch_from_archive": True, "archived_after": "2010-10-10" } app = Arthur(self.conn, async_mode=False) with self.assertRaises(ValueError) as ex: app.add_task(task_id, backend, category, backend_params, archive_params) self.assertEqual(ex.exception.args[0], "unknown not accepted in archive_args")
def test_reschedule_non_failed_task(self): """Check whether the method does nothing when re-scheduling a non failed task""" task_id = "arthur.task" backend = "git" category = "commit" backend_params = {"a": "a", "b": "b"} app = Arthur(self.conn, async_mode=False) app.add_task(task_id, backend, category, backend_params) # Force a different status t = app._tasks.tasks[0] t.status = TaskStatus.RUNNING app._tasks.update(task_id, t) result = app.reschedule_task('arthur.task') # The status should not have changed because # the task is a non failed one t = app._tasks.get(task_id) self.assertEqual(result, False) self.assertEqual(t.status, TaskStatus.RUNNING)
def test_task_wrong_fetch_from_archive(self): """Check whether an exception is thrown when fetch_from_archive parameter is not properly set""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} archive_params = { "fetch_from_archive": 100, "archived_after": "2010-10-10" } app = Arthur(self.conn, async_mode=False) with self.assertRaisesRegex(ValueError, INVALID_FETCH_FROM_ARCHIVE): app.add_task(task_id, backend, category, backend_params, archive_params) archive_params = {"archived_after": "2010-10-10"} with self.assertRaisesRegex( TypeError, re.compile(MISSING_FETCH_FROM_ARCHIVE_PARAM)): app.add_task(task_id, backend, category, backend_params, archive_params)
def test_task_default_archive_path(self): """Check whether a default archive path is added when not defined in the archive params""" task_id = "arthur.task" backend = "backend" category = None backend_params = {"a": "a", "b": "b"} archive_params = { "fetch_from_archive": True, "archived_after": "2010-10-10" } app = Arthur(self.conn, async_mode=False) app.add_task(task_id, backend, category, backend_params, archive_params) t = app._tasks.tasks[0] self.assertEqual(t.archive_args['fetch_from_archive'], archive_params["fetch_from_archive"]) self.assertEqual(t.archive_args['archived_after'], archive_params["archived_after"]) self.assertEqual(t.archive_args['archive_path'], os.path.expanduser(ARCHIVES_DEFAULT_PATH))
def test_add_task_no_archive(self): """Check when a task has no archive params""" task_id = "arthur.task" backend = "backend" category = "mock_category" backend_params = {"a": "a", "b": "b"} app = Arthur(self.conn, async_mode=False) initial_tasks = len(app._tasks.tasks) app.add_task(task_id, backend, category, backend_params) after_tasks = len(app._tasks.tasks) t = app._tasks.tasks[0] self.assertEqual(t.task_id, task_id) self.assertEqual(t.backend, backend) self.assertDictEqual(t.backend_args, backend_params) self.assertEqual(t.archiving_cfg, None) self.assertEqual(t.scheduling_cfg, None) self.assertEqual(initial_tasks, 0) self.assertEqual(after_tasks, 1)