def test_run_on_error(self):
"""Test whether an exception is thrown if a task fails"""
small_delay = 0
manager = TasksManager(self.backend_tasks,
"fake-section",
self.stopper,
self.config,
timer=small_delay)
with self.assertRaises(Exception):
manager.run()
def test_initialization(self):
"""Test whether attributes are initializated"""
small_delay = 0
first_backend = self.backends[list(self.backends.keys())[0]]
manager = TasksManager(self.backend_tasks,
first_backend,
self.stopper,
self.config,
timer=small_delay)
self.assertEqual(manager.config, self.config)
self.assertEqual(manager.stopper, self.stopper)
self.assertEqual(manager.tasks_cls, self.backend_tasks)
self.assertEqual(manager.backend_section, first_backend)
self.assertEqual(manager.timer, small_delay)
self.assertEqual(manager.tasks, [])
def test_add_task(self):
"""Test whether tasks are properly added"""
small_delay = 0
first_backend = list(self.backends.keys())[0]
manager = TasksManager(self.backend_tasks,
first_backend,
self.stopper,
self.config,
timer=small_delay)
self.assertEqual(manager.tasks, [])
for tc in manager.tasks_cls:
task = tc(manager.config)
task.set_backend_section(manager.backend_section)
manager.tasks.append(task)
self.assertEqual(len(manager.tasks), len(manager.tasks_cls))
def execute_batch_tasks(self,
tasks_cls,
big_delay=0,
small_delay=0,
wait_for_threads=True):
"""
Start a task manager per backend to complete the tasks.
:param task_cls: list of tasks classes to be executed
:param big_delay: seconds before global tasks are executed, should be days usually
:param small_delay: seconds before backend tasks are executed, should be minutes
:param wait_for_threads: boolean to set when threads are infinite or
should be synchronized in a meeting point
"""
def _split_tasks(tasks_cls):
"""
we internally distinguish between tasks executed by backend
and tasks executed with no specific backend. """
backend_t = []
global_t = []
for t in tasks_cls:
if t.is_backend_task(t):
backend_t.append(t)
else:
global_t.append(t)
return backend_t, global_t
logger.debug('Tasks Manager starting .. ')
backend_tasks, global_tasks = _split_tasks(tasks_cls)
logger.debug('backend_tasks = %s' % (backend_tasks))
logger.debug('global_tasks = %s' % (global_tasks))
threads = []
# stopper won't be set unless wait_for_threads is True
stopper = threading.Event()
# launching threads for tasks by backend
if len(backend_tasks) > 0:
repos_backend = self._get_repos_by_backend()
for backend in repos_backend:
# Start new Threads and add them to the threads list to complete
t = TasksManager(backend_tasks, backend, stopper, self.config,
small_delay)
threads.append(t)
t.start()
# launch thread for global tasks
if len(global_tasks) > 0:
# FIXME timer is applied to all global_tasks, does it make sense?
# All tasks are executed in the same thread sequentially
gt = TasksManager(global_tasks, "Global tasks", stopper,
self.config, big_delay)
threads.append(gt)
gt.start()
if big_delay > 0:
when = datetime.now() + timedelta(seconds=big_delay)
when_str = when.strftime('%a, %d %b %Y %H:%M:%S %Z')
logger.info("%s will be executed on %s" %
(global_tasks, when_str))
if wait_for_threads:
time.sleep(1) # Give enough time create and run all threads
stopper.set() # All threads must stop in the next iteration
logger.debug(
" Waiting for all threads to complete. This could take a while .."
)
# Wait for all threads to complete
for t in threads:
t.join()
# Checking for exceptions in threads to log them
self.__check_queue_for_errors()
logger.debug(" Task manager and all its tasks (threads) finished!")
def execute_batch_tasks(self,
tasks_cls,
big_delay=0,
small_delay=0,
wait_for_threads=True):
"""
Start a task manager per backend to complete the tasks.
:param task_cls: list of tasks classes to be executed
:param big_delay: seconds before global tasks are executed, should be days usually
:param small_delay: seconds before backend tasks are executed, should be minutes
:param wait_for_threads: boolean to set when threads are infinite or
should be synchronized in a meeting point
"""
# 把task分为两种
# backend_tasks 包括从backend获取数据，并作ecrich工作
# backend_tasks = [<class 'sirmordred.task_collection.TaskRawDataCollection'>, <class 'sirmordred.task_enrich.TaskEnrich'>]
# global_tasks = [<class 'sirmordred.task_projects.TaskProjects'>] 做project加载工作
def _split_tasks(tasks_cls):
"""
we internally distinguish between tasks executed by backend
and tasks executed with no specific backend. """
backend_t = []
global_t = []
for t in tasks_cls:
if t.is_backend_task(t):
backend_t.append(t)
else:
global_t.append(t)
return backend_t, global_t
backend_tasks, global_tasks = _split_tasks(tasks_cls)
logger.debug('backend_tasks = %s' % (backend_tasks))
logger.debug('global_tasks = %s' % (global_tasks))
# 多线程
threads = []
# update 为 false 时, wait_for_threads 为 true ,会去设置一个相当于贯穿多个线程之间的条件变量
# stopper won't be set unless wait_for_threads is True
stopper = threading.Event()
# launching threads for tasks by backend
if len(backend_tasks) > 0:
repos_backend = self._get_repos_by_backend()
for backend in repos_backend:
# 启动线程
# Start new Threads and add them to the threads list to complete
t = TasksManager(backend_tasks, backend, stopper, self.config,
small_delay)
threads.append(t)
t.start()
# launch thread for global tasks
if len(global_tasks) > 0:
# FIXME timer is applied to all global_tasks, does it make sense?
# All tasks are executed in the same thread sequentially
gt = TasksManager(global_tasks, "Global tasks", stopper,
self.config, big_delay)
threads.append(gt)
gt.start()
if big_delay > 0:
when = datetime.now() + timedelta(seconds=big_delay)
when_str = when.strftime('%a, %d %b %Y %H:%M:%S %Z')
logger.info("%s will be executed on %s" %
(' '.join([g.__name__
for g in global_tasks]), when_str))
# update 为 false 时, wait_for_threads 为 true ,stopper 变量申请完默认为 false
# stopper 为false时，其他线程t.start()，但是会阻塞等待，等到调用 set之后为 true，才会开始执行
# 可以去看 task_manager.py
if wait_for_threads:
time.sleep(1) # Give enough time create and run all threads
stopper.set() # All threads must stop in the next iteration
# Wait for all threads to complete
for t in threads:
t.join()
# Checking for exceptions in threads to log them
self.__check_queue_for_errors()
logger.debug(
"[thread:main] All threads (and their tasks) are finished")