def __init__(self, parent, task_config): self.id = util.short_unique_id() self.__parent = parent self.name = task_config.get('taskName') self.description = task_config.get('taskDescription') self.wd = task_config.get('workingDir') self.wait = task_config.get('wait') self.fail_tolerant = task_config.get('failTolerant') self.__preparation = task_config.get('preparation') self.__commands = task_config.get('commands') self.__children = [] logging.debug('Task - Created [%s] with ID [%s].', str(self.name), str(self.id)) logging.debug('Task - Description [%s].', str(self.description)) children_config = task_config.get('children') if children_config: for config in children_config: self.__children.append(Task(self, config)) self.sub_tasks = []
def __init__(self, job_config): self.__id = util.short_unique_id() self.name = job_config.get('jobName') self.description = job_config.get('jobDescription') self.wd = job_config.get('workingDir') self.timeout = job_config.get('timeout') self.__tasks = [] logging.debug('Job - Created [%s] with ID [%s].', str(self.name), str(self.__id)) logging.debug('Job - Description [%s].', str(self.description)) for task_config in job_config.get('tasks'): self.__tasks.append(Task(None, task_config)) # change working directory os.chdir(self.wd) logging.debug('Job - Will run in the following Working Directory [%s].', str(self.wd)) # ensure timeout signal.signal(signal.SIGALRM, self.__timeout) signal.alarm(self.timeout) # ensure nothing stays running if the tool blows for some reason atexit.register(self.__cleanup) self.multiprocessing_manager = ThreadingManager() self.multiprocessing_manager.start() self.pool = multiprocessing.Pool() self.thread_utils = self.multiprocessing_manager.ThreadingProperties() logging.debug('Job - Initializing a Processing Pool with [%s] cores.', str(multiprocessing.cpu_count()))
def monitor(thread_util, sub_tasks): unique_id = util.short_unique_id() exit_event = multiprocessing.Event() waiting_tasks = [] logging.debug('Manager - Starting ID [%s].', str(unique_id)) while not exit_event.is_set() or not thread_util.is_kill_event_set(): non_finished_tasks = [] if sub_tasks: for (sub_task, process) in sub_tasks: return_code = process.poll() if return_code is not None: # process finished thread_util.remove_process(sub_task.get_parent().id, process.pid) (std_out, std_err) = process.communicate() if return_code == 0: # sub_task finished successfully logging.info('Manager - FINISHED - Task [%s], SubTask [%s].', str(sub_task.get_parent().name), str(sub_task.id)) util.print_output(std_err, std_out) if sub_task.get_parent().wait: # should we wait for others to finish? logging.info('Manager - Waiting for other Tasks to finish.') waiting_tasks.append((sub_task, process)) continue else: # good to go if sub_task.get_parent().has_children(): logging.info('Manager - No need to wait for other processes to finish.') for (s, p) in sub_tasks: util.kill_process(p.pid) thread_util.remove_process(s.get_parent().id, p.pid) logging.info('Manager - Task has Children. Sending Tasks to Processing Queue.') for task in sub_task.get_parent().get_children(): thread_util.add_task(task) exit_event.set() break else: util.print_task_tree(sub_task.get_parent()) logging.info('Manager - Job Finished with success.') exit_event.set() thread_util.kill(0) exit(0) else: # failed tasks goes here logging.info('Manager - FINISHED - Task Failure [%s], SubTask [%s].', str(sub_task.get_parent().name), str(sub_task.id)) util.print_output(std_err, std_out) if sub_task.get_parent().fail_tolerant: logging.info('Manager - The Task is Fail Tolerant.') if thread_util.has_running_processes(sub_task.get_parent().id): # should we wait for others to finish? logging.info('Manager - Waiting for other Tasks to finish.') continue else: # good to go if sub_task.get_parent().has_children(): logging.info('Manager - No need to wait for other processes to finish.') for (s, p) in sub_tasks: util.kill_process(p.pid) thread_util.remove_process(s.get_parent().id, p.pid) logging.info('Manager - Task has Children. Sending Tasks to Processing Queue.') for task in sub_task.get_parent().get_children(): thread_util.add_task(task) exit_event.set() else: util.print_task_tree(sub_task.get_parent()) logging.info( "Manager - Job Finished with success, but Fail Tolerance has been applied.") exit_event.set() thread_util.kill(0) exit(0) elif sub_task.get_parent().wait: # hum this task failed and it seems to be waiting for # the output of another at the same level, the most probable scenario # is that it won't work from here on. Better to kill the Job now. logging.info('Manager - Job Finished with errors.') exit_event.set() thread_util.kill(1) exit(1) else: # hum we cannot proceed to the children tasks because this one failed # lets see if the Job has still tasks running if thread_util.has_running_processes(sub_task.get_parent().id): # OK fine, there are still other tasks at the same level running continue else: # seems we were waiting for this one to complete # better to kill this now logging.info('Manager - Job Finished with errors.') exit_event.set() thread_util.kill(2) exit(2) else: non_finished_tasks.append((sub_task, process)) else: # are there tasks waiting for others to finish? if waiting_tasks: for (sub_task, process) in waiting_tasks: if sub_task.get_parent().has_children(): logging.info('Manager - Task has Children. Sending Tasks to self.manager.') thread_util.add_task(sub_task.get_parent().get_children()) exit_event.set() break else: util.print_task_tree(sub_task.get_parent()) logging.info('Manager - Job Finished with success.') exit_event.set() thread_util.kill(0) exit(0) else: exit_event.set() break # remove elements that were already processed sub_tasks = non_finished_tasks logging.debug('Manager - Finished ID [%s].', str(unique_id)) return
def __create_copy(self, file): new_filename = '{0}-{1}-{2}'.format(self.name, util.short_unique_id(), os.path.basename(file)) shutil.copy2(file, new_filename) return new_filename
def __init__(self, parent_task): self.id = util.short_unique_id() self.name = self.id self.__parent_task = parent_task self.__commands = [] logging.info('SubTask - Created ID [%s] for Task [%s].', str(self.name), str(self.__parent_task.name))