def __init__(self, job_config): self.__id = util.short_unique_id() self.name = job_config.get('jobName') self.description = job_config.get('jobDescription') self.wd = job_config.get('workingDir') self.timeout = job_config.get('timeout') self.__tasks = [] logging.debug('Job - Created [%s] with ID [%s].', str(self.name), str(self.__id)) logging.debug('Job - Description [%s].', str(self.description)) for task_config in job_config.get('tasks'): self.__tasks.append(Task(None, task_config)) # change working directory os.chdir(self.wd) logging.debug('Job - Will run in the following Working Directory [%s].', str(self.wd)) # ensure timeout signal.signal(signal.SIGALRM, self.__timeout) signal.alarm(self.timeout) # ensure nothing stays running if the tool blows for some reason atexit.register(self.__cleanup) self.multiprocessing_manager = ThreadingManager() self.multiprocessing_manager.start() self.pool = multiprocessing.Pool() self.thread_utils = self.multiprocessing_manager.ThreadingProperties() logging.debug('Job - Initializing a Processing Pool with [%s] cores.', str(multiprocessing.cpu_count()))
class Job: """ Job constructor :param job_config """ def __init__(self, job_config): self.__id = util.short_unique_id() self.name = job_config.get('jobName') self.description = job_config.get('jobDescription') self.wd = job_config.get('workingDir') self.timeout = job_config.get('timeout') self.__tasks = [] logging.debug('Job - Created [%s] with ID [%s].', str(self.name), str(self.__id)) logging.debug('Job - Description [%s].', str(self.description)) for task_config in job_config.get('tasks'): self.__tasks.append(Task(None, task_config)) # change working directory os.chdir(self.wd) logging.debug('Job - Will run in the following Working Directory [%s].', str(self.wd)) # ensure timeout signal.signal(signal.SIGALRM, self.__timeout) signal.alarm(self.timeout) # ensure nothing stays running if the tool blows for some reason atexit.register(self.__cleanup) self.multiprocessing_manager = ThreadingManager() self.multiprocessing_manager.start() self.pool = multiprocessing.Pool() self.thread_utils = self.multiprocessing_manager.ThreadingProperties() logging.debug('Job - Initializing a Processing Pool with [%s] cores.', str(multiprocessing.cpu_count())) """ Get Tasks :return tasks """ def get_tasks(self): return self.__tasks """ Start the Job """ def start(self): logging.info('Job - Starting Job [%s].', str(self.name)) logging.info('Job - Sending Tasks to Processing Queue.') for task in self.__tasks: self.thread_utils.add_task(task) # runs while none of the threads notify to stop with an exit code while not self.thread_utils.is_kill_event_set(): try: task = self.thread_utils.get_task() logging.info('Job - Starting processing Task [%s].', str(task.name)) sub_tasks = task.get_sub_tasks() for sub_task in sub_tasks: self.pool.apply_async(worker.do, args=(self.thread_utils, sub_task,)) except Queue.Empty: # seems nothing is the queue for now time.sleep(1) pass exit(self.thread_utils.get_result_code()) def __timeout(self, signum, frame): logging.info('Job - Processing timed out after %s seconds.', str(self.timeout)) exit(5) def __cleanup(self): logging.debug('Job - Closing Processing Pool.') self.pool.close() logging.info('Job - Bye Bye.')