def __init__(self, parent, task_config):
self.id = util.short_unique_id()
self.__parent = parent
self.name = task_config.get('taskName')
self.description = task_config.get('taskDescription')
self.wd = task_config.get('workingDir')
self.wait = task_config.get('wait')
self.fail_tolerant = task_config.get('failTolerant')
self.__preparation = task_config.get('preparation')
self.__commands = task_config.get('commands')
self.__children = []
logging.debug('Task - Created [%s] with ID [%s].', str(self.name), str(self.id))
logging.debug('Task - Description [%s].', str(self.description))
children_config = task_config.get('children')
if children_config:
for config in children_config:
self.__children.append(Task(self, config))
self.sub_tasks = []
def __init__(self, job_config):
self.__id = util.short_unique_id()
self.name = job_config.get('jobName')
self.description = job_config.get('jobDescription')
self.wd = job_config.get('workingDir')
self.timeout = job_config.get('timeout')
self.__tasks = []
logging.debug('Job - Created [%s] with ID [%s].', str(self.name), str(self.__id))
logging.debug('Job - Description [%s].', str(self.description))
for task_config in job_config.get('tasks'):
self.__tasks.append(Task(None, task_config))
# change working directory
os.chdir(self.wd)
logging.debug('Job - Will run in the following Working Directory [%s].', str(self.wd))
# ensure timeout
signal.signal(signal.SIGALRM, self.__timeout)
signal.alarm(self.timeout)
# ensure nothing stays running if the tool blows for some reason
atexit.register(self.__cleanup)
self.multiprocessing_manager = ThreadingManager()
self.multiprocessing_manager.start()
self.pool = multiprocessing.Pool()
self.thread_utils = self.multiprocessing_manager.ThreadingProperties()
logging.debug('Job - Initializing a Processing Pool with [%s] cores.', str(multiprocessing.cpu_count()))
def monitor(thread_util, sub_tasks):
unique_id = util.short_unique_id()
exit_event = multiprocessing.Event()
waiting_tasks = []
logging.debug('Manager - Starting ID [%s].', str(unique_id))
while not exit_event.is_set() or not thread_util.is_kill_event_set():
non_finished_tasks = []
if sub_tasks:
for (sub_task, process) in sub_tasks:
return_code = process.poll()
if return_code is not None:
# process finished
thread_util.remove_process(sub_task.get_parent().id, process.pid)
(std_out, std_err) = process.communicate()
if return_code == 0:
# sub_task finished successfully
logging.info('Manager - FINISHED - Task [%s], SubTask [%s].', str(sub_task.get_parent().name),
str(sub_task.id))
util.print_output(std_err, std_out)
if sub_task.get_parent().wait:
# should we wait for others to finish?
logging.info('Manager - Waiting for other Tasks to finish.')
waiting_tasks.append((sub_task, process))
continue
else:
# good to go
if sub_task.get_parent().has_children():
logging.info('Manager - No need to wait for other processes to finish.')
for (s, p) in sub_tasks:
util.kill_process(p.pid)
thread_util.remove_process(s.get_parent().id, p.pid)
logging.info('Manager - Task has Children. Sending Tasks to Processing Queue.')
for task in sub_task.get_parent().get_children():
thread_util.add_task(task)
exit_event.set()
break
else:
util.print_task_tree(sub_task.get_parent())
logging.info('Manager - Job Finished with success.')
exit_event.set()
thread_util.kill(0)
exit(0)
else:
# failed tasks goes here
logging.info('Manager - FINISHED - Task Failure [%s], SubTask [%s].',
str(sub_task.get_parent().name), str(sub_task.id))
util.print_output(std_err, std_out)
if sub_task.get_parent().fail_tolerant:
logging.info('Manager - The Task is Fail Tolerant.')
if thread_util.has_running_processes(sub_task.get_parent().id):
# should we wait for others to finish?
logging.info('Manager - Waiting for other Tasks to finish.')
continue
else:
# good to go
if sub_task.get_parent().has_children():
logging.info('Manager - No need to wait for other processes to finish.')
for (s, p) in sub_tasks:
util.kill_process(p.pid)
thread_util.remove_process(s.get_parent().id, p.pid)
logging.info('Manager - Task has Children. Sending Tasks to Processing Queue.')
for task in sub_task.get_parent().get_children():
thread_util.add_task(task)
exit_event.set()
else:
util.print_task_tree(sub_task.get_parent())
logging.info(
"Manager - Job Finished with success, but Fail Tolerance has been applied.")
exit_event.set()
thread_util.kill(0)
exit(0)
elif sub_task.get_parent().wait:
# hum this task failed and it seems to be waiting for
# the output of another at the same level, the most probable scenario
# is that it won't work from here on. Better to kill the Job now.
logging.info('Manager - Job Finished with errors.')
exit_event.set()
thread_util.kill(1)
exit(1)
else:
# hum we cannot proceed to the children tasks because this one failed
# lets see if the Job has still tasks running
if thread_util.has_running_processes(sub_task.get_parent().id):
# OK fine, there are still other tasks at the same level running
continue
else:
# seems we were waiting for this one to complete
# better to kill this now
logging.info('Manager - Job Finished with errors.')
exit_event.set()
thread_util.kill(2)
exit(2)
else:
non_finished_tasks.append((sub_task, process))
else:
# are there tasks waiting for others to finish?
if waiting_tasks:
for (sub_task, process) in waiting_tasks:
if sub_task.get_parent().has_children():
logging.info('Manager - Task has Children. Sending Tasks to self.manager.')
thread_util.add_task(sub_task.get_parent().get_children())
exit_event.set()
break
else:
util.print_task_tree(sub_task.get_parent())
logging.info('Manager - Job Finished with success.')
exit_event.set()
thread_util.kill(0)
exit(0)
else:
exit_event.set()
break
# remove elements that were already processed
sub_tasks = non_finished_tasks
logging.debug('Manager - Finished ID [%s].', str(unique_id))
return
def __create_copy(self, file):
new_filename = '{0}-{1}-{2}'.format(self.name, util.short_unique_id(), os.path.basename(file))
shutil.copy2(file, new_filename)
return new_filename
def __init__(self, parent_task):
self.id = util.short_unique_id()
self.name = self.id
self.__parent_task = parent_task
self.__commands = []
logging.info('SubTask - Created ID [%s] for Task [%s].', str(self.name), str(self.__parent_task.name))
