def build(workdir, scheds_list, logger_queue: Queue): """ Return TemplateResolver with Scheduler model, appropriate for the system :param workdir: path to working directory :param scheds_list: known Scheduler list :param logger_queue: class instance that should be used for logging :return: TemplateResolver with appropriate Scheduler """ log_writer = NamedLogWriter(logger_queue, 'TemplateResolverBuilder') template_resolver = None executor = Executor() for sched in scheds_list: # type: Scheduler try: template_resolver = TemplateResolver(workdir, sched, logger_queue) caught = False cmd = template_resolver.get_hpc_stats_cmd() executor.execute_shell_command(cmd) except ShellExecutionError as error: log_writer.log('Scheduler {} wasn\'t found. Output: {}'.format( sched.get_name(), error)) caught = True if not caught: return template_resolver raise RuntimeError('Unknown scheduler or no scheduler was found')
def get_cur_shell_dir(): executor = Executor() if is_linux(): res = executor.execute_shell_command('pwd') else: res = executor.execute_shell_command('cd') res = '{}'.format(ShellPath(res)) return res
def __init__(self, streams_handler: StreamsHandler, template_resolver: TemplateResolver, io_handler: IOHandler, logger_queue: Queue, workdir: str, server_addr: str): # for writing files self.__io_handler = io_handler # for receiving/sending data to the web application using sockets self.__streams_handler = streams_handler self.__executor = Executor() # support classes self.__template_resolver = template_resolver self.__compilation_handler = CompilationHandler( template_resolver, logger_queue, workdir) self.__task_stats_handler = StatsCmdHandler(streams_handler, template_resolver, io_handler, logger_queue) self.__hpc_stats_handler = HPCStatsCmdHandler( template_resolver=template_resolver, hpc_stats_resolver=HPCStatsResolver( template_resolver.get_templ_sign(), logger_queue), logger_queue=logger_queue, server_addr=server_addr) self.__workdir = workdir self.__log_writer = NamedLogWriter(logger_queue, 'RunCmdHandler') super().__init__()
def __init__(self, host_n_port, writing_end_of_pipe, logger_queue: Queue, template_resolver=None, workdir='.'): self.__host = host_n_port[0] self.__port = host_n_port[1] self.__handler = RequestHandler(template_resolver, workdir, logger_queue, writing_end_of_pipe) self.__executor = Executor() self.__log_writer = NamedLogWriter(logger_queue, 'TaskServer') self.__log_writer.log('Created instance of TaskServer')
def __init__(self, server_addr: str, logger_queue: Queue, template_resolver: TemplateResolver, hpc_stats_handler: HPCStatsResolver): self.__server_addr = server_addr self.__template_resolver = template_resolver self.__hpc_stats_handler = hpc_stats_handler self.__log_writer = NamedLogWriter(logger_queue, 'HTTPSender') self.__executor = Executor() self.__log_writer.log( 'Created instance with server addr: {}'.format(server_addr))
def __init__(self, server_addr: str, logger_queue: Queue, template_resolver: TemplateResolver, reading_end_of_pipe: Connection, err_threshold=10): self.__err_count = 0 self.__err_threshold = err_threshold self.__reading_conn = reading_end_of_pipe self.__template_resolver = None self.sources = set() self.__senders = [] if server_addr is not None and template_resolver is not None: self.__template_resolver = template_resolver self.__stats_resolver = HPCStatsResolver(template_resolver.get_templ_sign(), logger_queue) self.__senders.append(HTTPSender(server_addr, logger_queue, template_resolver, self.__stats_resolver)) self.sources.add(server_addr) self.__executor = Executor() self.__logger_queue = logger_queue self.__log_writer = NamedLogWriter(logger_queue, 'StatusDaemon') self.__log_writer.log('Created instance of StatusDaemon')
class TaskServer: __server = None __host = None __port = None __handler = None def __init__(self, host_n_port, writing_end_of_pipe, logger_queue: Queue, template_resolver=None, workdir='.'): self.__host = host_n_port[0] self.__port = host_n_port[1] self.__handler = RequestHandler(template_resolver, workdir, logger_queue, writing_end_of_pipe) self.__executor = Executor() self.__log_writer = NamedLogWriter(logger_queue, 'TaskServer') self.__log_writer.log('Created instance of TaskServer') def main_loop(self): """ Synchronous method that starts the async method that will initialize the server :return: nothing """ self.__log_writer.log('Starting TaskServer') exception = self.__executor.wrap_async_call(self.__loop) self.__log_writer.log('{}'.format(exception), LogLevel.ERROR) async def __loop(self): """ Async method that actually initializes server :return: nothing """ self.__log_writer.log('_loop called', LogLevel.DEBUG) self.__server = await asyncio.start_server(self.__handler.safe_handle, self.__host, self.__port) await self.__server.wait_closed() self.__log_writer.log('_loop exited', LogLevel.DEBUG) def shutdown(self): if self.__server is not None: self.__server.close() self.__log_writer.log('Shutting down TaskServer')
class AsyncFileHandler: __executor = Executor() async def async_write_file(self, filename: str, data: bytes): if isinstance(data, type('str')): data = data.encode('utf-8') with open(filename, 'wb') as file: await self.__executor.async_execution( lambda f, to_write: f.write(to_write), file, data) return async def async_read_file(self, filename: str): with open(filename, 'rb') as file: res = await self.__executor.async_execution( lambda f: f.read(), file) return res.decode()
class BaseCmdHandler(AbstractCmdHandler): executor = Executor() @classmethod async def exec_shell_command(cls, command, args=None): """ Form and execute shell command :param command: command to execute :param args: additional arguments :return: command output """ if args is not None and len(args) > 0: cmd = command + ' ' + ' '.join(args) else: cmd = command return await cls.executor.async_exec_shell_command(cmd)
def __init__(self, workdir, logger): self.workdir = workdir self.__log_writer = NamedLogWriter(logger, 'IOHandler') self.__executor = Executor()
class CompilationHandler: __cmake_handler = None __executor = Executor() __cmake_version = '' def __init__(self, template_resolver: TemplateResolver, logger_queue: Queue, workdir: str): self.__workdir = workdir self.__template_resolver = template_resolver self.__log_writer = NamedLogWriter(logger_queue, CompilationHandler.__name__) if self.__check_cmake(): self.__cmake_handler = CMakeHandler(logger_queue, workdir, self.__cmake_version) def __check_cmake(self): """ Check whether the machine supports building projects with CMake :return: True if CMake is supported, False otherwise """ cmake_supported = True try: version = self.__executor.execute_shell_command('cmake --version') self.__cmake_version = version.split()[-1] except ShellExecutionError: cmake_supported = False return cmake_supported async def handle_compilation(self, task: Task, args: list): task.set_bin_name( os.path.join(task.path_to_task_bin(self.__workdir), task.get_name() + '.out')) compilation_commands = self.__get_compile_cmds(task, args) compilation_output = await self.__executor.async_exec_cmds_with_wrapping( commands=compilation_commands, dir_to_use=task.path_to_task_bin(self.__workdir), ) compilation_log_file = os.path.join( task.get_dir_name_for_task(self.__workdir), 'compilation.log') with open(compilation_log_file, 'w') as output_file: output_file.write(compilation_output) return @staticmethod def __is_c_compiler(compiler): c_compiler = 'cc' in compiler \ or '++' in compiler \ or 'xx' in compiler return c_compiler def __get_compile_cmds(self, task: Task, args: list): """ Get compilation command to compile all of the task source files. :param task: related task :param args: additional arguments :return: list of compilation commands """ compiler = task.get_compiler() if '' != compiler: bin_path = task.path_to_task_bin(self.__workdir) if not os.path.exists(bin_path): os.makedirs(bin_path) path = task.path_to_task_src(self.__workdir) if task.is_file_archive(): if self.__cmake_handler is not None and task.uses_cmake(): if not self.__cmake_handler.is_cmake_target(path): self.__cmake_handler.create_cmake_lists(task) commands = self.__cmake_handler.get_compilation_commands_using_cmake( task) else: commands = self.__no_cmake_compile_cmd( compiler, task, args) else: commands = self.__get_compile_cmd_for_single_file( compiler, task, args) self.__log_writer.log(json.dumps(commands, indent=4), level=LogLevel.DEBUG) return commands else: raise RuntimeError('No compiler is set for the task: {}'.format( task.get_name())) def __get_compile_cmd_for_single_file(self, compiler, task, args): files = os.path.join(task.path_to_task_src(self.__workdir), task.get_name()) command = self.__get_compile_cmd_for_args(compiler, files, task, args) return [command] def __no_cmake_compile_cmd(self, compiler, task: Task, args: list): files = self.__get_flat_archive_files(self.__is_c_compiler(compiler), task) command = self.__get_compile_cmd_for_args(compiler, files, task, args) return [command] @staticmethod def __get_compile_cmd_for_args(compiler, files, task, args): command = '{} {} -o {} {} >{} '.format(compiler, files, task.get_bin_name(), ' '.join(args), task.get_log_name()) return command def __get_flat_archive_files(self, c_compiler, task: Task): """ Get files list relevant for compilation :param c_compiler: whether the program would be compiled using C/C++ compiler :param task: task to handle :return: source files list """ path = task.path_to_task_src(self.__workdir) dirfiles = os.listdir(path) if c_compiler: files = list( filter(lambda x: '.cxx' in x or '.cpp' in x or '.c' in x, dirfiles)) else: files = list(filter(lambda x: '.f' in x, dirfiles)) files = list(map(lambda x: os.path.join(path, x), files)) files = ' '.join(files) return files
class StatusDaemon: __looping = None __stats_resolver = None def __init__(self, server_addr: str, logger_queue: Queue, template_resolver: TemplateResolver, reading_end_of_pipe: Connection, err_threshold=10): self.__err_count = 0 self.__err_threshold = err_threshold self.__reading_conn = reading_end_of_pipe self.__template_resolver = None self.sources = set() self.__senders = [] if server_addr is not None and template_resolver is not None: self.__template_resolver = template_resolver self.__stats_resolver = HPCStatsResolver(template_resolver.get_templ_sign(), logger_queue) self.__senders.append(HTTPSender(server_addr, logger_queue, template_resolver, self.__stats_resolver)) self.sources.add(server_addr) self.__executor = Executor() self.__logger_queue = logger_queue self.__log_writer = NamedLogWriter(logger_queue, 'StatusDaemon') self.__log_writer.log('Created instance of StatusDaemon') def main_loop(self): self.__log_writer.log('Starting StatusDaemon') self.__looping = True exception = self.__executor.wrap_async_call(self.__loop) self.__log_writer.log('{}'.format(exception), LogLevel.ERROR) async def __loop(self): self.__log_writer.log('_loop called', LogLevel.DEBUG) try: while self.__looping: try: if self.__reading_conn.poll(): self.__handle_incoming_data() response = await asyncio.gather(*[sender.get_and_send_hpc_stats() for sender in self.__senders]) self.__log_writer.log(response, LogLevel.DEBUG) self.__err_count = 0 except Exception as e: self.__handle_exception(e) await asyncio.sleep(59) finally: self.shutdown() def __handle_incoming_data(self): if self.__template_resolver is None: workdir = self.__reading_conn.recv() sched_json = self.__reading_conn.recv() sched = Scheduler(json.loads(sched_json)) self.__template_resolver = TemplateResolver(workdir, sched, self.__logger_queue) new_server_addr = self.__reading_conn.recv() if new_server_addr not in self.sources: new_sender = HTTPSender(new_server_addr, self.__logger_queue, self.__template_resolver, self.__stats_resolver) self.__senders.append(new_sender) def __handle_exception(self, e): self.__log_writer.log(e, LogLevel.ERROR) def shutdown(self): self.__looping = False self.__log_writer.log('Shutting down StatusDaemon')