async def handle(self, task: Task, args: list):
"""
Method for handling compilation and scheduling of the incoming task
:param task: incoming task
:param args: additional arguments for compilation
:raises CmdHandlerError
:return: nothing
"""
self.__set_task_log(task)
task = await self.__streams_handler.recv_file(task)
task.set_status(TaskStatus.COMPILING)
await self.__io_handler.save_task(task)
await self.__streams_handler.inform_client(task, 'Task is compiling')
try:
await self.__compilation_handler.handle_compilation(task, args)
if self.__template_resolver.sched_uses_desc():
await self.__create_task_passport(task)
await self.__run_task(task)
# Update HPC info on the web application side
await self.__hpc_stats_handler.handle(task, [])
except Exception as e:
raise CmdHandlerError(Command.RUN, e)
async def save_task(self, task: Task):
target_dir = task.path_to_task_etc(self.workdir)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
task_info = os.path.join(target_dir, self.info_filename)
await self.__async_rw.async_write_file(task_info,
task.to_json(indent=4))
return
def __extract_path_files_and_zip_res(self, task: Task, zip_name: str):
path = task.get_dir_name_for_task(self.workdir)
if not os.path.exists(path):
raise RuntimeError(
'Cannot get files for task {}. No directory is present for this task.'
.format(task.get_name()))
files = os.listdir(path)
zip_res = os.path.join(path, zip_name)
return path, files, zip_res
async def __cancel_task(self, task: Task, args: list):
"""
Handle 'cancel' command
:param task: related task
:param args: additional arguments
:return: None
"""
cancel_command = self.__template_resolver.get_cancel_for_task(task)
await self.exec_shell_command(cancel_command, args)
task.set_status(TaskStatus.CANCELLED)
return None
async def recv_file(self, task: Task) -> Task:
"""
Receive incoming file from the reader channel
:param task: related task
:return: task
"""
bufsize = task.get_filesize()
in_file = await self.__in_stream.read(bufsize)
recv_name = task.get_name()
await self.__io_handler.write_recvd_data(task, recv_name, in_file)
return task
async def inform_client(self,
task: Task,
message: str,
error: Exception = None) -> None:
"""
Send information to the server using async writer channel
:param task: related task
:param error: related error message
:param message: message to send
:return: nothing
"""
to_send = {'task_info': {}, 'message': '', 'error': ''}
if task is not None:
# Add info about the received task
task_msg = {
'name': task.get_name(),
'username': task.get_user(),
'status': task.get_str_status()
}
to_send['task_info'] = task_msg
to_send['message'] = message
levelname = LogLevel.DEBUG
if error is not None and error != 'None':
error = '{}'.format(error)
idx = error.find(self.__ERROR_DESC_START)
if idx >= 0:
error = error[idx + len(self.__ERROR_DESC_START):]
to_send['error'] = '{}'.format(error)
to_send['task_info']['status'] = 'ERROR'
levelname = LogLevel.ERROR
self.__log_writer.log('Request response: {}'.format(
json.dumps(to_send, indent=4)),
level=levelname)
try:
# Sending response to CKD web application
self.__out_stream.write(json.dumps(to_send, indent=4).encode())
await self.__out_stream.drain()
except Exception as e:
self.__log_writer.log(
'Error occurred while responding to the CKD application: {}'.
format(e))
self.__log_writer.log('Response sent.', level=LogLevel.DEBUG)
async def write_recvd_data(self, task: Task, recv_name: str,
recvd_data: bytes):
# Create directory for task files if it doesn't exist
srcdir = task.path_to_task_src(self.workdir)
if not os.path.exists(srcdir):
os.makedirs(srcdir)
# Write incoming file on disk
recv_file = os.path.join(srcdir, recv_name)
await self.__async_rw.async_write_file(recv_file, recvd_data)
if task.is_file_archive():
await self.__executor.async_execution(self.__handle_archive, task,
srcdir, recv_file)
return
def create_cmake_lists(self, task: Task):
"""
Generates CMakeLists.txt to make an archive into the CMake project
:param task: task to handle
:return: None
"""
path = task.path_to_task_src(self.__workdir)
cmake_lists_filename = os.path.join(path, self.__C_MAKE_LISTS_TXT__)
main_file = self.__get_main_file_in_project(path)
task_name = task.get_name()
file_contents = [
'cmake_minimum_required(VERSION {})'.format(self.__cmake_version),
'project({})'.format(task_name),
'SET(CMAKE_BUILD_TYPE Release)',
]
extension = main_file.split('.')[-1]
file_contents.append('file(GLOB SOURCES "./*.{}")'.format(extension))
file_contents.append(' '.join(
['add_executable('.format(task_name), '${SOURCES})']))
dirs = self.__filter_dirs(path)
include_dirs = list(filter(lambda x: 'include' in x, dirs))
include_dirs = set(include_dirs)
file_contents.append('include_directories({})'.format(
', '.join(include_dirs)))
dirs = set(dirs) - include_dirs
for dir_name in dirs:
file_contents.append('add_subdirectory({})'.format(dir_name))
file_contents = '\n'.join(file_contents)
file_contents = correct_line_breaks(file_contents)
with open(cmake_lists_filename, 'w') as cmake_lists_file:
cmake_lists_file.write(file_contents)
def parse_net_command(json_bytes_with_sep):
json_string = json_bytes_with_sep.rstrip(b'\0').decode()
dct = json.loads(json_string)
source = dct['source'].rstrip('/')
cmd = Command(dct['cmd'])
if dct['cmd'] != 'hpc_stats':
return source, cmd, Task(dct['task_info']), dct['args']
else:
return source, cmd, None, None
def __get_compile_cmds(self, task: Task, args: list):
"""
Get compilation command to compile all of the task source files.
:param task: related task
:param args: additional arguments
:return: list of compilation commands
"""
compiler = task.get_compiler()
if '' != compiler:
bin_path = task.path_to_task_bin(self.__workdir)
if not os.path.exists(bin_path):
os.makedirs(bin_path)
path = task.path_to_task_src(self.__workdir)
if task.is_file_archive():
if self.__cmake_handler is not None and task.uses_cmake():
if not self.__cmake_handler.is_cmake_target(path):
self.__cmake_handler.create_cmake_lists(task)
commands = self.__cmake_handler.get_compilation_commands_using_cmake(
task)
else:
commands = self.__no_cmake_compile_cmd(
compiler, task, args)
else:
commands = self.__get_compile_cmd_for_single_file(
compiler, task, args)
self.__log_writer.log(json.dumps(commands, indent=4),
level=LogLevel.DEBUG)
return commands
else:
raise RuntimeError('No compiler is set for the task: {}'.format(
task.get_name()))
async def get_sources(self, task: Task):
zip_name = 'sources.zip'
path, files, zip_res = self.__extract_path_files_and_zip_res(
task, zip_name)
if zip_name not in files:
path = task.path_to_task_src(self.workdir)
files = list(map(lambda x: os.path.join(path, x),
os.listdir(path)))
await self.__executor.async_execution(self.__write_zip_file, files,
zip_res)
return zip_res
async def write_task_passport(self, task: Task, passport: str):
path_to_passport = task.path_to_task_etc(self.workdir)
if not os.path.exists(path_to_passport):
os.makedirs(path_to_passport)
path_to_passport = os.path.join(path_to_passport,
self.passport_filename)
passport = correct_line_breaks(passport)
await self.__async_rw.async_write_file(path_to_passport, passport)
return path_to_passport
def get_compilation_commands_using_cmake(self, task: Task):
"""
Constructs commands to build CMake project and sets target executable name for task.
For now supports only project build using Unix Makefiles.
:param task: task to handle
:return: command list to build CMake project
"""
path = task.path_to_task_src(self.__workdir)
bin_path = task.path_to_task_bin(self.__workdir)
if not os.path.exists(bin_path):
os.makedirs(bin_path)
commands = [
# invoking cmake in the 'bin' directory so that all of the generated files are inside 'bin' directory
'cmake {} --warn-uninitialized --warn-unused-vars -Wno-dev'.format(
path),
# invoking build through the CMake build tool using appropriate tool for the system
'cmake --build .',
]
self.__log_writer.log(json.dumps(commands, indent=4), LogLevel.DEBUG)
with open(os.path.join(path, self.__C_MAKE_LISTS_TXT__),
'r') as cmake_file:
lines = cmake_file.read().splitlines()
lines = list(filter(lambda x: 'project' in x, lines))
# line = 'project(project_name)'
lines = list(filter(lambda x: len(x) > 0, lines))
lines = list(map(lambda x: re.split(r'[() ]', x), lines))
self.__log_writer.log('LINES: {}'.format(lines))
bin_name = os.path.join(bin_path, lines[0][1])
task.set_bin_name(bin_name)
return commands
def __handle_archive(task: Task, curdir: str, recv_file: str):
before = set(os.listdir(curdir))
if '.zip' in task.get_filename():
with zipfile.ZipFile(recv_file, mode='r') as zip_arc:
arc_type = 'zip'
zip_arc.extractall(path=curdir)
else:
with tarfile.open(name=recv_file, mode='r:*') as tar:
arc_type = 'tar'
tar.extractall(path=curdir)
after = set(os.listdir(curdir))
if after - (after ^ before) == set():
# No changes in the directory
raise RuntimeError('No extraction for {} archive'.format(arc_type))
# There is no need for the archive to stay any longer
os.remove(recv_file)
def get_passport(self, task: Task):
template = self.__sched.get_passport_template()
template = template.splitlines()
if 'mpi' in task.get_compiler().lower():
template = self.__mpi_adapt_passport_template(
template, '__NO_MPI__', '__MPI__')
else:
template = self.__mpi_adapt_passport_template(
template, '__MPI__', '__NO_MPI__')
template = '\n'.join(template)
passport = self.__substitute_template_params(template, task)
self.__log_writer.log('PASSPORT:\n{}'.format(passport),
level=LogLevel.DEBUG)
return passport
def __get_flat_archive_files(self, c_compiler, task: Task):
"""
Get files list relevant for compilation
:param c_compiler: whether the program would be compiled using C/C++ compiler
:param task: task to handle
:return: source files list
"""
path = task.path_to_task_src(self.__workdir)
dirfiles = os.listdir(path)
if c_compiler:
files = list(
filter(lambda x: '.cxx' in x or '.cpp' in x or '.c' in x,
dirfiles))
else:
files = list(filter(lambda x: '.f' in x, dirfiles))
files = list(map(lambda x: os.path.join(path, x), files))
files = ' '.join(files)
return files
async def handle_compilation(self, task: Task, args: list):
task.set_bin_name(
os.path.join(task.path_to_task_bin(self.__workdir),
task.get_name() + '.out'))
compilation_commands = self.__get_compile_cmds(task, args)
compilation_output = await self.__executor.async_exec_cmds_with_wrapping(
commands=compilation_commands,
dir_to_use=task.path_to_task_bin(self.__workdir),
)
compilation_log_file = os.path.join(
task.get_dir_name_for_task(self.__workdir), 'compilation.log')
with open(compilation_log_file, 'w') as output_file:
output_file.write(compilation_output)
return
async def restore_task(self, task: Task):
task_info = os.path.join(task.path_to_task_etc(self.workdir),
self.info_filename)
task = Task.from_json(await self.__async_rw.async_read_file(task_info))
return task
def __init_templ_dct(self, templ_dct, task: Task):
"""
Function to initialize template dictionary with methods of the related Task object
:param task: related task
:return: nothing
"""
templ_dct['__procnum__'] = task.get_procnum()
templ_dct['__walltime__'] = task.get_walltime()
templ_dct['__memory__'] = task.get_memory()
templ_dct['__filename__'] = task.get_filename()
templ_dct['__descname__'] = task.get_passport_name()
templ_dct['__jobid__'] = task.get_jobid()
templ_dct['__name__'] = task.get_name()
templ_dct['__user__'] = task.get_user()
templ_dct['__taskdir__'] = task.get_dir_name_for_task(self.workdir)
templ_dct['__binname__'] = task.get_bin_name()
templ_dct['__logname__'] = task.get_log_name()
templ_dct['__workdir__'] = self.__get_workdir()