def link_test_executable(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): project_directory = resources['project_directory'] pralinefile = resources['pralinefile'] organization = pralinefile['organization'] artifact = pralinefile['artifact'] version = pralinefile['version'] compiler = resources['compiler'] objects = resources['main_objects'] + resources['test_objects'] main_executable_object = resources['main_executable_object'] external_libraries_root = resources['external_libraries_root'] external_libraries_interfaces_root = resources['external_libraries_interfaces_root'] external_libraries = resources['external_libraries'] external_libraries_interfaces = resources['external_libraries_interfaces'] executables_root = join(project_directory, 'target', 'executables') symbols_tables_root = join(project_directory, 'target', 'symbols_tables') objects = [normalized_path(o) for o in objects] if main_executable_object and normalized_path(main_executable_object) in objects: objects.remove(normalized_path(main_executable_object)) (resources['test_executable'], resources['test_executable_symbols_table']) = link_executable_using_cache(file_system, compiler, executables_root, symbols_tables_root, external_libraries_root, external_libraries_interfaces_root, objects, external_libraries, external_libraries_interfaces, organization, artifact, version, cache, True)
def link_main_library(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): project_directory = resources['project_directory'] pralinefile = resources['pralinefile'] organization = pralinefile['organization'] artifact = pralinefile['artifact'] version = pralinefile['version'] compiler = resources['compiler'] main_objects = resources['main_objects'] external_libraries_root = resources['external_libraries_root'] external_libraries_interfaces_root = resources[ 'external_libraries_interfaces_root'] external_libraries = resources['external_libraries'] external_libraries_interfaces = resources['external_libraries_interfaces'] libraries_root = join(project_directory, 'target', 'libraries') libraries_interfaces_root = join(project_directory, 'target', 'libraries_interfaces') symbols_tables_root = join(project_directory, 'target', 'symbols_tables') (resources['main_library'], resources['main_library_interface'], resources['main_library_symbols_table']) = link_library_using_cache( file_system, compiler, libraries_root, libraries_interfaces_root, symbols_tables_root, external_libraries_root, external_libraries_interfaces_root, main_objects, external_libraries, external_libraries_interfaces, organization, artifact, version, cache)
def get_symbols_table(self, symbols_tables_root: str, name: str) -> str: if self.mode == 'debug': return join(symbols_tables_root, f'{name}.pdb') elif self.mode == 'release': return None else: raise RuntimeError(f"unrecognized compiler mode '{self.mode}'")
def get_package_extracted_contents( file_system: FileSystem, package_path: str, extraction_path: str) -> Dict[str, List[str]]: contents = { 'resources': [], 'headers': [], 'libraries': [], 'libraries_interfaces': [], 'symbols_tables': [], 'executables': [] } with file_system.open_tarfile(package_path, 'r:gz') as archive: for member in archive.getmembers(): if member.isfile(): extracted = False for root, files in contents.items(): if common_path( [normalized_path(member.name), normalized_path(root)]) == root: files.append(join(extraction_path, member.name)) extracted = True if not extracted and member.name != 'Pralinefile': raise InvalidPackageContentsError( f"unrecognized file '{member.name}' in package") return contents
def invoke_stage(target_stage: str, stages: Dict[str, Stage], file_system: FileSystem, program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy) -> None: resources = {} pipeline = create_pipeline(target_stage, stages, file_system, program_arguments, configuration) project_directory = file_system.get_working_directory() cache_path = join(project_directory, 'target', 'cache.pickle') for activation, stage_name in pipeline: stage = stages[stage_name] stage_resources = StageResources( stage_name, activation, { resource: resources[resource] for resource in stage.requirements[activation] }, stage.output) stage_program_arguments = get_stage_program_arguments( stage_name, program_arguments) if stage.cacheable: with Cache(file_system, cache_path) as cache: cache[stage_name] = stage_cache = cache.get(stage_name, {}) stage.invoker(file_system, stage_resources, stage_cache, stage_program_arguments, configuration, remote_proxy) else: stage.invoker(file_system, stage_resources, None, stage_program_arguments, configuration, remote_proxy) for resource in stage.output: if resource not in stage_resources: raise ResourceNotSuppliedError( f"stage '{stage_name}' didn't supply resource '{resource}'" ) resources.update(stage_resources.resources)
def unpack(file_system: FileSystem, package_path: str, extraction_path: str) -> Dict[str, List[str]]: contents = { 'resources': [], 'headers': [], 'libraries': [], 'libraries_interfaces': [], 'symbols_tables': [], 'executables': [] } with file_system.open_tarfile(package_path, 'r:gz') as archive: for member in archive.getmembers(): if member.isfile(): extracted = False for root, files in contents.items(): if common_path( [normalized_path(member.name), normalized_path(root)]) == root: archive.extract(member, extraction_path) files.append(join(extraction_path, member.name)) extracted = True if not extracted and member.name != 'Pralinefile': raise InvalidPackageContentsError( f"unrecognized file '{member.name}' in package") for header in contents['headers']: with file_system.open_file(header, 'rb') as f: text = f.read().decode() with file_system.open_file(header, 'wb') as f: f.write(text.replace('PRALINE_EXPORT', 'PRALINE_IMPORT').encode()) return contents
def has_headers_only(file_system: FileSystem, program_arguments: Dict[str, Any], configuration: Dict[str, Any]): sources_root = join(file_system.get_working_directory(), 'sources') files = file_system.files_in_directory(sources_root) return not program_arguments['global']['executable'] and all( f.endswith('.hpp') or f.endswith('.test.cpp') for f in files)
def compile_test_sources(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): project_directory = resources['project_directory'] compiler = resources['compiler'] header_roots = resources['headers_root'] external_headers_root = resources['external_headers_root'] sources_root = resources['test_sources_root'] if resources.activation == 0: headers = resources['formatted_headers'] sources = resources['formatted_test_sources'] elif resources.activation == 1: headers = resources['headers'] sources = resources['test_sources'] resources['test_objects_root'] = objects_root = join( project_directory, 'target', 'objects') resources['test_objects'] = compile_using_cache(file_system, compiler, header_roots, external_headers_root, sources_root, objects_root, headers, sources, cache)
def compile_main_sources(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): project_directory = resources['project_directory'] compiler = resources['compiler'] header_roots = resources['headers_root'] external_headers_root = resources['external_headers_root'] sources_root = resources['main_sources_root'] if resources.activation == 0: headers = resources['formatted_headers'] sources = resources['formatted_main_sources'] executable_source = resources['formatted_main_executable_source'] elif resources.activation == 1: headers = resources['headers'] sources = resources['main_sources'] executable_source = resources['main_executable_source'] resources['main_objects_root'] = objects_root = join(project_directory, 'target', 'objects') resources['main_objects'] = compile_using_cache(file_system, compiler, header_roots, external_headers_root, sources_root, objects_root, headers, sources, cache) if executable_source: resources['main_executable_object'] = compiler.get_yield_descriptor().get_object(sources_root, objects_root, executable_source) else: resources['main_executable_object'] = None
def package_main_library(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): project_directory = resources['project_directory'] pralinefile = resources['pralinefile'] compiler = resources['compiler'] resources_root = resources['resources_root'] resource_files = resources['resources'] headers_root = resources['headers_root'] main_library = resources['main_library'] main_library_interface = resources['main_library_interface'] main_library_symbols_table = resources['main_library_symbols_table'] if resources.activation in [0, 1]: headers = resources['formatted_headers'] elif resources.activation in [2, 3]: headers = resources['headers'] package_path = join( project_directory, 'target', get_package(pralinefile['organization'], pralinefile['artifact'], compiler.get_architecture(), compiler.get_platform(), compiler.get_name(), compiler.get_mode(), pralinefile['version'])) package_files = [(path, join('resources', relative_path(path, resources_root))) for path in resource_files] package_files.extend( (path, join('headers', relative_path(path, headers_root))) for path in headers) package_files.append((join(project_directory, 'Pralinefile'), 'Pralinefile')) package_files.append((main_library, f'libraries/{basename(main_library)}')) if main_library_interface: package_files.append( (main_library_interface, f'libraries_interfaces/{basename(main_library_interface)}')) if main_library_symbols_table: package_files.append( (main_library_symbols_table, f'symbols_tables/{basename(main_library_symbols_table)}')) pack(file_system, package_path, package_files, cache) resources['main_library_package'] = package_path
def has_non_executable_sources(file_system: FileSystem, program_arguments: Dict[str, Any], configuration: Dict[str, Any]): sources_root = join(file_system.get_working_directory(), 'sources') files = file_system.files_in_directory(sources_root) return (not program_arguments['global']['executable'] and all(basename(f) != 'executable.cpp' for f in files) and any( f.endswith('.cpp') and not f.endswith('.test.cpp') for f in files))
def check_unique(file_system, root, organization, artifact): if len(file_system.list_directory(root)) != 1: raise IllformedProjectError( f"'{root}' directory must only contain the '{organization}' organization directory" ) if len(file_system.list_directory(join(root, organization))) != 1: raise IllformedProjectError( f"'{join(root, organization)}' directory must only contain the '{artifact}' artifact directory" )
def solve_dependencies() -> Dict[str, str]: file_system.create_directory_if_missing(repository_path) payload = request.get_json() try: dependencies = get_package_dependencies_recursively( file_system, payload['package'], payload['dependencies'], repository_path) dependencies_with_hashes = { d: hash_archive(file_system, join(repository_path, d)) for d in dependencies } return jsonify(dependencies_with_hashes) except RuntimeError as exception: return Response(str(exception), status=400, mimetype='text/plain')
def load_clang_format(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): if 'clang-format-executable-path' in configuration: clang_format_executable = configuration['clang-format-executable-path'] if not file_system.is_file(clang_format_executable): raise ClangFormatConfigurationError(f"user supplied clang-format '{clang_format_executable}' is not a file") else: clang_format_executable = file_system.which('clang-format') if clang_format_executable is None: raise ClangFormatConfigurationError("coudn't find clang-format in path -- either supply it in the praline-client.config file or add it to the path environment variable") project_directory = resources['project_directory'] resources['clang_format_executable'] = clang_format_executable resources['clang_format_style_file'] = clang_format_style_file = join(project_directory, '.clang-format') file_system.create_file_if_missing(clang_format_style_file, clang_format_style_file_contents)
def package_headers_only(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): project_directory = resources['project_directory'] pralinefile = resources['pralinefile'] compiler = resources['compiler'] resources_root = resources['resources_root'] resource_files = resources['resources'] headers_root = resources['headers_root'] if resources.activation in [0, 1]: headers = resources['formatted_headers'] elif resources.activation in [2, 3]: headers = resources['headers'] package_path = join( project_directory, 'target', get_package(pralinefile['organization'], pralinefile['artifact'], compiler.get_architecture(), compiler.get_platform(), compiler.get_name(), compiler.get_mode(), pralinefile['version'])) file_system.create_directory_if_missing(join(project_directory, 'target')) package_files = [(path, join('resources', relative_path(path, resources_root))) for path in resource_files] package_files.extend( (path, join('headers', relative_path(path, headers_root))) for path in headers) package_files.append((join(project_directory, 'Pralinefile'), 'Pralinefile')) pack(file_system, package_path, package_files, cache) resources['headers_only_package'] = package_path
def load_test_sources(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): pralinefile = resources['pralinefile'] test_sources_root = resources['test_sources_root'] test_executable_source = join(test_sources_root, pralinefile['organization'], pralinefile['artifact'], 'executable.test.cpp') file_system.create_file_if_missing(test_executable_source, test_executable_contents) resources['test_sources'] = [ f for f in file_system.files_in_directory(test_sources_root) if f.endswith('.test.cpp') ]
def visitor(package): if package == root_package: dependencies = root_package_dependencies else: package_path = join(repository_path, package) dependencies = get_package_dependencies_from_archive( file_system, package_path, 'main') fixed_dependencies = [] for dependency in dependencies: matching_packages = get_matching_packages(dependency, candidate_packages) if not matching_packages: raise UnsatisfiableArtifactDependencyError( f"couldn't find any package matching '{dependency}' when solving dependencies for package '{package}'" ) fixed_dependencies.append(matching_packages) return cartesian_product(fixed_dependencies)
def load_pralinefile(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): resources[ 'project_directory'] = project_directory = file_system.get_working_directory( ) pralinefile_path = join(project_directory, 'Pralinefile') try: resources['pralinefile'] = pralinefile = pralinefile_from_path( file_system, pralinefile_path) except FileNotFoundError as e: raise FileNotFoundError( f"no Pralinefile was found in current working directory {project_directory}" ) from e architecture = file_system.get_architecture() if architecture not in pralinefile['architectures']: raise UnsupportedArchitectureError( f"system architecture '{architecture}' is not supported -- supported architectures for this project are {pralinefile['architectures']}" ) platform = file_system.get_platform() if platform not in pralinefile['platforms']: raise UnsupportedPlatformError( f"{platform} is not supported -- supported architectures are {pralinefile['platforms']}" ) mode = 'release' if program_arguments['global'][ 'release'] else pralinefile['modes'][0] logging_level = program_arguments['global']['logging_level'] matching_compilers = [ compiler for compiler in get_compilers(file_system, architecture, platform, mode, logging_level) if compiler.matches() ] compilers = [ compiler for compiler in matching_compilers if compiler.get_name() in pralinefile['compilers'] ] if not compilers: raise NoMatchingCompilerFoundError( f"no suitable compiler was found -- matching compilers are {[c.get_name() for c in matching_compilers]} while specified compilers are {pralinefile['compilers']}" ) resources['compiler'] = compilers[0]
def package(package) -> Response: file_system.create_directory_if_missing(repository_path) if request.method == 'GET': return send_from_directory(repository_path, package, as_attachment=True) elif request.method == 'PUT': package_path = join(repository_path, package) if file_system.exists(package_path): return Response( f"package '{package}' already exists -- increment the version and try again", status=409, mimetype='text/plain') with open(package_path, 'wb') as f: f.write(request.stream.read()) return Response(f"succesfully created package '{package}'", status=201, mimetype='text/plain')
def load_main_sources(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): main_sources_root = resources['main_sources_root'] pralinefile = resources['pralinefile'] main_executable_source = join(main_sources_root, pralinefile['organization'], pralinefile['artifact'], 'executable.cpp') if program_arguments['global']['executable']: resources['main_executable_source'] = main_executable_source file_system.create_file_if_missing(main_executable_source, main_executable_contents) elif file_system.is_file(main_executable_source): resources['main_executable_source'] = main_executable_source else: resources['main_executable_source'] = None resources['main_sources'] = [ f for f in file_system.files_in_directory(main_sources_root) if f.endswith('.cpp') and not f.endswith('.test.cpp') ]
def clean_up_package(file_system: FileSystem, package_path: str, extraction_path: str, logging_level: int): package = get_package_metadata(package_path) name = package['name'].replace(package_extension, '') organization = package['organization'] artifact = package['artifact'] architecture = package['architecture'] platform = package['platform'] compiler = package['compiler'] mode = package['mode'] yield_descriptor = get_compiler(file_system, compiler, architecture, platform, mode, logging_level).get_yield_descriptor() resources = join(extraction_path, 'resources', organization, artifact) headers = join(extraction_path, 'headers', organization, artifact) library = yield_descriptor.get_library(join(extraction_path, 'libraries'), name) library_interface = yield_descriptor.get_library_interface( join(extraction_path, 'libraries_interfaces'), name) symbols_table = yield_descriptor.get_symbols_table( join(extraction_path, 'symbols_tables'), name) executable = yield_descriptor.get_executable( join(extraction_path, 'executables'), name) if file_system.exists(resources): file_system.remove_directory_recursively(resources) if file_system.exists(headers): file_system.remove_directory_recursively(headers) if library and file_system.exists(library): file_system.remove_file(library) if library_interface and file_system.exists(library_interface): file_system.remove_file(library_interface) if symbols_table and file_system.exists(symbols_table): file_system.remove_file(symbols_table) if executable and file_system.exists(executable): file_system.remove_file(executable) if file_system.exists(package_path): file_system.remove_file(package_path)
def validate_project(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): project_directory = resources['project_directory'] resources['headers_root'] = join(project_directory, 'sources') resources['test_sources_root'] = join(project_directory, 'sources') resources['resources_root'] = resources_root = join( project_directory, 'resources') resources['main_sources_root'] = main_sources_root = join( project_directory, 'sources') pralinefile = resources['pralinefile'] organization = pralinefile['organization'] artifact = pralinefile['artifact'] file_system.create_directory_if_missing( join(resources_root, organization, artifact)) file_system.create_directory_if_missing( join(main_sources_root, organization, artifact)) check_unique(file_system, resources_root, organization, artifact) check_unique(file_system, main_sources_root, organization, artifact)
def get_object(self, sources_root: str, objects_root: str, source: str) -> str: name = relative_path(source, sources_root).replace(get_separator(), '-').replace('.cpp', '.o') return join(objects_root, name)
def get_library(self, libraries_root: str, name: str) -> str: return join(libraries_root, f'lib{name}.dylib')
def get_executable(self, executables_root: str, name: str) -> str: return join(executables_root, f'{name}.out')
def clean(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): target = join(resources['project_directory'], 'target') if file_system.exists(target): file_system.remove_directory_recursively(target)
def get_symbols_table(self, symbols_tables_root: str, name: str) -> str: return join(symbols_tables_root, f'{name}.pdb')
def get_library_interface(self, library_interfaces_root: str, name: str) -> str: return join(library_interfaces_root, f'{name}.lib')
def pull_dependencies(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy): project_directory = resources['project_directory'] external_root = join(project_directory, 'target', 'external') external_packages_root = join(external_root, 'packages') resources['external_resources_root'] = external_resources_root = join(external_root, 'resources') resources['external_headers_root'] = external_headers_root = join(external_root, 'headers') resources['external_executables_root'] = external_executables_root = join(external_root, 'executables') resources['external_libraries_root'] = external_libraries_root = join(external_root, 'libraries') resources['external_libraries_interfaces_root'] = external_libraries_interfaces_root = join(external_root, 'libraries_interfaces') resources['external_symbols_tables_root'] = external_symbols_tables_root = join(external_root, 'symbols_tables') file_system.create_directory_if_missing(external_packages_root) file_system.create_directory_if_missing(external_resources_root) file_system.create_directory_if_missing(external_headers_root) file_system.create_directory_if_missing(external_executables_root) file_system.create_directory_if_missing(external_libraries_root) file_system.create_directory_if_missing(external_libraries_interfaces_root) file_system.create_directory_if_missing(external_symbols_tables_root) external_resources = [] external_headers = [] external_libraries = [] external_libraries_interfaces = [] external_symbols_tables = [] external_executables = [] def extend_externals(contents): external_resources.extend(contents['resources']) external_headers.extend(contents['headers']) external_libraries.extend(contents['libraries']) external_libraries_interfaces.extend(contents['libraries_interfaces']) external_symbols_tables.extend(contents['symbols_tables']) external_executables.extend(contents['executables']) pralinefile = resources['pralinefile'] compiler = resources['compiler'] logging_level = program_arguments['global']['logging_level'] packages = remote_proxy.solve_dependencies(pralinefile, compiler.get_architecture(), compiler.get_platform(), compiler.get_name(), compiler.get_mode()) updated, removed, new_cache = key_delta(packages.keys(), lambda p: packages[p], cache) for package in removed: package_path = join(external_packages_root, package) clean_up_package(file_system, package_path, external_root, logging_level) for package in updated: package_path = join(external_packages_root, package) clean_up_package(file_system, package_path, external_root, logging_level) remote_proxy.pull_package(package_path) contents = unpack(file_system, package_path, external_root) extend_externals(contents) for package in packages: if package not in updated: package_path = join(external_packages_root, package) if not file_system.exists(package_path): remote_proxy.pull_package(package_path) contents = unpack(file_system, package_path, external_root) else: contents = get_package_extracted_contents(file_system, package_path, external_root) extend_externals(contents) resources['external_resources'] = external_resources resources['external_headers'] = external_headers resources['external_libraries'] = external_libraries resources['external_libraries_interfaces'] = external_libraries_interfaces resources['external_symbols_tables'] = external_symbols_tables resources['external_executables'] = external_executables cache.clear() cache.update(new_cache)
from praline.server.configuration import configuration import logging.config logging.config.dictConfig(configuration['logging']) from flask import Flask, send_from_directory, request, Response, jsonify from praline.common.file_system import FileSystem, join from praline.common.hashing import hash_archive from praline.common.package import get_package_dependencies_recursively from typing import Dict file_system = FileSystem() server = Flask(__name__, root_path=file_system.get_working_directory()) repository_path = join(configuration['repository'], 'packages') @server.route('/package/<package>', methods=['GET', 'PUT']) def package(package) -> Response: file_system.create_directory_if_missing(repository_path) if request.method == 'GET': return send_from_directory(repository_path, package, as_attachment=True) elif request.method == 'PUT': package_path = join(repository_path, package) if file_system.exists(package_path): return Response( f"package '{package}' already exists -- increment the version and try again", status=409,