def link_test_executable(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy):
project_directory = resources['project_directory']
pralinefile = resources['pralinefile']
organization = pralinefile['organization']
artifact = pralinefile['artifact']
version = pralinefile['version']
compiler = resources['compiler']
objects = resources['main_objects'] + resources['test_objects']
main_executable_object = resources['main_executable_object']
external_libraries_root = resources['external_libraries_root']
external_libraries_interfaces_root = resources['external_libraries_interfaces_root']
external_libraries = resources['external_libraries']
external_libraries_interfaces = resources['external_libraries_interfaces']
executables_root = join(project_directory, 'target', 'executables')
symbols_tables_root = join(project_directory, 'target', 'symbols_tables')
objects = [normalized_path(o) for o in objects]
if main_executable_object and normalized_path(main_executable_object) in objects:
objects.remove(normalized_path(main_executable_object))
(resources['test_executable'],
resources['test_executable_symbols_table']) = link_executable_using_cache(file_system,
compiler,
executables_root,
symbols_tables_root,
external_libraries_root,
external_libraries_interfaces_root,
objects,
external_libraries,
external_libraries_interfaces,
organization,
artifact,
version,
cache, True)
def link_main_library(file_system: FileSystem, resources: StageResources,
cache: Dict[str, Any], program_arguments: Dict[str, Any],
configuration: Dict[str,
Any], remote_proxy: RemoteProxy):
project_directory = resources['project_directory']
pralinefile = resources['pralinefile']
organization = pralinefile['organization']
artifact = pralinefile['artifact']
version = pralinefile['version']
compiler = resources['compiler']
main_objects = resources['main_objects']
external_libraries_root = resources['external_libraries_root']
external_libraries_interfaces_root = resources[
'external_libraries_interfaces_root']
external_libraries = resources['external_libraries']
external_libraries_interfaces = resources['external_libraries_interfaces']
libraries_root = join(project_directory, 'target', 'libraries')
libraries_interfaces_root = join(project_directory, 'target',
'libraries_interfaces')
symbols_tables_root = join(project_directory, 'target', 'symbols_tables')
(resources['main_library'], resources['main_library_interface'],
resources['main_library_symbols_table']) = link_library_using_cache(
file_system, compiler, libraries_root, libraries_interfaces_root,
symbols_tables_root, external_libraries_root,
external_libraries_interfaces_root, main_objects, external_libraries,
external_libraries_interfaces, organization, artifact, version, cache)
def get_symbols_table(self, symbols_tables_root: str, name: str) -> str:
if self.mode == 'debug':
return join(symbols_tables_root, f'{name}.pdb')
elif self.mode == 'release':
return None
else:
raise RuntimeError(f"unrecognized compiler mode '{self.mode}'")
def get_package_extracted_contents(
file_system: FileSystem, package_path: str,
extraction_path: str) -> Dict[str, List[str]]:
contents = {
'resources': [],
'headers': [],
'libraries': [],
'libraries_interfaces': [],
'symbols_tables': [],
'executables': []
}
with file_system.open_tarfile(package_path, 'r:gz') as archive:
for member in archive.getmembers():
if member.isfile():
extracted = False
for root, files in contents.items():
if common_path(
[normalized_path(member.name),
normalized_path(root)]) == root:
files.append(join(extraction_path, member.name))
extracted = True
if not extracted and member.name != 'Pralinefile':
raise InvalidPackageContentsError(
f"unrecognized file '{member.name}' in package")
return contents
def invoke_stage(target_stage: str, stages: Dict[str, Stage],
file_system: FileSystem, program_arguments: Dict[str, Any],
configuration: Dict[str,
Any], remote_proxy: RemoteProxy) -> None:
resources = {}
pipeline = create_pipeline(target_stage, stages, file_system,
program_arguments, configuration)
project_directory = file_system.get_working_directory()
cache_path = join(project_directory, 'target', 'cache.pickle')
for activation, stage_name in pipeline:
stage = stages[stage_name]
stage_resources = StageResources(
stage_name, activation, {
resource: resources[resource]
for resource in stage.requirements[activation]
}, stage.output)
stage_program_arguments = get_stage_program_arguments(
stage_name, program_arguments)
if stage.cacheable:
with Cache(file_system, cache_path) as cache:
cache[stage_name] = stage_cache = cache.get(stage_name, {})
stage.invoker(file_system, stage_resources, stage_cache,
stage_program_arguments, configuration,
remote_proxy)
else:
stage.invoker(file_system, stage_resources, None,
stage_program_arguments, configuration, remote_proxy)
for resource in stage.output:
if resource not in stage_resources:
raise ResourceNotSuppliedError(
f"stage '{stage_name}' didn't supply resource '{resource}'"
)
resources.update(stage_resources.resources)
def unpack(file_system: FileSystem, package_path: str,
extraction_path: str) -> Dict[str, List[str]]:
contents = {
'resources': [],
'headers': [],
'libraries': [],
'libraries_interfaces': [],
'symbols_tables': [],
'executables': []
}
with file_system.open_tarfile(package_path, 'r:gz') as archive:
for member in archive.getmembers():
if member.isfile():
extracted = False
for root, files in contents.items():
if common_path(
[normalized_path(member.name),
normalized_path(root)]) == root:
archive.extract(member, extraction_path)
files.append(join(extraction_path, member.name))
extracted = True
if not extracted and member.name != 'Pralinefile':
raise InvalidPackageContentsError(
f"unrecognized file '{member.name}' in package")
for header in contents['headers']:
with file_system.open_file(header, 'rb') as f:
text = f.read().decode()
with file_system.open_file(header, 'wb') as f:
f.write(text.replace('PRALINE_EXPORT', 'PRALINE_IMPORT').encode())
return contents
def has_headers_only(file_system: FileSystem, program_arguments: Dict[str,
Any],
configuration: Dict[str, Any]):
sources_root = join(file_system.get_working_directory(), 'sources')
files = file_system.files_in_directory(sources_root)
return not program_arguments['global']['executable'] and all(
f.endswith('.hpp') or f.endswith('.test.cpp') for f in files)
def compile_test_sources(file_system: FileSystem, resources: StageResources,
cache: Dict[str, Any], program_arguments: Dict[str,
Any],
configuration: Dict[str,
Any], remote_proxy: RemoteProxy):
project_directory = resources['project_directory']
compiler = resources['compiler']
header_roots = resources['headers_root']
external_headers_root = resources['external_headers_root']
sources_root = resources['test_sources_root']
if resources.activation == 0:
headers = resources['formatted_headers']
sources = resources['formatted_test_sources']
elif resources.activation == 1:
headers = resources['headers']
sources = resources['test_sources']
resources['test_objects_root'] = objects_root = join(
project_directory, 'target', 'objects')
resources['test_objects'] = compile_using_cache(file_system, compiler,
header_roots,
external_headers_root,
sources_root, objects_root,
headers, sources, cache)
def compile_main_sources(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy):
project_directory = resources['project_directory']
compiler = resources['compiler']
header_roots = resources['headers_root']
external_headers_root = resources['external_headers_root']
sources_root = resources['main_sources_root']
if resources.activation == 0:
headers = resources['formatted_headers']
sources = resources['formatted_main_sources']
executable_source = resources['formatted_main_executable_source']
elif resources.activation == 1:
headers = resources['headers']
sources = resources['main_sources']
executable_source = resources['main_executable_source']
resources['main_objects_root'] = objects_root = join(project_directory, 'target', 'objects')
resources['main_objects'] = compile_using_cache(file_system,
compiler,
header_roots,
external_headers_root,
sources_root,
objects_root,
headers,
sources,
cache)
if executable_source:
resources['main_executable_object'] = compiler.get_yield_descriptor().get_object(sources_root, objects_root, executable_source)
else:
resources['main_executable_object'] = None
def package_main_library(file_system: FileSystem, resources: StageResources,
cache: Dict[str, Any], program_arguments: Dict[str,
Any],
configuration: Dict[str,
Any], remote_proxy: RemoteProxy):
project_directory = resources['project_directory']
pralinefile = resources['pralinefile']
compiler = resources['compiler']
resources_root = resources['resources_root']
resource_files = resources['resources']
headers_root = resources['headers_root']
main_library = resources['main_library']
main_library_interface = resources['main_library_interface']
main_library_symbols_table = resources['main_library_symbols_table']
if resources.activation in [0, 1]:
headers = resources['formatted_headers']
elif resources.activation in [2, 3]:
headers = resources['headers']
package_path = join(
project_directory, 'target',
get_package(pralinefile['organization'], pralinefile['artifact'],
compiler.get_architecture(), compiler.get_platform(),
compiler.get_name(), compiler.get_mode(),
pralinefile['version']))
package_files = [(path,
join('resources', relative_path(path, resources_root)))
for path in resource_files]
package_files.extend(
(path, join('headers', relative_path(path, headers_root)))
for path in headers)
package_files.append((join(project_directory,
'Pralinefile'), 'Pralinefile'))
package_files.append((main_library, f'libraries/{basename(main_library)}'))
if main_library_interface:
package_files.append(
(main_library_interface,
f'libraries_interfaces/{basename(main_library_interface)}'))
if main_library_symbols_table:
package_files.append(
(main_library_symbols_table,
f'symbols_tables/{basename(main_library_symbols_table)}'))
pack(file_system, package_path, package_files, cache)
resources['main_library_package'] = package_path
def has_non_executable_sources(file_system: FileSystem,
program_arguments: Dict[str, Any],
configuration: Dict[str, Any]):
sources_root = join(file_system.get_working_directory(), 'sources')
files = file_system.files_in_directory(sources_root)
return (not program_arguments['global']['executable']
and all(basename(f) != 'executable.cpp' for f in files) and any(
f.endswith('.cpp') and not f.endswith('.test.cpp')
for f in files))
def check_unique(file_system, root, organization, artifact):
if len(file_system.list_directory(root)) != 1:
raise IllformedProjectError(
f"'{root}' directory must only contain the '{organization}' organization directory"
)
if len(file_system.list_directory(join(root, organization))) != 1:
raise IllformedProjectError(
f"'{join(root, organization)}' directory must only contain the '{artifact}' artifact directory"
)
def solve_dependencies() -> Dict[str, str]:
file_system.create_directory_if_missing(repository_path)
payload = request.get_json()
try:
dependencies = get_package_dependencies_recursively(
file_system, payload['package'], payload['dependencies'],
repository_path)
dependencies_with_hashes = {
d: hash_archive(file_system, join(repository_path, d))
for d in dependencies
}
return jsonify(dependencies_with_hashes)
except RuntimeError as exception:
return Response(str(exception), status=400, mimetype='text/plain')
def load_clang_format(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy):
if 'clang-format-executable-path' in configuration:
clang_format_executable = configuration['clang-format-executable-path']
if not file_system.is_file(clang_format_executable):
raise ClangFormatConfigurationError(f"user supplied clang-format '{clang_format_executable}' is not a file")
else:
clang_format_executable = file_system.which('clang-format')
if clang_format_executable is None:
raise ClangFormatConfigurationError("coudn't find clang-format in path -- either supply it in the praline-client.config file or add it to the path environment variable")
project_directory = resources['project_directory']
resources['clang_format_executable'] = clang_format_executable
resources['clang_format_style_file'] = clang_format_style_file = join(project_directory, '.clang-format')
file_system.create_file_if_missing(clang_format_style_file, clang_format_style_file_contents)
def package_headers_only(file_system: FileSystem, resources: StageResources,
cache: Dict[str, Any], program_arguments: Dict[str,
Any],
configuration: Dict[str,
Any], remote_proxy: RemoteProxy):
project_directory = resources['project_directory']
pralinefile = resources['pralinefile']
compiler = resources['compiler']
resources_root = resources['resources_root']
resource_files = resources['resources']
headers_root = resources['headers_root']
if resources.activation in [0, 1]:
headers = resources['formatted_headers']
elif resources.activation in [2, 3]:
headers = resources['headers']
package_path = join(
project_directory, 'target',
get_package(pralinefile['organization'], pralinefile['artifact'],
compiler.get_architecture(), compiler.get_platform(),
compiler.get_name(), compiler.get_mode(),
pralinefile['version']))
file_system.create_directory_if_missing(join(project_directory, 'target'))
package_files = [(path,
join('resources', relative_path(path, resources_root)))
for path in resource_files]
package_files.extend(
(path, join('headers', relative_path(path, headers_root)))
for path in headers)
package_files.append((join(project_directory,
'Pralinefile'), 'Pralinefile'))
pack(file_system, package_path, package_files, cache)
resources['headers_only_package'] = package_path
def load_test_sources(file_system: FileSystem, resources: StageResources,
cache: Dict[str, Any], program_arguments: Dict[str, Any],
configuration: Dict[str,
Any], remote_proxy: RemoteProxy):
pralinefile = resources['pralinefile']
test_sources_root = resources['test_sources_root']
test_executable_source = join(test_sources_root,
pralinefile['organization'],
pralinefile['artifact'],
'executable.test.cpp')
file_system.create_file_if_missing(test_executable_source,
test_executable_contents)
resources['test_sources'] = [
f for f in file_system.files_in_directory(test_sources_root)
if f.endswith('.test.cpp')
]
def visitor(package):
if package == root_package:
dependencies = root_package_dependencies
else:
package_path = join(repository_path, package)
dependencies = get_package_dependencies_from_archive(
file_system, package_path, 'main')
fixed_dependencies = []
for dependency in dependencies:
matching_packages = get_matching_packages(dependency,
candidate_packages)
if not matching_packages:
raise UnsatisfiableArtifactDependencyError(
f"couldn't find any package matching '{dependency}' when solving dependencies for package '{package}'"
)
fixed_dependencies.append(matching_packages)
return cartesian_product(fixed_dependencies)
def load_pralinefile(file_system: FileSystem, resources: StageResources,
cache: Dict[str, Any], program_arguments: Dict[str, Any],
configuration: Dict[str, Any], remote_proxy: RemoteProxy):
resources[
'project_directory'] = project_directory = file_system.get_working_directory(
)
pralinefile_path = join(project_directory, 'Pralinefile')
try:
resources['pralinefile'] = pralinefile = pralinefile_from_path(
file_system, pralinefile_path)
except FileNotFoundError as e:
raise FileNotFoundError(
f"no Pralinefile was found in current working directory {project_directory}"
) from e
architecture = file_system.get_architecture()
if architecture not in pralinefile['architectures']:
raise UnsupportedArchitectureError(
f"system architecture '{architecture}' is not supported -- supported architectures for this project are {pralinefile['architectures']}"
)
platform = file_system.get_platform()
if platform not in pralinefile['platforms']:
raise UnsupportedPlatformError(
f"{platform} is not supported -- supported architectures are {pralinefile['platforms']}"
)
mode = 'release' if program_arguments['global'][
'release'] else pralinefile['modes'][0]
logging_level = program_arguments['global']['logging_level']
matching_compilers = [
compiler for compiler in get_compilers(file_system, architecture,
platform, mode, logging_level)
if compiler.matches()
]
compilers = [
compiler for compiler in matching_compilers
if compiler.get_name() in pralinefile['compilers']
]
if not compilers:
raise NoMatchingCompilerFoundError(
f"no suitable compiler was found -- matching compilers are {[c.get_name() for c in matching_compilers]} while specified compilers are {pralinefile['compilers']}"
)
resources['compiler'] = compilers[0]
def package(package) -> Response:
file_system.create_directory_if_missing(repository_path)
if request.method == 'GET':
return send_from_directory(repository_path,
package,
as_attachment=True)
elif request.method == 'PUT':
package_path = join(repository_path, package)
if file_system.exists(package_path):
return Response(
f"package '{package}' already exists -- increment the version and try again",
status=409,
mimetype='text/plain')
with open(package_path, 'wb') as f:
f.write(request.stream.read())
return Response(f"succesfully created package '{package}'",
status=201,
mimetype='text/plain')
def load_main_sources(file_system: FileSystem, resources: StageResources,
cache: Dict[str, Any], program_arguments: Dict[str, Any],
configuration: Dict[str,
Any], remote_proxy: RemoteProxy):
main_sources_root = resources['main_sources_root']
pralinefile = resources['pralinefile']
main_executable_source = join(main_sources_root,
pralinefile['organization'],
pralinefile['artifact'], 'executable.cpp')
if program_arguments['global']['executable']:
resources['main_executable_source'] = main_executable_source
file_system.create_file_if_missing(main_executable_source,
main_executable_contents)
elif file_system.is_file(main_executable_source):
resources['main_executable_source'] = main_executable_source
else:
resources['main_executable_source'] = None
resources['main_sources'] = [
f for f in file_system.files_in_directory(main_sources_root)
if f.endswith('.cpp') and not f.endswith('.test.cpp')
]
def clean_up_package(file_system: FileSystem, package_path: str,
extraction_path: str, logging_level: int):
package = get_package_metadata(package_path)
name = package['name'].replace(package_extension, '')
organization = package['organization']
artifact = package['artifact']
architecture = package['architecture']
platform = package['platform']
compiler = package['compiler']
mode = package['mode']
yield_descriptor = get_compiler(file_system, compiler, architecture,
platform, mode,
logging_level).get_yield_descriptor()
resources = join(extraction_path, 'resources', organization, artifact)
headers = join(extraction_path, 'headers', organization, artifact)
library = yield_descriptor.get_library(join(extraction_path, 'libraries'),
name)
library_interface = yield_descriptor.get_library_interface(
join(extraction_path, 'libraries_interfaces'), name)
symbols_table = yield_descriptor.get_symbols_table(
join(extraction_path, 'symbols_tables'), name)
executable = yield_descriptor.get_executable(
join(extraction_path, 'executables'), name)
if file_system.exists(resources):
file_system.remove_directory_recursively(resources)
if file_system.exists(headers):
file_system.remove_directory_recursively(headers)
if library and file_system.exists(library):
file_system.remove_file(library)
if library_interface and file_system.exists(library_interface):
file_system.remove_file(library_interface)
if symbols_table and file_system.exists(symbols_table):
file_system.remove_file(symbols_table)
if executable and file_system.exists(executable):
file_system.remove_file(executable)
if file_system.exists(package_path):
file_system.remove_file(package_path)
def validate_project(file_system: FileSystem, resources: StageResources,
cache: Dict[str, Any], program_arguments: Dict[str, Any],
configuration: Dict[str, Any], remote_proxy: RemoteProxy):
project_directory = resources['project_directory']
resources['headers_root'] = join(project_directory, 'sources')
resources['test_sources_root'] = join(project_directory, 'sources')
resources['resources_root'] = resources_root = join(
project_directory, 'resources')
resources['main_sources_root'] = main_sources_root = join(
project_directory, 'sources')
pralinefile = resources['pralinefile']
organization = pralinefile['organization']
artifact = pralinefile['artifact']
file_system.create_directory_if_missing(
join(resources_root, organization, artifact))
file_system.create_directory_if_missing(
join(main_sources_root, organization, artifact))
check_unique(file_system, resources_root, organization, artifact)
check_unique(file_system, main_sources_root, organization, artifact)
def get_object(self, sources_root: str, objects_root: str,
source: str) -> str:
name = relative_path(source,
sources_root).replace(get_separator(),
'-').replace('.cpp', '.o')
return join(objects_root, name)
def get_library(self, libraries_root: str, name: str) -> str:
return join(libraries_root, f'lib{name}.dylib')
def get_executable(self, executables_root: str, name: str) -> str:
return join(executables_root, f'{name}.out')
def clean(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy):
target = join(resources['project_directory'], 'target')
if file_system.exists(target):
file_system.remove_directory_recursively(target)
def get_symbols_table(self, symbols_tables_root: str, name: str) -> str:
return join(symbols_tables_root, f'{name}.pdb')
def get_library_interface(self, library_interfaces_root: str,
name: str) -> str:
return join(library_interfaces_root, f'{name}.lib')
def pull_dependencies(file_system: FileSystem, resources: StageResources, cache: Dict[str, Any], program_arguments: Dict[str, Any], configuration: Dict[str, Any], remote_proxy: RemoteProxy):
project_directory = resources['project_directory']
external_root = join(project_directory, 'target', 'external')
external_packages_root = join(external_root, 'packages')
resources['external_resources_root'] = external_resources_root = join(external_root, 'resources')
resources['external_headers_root'] = external_headers_root = join(external_root, 'headers')
resources['external_executables_root'] = external_executables_root = join(external_root, 'executables')
resources['external_libraries_root'] = external_libraries_root = join(external_root, 'libraries')
resources['external_libraries_interfaces_root'] = external_libraries_interfaces_root = join(external_root, 'libraries_interfaces')
resources['external_symbols_tables_root'] = external_symbols_tables_root = join(external_root, 'symbols_tables')
file_system.create_directory_if_missing(external_packages_root)
file_system.create_directory_if_missing(external_resources_root)
file_system.create_directory_if_missing(external_headers_root)
file_system.create_directory_if_missing(external_executables_root)
file_system.create_directory_if_missing(external_libraries_root)
file_system.create_directory_if_missing(external_libraries_interfaces_root)
file_system.create_directory_if_missing(external_symbols_tables_root)
external_resources = []
external_headers = []
external_libraries = []
external_libraries_interfaces = []
external_symbols_tables = []
external_executables = []
def extend_externals(contents):
external_resources.extend(contents['resources'])
external_headers.extend(contents['headers'])
external_libraries.extend(contents['libraries'])
external_libraries_interfaces.extend(contents['libraries_interfaces'])
external_symbols_tables.extend(contents['symbols_tables'])
external_executables.extend(contents['executables'])
pralinefile = resources['pralinefile']
compiler = resources['compiler']
logging_level = program_arguments['global']['logging_level']
packages = remote_proxy.solve_dependencies(pralinefile,
compiler.get_architecture(),
compiler.get_platform(),
compiler.get_name(),
compiler.get_mode())
updated, removed, new_cache = key_delta(packages.keys(), lambda p: packages[p], cache)
for package in removed:
package_path = join(external_packages_root, package)
clean_up_package(file_system, package_path, external_root, logging_level)
for package in updated:
package_path = join(external_packages_root, package)
clean_up_package(file_system, package_path, external_root, logging_level)
remote_proxy.pull_package(package_path)
contents = unpack(file_system, package_path, external_root)
extend_externals(contents)
for package in packages:
if package not in updated:
package_path = join(external_packages_root, package)
if not file_system.exists(package_path):
remote_proxy.pull_package(package_path)
contents = unpack(file_system, package_path, external_root)
else:
contents = get_package_extracted_contents(file_system, package_path, external_root)
extend_externals(contents)
resources['external_resources'] = external_resources
resources['external_headers'] = external_headers
resources['external_libraries'] = external_libraries
resources['external_libraries_interfaces'] = external_libraries_interfaces
resources['external_symbols_tables'] = external_symbols_tables
resources['external_executables'] = external_executables
cache.clear()
cache.update(new_cache)
from praline.server.configuration import configuration
import logging.config
logging.config.dictConfig(configuration['logging'])
from flask import Flask, send_from_directory, request, Response, jsonify
from praline.common.file_system import FileSystem, join
from praline.common.hashing import hash_archive
from praline.common.package import get_package_dependencies_recursively
from typing import Dict
file_system = FileSystem()
server = Flask(__name__, root_path=file_system.get_working_directory())
repository_path = join(configuration['repository'], 'packages')
@server.route('/package/<package>', methods=['GET', 'PUT'])
def package(package) -> Response:
file_system.create_directory_if_missing(repository_path)
if request.method == 'GET':
return send_from_directory(repository_path,
package,
as_attachment=True)
elif request.method == 'PUT':
package_path = join(repository_path, package)
if file_system.exists(package_path):
return Response(
f"package '{package}' already exists -- increment the version and try again",
status=409,
