def topological_order(root_dir, whitelisted=None, blacklisted=None, underlay_workspaces=None): ''' Crawls the filesystem to find projects and uses their dependencies to return a topologically order list. :param root_dir: The path to search in, ``str`` :param whitelisted: A list of whitelisted project names, ``list`` :param blacklisted: A list of blacklisted project names, ``list`` :param underlay_workspaces: A list of underlay workspaces of projects which might provide dependencies in case of partial workspaces, ``list`` :returns: A list of tuples containing the relative path and a ``Project`` object, ``list`` ''' projects = find_projects(root_dir) # find projects in underlayed workspaces underlay_workspaces = {} if underlay_workspaces: for workspace in reversed(underlay_workspaces): for path, project in find_projects(workspace).items(): underlay_projects[project.name] = (path, project) return topological_order_projects(projects, whitelisted, blacklisted, underlay_projects=dict(underlay_projects.values()))
def test_validate_metaproject(self): pkgs_dict = find_projects(test_data_dir) for path, project in pkgs_dict.items(): path = os.path.join(test_data_dir, path) assert project.name in test_expectations, 'Unknown test %s' % project.name exc, excreg, warnreg = test_expectations[project.name] with assert_warning(warnreg): if exc is not None: if excreg is not None: with self.assertRaisesRegexp(exc, excreg): _validate_metaproject(path, project) else: with self.assertRaises(exc): _validate_metaproject(path, project) else: _validate_metaproject(path, project)
def find_in_workspaces(search_dirs=None, project=None, path=None, _workspaces=get_workspaces(), considered_paths=None, first_matching_workspace_only=False, first_match_only=False): ''' Find all paths which match the search criteria. All workspaces are searched in order. Each workspace, each search_in subfolder, the project name and the path are concatenated to define a candidate path. If the candidate path exists it is appended to the result list. Note: the search might return multiple paths for 'share' from devel- and source-space. :param search_dir: The list of subfolders to search in (default contains all valid values: 'bin', 'etc', 'lib', 'libexec', 'share'), ``list`` :param project: The project name to search for (optional, not possible with the global search_in folders 'bin' and 'lib'), ``str`` :param path: The path, ``str`` :param _workspaces: (optional, used for unit tests), the list of workspaces to use. :param considered_paths: If not None, function will append all path that were searched :param first_matching_workspace_only: if True returns all results found for first workspace with results :param first_match_only: if True returns first path found (supercedes first_matching_workspace_only) :raises ValueError: if search_dirs contains an invalid folder name :returns: List of paths ''' search_dirs = _get_valid_search_dirs(search_dirs, project) paths = [] existing_paths = [] try: for workspace in (_workspaces or []): for sub in search_dirs: # search in workspace p = os.path.join(workspace, sub if sub != 'libexec' else 'lib') if project: p = os.path.join(p, project) if path: p = os.path.join(p, path) paths.append(p) if os.path.exists(p): existing_paths.append(p) if first_match_only: raise StopIteration # for search in share also consider source spaces if project is not None and sub == 'share': source_paths = get_source_paths(workspace) for source_path in source_paths: packages = find_projects(source_path) matching_packages = [p for p, pkg in packages.items() if pkg.name == project] if matching_packages: p = os.path.join(source_path, matching_packages[0]) if path is not None: p = os.path.join(p, path) paths.append(p) if os.path.exists(p): existing_paths.append(p) if first_match_only: raise StopIteration if first_matching_workspace_only and existing_paths: break except StopIteration: pass if considered_paths is not None: considered_paths.extend(paths) return existing_paths
def build_workspace_isolated( workspace='.', sourcespace=None, buildspace=None, develspace=None, installspace=None, merge=False, install=False, force_cmake=False, colorize=True, build_packages=None, quiet=False, cmake_args=None, make_args=None, alpine_make_args=None, continue_from_pkg=False, only_pkg_with_deps=None, destdir=None ): ''' Runs ``cmake``, ``make`` and optionally ``make install`` for all alpine projects in sourcespace_dir. It creates several folders in the current working directory. For non-alpine projects it runs ``cmake``, ``make`` and ``make install`` for each, installing it to the devel space or install space if the ``install`` option is specified. :param workspace: path to the current workspace, ``str`` :param sourcespace: workspace folder containing alpine projects, ``str`` :param buildspace: path to build space location, ``str`` :param develspace: path to devel space location, ``str`` :param installspace: path to install space (CMAKE_INSTALL_PREFIX), ``str`` :param merge: if True, build each alpine project into the same devel space (not affecting plain cmake packages), ``bool`` :param install: if True, install all packages to the install space, ``bool`` :param force_cmake: (optional), if True calls cmake explicitly for each package, ``bool`` :param colorize: if True, colorize cmake output and other messages, ``bool`` :param build_packages: specific packages to build (all parent packages in the topological order must have been built before), ``str`` :param quiet: if True, hides some build output, ``bool`` :param cmake_args: additional arguments for cmake, ``[str]`` :param make_args: additional arguments for make, ``[str]`` :param alpine_make_args: additional arguments for make but only for alpine packages, ``[str]`` :param continue_from_pkg: indicates whether or not cmi should continue when a package is reached, ``bool`` :param only_pkg_with_deps: only consider the specific packages and their recursive dependencies and ignore all other packages in the workspace, ``[str]`` :param destdir: define DESTDIR for cmake/invocation, ``string`` ''' if not colorize: disable_ANSI_colors() # Check workspace existance if not os.path.exists(workspace): sys.exit("Workspace path '{0}' does not exist.".format(workspace)) workspace = os.path.abspath(workspace) # Check source space existance if sourcespace is None: ws_sourcespace = os.path.join(workspace, 'src') if not os.path.exists(ws_sourcespace): sys.exit("Could not find source space: {0}".format(sourcespace)) sourcespace = ws_sourcespace sourcespace = os.path.abspath(sourcespace) print('Base path: ' + str(workspace)) print('Source space: ' + str(sourcespace)) # Check build space if buildspace is None: buildspace = os.path.join(workspace, 'build_isolated') buildspace = os.path.abspath(buildspace) if not os.path.exists(buildspace): os.mkdir(buildspace) print('Build space: ' + str(buildspace)) # Check devel space if develspace is None: develspace = os.path.join(workspace, 'devel_isolated') develspace = os.path.abspath(develspace) print('Devel space: ' + str(develspace)) # Check install space if installspace is None: installspace = os.path.join(workspace, 'install_isolated') installspace = os.path.abspath(installspace) print('Install space: ' + str(installspace)) if cmake_args: print("Additional CMake Arguments: " + " ".join(cmake_args)) else: cmake_args = [] if make_args: print("Additional make Arguments: " + " ".join(make_args)) else: make_args = [] if alpine_make_args: print("Additional make Arguments for alpine projects: " + " ".join(alpine_make_args)) else: alpine_make_args = [] # Find packages packages = find_projects(sourcespace, exclude_subspaces=True) if not packages: print(fmt("@{yf}No packages found in source space: %s@|" % sourcespace)) # whitelist packages and their dependencies in workspace if only_pkg_with_deps: project_names = [p.name for p in packages.values()] unknown_packages = [name for name in only_pkg_with_deps if name not in project_names] if unknown_packages: sys.exit('Projects not found in the workspace: %s' % ', '.join(unknown_packages)) whitelist_pkg_names = get_project_names_with_recursive_dependencies(packages, only_pkg_with_deps) print('Whitelisted packages: %s' % ', '.join(sorted(whitelist_pkg_names))) packages = {path: p for path, p in packages.items() if p.name in whitelist_pkg_names} # verify that specified package exists in workspace if build_packages: packages_by_name = {p.name: path for path, p in packages.items()} unknown_packages = [p for p in build_packages if p not in packages_by_name] if unknown_packages: sys.exit('Projects not found in the workspace: %s' % ', '.join(unknown_packages)) # Report topological ordering ordered_projects = topological_order_projects(packages) unknown_build_types = [] msg = [] msg.append('@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' + ('~' * len(str(len(ordered_projects))))) msg.append('@{pf}~~@| traversing %d projects in topological order:' % len(ordered_projects)) for path, package in ordered_projects: export_tags = [e.tagname for e in package.exports] if 'build_type' in export_tags: build_type_tag = [e.content for e in package.exports if e.tagname == 'build_type'][0] else: build_type_tag = 'alpine' if build_type_tag == 'alpine': msg.append('@{pf}~~@| - @!@{bf}' + package.name + '@|') elif build_type_tag == 'cmake': msg.append( '@{pf}~~@| - @!@{bf}' + package.name + '@|' + ' (@!@{cf}plain cmake@|)' ) else: msg.append( '@{pf}~~@| - @!@{bf}' + package.name + '@|' + ' (@{rf}unknown@|)' ) unknown_build_types.append(package) msg.append('@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' + ('~' * len(str(len(ordered_projects))))) for index in range(len(msg)): msg[index] = fmt(msg[index]) print('\n'.join(msg)) # Error if there are packages with unknown build_types if unknown_build_types: print(colorize_line('Error: Projects with unknown build types exist')) sys.exit('Can not build workspace with projects of unknown build_type') # Check to see if the workspace has changed if not force_cmake and cmake_input_changed(packages, buildspace, cmake_args=cmake_args, filename='alpine_make_isolated'): print('The packages or cmake arguments have changed, forcing cmake invocation') force_cmake = True # Build packages pkg_develspace = None last_env = None for index, path_package in enumerate(ordered_projects): path, package = path_package if merge: pkg_develspace = develspace else: pkg_develspace = os.path.join(develspace, package.name) if not build_packages or package.name in build_packages: if continue_from_pkg and build_packages and package.name in build_packages: build_packages = None try: print() last_env = build_package( path, package, workspace, buildspace, pkg_develspace, installspace, install, force_cmake, quiet, last_env, cmake_args, make_args, alpine_make_args, destdir=destdir, number=index + 1, of=len(ordered_projects) ) except subprocess.CalledProcessError as e: _print_build_error(package, e) # Let users know how to reproduce # First add the cd to the build folder of the package cmd = 'cd ' + os.path.join(buildspace, package.name) + ' && ' # Then reproduce the command called cmd += ' '.join(e.cmd) if isinstance(e.cmd, list) else e.cmd print(fmt("\n@{rf}Reproduce this error by running:")) print(fmt("@{gf}@!==> @|") + cmd + "\n") sys.exit('Command failed, exiting.') except Exception as e: print("Unhandled exception of type '{0}':".format(type(e).__name__)) import traceback traceback.print_exc() _print_build_error(package, e) sys.exit('Command failed, exiting.') else: cprint("Skipping project: '@!@{bf}" + package.name + "@|'") last_env = get_new_env(package, pkg_develspace, installspace, install, last_env, destdir) # Provide a top level devel space environment setup script if not os.path.exists(develspace): os.makedirs(develspace) if not build_packages: generated_env_sh = os.path.join(develspace, 'env.sh') generated_setup_util_py = os.path.join(develspace, '_setup_util.py') if not merge and pkg_develspace: # generate env.sh and setup.sh|bash|zsh which relay to last devel space with open(generated_env_sh, 'w') as f: f.write("""\ #!/usr/bin/env sh # generated from alpine.builder module {0} "$@" """.format(os.path.join(pkg_develspace, 'env.sh'))) os.chmod(generated_env_sh, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) for shell in ['sh', 'bash', 'zsh']: with open(os.path.join(develspace, 'setup.%s' % shell), 'w') as f: f.write("""\ #!/usr/bin/env {1} # generated from alpine.builder module . "{0}/setup.{1}" """.format(pkg_develspace, shell)) # remove _setup_util.py file which might have been generated for an empty devel space before if os.path.exists(generated_setup_util_py): os.remove(generated_setup_util_py) elif not pkg_develspace: # generate env.sh and setup.sh|bash|zsh for an empty devel space if 'CMAKE_PREFIX_PATH' in os.environ.keys(): variables = { 'ALPINE_GLOBAL_BIN_DESTINATION': 'bin', 'ALPINE_GLOBAL_LIB_DESTINATION': 'lib', 'CMAKE_PREFIX_PATH_AS_IS': ';'.join(os.environ['CMAKE_PREFIX_PATH'].split(os.pathsep)), 'PYTHON_INSTALL_DIR': get_python_install_dir(), } with open(generated_setup_util_py, 'w') as f: f.write(configure_file(os.path.join(get_cmake_path(), 'templates', '_setup_util.py.in'), variables)) os.chmod(generated_setup_util_py, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) else: sys.exit("Unable to process CMAKE_PREFIX_PATH from environment. Cannot generate environment files.") variables = {'SETUP_FILENAME': 'setup'} with open(generated_env_sh, 'w') as f: f.write(configure_file(os.path.join(get_cmake_path(), 'templates', 'env.sh.in'), variables)) os.chmod(generated_env_sh, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) variables = {'SETUP_DIR': develspace} for shell in ['sh', 'bash', 'zsh']: with open(os.path.join(develspace, 'setup.%s' % shell), 'w') as f: f.write(configure_file(os.path.join(get_cmake_path(), 'templates', 'setup.%s.in' % shell), variables))