def configure(self, build_graph: Graph): config = "" node_ids = {} id = 0 for layer in build_graph.layers(): for node in layer: for artifact in node.artifacts(): config += f"path {artifact.path} #{id}\n" if artifact.dependencies: config += f"deps {' '.join([str(node_ids[node]) for node in artifact.dependencies])}\n" for cmd in artifact.commands: config += "run" for arg in cmd: config += f' "{arg}"' config += '\n' for cmd in artifact.always: config += "always" for arg in cmd: config += f' "{arg}"' config += '\n' # Only the id for the final artifact is used by other nodes node_ids[node] = id id += 1 G_LOGGER.info(f"Generating configuration files in build directory: {self.build_dir}") with open(self.config_file, "w") as f: G_LOGGER.debug(f"Writing {self.config_file}") f.write(config)
def __init__(self, path: str, inputs: List["Node"] = []): self.path = path self.inputs: List[Node] = [] self.outputs: List[Node] = [] G_LOGGER.debug( f"Constructing {type(self)} with path: {self.path}, with {len(inputs)} inputs: {inputs}" ) for inp in inputs: self.add_input(inp)
def build(self, nodes: List[Node]) -> (subprocess.CompletedProcess, float): # Early exit if no targets were provided if not nodes: G_LOGGER.debug(f"No targets specified, skipping build.") return subprocess.CompletedProcess(args=[], returncode=0, stdout=b"", stderr=b"No targets specified"), 0 paths = [node.path for node in nodes] cmd = ["rbuild", "--threads", str(multiprocessing.cpu_count()), f"{self.config_file}"] + paths G_LOGGER.verbose(f"Build command: {' '.join(cmd)}\nTarget file paths: {paths}") return utils.time_subprocess(cmd)
def load(path: str = None) -> "Project": f""" Load a project from the specified path. :param path: The path from which to load the project. Defaults to {os.path.abspath(os.path.join("build", Project.DEFAULT_SAVED_PROJECT_NAME))} :returns: The loaded project. """ path = path or os.path.abspath( os.path.join("build", Project.DEFAULT_SAVED_PROJECT_NAME)) G_LOGGER.debug(f"Loading project from {path}") with open(path, "rb") as f: return pickle.load(f)
def _run_linked_node(self, node: LinkedNode, *args, **kwargs) -> subprocess.CompletedProcess: loader_path = os.environ.get(paths.loader_path_env_var(), "") G_LOGGER.verbose(f"Running linked node: {node}") for lib_dir in node.lib_dirs: loader_path += f"{os.path.pathsep}{lib_dir}" G_LOGGER.debug(f"Using loader paths: {loader_path}") G_LOGGER.log( f"{paths.loader_path_env_var()}={loader_path} {node.path}\n", colors=[Color.BOLD, Color.GREEN]) env = copy.copy(os.environ) env[paths.loader_path_env_var()] = loader_path return subprocess.run([node.path], *args, env=env, **kwargs)
def select_nodes(targets: List[ProjectTarget], profile_names: List[str]) -> List[Node]: # Create all required profile build directories and populate nodes. nodes = [] for prof_name in profile_names: if prof_name not in self.profiles: G_LOGGER.critical( f"Profile {prof_name} does not exist in the project. Available profiles: {self.all_profile_names()}" ) # Populate nodes. for target in targets: if prof_name in target: node = target[prof_name] G_LOGGER.verbose( f"For target: {target}, profile: {prof_name}, found path: {node.path}" ) nodes.append(node) else: G_LOGGER.debug( f"Skipping target: {target.name} for profile: {prof_name}, as it does not exist." ) return nodes
def link( self, input_paths: List[str], output_path: str, libs: List[str] = [], lib_dirs: List[str] = [], flags: BuildFlags = BuildFlags()) -> List[str]: G_LOGGER.debug(f"self.ldef: {self.ldef}") linker_flags = self.ldef.parse_flags(flags) lib_dirs = [self.ldef.lib_dir(dir) for dir in lib_dirs] # In libs, absolute paths are not prepended with the lib prefix (e.g. -l) libs = [ lib if os.path.isabs(lib) else self.ldef.lib(lib) for lib in libs ] # The full command. cmd = [self.ldef.executable() ] + input_paths + libs + linker_flags + lib_dirs + [ self.ldef.output(output_path) ] G_LOGGER.debug( f"Linking: input_paths: {input_paths}, libs: {libs}, linker_flags: {linker_flags}, lib_dirs: {lib_dirs}" ) G_LOGGER.verbose(f"Link Command: {' '.join(cmd)}") return cmd
def __init__(self, root_dir: str, dirs: Set[str]=set(), exclude_dirs: Set[str]=set(), writable_dirs: Set[str]=set()): self.exclude_dirs: Set[str] = exclude_dirs # writable_dirs are the only locations to which FileManager is allowed to write. self.writable_dirs: Set[str] = writable_dirs G_LOGGER.verbose(f"Excluded directories: {exclude_dirs}. Writable directories: {writable_dirs}") # Include dirs/"header" files are only considered when searching for includes. self.include_dirs: List[str] = [] self.header_files: List[str] = [] # List to enable header priority self.files: Set[str] = set() self.root_dir = os.path.abspath(root_dir) if not os.path.isdir(self.root_dir): G_LOGGER.critical(f"Root Directory: {self.root_dir} does not exist, or is not a directory.") self.add_dir(self.root_dir) # Remove directories that are within exclude_dirs after converting all directories to abspaths. for dir in dirs: self.add_dir(dir) G_LOGGER.debug(f"Found {len(self.files)} files") G_LOGGER.verbose(f"{self.files}") # Keep track of all files relevant to building the project. self.graph = Graph()
def scan(self, node: str) -> None: G_LOGGER.debug(f"Scanning {node.path}") # Finds the file path for the file included in `include_path` by the `included_token` token. # This always returns an absolute path, since self.find always returns absolute paths. def disambiguate_included_file(included_token: str, include_path: str) -> str: # TODO: Handle paths that start with ../ # Such paths should always be relative to the file itself, otherwise it's an error. if include_path.startswith(os.path.pardir): raise NotImplementedError(f"FileManager does not currently support includes containing {os.path.pardir}") candidates = self.find(included_token, search_include_dirs=True) if len(candidates) == 0: return None # TODO: Move this into misc.paths, add test case. # Determines how "close together" files are. Smaller numbers mean they are further apart in the tree. def file_proximity(path_a: str, path_b: str) -> int: return len(os.path.commonpath([path_a, path_b]).split(os.path.sep)) # Return the path that is closest to the including file closest_path = max(candidates, key=lambda candidate: file_proximity(candidate, include_path)) if len(candidates) > 1: G_LOGGER.warning(f"For {include_path}, found multiple possible headers, but determined that {closest_path} best matches include for {included_token}. If this is not the case, please provide a longer path in the include to disambiguate, or manually provide the correct include directories. Note, candidates were: {candidates}") return closest_path # TODO: Handle relative paths in included here. # TODO: FIXME: This will not work if the include has escaped characters in it. def get_path_include_dir(included_path: str, included_token: str) -> str: # included_path is the full path of the included file. # included_token is the token used to include the file. # Absolute paths do not require include directories. if os.path.isabs(included_token): return None include_dir = included_path[:-len(included_token)] if not os.path.isdir(include_dir): # It would be completely ridiculous if this actually displays ever. G_LOGGER.critical(f"While attempting to find include dir to use for {included_path} (Note: included in {path}), found that {include_dir} does not exist!") return os.path.abspath(include_dir) # Find all included files in this file. If they are in the project, recurse over them. # Otherwise, assume they are external headers. include_dirs = set() external_includes = set() path = node.path included_files = _find_included(path) for included in included_files: # Determines the most likely file path based on an include. included_path = disambiguate_included_file(included, path) if included_path: G_LOGGER.verbose(f"For included token {included}, found path: {included_path}") # The include dir for a path for path depends on how exactly the path was included in path. include_dir = get_path_include_dir(included_path, included) if include_dir: G_LOGGER.verbose(f"For path {included_path}, using include dir: {include_dir}") include_dirs.add(include_dir) # Also recurse over any include directories needed for the path itself included_path_node = self.source(included_path) if included_path_node.include_dirs is None: G_LOGGER.verbose(f"{included_path_node} does not specify include directories. Scanning file.") self.scan(included_path_node) include_dirs.update(included_path_node.include_dirs) node.add_input(included_path_node) else: external_includes.add(included) if external_includes: G_LOGGER.debug(f"For {path}, could not find headers: {external_includes}. Assuming they are external. If this is not the case, please add the appropriate directories to the project definition.") include_dirs = sorted(include_dirs) node.include_dirs = include_dirs G_LOGGER.debug(f"For {path}, found include dirs: {include_dirs}") G_LOGGER.verbose(f"Updated source graph to: {self.graph}")
def build(self, targets: List[ProjectTarget] = None, profile_names: List[str] = None) -> float: """ Builds the specified targets for this project. Configuration should be run prior to calling this function. :param targets: The targets to build. Defaults to all targets. :param profile_names: The profiles for which to build the targets. Defaults to all profiles. :returns: Time elapsed during the build. """ targets = utils.default_value(targets, self.all_targets()) profile_names = utils.default_value(profile_names, self.all_profile_names()) G_LOGGER.info( f"Building targets: {[target.name for target in targets]} for profiles: {profile_names}" ) G_LOGGER.debug(f"Targets: {targets}") def select_nodes(targets: List[ProjectTarget], profile_names: List[str]) -> List[Node]: # Create all required profile build directories and populate nodes. nodes = [] for prof_name in profile_names: if prof_name not in self.profiles: G_LOGGER.critical( f"Profile {prof_name} does not exist in the project. Available profiles: {self.all_profile_names()}" ) # Populate nodes. for target in targets: if prof_name in target: node = target[prof_name] G_LOGGER.verbose( f"For target: {target}, profile: {prof_name}, found path: {node.path}" ) nodes.append(node) else: G_LOGGER.debug( f"Skipping target: {target.name} for profile: {prof_name}, as it does not exist." ) return nodes nodes = select_nodes(targets, profile_names) if not nodes: return # Create all required build directories. self.files.mkdir(self.common_build_dir) profile_build_dirs = [ self.profiles[prof_name].build_dir for prof_name in profile_names ] [self.files.mkdir(dir) for dir in profile_build_dirs] G_LOGGER.verbose( f"Created build directories: {self.common_build_dir}, {profile_build_dirs}" ) if not self.backend: G_LOGGER.critical( f"Backend has not been configured. Please call `configure()` prior to attempting to build" ) status, time_elapsed = self.backend.build(nodes) if status.returncode: G_LOGGER.critical( f"Failed with to build. Reconfiguring the project or running a clean build may resolve this." ) G_LOGGER.info( f"Built {plural('target', len(targets))} for {plural('profile', len(profile_names))} in {time_elapsed} seconds." ) return time_elapsed
def _target( self, name: str, ext_path: str, sources: List[str], flags: BuildFlags, libs: List[Union[DependencyLibrary, ProjectTarget, Library]], compiler: compiler.Compiler, include_dirs: List[str], linker: linker.Linker, depends: List[Dependency], internal: bool, is_lib: bool, ) -> ProjectTarget: if not all([ isinstance(lib, ProjectTarget) or isinstance(lib, Library) or isinstance(lib, DependencyLibrary) for lib in libs ]): G_LOGGER.critical( f"Libraries must be instances of either sbuildr.Library, sbuildr.dependencies.DependencyLibrary or sbuildr.ProjectTarget" ) if os.path.basename(ext_path) != ext_path: G_LOGGER.critical( f"Target: {ext_path} looks like a path. Target names should not contain characters that are unsupported by the filesystem." ) dependencies: List[Dependency] = [] + depends # Create copy for lib in libs: if isinstance(lib, DependencyLibrary): dependencies.append(lib.dependency) # Add all Library targets from dependencies to the file manager's graph, since they are independent of profiles # TODO: Add `library` function to FileManager self.files.graph.add(lib.library) G_LOGGER.verbose(f"Adding {lib.library} to file manager.") # Inherit dependencies from any input libraries as well [ dependencies.extend(lib.dependencies) for lib in libs if isinstance(lib, ProjectTarget) ] libs: List[Union[ProjectTarget, Library]] = [ lib.library if isinstance(lib, DependencyLibrary) else lib for lib in libs ] source_nodes: List[CompiledNode] = [ self.files.source(path) for path in sources ] G_LOGGER.verbose( f"For sources: {sources}, found source paths: {source_nodes}") target = ProjectTarget(name=name, internal=internal, is_lib=is_lib, dependencies=dependencies) for profile_name, profile in self.profiles.items(): # Convert all libraries to nodes. These will be inputs to the target. # Profile will later convert them to library names and directories. lib_nodes: List[Library] = [ lib[profile_name] if isinstance(lib, ProjectTarget) else lib for lib in libs ] input_nodes = [lib for lib in lib_nodes] G_LOGGER.verbose( f"Library inputs for target: {name} are: {input_nodes}") # Per-target flags always overwrite profile flags. flags = profile.flags + flags # First, add or retrieve object nodes for each source. for source_node in source_nodes: obj_path = os.path.join( self.common_build_dir, f"{os.path.splitext(os.path.basename(source_node.path))[0]}.o" ) # User defined includes are always prepended the ones deduced for SourceNodes. obj_node = CompiledNode(obj_path, source_node, compiler, include_dirs, flags) input_nodes.append(profile.graph.add(obj_node)) # Hard links are needed because during linkage, the library must have a clean name. hashed_path = os.path.join(self.common_build_dir, ext_path) path = os.path.join(profile.build_dir, paths.insert_suffix(ext_path, profile.suffix)) target[profile_name] = profile.graph.add( LinkedNode(path, input_nodes, linker, hashed_path=hashed_path, flags=flags)) G_LOGGER.debug( f"Adding target: {name}, with hashed path: {hashed_path}, public path: {path} to profile: {profile_name}" ) return target
from sbuildr.project.project import Project from sbuildr.project.profile import Profile from sbuildr.tools.flags import BuildFlags from sbuildr.tools import compiler, linker from sbuildr.graph.node import Library from sbuildr.logger import G_LOGGER, SBuildrException, Verbosity __version__ = "0.6.4" G_LOGGER.debug(f"Loading SBuildr {__version__} from {__path__}")