def rm_kconfig_include(path): """Remove a path from Kconfig files This function finds the given path in a 'source' statement in a Kconfig file and removes that line from the file. This is needed because the path is going to be removed, so any reference to it will cause a problem with Kconfig parsing. The changes are made locally and then added to the git staging area. Args: path: Path to search for and remove """ cmd = ['git', 'grep', path] stdout = command.run_pipe([cmd], capture=True, raise_on_error=False).stdout if not stdout: return fname = stdout.split(':')[0] print("Fixing up '%s' to remove reference to '%s'" % (fname, path)) cmd = ['sed', '-i', '\|%s|d' % path, fname] stdout = command.run_pipe([cmd], capture=True).stdout cmd = ['git', 'add', fname] stdout = command.run_pipe([cmd], capture=True).stdout
def count_commits_to_branch(branch): """Returns number of commits between HEAD and the tracking branch. This looks back to the tracking branch and works out the number of commits since then. Args: branch: Branch to count from (None for current branch) Return: Number of patches that exist on top of the branch """ if branch: us, msg = get_upstream('.git', branch) rev_range = '%s..%s' % (us, branch) else: rev_range = '@{upstream}..' pipe = [log_cmd(rev_range, oneline=True)] result = command.run_pipe(pipe, capture=True, capture_stderr=True, oneline=True, raise_on_error=False) if result.return_code: raise ValueError('Failed to determine upstream: %s' % result.stderr.strip()) patch_count = len(result.stdout.splitlines()) return patch_count
def guess_upstream(git_dir, branch): """Tries to guess the upstream for a branch This lists out top commits on a branch and tries to find a suitable upstream. It does this by looking for the first commit where 'git name-rev' returns a plain branch name, with no ! or ^ modifiers. Args: git_dir: Git directory containing repo branch: Name of branch Returns: Tuple: Name of upstream branch (e.g. 'upstream/master') or None if none Warning/error message, or None if none """ pipe = [log_cmd(branch, git_dir=git_dir, oneline=True, count=100)] result = command.run_pipe(pipe, capture=True, capture_stderr=True, raise_on_error=False) if result.return_code: return None, "Branch '%s' not found" % branch for line in result.stdout.splitlines()[1:]: commit_hash = line.split(' ')[0] name = name_revision(commit_hash) if '~' not in name and '^' not in name: if name.startswith('remotes/'): name = name[8:] return name, "Guessing upstream as '%s'" % name return None, "Cannot find a suitable upstream for branch '%s'" % branch
def prune_worktrees(git_dir): """Remove administrative files for deleted worktrees Args: git_dir: The repository whose deleted worktrees should be pruned """ pipe = ['git', '--git-dir', git_dir, 'worktree', 'prune'] result = command.run_pipe([pipe], capture=True, capture_stderr=True) if result.return_code != 0: raise OSError('git worktree prune: %s' % result.stderr)
def setup(): """Set up git utils, by reading the alias files.""" # Check for a git alias file also global use_no_decorate alias_fname = get_alias_file() if alias_fname: settings.ReadGitAliases(alias_fname) cmd = log_cmd(None, count=0) use_no_decorate = (command.run_pipe([cmd], raise_on_error=False).return_code == 0)
def count_commits(commit_range): """Returns the number of commits in the given range. Args: commit_range: Range of commits to count (e.g. 'HEAD..base') Return: Number of patches that exist on top of the branch """ pipe = [log_cmd(commit_range, oneline=True), ['wc', '-l']] stdout = command.run_pipe(pipe, capture=True, oneline=True).stdout patch_count = int(stdout) return patch_count
def run_result(name, *args, **kwargs): """Run a tool with some arguments This runs a 'tool', which is a program used by binman to process files and perhaps produce some output. Tools can be located on the PATH or in a search path. Args: name: Command name to run args: Arguments to the tool for_host: True to resolve the command to the version for the host for_target: False to run the command as-is, without resolving it to the version for the compile target raise_on_error: Raise an error if the command fails (True by default) Returns: CommandResult object """ try: binary = kwargs.get('binary') for_host = kwargs.get('for_host', False) for_target = kwargs.get('for_target', not for_host) raise_on_error = kwargs.get('raise_on_error', True) env = get_env_with_path() if for_target: name, extra_args = get_target_compile_tool(name) args = tuple(extra_args) + args elif for_host: name, extra_args = get_host_compile_tool(name) args = tuple(extra_args) + args name = os.path.expanduser(name) # Expand paths containing ~ all_args = (name, ) + args result = command.run_pipe([all_args], capture=True, capture_stderr=True, env=env, raise_on_error=False, binary=binary) if result.return_code: if raise_on_error: raise ValueError("Error %d running '%s': %s" % (result.return_code, ' '.join(all_args), result.stderr or result.stdout)) return result except ValueError: if env and not path_has_file(env['PATH'], name): msg = "Please install tool '%s'" % name package = packages.get(name) if package: msg += " (e.g. from package '%s')" % package raise ValueError(msg) raise
def clone(git_dir, output_dir): """Checkout the selected commit for this build Args: commit_hash: Commit hash to check out """ pipe = ['git', 'clone', git_dir, '.'] result = command.run_pipe([pipe], capture=True, cwd=output_dir, capture_stderr=True) if result.return_code != 0: raise OSError('git clone: %s' % result.stderr)
def name_revision(commit_hash): """Gets the revision name for a commit Args: commit_hash: Commit hash to look up Return: Name of revision, if any, else None """ pipe = ['git', 'name-rev', commit_hash] stdout = command.run_pipe([pipe], capture=True, oneline=True).stdout # We expect a commit, a space, then a revision name name = stdout.split(' ')[1].strip() return name
def check_worktree_is_available(git_dir): """Check if git-worktree functionality is available Args: git_dir: The repository to test in Returns: True if git-worktree commands will work, False otherwise. """ pipe = ['git', '--git-dir', git_dir, 'worktree', 'list'] result = command.run_pipe([pipe], capture=True, capture_stderr=True, raise_on_error=False) return result.return_code == 0
def fetch(git_dir=None, work_tree=None): """Fetch from the origin repo Args: commit_hash: Commit hash to check out """ pipe = ['git'] if git_dir: pipe.extend(['--git-dir', git_dir]) if work_tree: pipe.extend(['--work-tree', work_tree]) pipe.append('fetch') result = command.run_pipe([pipe], capture=True, capture_stderr=True) if result.return_code != 0: raise OSError('git fetch: %s' % result.stderr)
def run_cmd_result(self, *args, binary=False, raise_on_error=True): """Run the bintool using command-line arguments Args: args (list of str): Arguments to provide, in addition to the bintool name binary (bool): True to return output as bytes instead of str raise_on_error (bool): True to raise a ValueError exception if the tool returns a non-zero return code Returns: CommandResult: Resulting output from the bintool, or None if the tool is not present """ if self.name in self.missing_list: return None name = os.path.expanduser(self.name) # Expand paths containing ~ all_args = (name, ) + args env = tools.get_env_with_path() tout.detail(f"bintool: {' '.join(all_args)}") result = command.run_pipe([all_args], capture=True, capture_stderr=True, env=env, raise_on_error=False, binary=binary) if result.return_code: # Return None if the tool was not found. In this case there is no # output from the tool and it does not appear on the path. We still # try to run it (as above) since RunPipe() allows faking the tool's # output if not any([result.stdout, result.stderr, tools.tool_find(name)]): tout.info(f"bintool '{name}' not found") return None if raise_on_error: tout.info(f"bintool '{name}' failed") raise ValueError("Error %d running '%s': %s" % (result.return_code, ' '.join(all_args), result.stderr or result.stdout)) if result.stdout: tout.debug(result.stdout) if result.stderr: tout.debug(result.stderr) return result
def get_list(commit_range, git_dir=None, count=None): """Get a log of a list of comments This returns the output of 'git log' for the selected commits Args: commit_range (str): Range of commits to count (e.g. 'HEAD..base') git_dir (str): Path to git repositiory (None to use default) count (int): Number of commits to list, or None for no limit Returns str: String containing the contents of the git log """ params = gitutil.log_cmd(commit_range, reverse=True, count=count, git_dir=git_dir) return command.run_pipe([params], capture=True).stdout
def add_worktree(git_dir, output_dir, commit_hash=None): """Create and checkout a new git worktree for this build Args: git_dir: The repository to checkout the worktree from output_dir: Path for the new worktree commit_hash: Commit hash to checkout """ # We need to pass --detach to avoid creating a new branch pipe = ['git', '--git-dir', git_dir, 'worktree', 'add', '.', '--detach'] if commit_hash: pipe.append(commit_hash) result = command.run_pipe([pipe], capture=True, cwd=output_dir, capture_stderr=True) if result.return_code != 0: raise OSError('git worktree add: %s' % result.stderr)
def count_commits_in_range(git_dir, range_expr): """Returns the number of commits in the given range. Args: git_dir: Directory containing git repo range_expr: Range to check Return: Number of patches that exist in the supplied range or None if none were found """ pipe = [log_cmd(range_expr, git_dir=git_dir, oneline=True)] result = command.run_pipe(pipe, capture=True, capture_stderr=True, raise_on_error=False) if result.return_code: return None, "Range '%s' not found or is invalid" % range_expr patch_count = len(result.stdout.splitlines()) return patch_count, None
def checkout(commit_hash, git_dir=None, work_tree=None, force=False): """Checkout the selected commit for this build Args: commit_hash: Commit hash to check out """ pipe = ['git'] if git_dir: pipe.extend(['--git-dir', git_dir]) if work_tree: pipe.extend(['--work-tree', work_tree]) pipe.append('checkout') if force: pipe.append('-f') pipe.append(commit_hash) result = command.run_pipe([pipe], capture=True, raise_on_error=False, capture_stderr=True) if result.return_code != 0: raise OSError('git checkout (%s): %s' % (pipe, result.stderr))
def rm_board(board): """Create a commit which removes a single board This looks up the MAINTAINERS file to file files that need to be removed, then removes pieces from the Kconfig files that mention the board. Args: board: Board name to remove """ # Find all MAINTAINERS and Kconfig files which mention the board cmd = ['git', 'grep', '-l', board] stdout = command.run_pipe([cmd], capture=True).stdout maintain = [] kconfig = [] for line in stdout.splitlines(): line = line.strip() if 'MAINTAINERS' in line: if line not in maintain: maintain.append(line) elif 'Kconfig' in line: kconfig.append(line) paths = [] cc = [] # Look through the MAINTAINERS file to find things to remove for fname in maintain: with open(fname) as fd: for line in fd: line = line.strip() fields = re.split('[ \t]', line, 1) if len(fields) == 2: if fields[0] == 'M:': cc.append(fields[1]) elif fields[0] == 'F:': paths.append(fields[1].strip()) # Expand any wildcards in the MAINTAINERS file real = [] for path in paths: if path[-1] == '/': path = path[:-1] if '*' in path: globbed = glob.glob(path) print("Expanded '%s' to '%s'" % (path, globbed)) real += globbed else: real.append(path) # Search for Kconfig files in the resulting list. Remove any 'source' lines # which reference Kconfig files we want to remove for path in real: cmd = ['find', path] stdout = (command.run_pipe([cmd], capture=True, raise_on_error=False).stdout) for fname in stdout.splitlines(): if fname.endswith('Kconfig'): rm_kconfig_include(fname) # Remove unwanted files cmd = ['git', 'rm', '-r'] + real stdout = command.run_pipe([cmd], capture=True).stdout ## Change the messages as needed msg = '''arm: Remove %s board This board has not been converted to CONFIG_DM_MMC by the deadline. Remove it. ''' % board for name in cc: msg += 'Patch-cc: %s\n' % name # Create the commit cmd = ['git', 'commit', '-s', '-m', msg] stdout = command.run_pipe([cmd], capture=True).stdout # Check if the board is mentioned anywhere else. The user will need to deal # with this cmd = ['git', 'grep', '-il', board] print(command.run_pipe([cmd], capture=True, raise_on_error=False).stdout) print(' '.join(cmd))
def _RunBuildman(self, *args): return command.run_pipe([[self._buildman_pathname] + list(args)], capture=True, capture_stderr=True)
def _WriteResult(self, result, keep_outputs, work_in_output): """Write a built result to the output directory. Args: result: CommandResult object containing result to write keep_outputs: True to store the output binaries, False to delete them work_in_output: Use the output directory as the work directory and don't write to a separate output directory. """ # If we think this might have been aborted with Ctrl-C, record the # failure but not that we are 'done' with this board. A retry may fix # it. maybe_aborted = result.stderr and 'No child processes' in result.stderr if result.return_code >= 0 and result.already_done: return # Write the output and stderr output_dir = self.builder._GetOutputDir(result.commit_upto) Mkdir(output_dir) build_dir = self.builder.GetBuildDir(result.commit_upto, result.brd.target) Mkdir(build_dir) outfile = os.path.join(build_dir, 'log') with open(outfile, 'w') as fd: if result.stdout: fd.write(result.stdout) errfile = self.builder.GetErrFile(result.commit_upto, result.brd.target) if result.stderr: with open(errfile, 'w') as fd: fd.write(result.stderr) elif os.path.exists(errfile): os.remove(errfile) # Fatal error if result.return_code < 0: return if result.toolchain: # Write the build result and toolchain information. done_file = self.builder.GetDoneFile(result.commit_upto, result.brd.target) with open(done_file, 'w') as fd: if maybe_aborted: # Special code to indicate we need to retry fd.write('%s' % RETURN_CODE_RETRY) else: fd.write('%s' % result.return_code) with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: print('gcc', result.toolchain.gcc, file=fd) print('path', result.toolchain.path, file=fd) print('cross', result.toolchain.cross, file=fd) print('arch', result.toolchain.arch, file=fd) fd.write('%s' % result.return_code) # Write out the image and function size information and an objdump env = result.toolchain.MakeEnvironment(self.builder.full_path) with open(os.path.join(build_dir, 'out-env'), 'wb') as fd: for var in sorted(env.keys()): fd.write(b'%s="%s"' % (var, env[var])) lines = [] for fname in BASE_ELF_FILENAMES: cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] nm_result = command.run_pipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) if nm_result.stdout: nm = self.builder.GetFuncSizesFile(result.commit_upto, result.brd.target, fname) with open(nm, 'w') as fd: print(nm_result.stdout, end=' ', file=fd) cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] dump_result = command.run_pipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) rodata_size = '' if dump_result.stdout: objdump = self.builder.GetObjdumpFile(result.commit_upto, result.brd.target, fname) with open(objdump, 'w') as fd: print(dump_result.stdout, end=' ', file=fd) for line in dump_result.stdout.splitlines(): fields = line.split() if len(fields) > 5 and fields[1] == '.rodata': rodata_size = fields[2] cmd = ['%ssize' % self.toolchain.cross, fname] size_result = command.run_pipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) if size_result.stdout: lines.append(size_result.stdout.splitlines()[1] + ' ' + rodata_size) # Extract the environment from U-Boot and dump it out cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary', '-j', '.rodata.default_environment', 'env/built-in.o', 'uboot.env'] command.run_pipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) ubootenv = os.path.join(result.out_dir, 'uboot.env') if not work_in_output: self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env']) # Write out the image sizes file. This is similar to the output # of binutil's 'size' utility, but it omits the header line and # adds an additional hex value at the end of each line for the # rodata size if len(lines): sizes = self.builder.GetSizesFile(result.commit_upto, result.brd.target) with open(sizes, 'w') as fd: print('\n'.join(lines), file=fd) if not work_in_output: # Write out the configuration files, with a special case for SPL for dirname in ['', 'spl', 'tpl']: self.CopyFiles( result.out_dir, build_dir, dirname, ['u-boot.cfg', 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config', 'include/autoconf.mk', 'include/generated/autoconf.h']) # Now write the actual build output if keep_outputs: self.CopyFiles( result.out_dir, build_dir, '', ['u-boot*', '*.bin', '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk', 'spl/u-boot-spl*'])
def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, arch=None, override_toolchain=None): """Create a new toolchain object. Args: fname: Filename of the gcc component test: True to run the toolchain to test it verbose: True to print out the information priority: Priority to use for this toolchain, or PRIORITY_CALC to calculate it """ self.gcc = fname self.path = os.path.dirname(fname) self.override_toolchain = override_toolchain # Find the CROSS_COMPILE prefix to use for U-Boot. For example, # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. basename = os.path.basename(fname) pos = basename.rfind('-') self.cross = basename[:pos + 1] if pos != -1 else '' # The architecture is the first part of the name pos = self.cross.find('-') if arch: self.arch = arch else: self.arch = self.cross[:pos] if pos != -1 else 'sandbox' if self.arch == 'sandbox' and override_toolchain: self.gcc = override_toolchain env = self.MakeEnvironment(False) # As a basic sanity check, run the C compiler with --version cmd = [fname, '--version'] if priority == PRIORITY_CALC: self.priority = self.GetPriority(fname) else: self.priority = priority if test: result = command.run_pipe([cmd], capture=True, env=env, raise_on_error=False) self.ok = result.return_code == 0 if verbose: print('Tool chain test: ', end=' ') if self.ok: print("OK, arch='%s', priority %d" % (self.arch, self.priority)) else: print('BAD') print('Command: ', cmd) print(result.stdout) print(result.stderr) else: self.ok = True