def rm_kconfig_include(path): """Remove a path from Kconfig files This function finds the given path in a 'source' statement in a Kconfig file and removes that line from the file. This is needed because the path is going to be removed, so any reference to it will cause a problem with Kconfig parsing. The changes are made locally and then added to the git staging area. Args: path: Path to search for and remove """ cmd = ['git', 'grep', path] stdout = command.RunPipe([cmd], capture=True, raise_on_error=False).stdout if not stdout: return fname = stdout.split(':')[0] print("Fixing up '%s' to remove reference to '%s'" % (fname, path)) cmd = ['sed', '-i', '\|%s|d' % path, fname] stdout = command.RunPipe([cmd], capture=True).stdout cmd = ['git', 'add', fname] stdout = command.RunPipe([cmd], capture=True).stdout
def GetMetaDataForList(commit_range, git_dir=None, count=None, series = Series()): """Reads out patch series metadata from the commits This does a 'git log' on the relevant commits and pulls out the tags we are interested in. Args: commit_range: Range of commits to count (e.g. 'HEAD..base') git_dir: Path to git repositiory (None to use default) count: Number of commits to list, or None for no limit series: Series object to add information into. By default a new series is started. Returns: A Series object containing information about the commits. """ params = ['git', 'log', '--no-color', '--reverse', '--no-decorate', commit_range] if count is not None: params[2:2] = ['-n%d' % count] if git_dir: params[1:1] = ['--git-dir', git_dir] pipe = [params] stdout = command.RunPipe(pipe, capture=True).stdout ps = PatchStream(series, is_log=True) for line in stdout.splitlines(): ps.ProcessLine(line) ps.Finalize() return series
def Run(name, *args): """Run a tool with some arguments This runs a 'tool', which is a program used by binman to process files and perhaps produce some output. Tools can be located on the PATH or in a search path. Args: name: Command name to run args: Arguments to the tool Returns: CommandResult object """ try: env = None if tool_search_paths: env = dict(os.environ) env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH'] all_args = (name,) + args result = command.RunPipe([all_args], capture=True, capture_stderr=True, env=env, raise_on_error=False) if result.return_code: raise Exception("Error %d running '%s': %s" % (result.return_code,' '.join(all_args), result.stderr)) return result.stdout except: if env and not PathHasFile(env['PATH'], name): msg = "Please install tool '%s'" % name package = packages.get(name) if package: msg += " (e.g. from package '%s')" % package raise ValueError(msg) raise
def __init__(self, fname, test, verbose=False): """Create a new toolchain object. Args: fname: Filename of the gcc component test: True to run the toolchain to test it """ self.gcc = fname self.path = os.path.dirname(fname) self.cross = os.path.basename(fname)[:-3] pos = self.cross.find('-') self.arch = self.cross[:pos] if pos != -1 else 'sandbox' env = self.MakeEnvironment() # As a basic sanity check, run the C compiler with --version cmd = [fname, '--version'] if test: result = command.RunPipe([cmd], capture=True, env=env, raise_on_error=False) self.ok = result.return_code == 0 if verbose: print 'Tool chain test: ', if self.ok: print 'OK' else: print 'BAD' print 'Command: ', cmd print result.stdout print result.stderr else: self.ok = True self.priority = self.GetPriority(fname)
def GuessUpstream(git_dir, branch): """Tries to guess the upstream for a branch This lists out top commits on a branch and tries to find a suitable upstream. It does this by looking for the first commit where 'git name-rev' returns a plain branch name, with no ! or ^ modifiers. Args: git_dir: Git directory containing repo branch: Name of branch Returns: Tuple: Name of upstream branch (e.g. 'upstream/master') or None if none Warning/error message, or None if none """ pipe = [LogCmd(branch, git_dir=git_dir, oneline=True, count=100)] result = command.RunPipe(pipe, capture=True, capture_stderr=True, raise_on_error=False) if result.return_code: return None, "Branch '%s' not found" % branch for line in result.stdout.splitlines()[1:]: commit_hash = line.split(' ')[0] name = NameRevision(commit_hash) if '~' not in name and '^' not in name: if name.startswith('remotes/'): name = name[8:] return name, "Guessing upstream as '%s'" % name return None, "Cannot find a suitable upstream for branch '%s'" % branch
def ObtainContents(self): fname = tools.GetInputFilename('spl/u-boot-spl') args = [['nm', fname], ['grep', '__bss_size']] out = command.RunPipe(args, capture=True).stdout.splitlines() bss_size = int(out[0].split()[0], 16) self.data = chr(0) * bss_size self.contents_size = bss_size
def GetMetaDataForList(commit_range, git_dir=None, count=None, series=None, allow_overwrite=False): """Reads out patch series metadata from the commits This does a 'git log' on the relevant commits and pulls out the tags we are interested in. Args: commit_range: Range of commits to count (e.g. 'HEAD..base') git_dir: Path to git repositiory (None to use default) count: Number of commits to list, or None for no limit series: Series object to add information into. By default a new series is started. allow_overwrite: Allow tags to overwrite an existing tag Returns: A Series object containing information about the commits. """ if not series: series = Series() series.allow_overwrite = allow_overwrite params = gitutil.LogCmd(commit_range, reverse=True, count=count, git_dir=git_dir) stdout = command.RunPipe([params], capture=True).stdout ps = PatchStream(series, is_log=True) for line in stdout.splitlines(): ps.ProcessLine(line) ps.Finalize() return series
def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, arch=None): """Create a new toolchain object. Args: fname: Filename of the gcc component test: True to run the toolchain to test it verbose: True to print out the information priority: Priority to use for this toolchain, or PRIORITY_CALC to calculate it """ self.gcc = fname self.path = os.path.dirname(fname) # Find the CROSS_COMPILE prefix to use for U-Boot. For example, # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. basename = os.path.basename(fname) pos = basename.rfind('-') self.cross = basename[:pos + 1] if pos != -1 else '' # The architecture is the first part of the name pos = self.cross.find('-') if arch: self.arch = arch else: self.arch = self.cross[:pos] if pos != -1 else 'sandbox' env = self.MakeEnvironment(False) # As a basic sanity check, run the C compiler with --version cmd = [fname, '--version'] if priority == PRIORITY_CALC: self.priority = self.GetPriority(fname) else: self.priority = priority if test: result = command.RunPipe([cmd], capture=True, env=env, raise_on_error=False) self.ok = result.return_code == 0 if verbose: print 'Tool chain test: ', if self.ok: print "OK, arch='%s', priority %d" % (self.arch, self.priority) else: print 'BAD' print 'Command: ', cmd print result.stdout print result.stderr else: self.ok = True
def Setup(): """Set up git utils, by reading the alias files.""" # Check for a git alias file also alias_fname = GetAliasFile() if alias_fname: settings.ReadGitAliases(alias_fname) cmd = LogCmd(None, count=0) use_no_decorate = (command.RunPipe([cmd], raise_on_error=False).return_code == 0)
def Clone(git_dir, output_dir): """Checkout the selected commit for this build Args: commit_hash: Commit hash to check out """ pipe = ['git', 'clone', git_dir, '.'] result = command.RunPipe([pipe], capture=True, cwd=output_dir) if result.return_code != 0: raise OSError, 'git clone: %s' % result.stderr
def CountCommits(commit_range): """Returns the number of commits in the given range. Args: commit_range: Range of commits to count (e.g. 'HEAD..base') Return: Number of patches that exist on top of the branch """ pipe = [LogCmd(commit_range, oneline=True), ['wc', '-l']] stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout patch_count = int(stdout) return patch_count
def ObtainContents(self): # Figure out where to put the microcode pointer fname = tools.GetInputFilename(self.elf_fname) args = [['nm', fname], ['grep', '-w', '_dt_ucode_base_size']] out = (command.RunPipe(args, capture=True, raise_on_error=False).stdout.splitlines()) if len(out) == 1: self.target_pos = int(out[0].split()[0], 16) elif not fdt_util.GetBool(self._node, 'optional-ucode'): self.Raise('Cannot locate _dt_ucode_base_size symbol in u-boot') return Entry_blob.ObtainContents(self)
def _RunBinman(self, *args, **kwargs): """Run binman using the command line Args: Arguments to pass, as a list of strings kwargs: Arguments to pass to Command.RunPipe() """ result = command.RunPipe([[self._binman_pathname] + list(args)], capture=True, capture_stderr=True, raise_on_error=False) if result.return_code and kwargs.get('raise_on_error', True): raise Exception("Error running '%s': %s" % (' '.join(args), result.stdout + result.stderr)) return result
def CountCommitsToBranch(): """Returns number of commits between HEAD and the tracking branch. This looks back to the tracking branch and works out the number of commits since then. Return: Number of patches that exist on top of the branch """ pipe = [LogCmd('@{upstream}..', oneline=True), ['wc', '-l']] stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout patch_count = int(stdout) return patch_count
def Make(self, commit, brd, stage, cwd, *args, **kwargs): """Run make Args: commit: Commit object that is being built brd: Board object that is being built stage: Stage that we are at (mrproper, config, build) cwd: Directory where make should be run args: Arguments to pass to make kwargs: Arguments to pass to command.RunPipe() """ cmd = [self.gnu_make] + list(args) result = command.RunPipe([cmd], capture=True, capture_stderr=True, cwd=cwd, raise_on_error=False, **kwargs) return result
def Fetch(git_dir=None, work_tree=None): """Fetch from the origin repo Args: commit_hash: Commit hash to check out """ pipe = ['git'] if git_dir: pipe.extend(['--git-dir', git_dir]) if work_tree: pipe.extend(['--work-tree', work_tree]) pipe.append('fetch') result = command.RunPipe([pipe], capture=True) if result.return_code != 0: raise OSError, 'git fetch: %s' % result.stderr
def NameRevision(commit_hash): """Gets the revision name for a commit Args: commit_hash: Commit hash to look up Return: Name of revision, if any, else None """ pipe = ['git', 'name-rev', commit_hash] stdout = command.RunPipe([pipe], capture=True, oneline=True).stdout # We expect a commit, a space, then a revision name name = stdout.split(' ')[1].strip() return name
def CountCommitsInBranch(git_dir, branch, include_upstream=False): """Returns the number of commits in the given branch. Args: git_dir: Directory containing git repo branch: Name of branch Return: Number of patches that exist on top of the branch, or None if the branch does not exist. """ range_expr = GetRangeInBranch(git_dir, branch, include_upstream) if not range_expr: return None pipe = [LogCmd(range_expr, git_dir=git_dir, oneline=True), ['wc', '-l']] result = command.RunPipe(pipe, capture=True, oneline=True) patch_count = int(result.stdout) return patch_count
def CountCommitsInRange(git_dir, range_expr): """Returns the number of commits in the given range. Args: git_dir: Directory containing git repo range_expr: Range to check Return: Number of patches that exist in the supplied rangem or None if none were found """ pipe = [LogCmd(range_expr, git_dir=git_dir, oneline=True)] result = command.RunPipe(pipe, capture=True, capture_stderr=True, raise_on_error=False) if result.return_code: return None, "Range '%s' not found or is invalid" % range_expr patch_count = len(result.stdout.splitlines()) return patch_count, None
def CountCommitsInBranch(git_dir, branch, include_upstream=False): """Returns the number of commits in the given branch. Args: git_dir: Directory containing git repo branch: Name of branch Return: Number of patches that exist on top of the branch """ range_expr = GetRangeInBranch(git_dir, branch, include_upstream) pipe = [[ 'git', '--git-dir', git_dir, 'log', '--oneline', '--no-decorate', range_expr ], ['wc', '-l']] result = command.RunPipe(pipe, capture=True, oneline=True) patch_count = int(result.stdout) return patch_count
def GetMetaData(start, count): """Reads out patch series metadata from the commits This does a 'git log' on the relevant commits and pulls out the tags we are interested in. Args: start: Commit to start from: 0=HEAD, 1=next one, etc. count: Number of commits to list """ pipe = [['git', 'log', '--reverse', 'HEAD~%d' % start, '-n%d' % count]] stdout = command.RunPipe(pipe, capture=True) series = Series() ps = PatchStream(series, is_log=True) for line in stdout.splitlines(): ps.ProcessLine(line) ps.Finalize() return series
def Checkout(commit_hash, git_dir=None, work_tree=None, force=False): """Checkout the selected commit for this build Args: commit_hash: Commit hash to check out """ pipe = ['git'] if git_dir: pipe.extend(['--git-dir', git_dir]) if work_tree: pipe.extend(['--work-tree', work_tree]) pipe.append('checkout') if force: pipe.append('-f') pipe.append(commit_hash) result = command.RunPipe([pipe], capture=True, raise_on_error=False) if result.return_code != 0: raise OSError, 'git checkout (%s): %s' % (pipe, result.stderr)
def cbfstool(fname, *cbfs_args, **kwargs): """Run cbfstool with provided arguments If the tool fails then this function raises an exception and prints out the output and stderr. Args: fname: Filename of CBFS *cbfs_args: List of arguments to pass to cbfstool Returns: CommandResult object containing the results """ args = ['cbfstool', fname] + list(cbfs_args) if kwargs.get('base') is not None: args += ['-b', '%#x' % kwargs['base']] result = command.RunPipe([args], capture=not VERBOSE, capture_stderr=not VERBOSE, raise_on_error=False) if result.return_code: print(result.stderr, file=sys.stderr) raise Exception("Failed to run (error %d): '%s'" % (result.return_code, ' '.join(args)))
def rm_board(board): """Create a commit which removes a single board This looks up the MAINTAINERS file to file files that need to be removed, then removes pieces from the Kconfig files that mention the board. Args: board: Board name to remove """ # Find all MAINTAINERS and Kconfig files which mention the board cmd = ['git', 'grep', '-l', board] stdout = command.RunPipe([cmd], capture=True).stdout maintain = [] kconfig = [] for line in stdout.splitlines(): line = line.strip() if 'MAINTAINERS' in line: if line not in maintain: maintain.append(line) elif 'Kconfig' in line: kconfig.append(line) paths = [] cc = [] # Look through the MAINTAINERS file to find things to remove for fname in maintain: with open(fname) as fd: for line in fd: line = line.strip() fields = re.split('[ \t]', line, 1) if len(fields) == 2: if fields[0] == 'M:': cc.append(fields[1]) elif fields[0] == 'F:': paths.append(fields[1].strip()) # Expand any wildcards in the MAINTAINERS file real = [] for path in paths: if path[-1] == '/': path = path[:-1] if '*' in path: globbed = glob.glob(path) print("Expanded '%s' to '%s'" % (path, globbed)) real += globbed else: real.append(path) # Search for Kconfig files in the resulting list. Remove any 'source' lines # which reference Kconfig files we want to remove for path in real: cmd = ['find', path] stdout = (command.RunPipe([cmd], capture=True, raise_on_error=False).stdout) for fname in stdout.splitlines(): if fname.endswith('Kconfig'): rm_kconfig_include(fname) # Remove unwanted files cmd = ['git', 'rm', '-r'] + real stdout = command.RunPipe([cmd], capture=True).stdout ## Change the messages as needed msg = '''arm: Remove %s board This board has not been converted to CONFIG_DM_MMC by the deadline. Remove it. ''' % board for name in cc: msg += 'Patch-cc: %s\n' % name # Create the commit cmd = ['git', 'commit', '-s', '-m', msg] stdout = command.RunPipe([cmd], capture=True).stdout # Check if the board is mentioned anywhere else. The user will need to deal # with this cmd = ['git', 'grep', '-il', board] print(command.RunPipe([cmd], capture=True, raise_on_error=False).stdout) print(' '.join(cmd))
def _RunBuildman(self, *args): return command.RunPipe([[self._buildman_pathname] + list(args)], capture=True, capture_stderr=True)
def _WriteResult(self, result, keep_outputs): """Write a built result to the output directory. Args: result: CommandResult object containing result to write keep_outputs: True to store the output binaries, False to delete them """ # Fatal error if result.return_code < 0: return # Aborted? if result.stderr and 'No child processes' in result.stderr: return if result.already_done: return # Write the output and stderr output_dir = self.builder._GetOutputDir(result.commit_upto) Mkdir(output_dir) build_dir = self.builder.GetBuildDir(result.commit_upto, result.brd.target) Mkdir(build_dir) outfile = os.path.join(build_dir, 'log') with open(outfile, 'w') as fd: if result.stdout: fd.write(result.stdout) errfile = self.builder.GetErrFile(result.commit_upto, result.brd.target) if result.stderr: with open(errfile, 'w') as fd: fd.write(result.stderr) elif os.path.exists(errfile): os.remove(errfile) if result.toolchain: # Write the build result and toolchain information. done_file = self.builder.GetDoneFile(result.commit_upto, result.brd.target) with open(done_file, 'w') as fd: fd.write('%s' % result.return_code) with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: print >> fd, 'gcc', result.toolchain.gcc print >> fd, 'path', result.toolchain.path print >> fd, 'cross', result.toolchain.cross print >> fd, 'arch', result.toolchain.arch fd.write('%s' % result.return_code) with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: print >> fd, 'gcc', result.toolchain.gcc print >> fd, 'path', result.toolchain.path # Write out the image and function size information and an objdump env = result.toolchain.MakeEnvironment() lines = [] for fname in ['u-boot', 'spl/u-boot-spl']: cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] nm_result = command.RunPipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) if nm_result.stdout: nm = self.builder.GetFuncSizesFile(result.commit_upto, result.brd.target, fname) with open(nm, 'w') as fd: print >> fd, nm_result.stdout, cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] dump_result = command.RunPipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) rodata_size = '' if dump_result.stdout: objdump = self.builder.GetObjdumpFile( result.commit_upto, result.brd.target, fname) with open(objdump, 'w') as fd: print >> fd, dump_result.stdout, for line in dump_result.stdout.splitlines(): fields = line.split() if len(fields) > 5 and fields[1] == '.rodata': rodata_size = fields[2] cmd = ['%ssize' % self.toolchain.cross, fname] size_result = command.RunPipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) if size_result.stdout: lines.append(size_result.stdout.splitlines()[1] + ' ' + rodata_size) # Write out the image sizes file. This is similar to the output # of binutil's 'size' utility, but it omits the header line and # adds an additional hex value at the end of each line for the # rodata size if len(lines): sizes = self.builder.GetSizesFile(result.commit_upto, result.brd.target) with open(sizes, 'w') as fd: print >> fd, '\n'.join(lines) # Now write the actual build output if keep_outputs: patterns = [ 'u-boot', '*.bin', 'u-boot.dtb', '*.map', '*.img', 'include/autoconf.mk', 'spl/u-boot-spl', 'spl/u-boot-spl.bin' ] for pattern in patterns: file_list = glob.glob(os.path.join(result.out_dir, pattern)) for fname in file_list: shutil.copy(fname, build_dir)
def _WriteResult(self, result, keep_outputs): """Write a built result to the output directory. Args: result: CommandResult object containing result to write keep_outputs: True to store the output binaries, False to delete them """ # Fatal error if result.return_code < 0: return # If we think this might have been aborted with Ctrl-C, record the # failure but not that we are 'done' with this board. A retry may fix # it. maybe_aborted = result.stderr and 'No child processes' in result.stderr if result.already_done: return # Write the output and stderr output_dir = self.builder._GetOutputDir(result.commit_upto) Mkdir(output_dir) build_dir = self.builder.GetBuildDir(result.commit_upto, result.brd.target) Mkdir(build_dir) outfile = os.path.join(build_dir, 'log') with open(outfile, 'w') as fd: if result.stdout: # We don't want unicode characters in log files fd.write( result.stdout.decode('UTF-8').encode('ASCII', 'replace')) errfile = self.builder.GetErrFile(result.commit_upto, result.brd.target) if result.stderr: with open(errfile, 'w') as fd: # We don't want unicode characters in log files fd.write( result.stderr.decode('UTF-8').encode('ASCII', 'replace')) elif os.path.exists(errfile): os.remove(errfile) if result.toolchain: # Write the build result and toolchain information. done_file = self.builder.GetDoneFile(result.commit_upto, result.brd.target) with open(done_file, 'w') as fd: if maybe_aborted: # Special code to indicate we need to retry fd.write('%s' % RETURN_CODE_RETRY) else: fd.write('%s' % result.return_code) with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: print >> fd, 'gcc', result.toolchain.gcc print >> fd, 'path', result.toolchain.path print >> fd, 'cross', result.toolchain.cross print >> fd, 'arch', result.toolchain.arch fd.write('%s' % result.return_code) # Write out the image and function size information and an objdump env = result.toolchain.MakeEnvironment(self.builder.full_path) lines = [] for fname in ['u-boot', 'spl/u-boot-spl']: cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] nm_result = command.RunPipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) if nm_result.stdout: nm = self.builder.GetFuncSizesFile(result.commit_upto, result.brd.target, fname) with open(nm, 'w') as fd: print >> fd, nm_result.stdout, cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] dump_result = command.RunPipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) rodata_size = '' if dump_result.stdout: objdump = self.builder.GetObjdumpFile( result.commit_upto, result.brd.target, fname) with open(objdump, 'w') as fd: print >> fd, dump_result.stdout, for line in dump_result.stdout.splitlines(): fields = line.split() if len(fields) > 5 and fields[1] == '.rodata': rodata_size = fields[2] cmd = ['%ssize' % self.toolchain.cross, fname] size_result = command.RunPipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) if size_result.stdout: lines.append(size_result.stdout.splitlines()[1] + ' ' + rodata_size) # Extract the environment from U-Boot and dump it out cmd = [ '%sobjcopy' % self.toolchain.cross, '-O', 'binary', '-j', '.rodata.default_environment', 'env/built-in.o', 'uboot.env' ] command.RunPipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) ubootenv = os.path.join(result.out_dir, 'uboot.env') self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env']) # Write out the image sizes file. This is similar to the output # of binutil's 'size' utility, but it omits the header line and # adds an additional hex value at the end of each line for the # rodata size if len(lines): sizes = self.builder.GetSizesFile(result.commit_upto, result.brd.target) with open(sizes, 'w') as fd: print >> fd, '\n'.join(lines) # Write out the configuration files, with a special case for SPL for dirname in ['', 'spl', 'tpl']: self.CopyFiles(result.out_dir, build_dir, dirname, [ 'u-boot.cfg', 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config', 'include/autoconf.mk', 'include/generated/autoconf.h' ]) # Now write the actual build output if keep_outputs: self.CopyFiles(result.out_dir, build_dir, '', [ 'u-boot*', '*.bin', '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk', 'spl/u-boot-spl*' ])