def rm_kconfig_include(path):
"""Remove a path from Kconfig files
This function finds the given path in a 'source' statement in a Kconfig
file and removes that line from the file. This is needed because the path
is going to be removed, so any reference to it will cause a problem with
Kconfig parsing.
The changes are made locally and then added to the git staging area.
Args:
path: Path to search for and remove
"""
cmd = ['git', 'grep', path]
stdout = command.RunPipe([cmd], capture=True, raise_on_error=False).stdout
if not stdout:
return
fname = stdout.split(':')[0]
print("Fixing up '%s' to remove reference to '%s'" % (fname, path))
cmd = ['sed', '-i', '\|%s|d' % path, fname]
stdout = command.RunPipe([cmd], capture=True).stdout
cmd = ['git', 'add', fname]
stdout = command.RunPipe([cmd], capture=True).stdout
def CountCommitsToBranch(branch):
"""Returns number of commits between HEAD and the tracking branch.
This looks back to the tracking branch and works out the number of commits
since then.
Args:
branch: Branch to count from (None for current branch)
Return:
Number of patches that exist on top of the branch
"""
if branch:
us, msg = GetUpstream('.git', branch)
rev_range = '%s..%s' % (us, branch)
else:
rev_range = '@{upstream}..'
pipe = [LogCmd(rev_range, oneline=True)]
result = command.RunPipe(pipe, capture=True, capture_stderr=True,
oneline=True, raise_on_error=False)
if result.return_code:
raise ValueError('Failed to determine upstream: %s' %
result.stderr.strip())
patch_count = len(result.stdout.splitlines())
return patch_count
def GetMetaDataForList(commit_range,
git_dir=None,
count=None,
series=None,
allow_overwrite=False):
"""Reads out patch series metadata from the commits
This does a 'git log' on the relevant commits and pulls out the tags we
are interested in.
Args:
commit_range: Range of commits to count (e.g. 'HEAD..base')
git_dir: Path to git repositiory (None to use default)
count: Number of commits to list, or None for no limit
series: Series object to add information into. By default a new series
is started.
allow_overwrite: Allow tags to overwrite an existing tag
Returns:
A Series object containing information about the commits.
"""
if not series:
series = Series()
series.allow_overwrite = allow_overwrite
params = gitutil.LogCmd(commit_range,
reverse=True,
count=count,
git_dir=git_dir)
stdout = command.RunPipe([params], capture=True).stdout
ps = PatchStream(series, is_log=True)
for line in stdout.splitlines():
ps.ProcessLine(line)
ps.Finalize()
return series
def GuessUpstream(git_dir, branch):
"""Tries to guess the upstream for a branch
This lists out top commits on a branch and tries to find a suitable
upstream. It does this by looking for the first commit where
'git name-rev' returns a plain branch name, with no ! or ^ modifiers.
Args:
git_dir: Git directory containing repo
branch: Name of branch
Returns:
Tuple:
Name of upstream branch (e.g. 'upstream/master') or None if none
Warning/error message, or None if none
"""
pipe = [LogCmd(branch, git_dir=git_dir, oneline=True, count=100)]
result = command.RunPipe(pipe,
capture=True,
capture_stderr=True,
raise_on_error=False)
if result.return_code:
return None, "Branch '%s' not found" % branch
for line in result.stdout.splitlines()[1:]:
commit_hash = line.split(' ')[0]
name = NameRevision(commit_hash)
if '~' not in name and '^' not in name:
if name.startswith('remotes/'):
name = name[8:]
return name, "Guessing upstream as '%s'" % name
return None, "Cannot find a suitable upstream for branch '%s'" % branch
def PruneWorktrees(git_dir):
"""Remove administrative files for deleted worktrees
Args:
git_dir: The repository whose deleted worktrees should be pruned
"""
pipe = ['git', '--git-dir', git_dir, 'worktree', 'prune']
result = command.RunPipe([pipe], capture=True, capture_stderr=True)
if result.return_code != 0:
raise OSError('git worktree prune: %s' % result.stderr)
def Clone(git_dir, output_dir):
"""Checkout the selected commit for this build
Args:
commit_hash: Commit hash to check out
"""
pipe = ['git', 'clone', git_dir, '.']
result = command.RunPipe([pipe], capture=True, cwd=output_dir,
capture_stderr=True)
if result.return_code != 0:
raise OSError('git clone: %s' % result.stderr)
def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
arch=None, override_toolchain=None):
"""Create a new toolchain object.
Args:
fname: Filename of the gcc component
test: True to run the toolchain to test it
verbose: True to print out the information
priority: Priority to use for this toolchain, or PRIORITY_CALC to
calculate it
"""
self.gcc = fname
self.path = os.path.dirname(fname)
self.override_toolchain = override_toolchain
# Find the CROSS_COMPILE prefix to use for U-Boot. For example,
# 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
basename = os.path.basename(fname)
pos = basename.rfind('-')
self.cross = basename[:pos + 1] if pos != -1 else ''
# The architecture is the first part of the name
pos = self.cross.find('-')
if arch:
self.arch = arch
else:
self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
if self.arch == 'sandbox' and override_toolchain:
self.gcc = override_toolchain
env = self.MakeEnvironment(False)
# As a basic sanity check, run the C compiler with --version
cmd = [fname, '--version']
if priority == PRIORITY_CALC:
self.priority = self.GetPriority(fname)
else:
self.priority = priority
if test:
result = command.RunPipe([cmd], capture=True, env=env,
raise_on_error=False)
self.ok = result.return_code == 0
if verbose:
print('Tool chain test: ', end=' ')
if self.ok:
print("OK, arch='%s', priority %d" % (self.arch,
self.priority))
else:
print('BAD')
print('Command: ', cmd)
print(result.stdout)
print(result.stderr)
else:
self.ok = True
def Setup():
"""Set up git utils, by reading the alias files."""
# Check for a git alias file also
global use_no_decorate
alias_fname = GetAliasFile()
if alias_fname:
settings.ReadGitAliases(alias_fname)
cmd = LogCmd(None, count=0)
use_no_decorate = (command.RunPipe([cmd],
raise_on_error=False).return_code == 0)
def CountCommits(commit_range):
"""Returns the number of commits in the given range.
Args:
commit_range: Range of commits to count (e.g. 'HEAD..base')
Return:
Number of patches that exist on top of the branch
"""
pipe = [LogCmd(commit_range, oneline=True), ['wc', '-l']]
stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
patch_count = int(stdout)
return patch_count
def Run(name, *args, **kwargs):
"""Run a tool with some arguments
This runs a 'tool', which is a program used by binman to process files and
perhaps produce some output. Tools can be located on the PATH or in a
search path.
Args:
name: Command name to run
args: Arguments to the tool
for_host: True to resolve the command to the version for the host
for_target: False to run the command as-is, without resolving it
to the version for the compile target
Returns:
CommandResult object
"""
try:
binary = kwargs.get('binary')
for_host = kwargs.get('for_host', False)
for_target = kwargs.get('for_target', not for_host)
env = None
if tool_search_paths:
env = dict(os.environ)
env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH']
if for_target:
name, extra_args = GetTargetCompileTool(name)
args = tuple(extra_args) + args
elif for_host:
name, extra_args = GetHostCompileTool(name)
args = tuple(extra_args) + args
name = os.path.expanduser(name) # Expand paths containing ~
all_args = (name, ) + args
result = command.RunPipe([all_args],
capture=True,
capture_stderr=True,
env=env,
raise_on_error=False,
binary=binary)
if result.return_code:
raise ValueError(
"Error %d running '%s': %s" %
(result.return_code, ' '.join(all_args), result.stderr))
return result.stdout
except:
if env and not PathHasFile(env['PATH'], name):
msg = "Please install tool '%s'" % name
package = packages.get(name)
if package:
msg += " (e.g. from package '%s')" % package
raise ValueError(msg)
raise
def CheckWorktreeIsAvailable(git_dir):
"""Check if git-worktree functionality is available
Args:
git_dir: The repository to test in
Returns:
True if git-worktree commands will work, False otherwise.
"""
pipe = ['git', '--git-dir', git_dir, 'worktree', 'list']
result = command.RunPipe([pipe], capture=True, capture_stderr=True,
raise_on_error=False)
return result.return_code == 0
def CountCommitsToBranch():
"""Returns number of commits between HEAD and the tracking branch.
This looks back to the tracking branch and works out the number of commits
since then.
Return:
Number of patches that exist on top of the branch
"""
pipe = [LogCmd('@{upstream}..', oneline=True), ['wc', '-l']]
stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
patch_count = int(stdout)
return patch_count
def Fetch(git_dir=None, work_tree=None):
"""Fetch from the origin repo
Args:
commit_hash: Commit hash to check out
"""
pipe = ['git']
if git_dir:
pipe.extend(['--git-dir', git_dir])
if work_tree:
pipe.extend(['--work-tree', work_tree])
pipe.append('fetch')
result = command.RunPipe([pipe], capture=True, capture_stderr=True)
if result.return_code != 0:
raise OSError('git fetch: %s' % result.stderr)
def NameRevision(commit_hash):
"""Gets the revision name for a commit
Args:
commit_hash: Commit hash to look up
Return:
Name of revision, if any, else None
"""
pipe = ['git', 'name-rev', commit_hash]
stdout = command.RunPipe([pipe], capture=True, oneline=True).stdout
# We expect a commit, a space, then a revision name
name = stdout.split(' ')[1].strip()
return name
def get_list(commit_range, git_dir=None, count=None):
"""Get a log of a list of comments
This returns the output of 'git log' for the selected commits
Args:
commit_range (str): Range of commits to count (e.g. 'HEAD..base')
git_dir (str): Path to git repositiory (None to use default)
count (int): Number of commits to list, or None for no limit
Returns
str: String containing the contents of the git log
"""
params = gitutil.LogCmd(commit_range, reverse=True, count=count,
git_dir=git_dir)
return command.RunPipe([params], capture=True).stdout
def AddWorktree(git_dir, output_dir, commit_hash=None):
"""Create and checkout a new git worktree for this build
Args:
git_dir: The repository to checkout the worktree from
output_dir: Path for the new worktree
commit_hash: Commit hash to checkout
"""
# We need to pass --detach to avoid creating a new branch
pipe = ['git', '--git-dir', git_dir, 'worktree', 'add', '.', '--detach']
if commit_hash:
pipe.append(commit_hash)
result = command.RunPipe([pipe], capture=True, cwd=output_dir,
capture_stderr=True)
if result.return_code != 0:
raise OSError('git worktree add: %s' % result.stderr)
def CountCommitsInRange(git_dir, range_expr):
"""Returns the number of commits in the given range.
Args:
git_dir: Directory containing git repo
range_expr: Range to check
Return:
Number of patches that exist in the supplied range or None if none
were found
"""
pipe = [LogCmd(range_expr, git_dir=git_dir, oneline=True)]
result = command.RunPipe(pipe, capture=True, capture_stderr=True,
raise_on_error=False)
if result.return_code:
return None, "Range '%s' not found or is invalid" % range_expr
patch_count = len(result.stdout.splitlines())
return patch_count, None
def run_cmd_result(self, *args, binary=False, raise_on_error=True):
"""Run the bintool using command-line arguments
Args:
args (list of str): Arguments to provide, in addition to the bintool
name
binary (bool): True to return output as bytes instead of str
raise_on_error (bool): True to raise a ValueError exception if the
tool returns a non-zero return code
Returns:
CommandResult: Resulting output from the bintool, or None if the
tool is not present
"""
if self.name in self.missing_list:
return None
name = os.path.expanduser(self.name) # Expand paths containing ~
all_args = (name, ) + args
env = tools.get_env_with_path()
tout.Detail(f"bintool: {' '.join(all_args)}")
result = command.RunPipe([all_args],
capture=True,
capture_stderr=True,
env=env,
raise_on_error=False,
binary=binary)
if result.return_code:
# Return None if the tool was not found. In this case there is no
# output from the tool and it does not appear on the path. We still
# try to run it (as above) since RunPipe() allows faking the tool's
# output
if not any([result.stdout, result.stderr, tools.tool_find(name)]):
tout.Info(f"bintool '{name}' not found")
return None
if raise_on_error:
tout.Info(f"bintool '{name}' failed")
raise ValueError("Error %d running '%s': %s" %
(result.return_code, ' '.join(all_args),
result.stderr or result.stdout))
if result.stdout:
tout.Debug(result.stdout)
if result.stderr:
tout.Debug(result.stderr)
return result
def Checkout(commit_hash, git_dir=None, work_tree=None, force=False):
"""Checkout the selected commit for this build
Args:
commit_hash: Commit hash to check out
"""
pipe = ['git']
if git_dir:
pipe.extend(['--git-dir', git_dir])
if work_tree:
pipe.extend(['--work-tree', work_tree])
pipe.append('checkout')
if force:
pipe.append('-f')
pipe.append(commit_hash)
result = command.RunPipe([pipe], capture=True, raise_on_error=False,
capture_stderr=True)
if result.return_code != 0:
raise OSError('git checkout (%s): %s' % (pipe, result.stderr))
def CountCommitsToBranch(branch):
"""Returns number of commits between HEAD and the tracking branch.
This looks back to the tracking branch and works out the number of commits
since then.
Args:
branch: Branch to count from (None for current branch)
Return:
Number of patches that exist on top of the branch
"""
if branch:
us, msg = GetUpstream('.git', branch)
rev_range = '%s..%s' % (us, branch)
else:
rev_range = '@{upstream}..'
pipe = [LogCmd(rev_range, oneline=True), ['wc', '-l']]
stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
patch_count = int(stdout)
return patch_count
def Run(name, *args, **kwargs):
"""Run a tool with some arguments
This runs a 'tool', which is a program used by binman to process files and
perhaps produce some output. Tools can be located on the PATH or in a
search path.
Args:
name: Command name to run
args: Arguments to the tool
Returns:
CommandResult object
"""
try:
binary = kwargs.get('binary')
env = None
if tool_search_paths:
env = dict(os.environ)
env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH']
all_args = (name, ) + args
result = command.RunPipe([all_args],
capture=True,
capture_stderr=True,
env=env,
raise_on_error=False,
binary=binary)
if result.return_code:
raise Exception(
"Error %d running '%s': %s" %
(result.return_code, ' '.join(all_args), result.stderr))
return result.stdout
except:
if env and not PathHasFile(env['PATH'], name):
msg = "Please install tool '%s'" % name
package = packages.get(name)
if package:
msg += " (e.g. from package '%s')" % package
raise ValueError(msg)
raise
def fiptool(fname, *fip_args):
"""Run fiptool with provided arguments
If the tool fails then this function raises an exception and prints out the
output and stderr.
Args:
fname (str): Filename of FIP
*fip_args: List of arguments to pass to fiptool
Returns:
CommandResult: object containing the results
Raises:
ValueError: the tool failed to run
"""
args = ['fiptool', fname] + list(fip_args)
result = command.RunPipe([args], capture=not VERBOSE,
capture_stderr=not VERBOSE, raise_on_error=False)
if result.return_code:
print(result.stderr, file=sys.stderr)
raise ValueError("Failed to run (error %d): '%s'" %
(result.return_code, ' '.join(args)))
return result
def cbfstool(fname, *cbfs_args, **kwargs):
"""Run cbfstool with provided arguments
If the tool fails then this function raises an exception and prints out the
output and stderr.
Args:
fname: Filename of CBFS
*cbfs_args: List of arguments to pass to cbfstool
Returns:
CommandResult object containing the results
"""
args = ['cbfstool', fname] + list(cbfs_args)
if kwargs.get('base') is not None:
args += ['-b', '%#x' % kwargs['base']]
result = command.RunPipe([args],
capture=not VERBOSE,
capture_stderr=not VERBOSE,
raise_on_error=False)
if result.return_code:
print(result.stderr, file=sys.stderr)
raise Exception("Failed to run (error %d): '%s'" %
(result.return_code, ' '.join(args)))
def _WriteResult(self, result, keep_outputs, work_in_output):
"""Write a built result to the output directory.
Args:
result: CommandResult object containing result to write
keep_outputs: True to store the output binaries, False
to delete them
work_in_output: Use the output directory as the work directory and
don't write to a separate output directory.
"""
# Fatal error
if result.return_code < 0:
return
# If we think this might have been aborted with Ctrl-C, record the
# failure but not that we are 'done' with this board. A retry may fix
# it.
maybe_aborted = result.stderr and 'No child processes' in result.stderr
if result.already_done:
return
# Write the output and stderr
output_dir = self.builder._GetOutputDir(result.commit_upto)
Mkdir(output_dir)
build_dir = self.builder.GetBuildDir(result.commit_upto,
result.brd.target)
Mkdir(build_dir)
outfile = os.path.join(build_dir, 'log')
with open(outfile, 'w') as fd:
if result.stdout:
fd.write(result.stdout)
errfile = self.builder.GetErrFile(result.commit_upto,
result.brd.target)
if result.stderr:
with open(errfile, 'w') as fd:
fd.write(result.stderr)
elif os.path.exists(errfile):
os.remove(errfile)
if result.toolchain:
# Write the build result and toolchain information.
done_file = self.builder.GetDoneFile(result.commit_upto,
result.brd.target)
with open(done_file, 'w') as fd:
if maybe_aborted:
# Special code to indicate we need to retry
fd.write('%s' % RETURN_CODE_RETRY)
else:
fd.write('%s' % result.return_code)
with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
print('gcc', result.toolchain.gcc, file=fd)
print('path', result.toolchain.path, file=fd)
print('cross', result.toolchain.cross, file=fd)
print('arch', result.toolchain.arch, file=fd)
fd.write('%s' % result.return_code)
# Write out the image and function size information and an objdump
env = result.toolchain.MakeEnvironment(self.builder.full_path)
with open(os.path.join(build_dir, 'out-env'), 'w') as fd:
for var in sorted(env.keys()):
print('%s="%s"' % (var, env[var]), file=fd)
lines = []
for fname in BASE_ELF_FILENAMES:
cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
nm_result = command.RunPipe([cmd],
capture=True,
capture_stderr=True,
cwd=result.out_dir,
raise_on_error=False,
env=env)
if nm_result.stdout:
nm = self.builder.GetFuncSizesFile(result.commit_upto,
result.brd.target,
fname)
with open(nm, 'w') as fd:
print(nm_result.stdout, end=' ', file=fd)
cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
dump_result = command.RunPipe([cmd],
capture=True,
capture_stderr=True,
cwd=result.out_dir,
raise_on_error=False,
env=env)
rodata_size = ''
if dump_result.stdout:
objdump = self.builder.GetObjdumpFile(
result.commit_upto, result.brd.target, fname)
with open(objdump, 'w') as fd:
print(dump_result.stdout, end=' ', file=fd)
for line in dump_result.stdout.splitlines():
fields = line.split()
if len(fields) > 5 and fields[1] == '.rodata':
rodata_size = fields[2]
cmd = ['%ssize' % self.toolchain.cross, fname]
size_result = command.RunPipe([cmd],
capture=True,
capture_stderr=True,
cwd=result.out_dir,
raise_on_error=False,
env=env)
if size_result.stdout:
lines.append(size_result.stdout.splitlines()[1] + ' ' +
rodata_size)
# Extract the environment from U-Boot and dump it out
cmd = [
'%sobjcopy' % self.toolchain.cross, '-O', 'binary', '-j',
'.rodata.default_environment', 'env/built-in.o', 'uboot.env'
]
command.RunPipe([cmd],
capture=True,
capture_stderr=True,
cwd=result.out_dir,
raise_on_error=False,
env=env)
ubootenv = os.path.join(result.out_dir, 'uboot.env')
if not work_in_output:
self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env'])
# Write out the image sizes file. This is similar to the output
# of binutil's 'size' utility, but it omits the header line and
# adds an additional hex value at the end of each line for the
# rodata size
if len(lines):
sizes = self.builder.GetSizesFile(result.commit_upto,
result.brd.target)
with open(sizes, 'w') as fd:
print('\n'.join(lines), file=fd)
if not work_in_output:
# Write out the configuration files, with a special case for SPL
for dirname in ['', 'spl', 'tpl']:
self.CopyFiles(result.out_dir, build_dir, dirname, [
'u-boot.cfg', 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg',
'.config', 'include/autoconf.mk',
'include/generated/autoconf.h'
])
# Now write the actual build output
if keep_outputs:
self.CopyFiles(result.out_dir, build_dir, '', [
'u-boot*', '*.bin', '*.map', '*.img', 'MLO', 'SPL',
'include/autoconf.mk', 'spl/u-boot-spl*'
])
def rm_board(board):
"""Create a commit which removes a single board
This looks up the MAINTAINERS file to file files that need to be removed,
then removes pieces from the Kconfig files that mention the board.
Args:
board: Board name to remove
"""
# Find all MAINTAINERS and Kconfig files which mention the board
cmd = ['git', 'grep', '-l', board]
stdout = command.RunPipe([cmd], capture=True).stdout
maintain = []
kconfig = []
for line in stdout.splitlines():
line = line.strip()
if 'MAINTAINERS' in line:
if line not in maintain:
maintain.append(line)
elif 'Kconfig' in line:
kconfig.append(line)
paths = []
cc = []
# Look through the MAINTAINERS file to find things to remove
for fname in maintain:
with open(fname) as fd:
for line in fd:
line = line.strip()
fields = re.split('[ \t]', line, 1)
if len(fields) == 2:
if fields[0] == 'M:':
cc.append(fields[1])
elif fields[0] == 'F:':
paths.append(fields[1].strip())
# Expand any wildcards in the MAINTAINERS file
real = []
for path in paths:
if path[-1] == '/':
path = path[:-1]
if '*' in path:
globbed = glob.glob(path)
print("Expanded '%s' to '%s'" % (path, globbed))
real += globbed
else:
real.append(path)
# Search for Kconfig files in the resulting list. Remove any 'source' lines
# which reference Kconfig files we want to remove
for path in real:
cmd = ['find', path]
stdout = (command.RunPipe([cmd], capture=True,
raise_on_error=False).stdout)
for fname in stdout.splitlines():
if fname.endswith('Kconfig'):
rm_kconfig_include(fname)
# Remove unwanted files
cmd = ['git', 'rm', '-r'] + real
stdout = command.RunPipe([cmd], capture=True).stdout
## Change the messages as needed
msg = '''arm: Remove %s board
This board has not been converted to CONFIG_DM_MMC by the deadline.
Remove it.
''' % board
for name in cc:
msg += 'Patch-cc: %s\n' % name
# Create the commit
cmd = ['git', 'commit', '-s', '-m', msg]
stdout = command.RunPipe([cmd], capture=True).stdout
# Check if the board is mentioned anywhere else. The user will need to deal
# with this
cmd = ['git', 'grep', '-il', board]
print(command.RunPipe([cmd], capture=True, raise_on_error=False).stdout)
print(' '.join(cmd))
def _RunBuildman(self, *args):
return command.RunPipe([[self._buildman_pathname] + list(args)],
capture=True,
capture_stderr=True)
