def collect_patches(series, series_id, url, rest_api=call_rest_api):
"""Collect patch information about a series from patchwork
Uses the Patchwork REST API to collect information provided by patchwork
about the status of each patch.
Args:
series (Series): Series object corresponding to the local branch
containing the series
series_id (str): Patch series ID number
url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
rest_api (function): API function to call to access Patchwork, for
testing
Returns:
list: List of patches sorted by sequence number, each a Patch object
Raises:
ValueError: if the URL could not be read or the web page does not follow
the expected structure
"""
data = rest_api(url, 'series/%s/' % series_id)
# Get all the rows, which are patches
patch_dict = data['patches']
count = len(patch_dict)
num_commits = len(series.commits)
if count != num_commits:
tout.Warning('Warning: Patchwork reports %d patches, series has %d' %
(count, num_commits))
patches = []
# Work through each row (patch) one at a time, collecting the information
warn_count = 0
for pw_patch in patch_dict:
patch = Patch(pw_patch['id'])
patch.parse_subject(pw_patch['name'])
patches.append(patch)
if warn_count > 1:
tout.Warning(' (total of %d warnings)' % warn_count)
# Sort patches by patch number
patches = sorted(patches, key=lambda x: x.seq)
return patches
def check_patchwork_status(series,
series_id,
branch,
dest_branch,
force,
show_comments,
url,
rest_api=call_rest_api,
test_repo=None):
"""Check the status of a series on Patchwork
This finds review tags and comments for a series in Patchwork, displaying
them to show what is new compared to the local series.
Args:
series (Series): Series object for the existing branch
series_id (str): Patch series ID number
branch (str): Existing branch to update, or None
dest_branch (str): Name of new branch to create, or None
force (bool): True to force overwriting dest_branch if it exists
show_comments (bool): True to show the comments on each patch
url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
rest_api (function): API function to call to access Patchwork, for
testing
test_repo (pygit2.Repository): Repo to use (use None unless testing)
"""
patches = collect_patches(series, series_id, url, rest_api)
col = terminal.Color()
count = len(series.commits)
new_rtag_list = [None] * count
review_list = [None] * count
patch_for_commit, _, warnings = compare_with_series(series, patches)
for warn in warnings:
tout.Warning(warn)
patch_list = [patch_for_commit.get(c) for c in range(len(series.commits))]
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
futures = executor.map(find_new_responses, repeat(new_rtag_list),
repeat(review_list), range(count),
series.commits, patch_list, repeat(url),
repeat(rest_api))
for fresponse in futures:
if fresponse:
raise fresponse.exception()
num_to_add = 0
for seq, cmt in enumerate(series.commits):
patch = patch_for_commit.get(seq)
if not patch:
continue
terminal.Print('%3d %s' % (patch.seq, patch.subject[:50]),
colour=col.BLUE)
cmt = series.commits[seq]
base_rtags = cmt.rtags
new_rtags = new_rtag_list[seq]
indent = ' ' * 2
show_responses(base_rtags, indent, False)
num_to_add += show_responses(new_rtags, indent, True)
if show_comments:
for review in review_list[seq]:
terminal.Print('Review: %s' % review.meta, colour=col.RED)
for snippet in review.snippets:
for line in snippet:
quoted = line.startswith('>')
terminal.Print(' %s' % line,
colour=col.MAGENTA if quoted else None)
terminal.Print()
terminal.Print(
"%d new response%s available in patchwork%s" %
(num_to_add, 's' if num_to_add != 1 else '',
'' if dest_branch else ' (use -d to write them to a new branch)'))
if dest_branch:
num_added = create_branch(series, new_rtag_list, branch, dest_branch,
force, test_repo)
terminal.Print(
"%d response%s added from patchwork into new branch '%s'" %
(num_added, 's' if num_added != 1 else '', dest_branch))
def Binman(args):
"""The main control code for binman
This assumes that help and test options have already been dealt with. It
deals with the core task of building images.
Args:
args: Command line arguments Namespace object
"""
global Image
global state
if args.full_help:
pager = os.getenv('PAGER')
if not pager:
pager = 'more'
fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
'README')
command.Run(pager, fname)
return 0
# Put these here so that we can import this module without libfdt
from binman.image import Image
from binman import state
if args.cmd in ['ls', 'extract', 'replace']:
try:
tout.Init(args.verbosity)
tools.PrepareOutputDir(None)
if args.cmd == 'ls':
ListEntries(args.image, args.paths)
if args.cmd == 'extract':
ExtractEntries(args.image, args.filename, args.outdir, args.paths,
not args.uncompressed)
if args.cmd == 'replace':
ReplaceEntries(args.image, args.filename, args.indir, args.paths,
do_compress=not args.compressed,
allow_resize=not args.fix_size, write_map=args.map)
except:
raise
finally:
tools.FinaliseOutputDir()
return 0
# Try to figure out which device tree contains our image description
if args.dt:
dtb_fname = args.dt
else:
board = args.board
if not board:
raise ValueError('Must provide a board to process (use -b <board>)')
board_pathname = os.path.join(args.build_dir, board)
dtb_fname = os.path.join(board_pathname, 'u-boot.dtb')
if not args.indir:
args.indir = ['.']
args.indir.append(board_pathname)
try:
tout.Init(args.verbosity)
elf.debug = args.debug
cbfs_util.VERBOSE = args.verbosity > 2
state.use_fake_dtb = args.fake_dtb
try:
tools.SetInputDirs(args.indir)
tools.PrepareOutputDir(args.outdir, args.preserve)
tools.SetToolPaths(args.toolpath)
state.SetEntryArgs(args.entry_arg)
images = PrepareImagesAndDtbs(dtb_fname, args.image,
args.update_fdt)
missing = False
for image in images.values():
missing |= ProcessImage(image, args.update_fdt, args.map,
allow_missing=args.allow_missing)
# Write the updated FDTs to our output files
for dtb_item in state.GetAllFdts():
tools.WriteFile(dtb_item._fname, dtb_item.GetContents())
if missing:
tout.Warning("Some images are invalid")
finally:
tools.FinaliseOutputDir()
finally:
tout.Uninit()
return 0
def ProcessImage(image, update_fdt, write_map, get_contents=True,
allow_resize=True, allow_missing=False):
"""Perform all steps for this image, including checking and # writing it.
This means that errors found with a later image will be reported after
earlier images are already completed and written, but that does not seem
important.
Args:
image: Image to process
update_fdt: True to update the FDT wth entry offsets, etc.
write_map: True to write a map file
get_contents: True to get the image contents from files, etc., False if
the contents is already present
allow_resize: True to allow entries to change size (this does a re-pack
of the entries), False to raise an exception
allow_missing: Allow blob_ext objects to be missing
Returns:
True if one or more external blobs are missing, False if all are present
"""
if get_contents:
image.SetAllowMissing(allow_missing)
image.GetEntryContents()
image.GetEntryOffsets()
# We need to pack the entries to figure out where everything
# should be placed. This sets the offset/size of each entry.
# However, after packing we call ProcessEntryContents() which
# may result in an entry changing size. In that case we need to
# do another pass. Since the device tree often contains the
# final offset/size information we try to make space for this in
# AddMissingProperties() above. However, if the device is
# compressed we cannot know this compressed size in advance,
# since changing an offset from 0x100 to 0x104 (for example) can
# alter the compressed size of the device tree. So we need a
# third pass for this.
passes = 5
for pack_pass in range(passes):
try:
image.PackEntries()
image.CheckSize()
image.CheckEntries()
except Exception as e:
if write_map:
fname = image.WriteMap()
print("Wrote map file '%s' to show errors" % fname)
raise
image.SetImagePos()
if update_fdt:
image.SetCalculatedProperties()
for dtb_item in state.GetAllFdts():
dtb_item.Sync()
dtb_item.Flush()
image.WriteSymbols()
sizes_ok = image.ProcessEntryContents()
if sizes_ok:
break
image.ResetForPack()
tout.Info('Pack completed after %d pass(es)' % (pack_pass + 1))
if not sizes_ok:
image.Raise('Entries changed size after packing (tried %s passes)' %
passes)
image.BuildImage()
if write_map:
image.WriteMap()
missing_list = []
image.CheckMissing(missing_list)
if missing_list:
tout.Warning("Image '%s' is missing external blobs and is non-functional: %s" %
(image.name, ' '.join([e.name for e in missing_list])))
return bool(missing_list)
def ReplaceEntries(image_fname, input_fname, indir, entry_paths,
do_compress=True, allow_resize=True, write_map=False):
"""Replace the data from one or more entries from input files
Args:
image_fname: Image filename to process
input_fname: Single input ilename to use if replacing one file, None
otherwise
indir: Input directory to use (for any number of files), else None
entry_paths: List of entry paths to extract
do_compress: True if the input data is uncompressed and may need to be
compressed if the entry requires it, False if the data is already
compressed.
write_map: True to write a map file
Returns:
List of EntryInfo records that were written
"""
image = Image.FromFile(image_fname)
# Replace an entry from a single file, as a special case
if input_fname:
if not entry_paths:
raise ValueError('Must specify an entry path to read with -f')
if len(entry_paths) != 1:
raise ValueError('Must specify exactly one entry path to write with -f')
entry = image.FindEntryPath(entry_paths[0])
data = tools.ReadFile(input_fname)
tout.Notice("Read %#x bytes from file '%s'" % (len(data), input_fname))
WriteEntryToImage(image, entry, data, do_compress=do_compress,
allow_resize=allow_resize, write_map=write_map)
return
# Otherwise we will input from a path given by the entry path of each entry.
# This means that files must appear in subdirectories if they are part of
# a sub-section.
einfos = image.GetListEntries(entry_paths)[0]
tout.Notice("Replacing %d matching entries in image '%s'" %
(len(einfos), image_fname))
BeforeReplace(image, allow_resize)
for einfo in einfos:
entry = einfo.entry
if entry.GetEntries():
tout.Info("Skipping section entry '%s'" % entry.GetPath())
continue
path = entry.GetPath()[1:]
fname = os.path.join(indir, path)
if os.path.exists(fname):
tout.Notice("Write entry '%s' from file '%s'" %
(entry.GetPath(), fname))
data = tools.ReadFile(fname)
ReplaceOneEntry(image, entry, data, do_compress, allow_resize)
else:
tout.Warning("Skipping entry '%s' from missing file '%s'" %
(entry.GetPath(), fname))
AfterReplace(image, allow_resize=allow_resize, write_map=write_map)
return image
def _ShowBlobHelp(path, text):
tout.Warning('\n%s:' % path)
for line in text.splitlines():
tout.Warning(' %s' % line)
def Binman(args):
"""The main control code for binman
This assumes that help and test options have already been dealt with. It
deals with the core task of building images.
Args:
args: Command line arguments Namespace object
"""
global Image
global state
if args.full_help:
tools.PrintFullHelp(
os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README.rst')
)
return 0
# Put these here so that we can import this module without libfdt
from binman.image import Image
from binman import state
if args.cmd in ['ls', 'extract', 'replace']:
try:
tout.Init(args.verbosity)
tools.PrepareOutputDir(None)
if args.cmd == 'ls':
ListEntries(args.image, args.paths)
if args.cmd == 'extract':
ExtractEntries(args.image, args.filename, args.outdir, args.paths,
not args.uncompressed)
if args.cmd == 'replace':
ReplaceEntries(args.image, args.filename, args.indir, args.paths,
do_compress=not args.compressed,
allow_resize=not args.fix_size, write_map=args.map)
except:
raise
finally:
tools.FinaliseOutputDir()
return 0
# Try to figure out which device tree contains our image description
if args.dt:
dtb_fname = args.dt
else:
board = args.board
if not board:
raise ValueError('Must provide a board to process (use -b <board>)')
board_pathname = os.path.join(args.build_dir, board)
dtb_fname = os.path.join(board_pathname, 'u-boot.dtb')
if not args.indir:
args.indir = ['.']
args.indir.append(board_pathname)
try:
tout.Init(args.verbosity)
elf.debug = args.debug
cbfs_util.VERBOSE = args.verbosity > 2
state.use_fake_dtb = args.fake_dtb
# Normally we replace the 'u-boot' etype with 'u-boot-expanded', etc.
# When running tests this can be disabled using this flag. When not
# updating the FDT in image, it is not needed by binman, but we use it
# for consistency, so that the images look the same to U-Boot at
# runtime.
use_expanded = not args.no_expanded
try:
tools.SetInputDirs(args.indir)
tools.PrepareOutputDir(args.outdir, args.preserve)
tools.SetToolPaths(args.toolpath)
state.SetEntryArgs(args.entry_arg)
state.SetThreads(args.threads)
images = PrepareImagesAndDtbs(dtb_fname, args.image,
args.update_fdt, use_expanded)
if args.test_section_timeout:
# Set the first image to timeout, used in testThreadTimeout()
images[list(images.keys())[0]].test_section_timeout = True
missing = False
for image in images.values():
missing |= ProcessImage(image, args.update_fdt, args.map,
allow_missing=args.allow_missing)
# Write the updated FDTs to our output files
for dtb_item in state.GetAllFdts():
tools.WriteFile(dtb_item._fname, dtb_item.GetContents())
if missing:
tout.Warning("\nSome images are invalid")
# Use this to debug the time take to pack the image
#state.TimingShow()
finally:
tools.FinaliseOutputDir()
finally:
tout.Uninit()
return 0
