def check_call(self, args, cwd=None):
cmd_str = util.quote_sh_list(args)
log.dbg("running '{}'".format(cmd_str),
'in',
cwd or os.getcwd(),
level=log.VERBOSE_VERY)
subprocess.check_call(args, cwd=cwd)
def print_loot(self, name, project, z_project, args):
# Print a list of out of tree outstanding patches in the given
# project.
#
# name: project name
# project: the west.manifest.Project instance in the NCS manifest
# z_project: the Project instance in the upstream manifest
name_path = _name_and_path(project)
# Get the upstream revision of the project. The zephyr project
# has to be treated as a special case.
if name == 'zephyr':
z_rev = self.zephyr_rev
else:
z_rev = z_project.revision
try:
nsha = project.sha(project.revision)
project.git('cat-file -e ' + nsha)
except subprocess.CalledProcessError:
log.wrn(f"{name_path}: can't get loot; please run "
f'"west update" (need revision {project.revision})')
return
try:
zsha = z_project.sha(z_rev)
z_project.git('cat-file -e ' + zsha)
except subprocess.CalledProcessError:
log.wrn(f"{name_path}: can't get loot; please fetch upstream URL "
f'{z_project.url} (need revision {z_project.revision})')
return
try:
analyzer = nwh.RepoAnalyzer(project, z_project, project.revision,
z_rev)
except nwh.InvalidRepositoryError as ire:
log.die(f"{name_path}: {str(ire)}")
try:
loot = analyzer.downstream_outstanding
except nwh.UnknownCommitsError as uce:
log.die(f'{name_path}: unknown commits: {str(uce)}')
if not loot and log.VERBOSE <= log.VERBOSE_NONE:
# Don't print output if there's no loot unless verbose
# mode is on.
return
log.banner(name_path)
log.inf(f'NCS commit: {nsha}, upstream commit: {zsha}')
log.inf('OOT patches: ' + (f'{len(loot)} total' if loot else 'none') +
(', output limited by --file' if args.files else ''))
for c in loot:
if args.files and not nwh.commit_affects_files(c, args.files):
log.dbg(f"skipping {c.oid}; it doesn't affect file filter",
level=log.VERBOSE_VERY)
continue
if args.sha_only:
log.inf(str(c.oid))
else:
log.inf(f'- {c.oid} {nwh.commit_shortlog(c)}')
def _setup_build_dir(self):
# Initialize build_dir and created_build_dir attributes.
# If we created the build directory, we must run CMake.
log.dbg('setting up build directory', level=log.VERBOSE_EXTREME)
# The CMake Cache has not been loaded yet, so this is safe
board, _ = self._find_board()
source_dir = self._find_source_dir()
app = os.path.split(source_dir)[1]
build_dir = find_build_dir(self.args.build_dir,
board=board,
source_dir=source_dir,
app=app)
if not build_dir:
log.die('Unable to determine a default build folder. Check '
'your build.dir-fmt configuration option')
if os.path.exists(build_dir):
if not os.path.isdir(build_dir):
log.die(
'build directory {} exists and is not a directory'.format(
build_dir))
else:
os.makedirs(build_dir, exist_ok=False)
self.created_build_dir = True
self.run_cmake = True
self.build_dir = build_dir
def do_run(self, command, **kwargs):
reset = False
if not self.find_device():
reset = True
log.dbg('Device not found, waiting for it',
level=log.VERBOSE_EXTREME)
# Use of print() here is advised. We don't want to lose
# this information in a separate log -- this is
# interactive and requires a terminal.
print('Please reset your board to switch to DFU mode...')
while not self.find_device():
time.sleep(0.1)
cmd = list(self.cmd)
if self.dfuse:
# http://dfu-util.sourceforge.net/dfuse.html
dcfg = self.dfuse_config
addr_opts = hex(dcfg.address) + ':' + dcfg.options
cmd.extend(['-s', addr_opts])
cmd.extend(['-a', self.alt, '-D', self.img])
self.check_call(cmd)
if self.dfuse and 'leave' in dcfg.options.split(':'):
# Normal DFU devices generally need to be reset to switch
# back to the flashed program.
#
# DfuSe targets do as well, except when 'leave' is given
# as an option.
reset = False
if reset:
print('Now reset your board again to switch back to runtime mode.')
def edt_flash_node(b, cache):
# Get the EDT Node corresponding to the zephyr,flash chosen DT
# node.
# Retrieve the list of devicetree bindings from cache.
try:
bindings = cache.get_list('CACHED_DTS_ROOT_BINDINGS')
log.dbg('DTS bindings:', bindings, level=log.VERBOSE_VERY)
except KeyError:
log.die('CMake cache has no CACHED_DTS_ROOT_BINDINGS.'
'\n Try again after re-building your application.')
# Ensure the build directory has a compiled DTS file
# where we expect it to be.
dts = b / 'zephyr' / (cache['CACHED_BOARD'] + '.dts.pre.tmp')
if not dts.is_file():
log.die("can't find DTS; expected:", dts)
log.dbg('DTS file:', dts, level=log.VERBOSE_VERY)
# Parse the devicetree using bindings from cache.
try:
edt = edtlib.EDT(dts, bindings)
except edtlib.EDTError as e:
log.die("can't parse devicetree:", e)
# By convention, the zephyr,flash chosen node contains the
# partition information about the zephyr image to sign.
flash = edt.chosen_node('zephyr,flash')
if not flash:
log.die('devicetree has no chosen zephyr,flash node;',
"can't infer flash write block or image-0 slot sizes")
return flash
def get_board_snr_from_user(self):
snrs = self.check_output(['nrfjprog', '--ids'])
snrs = snrs.decode(sys.getdefaultencoding()).strip().splitlines()
if len(snrs) == 0:
raise RuntimeError('"nrfjprog --ids" did not find a board; Is the board connected?')
elif len(snrs) == 1:
board_snr = snrs[0]
if board_snr == '0':
raise RuntimeError('"nrfjprog --ids" returned 0; is a debugger already connected?')
return board_snr
log.dbg("Refusing the temptation to guess a board",
level=log.VERBOSE_EXTREME)
# Use of print() here is advised. We don't want to lose
# this information in a separate log -- this is
# interactive and requires a terminal.
print('There are multiple boards connected.')
for i, snr in enumerate(snrs, 1):
print('{}. {}'.format(i, snr))
p = 'Please select one with desired serial number (1-{}): '.format(
len(snrs))
while True:
value = input(p)
try:
value = int(value)
except ValueError:
continue
if 1 <= value <= len(snrs):
break
return snrs[value - 1]
def find_build_dir(dir, **kwargs):
'''Heuristic for finding a build directory.
The default build directory is computed by reading the build.dir-fmt
configuration option, defaulting to DEFAULT_BUILD_DIR if not set. It might
be None if the build.dir-fmt configuration option is set but cannot be
resolved.
If the given argument is truthy, it is returned. Otherwise, if
the default build folder is a build directory, it is returned.
Next, if the current working directory is a build directory, it is
returned. Finally, the default build directory is returned (may be None).
'''
if dir:
build_dir = dir
else:
cwd = os.getcwd()
default = config.get('build', 'dir-fmt', fallback=DEFAULT_BUILD_DIR)
try:
default = _resolve_build_dir(default, cwd, **kwargs)
log.dbg('config dir-fmt: {}'.format(default),
level=log.VERBOSE_EXTREME)
except KeyError:
default = None
if default and is_zephyr_build(default):
build_dir = default
elif is_zephyr_build(cwd):
build_dir = cwd
else:
build_dir = default
log.dbg('build dir: {}'.format(build_dir), level=log.VERBOSE_EXTREME)
if build_dir:
return os.path.abspath(build_dir)
else:
return None
def getExpressionData(filePath, numLines):
"""
Scans the specified file for the first SPDX-License-Identifier:
tag in the file.
Arguments:
- filePath: path to file to scan.
- numLines: number of lines to scan for an expression before
giving up. If 0, will scan the entire file.
Returns: parsed expression if found; None if not found.
"""
log.dbg(f" - getting licenses for {filePath}")
with open(filePath, "r") as f:
try:
lineno = 0
for line in f:
lineno += 1
if lineno > numLines > 0:
break
expression = parseLineForExpression(line)
if expression is not None:
return expression
except UnicodeDecodeError:
# invalid UTF-8 content
return None
# if we get here, we didn't find an expression
return None
def do_run(self, args, ignored):
self.args = args # Avoid having to pass them around
log.dbg('args:', args, level=log.VERBOSE_EXTREME)
self._sanity_precheck()
self._setup_build_dir()
if is_zephyr_build(self.build_dir):
self._update_cache()
if self.args.cmake or self.args.cmake_opts:
self.run_cmake = True
else:
self.run_cmake = True
self._setup_source_dir()
self._sanity_check()
log.inf('source directory: {}'.format(self.source_dir), colorize=True)
log.inf('build directory: {}{}'.
format(self.build_dir,
(' (created)' if self.created_build_dir
else '')),
colorize=True)
if self.cmake_cache:
board = self.cmake_cache.get('CACHED_BOARD')
elif self.args.board:
board = self.args.board
else:
board = 'UNKNOWN' # shouldn't happen
log.inf('BOARD:', board, colorize=True)
self._run_cmake(self.args.cmake_opts)
self._sanity_check()
self._update_cache()
extra_args = ['--target', args.target] if args.target else []
cmake.run_build(self.build_dir, extra_args=extra_args)
def run_cmake(args, cwd=None, capture_output=False):
'''Run cmake to (re)generate a build system.
If capture_output is set to True, returns the output of the command instead
of displaying it on stdout/stderr..'''
cmake = shutil.which('cmake')
if cmake is None:
log.die('CMake is not installed or cannot be found; cannot build.')
cmd = [cmake] + args
kwargs = dict()
if capture_output:
kwargs['stdout'] = subprocess.PIPE
# CMake sends the output of message() to stderr unless it's STATUS
kwargs['stderr'] = subprocess.STDOUT
if cwd:
kwargs['cwd'] = cwd
log.dbg('Running CMake:', quote_sh_list(cmd), level=log.VERBOSE_NORMAL)
p = subprocess.Popen(cmd, **kwargs)
out, err = p.communicate()
if p.returncode == 0:
if out:
return out.decode(sys.getdefaultencoding()).splitlines()
else:
return None
else:
# A real error occurred, raise an exception
raise subprocess.CalledProcessError(p.returncode, p.args)
def sign(self, command, build_dir, bcfg, formats):
if not formats:
return
args = command.args
b = pathlib.Path(build_dir)
cache = CMakeCache.from_build_dir(build_dir)
tool_path = self.find_imgtool(command, args)
# The vector table offset is set in Kconfig:
vtoff = self.get_cfg(command, bcfg, 'CONFIG_TEXT_SECTION_OFFSET')
# Flash device write alignment and the partition's slot size
# come from devicetree:
flash = self.edt_flash_node(b, cache)
align, addr, size = self.edt_flash_params(flash)
runner_config = cached_runner_config(build_dir, cache)
if 'bin' in formats:
in_bin = runner_config.bin_file
if not in_bin:
log.die("can't find unsigned .bin to sign")
else:
in_bin = None
if 'hex' in formats:
in_hex = runner_config.hex_file
if not in_hex:
log.die("can't find unsigned .hex to sign")
else:
in_hex = None
log.banner('image configuration:')
log.inf('partition offset: {0} (0x{0:x})'.format(addr))
log.inf('partition size: {0} (0x{0:x})'.format(size))
log.inf('text section offset: {0} (0x{0:x})'.format(vtoff))
# Base sign command.
#
# We provide a default --version in case the user is just
# messing around and doesn't want to set one. It will be
# overridden if there is a --version in args.tool_args.
sign_base = [tool_path, 'sign',
'--version', '0.0.0+0',
'--align', str(align),
'--header-size', str(vtoff),
'--slot-size', str(size)]
sign_base.extend(args.tool_args)
log.banner('signed binaries:')
if in_bin:
out_bin = args.sbin or str(b / 'zephyr' / 'zephyr.signed.bin')
sign_bin = sign_base + [in_bin, out_bin]
log.inf('bin: {}'.format(out_bin))
log.dbg(quote_sh_list(sign_bin))
subprocess.check_call(sign_bin)
if in_hex:
out_hex = args.shex or str(b / 'zephyr' / 'zephyr.signed.hex')
sign_hex = sign_base + [in_hex, out_hex]
log.inf('hex: {}'.format(out_hex))
log.dbg(quote_sh_list(sign_hex))
subprocess.check_call(sign_hex)
def create(cls, cfg, args):
daparg = os.environ.get('PYOCD_DAPARG')
if daparg:
log.wrn('Setting PYOCD_DAPARG in the environment is',
'deprecated; use the --daparg option instead.')
if args.daparg is None:
log.dbg('Missing --daparg set to {} from environment'.format(
daparg),
level=log.VERBOSE_VERY)
args.daparg = daparg
build_conf = BuildConfiguration(cfg.build_dir)
flash_addr = cls.get_flash_address(args, build_conf)
return PyOcdBinaryRunner(cfg,
args.target,
flashtool=args.flashtool,
flash_addr=flash_addr,
flashtool_opts=args.flashtool_opt,
gdbserver=args.gdbserver,
gdb_port=args.gdb_port,
tui=args.tui,
board_id=args.board_id,
daparg=args.daparg,
frequency=args.frequency)
def getCodemodel(self):
log.dbg("getting codemodel from CMake API reply files")
# make sure the reply directory exists
cmakeReplyDirPath = os.path.join(self.cfg.buildDir, ".cmake", "api",
"v1", "reply")
if not os.path.exists(cmakeReplyDirPath):
log.err(
f'cmake api reply directory {cmakeReplyDirPath} does not exist'
)
log.err('was query directory created before cmake build ran?')
return None
if not os.path.isdir(cmakeReplyDirPath):
log.err(
f'cmake api reply directory {cmakeReplyDirPath} exists but is not a directory'
)
return None
# find file with "index" prefix; there should only be one
indexFilePath = ""
for f in os.listdir(cmakeReplyDirPath):
if f.startswith("index"):
indexFilePath = os.path.join(cmakeReplyDirPath, f)
break
if indexFilePath == "":
# didn't find it
log.err(
f'cmake api reply index file not found in {cmakeReplyDirPath}')
return None
# parse it
return parseReply(indexFilePath)
def parseCMakeCacheFile(filePath):
log.dbg(f"parsing CMake cache file at {filePath}")
kv = {}
try:
with open(filePath, "r") as f:
# should be a short file, so we'll use readlines
lines = f.readlines()
# walk through and look for non-comment, non-empty lines
for line in lines:
sline = line.strip()
if sline == "":
continue
if sline.startswith("#") or sline.startswith("//"):
continue
# parse out : and = characters
pline1 = sline.split(":", maxsplit=1)
if len(pline1) != 2:
continue
pline2 = pline1[1].split("=", maxsplit=1)
if len(pline2) != 2:
continue
kv[pline1[0]] = pline2[1]
return kv
except OSError as e:
log.err(f"Error loading {filePath}: {str(e)}")
return {}
def _ensure_min_version(cmake, dry_run):
cmd = [cmake, '--version']
if dry_run:
log.inf('Dry run:', quote_sh_list(cmd))
return
try:
version_out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
except subprocess.CalledProcessError as cpe:
log.die('cannot get cmake version:', str(cpe))
decoded = version_out.decode('utf-8')
lines = decoded.splitlines()
if not lines:
log.die('can\'t get cmake version: ' +
'unexpected "cmake --version" output:\n{}\n'.format(decoded) +
'Please install CMake ' + _MIN_CMAKE_VERSION_STR +
' or higher (https://cmake.org/download/).')
version = lines[0].split()[2]
if '-' in version:
# Handle semver cases like "3.19.20210206-g1e50ab6"
# which Kitware uses for prerelease versions.
version = version.split('-', 1)[0]
if packaging.version.parse(version) < _MIN_CMAKE_VERSION:
log.die(
'cmake version', version,
'is less than minimum version {};'.format(_MIN_CMAKE_VERSION_STR),
'please update your CMake (https://cmake.org/download/).')
else:
log.dbg('cmake version', version, 'is OK; minimum version is',
_MIN_CMAKE_VERSION_STR)
def getHashes(filePath):
"""
Scan for and return hashes.
Arguments:
- filePath: path to file to scan.
Returns: tuple of (SHA1, SHA256, MD5) hashes for filePath, or
None if file is not found.
"""
hSHA1 = hashlib.sha1()
hSHA256 = hashlib.sha256()
hMD5 = hashlib.md5()
log.dbg(f" - getting hashes for {filePath}")
try:
with open(filePath, 'rb') as f:
buf = f.read()
hSHA1.update(buf)
hSHA256.update(buf)
hMD5.update(buf)
except OSError:
return None
return (hSHA1.hexdigest(), hSHA256.hexdigest(), hMD5.hexdigest())
def getCIncludes(compilerPath, srcFile, tcg):
log.dbg(f" - getting includes for {srcFile}")
# prepare fragments
fragments = [fr for fr in tcg.compileCommandFragments if fr.strip() != ""]
# prepare include arguments
includes = ["-I" + incl.path for incl in tcg.includes]
# prepare defines
defines = ["-D" + d.define for d in tcg.defines]
# prepare command invocation
cmd = [compilerPath, "-E", "-H"
] + fragments + includes + defines + [srcFile]
cp = run(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True)
if cp.returncode != 0:
log.dbg(
f" - calling {compilerPath} failed with error code {cp.returncode}"
)
return []
else:
# response will be in cp.stderr, not cp.stdout
return extractIncludes(cp.stderr)
def edt_flash_node(b, quiet=False):
# Get the EDT Node corresponding to the zephyr,flash chosen DT
# node; 'b' is the build directory as a pathlib object.
# Ensure the build directory has a compiled DTS file
# where we expect it to be.
dts = b / 'zephyr' / 'zephyr.dts'
if not quiet:
log.dbg('DTS file:', dts, level=log.VERBOSE_VERY)
edt_pickle = b / 'zephyr' / 'edt.pickle'
if not edt_pickle.is_file():
log.die("can't load devicetree; expected to find:", edt_pickle)
# Load the devicetree.
with open(edt_pickle, 'rb') as f:
edt = pickle.load(f)
# By convention, the zephyr,flash chosen node contains the
# partition information about the zephyr image to sign.
flash = edt.chosen_node('zephyr,flash')
if not flash:
log.die('devicetree has no chosen zephyr,flash node;',
"can't infer flash write block or image-0 slot sizes")
return flash
def _post_checkout_help(project, branch, sha, is_ancestor):
# Print helpful information to the user about a project that
# might have just left a branch behind.
if branch is None:
# If there was no branch checked out, there are no
# additional diagnostics that need emitting.
return
rel = relpath(project.abspath)
if is_ancestor:
# If the branch we just left behind is a descendant of
# the new HEAD (e.g. if this is a topic branch the
# user is working on and the remote hasn't changed),
# print a message that makes it easy to get back,
# no matter where in the installation os.getcwd() is.
log.wrn(project.format(
'left behind {name} branch "{b}"; to switch '
'back to it (fast forward), use: git -C {rp} checkout {b}',
b=branch, rp=rel))
log.dbg('(To do this automatically in the future,',
'use "west update --keep-descendants".)')
else:
# Tell the user how they could rebase by hand, and
# point them at west update --rebase.
log.wrn(project.format(
'left behind {name} branch "{b}"; '
'to rebase onto the new HEAD: git -C {rp} rebase {sh} {b}',
b=branch, rp=rel, sh=sha))
log.dbg('(To do this automatically in the future,',
'use "west update --rebase".)')
def is_zephyr_build(path):
'''Return true if and only if `path` appears to be a valid Zephyr
build directory.
"Valid" means the given path is a directory which contains a CMake
cache with a 'ZEPHYR_BASE' or 'ZEPHYR_TOOLCHAIN_VARIANT' variable.
(The check for ZEPHYR_BASE introduced sometime after Zephyr 2.4 to
fix https://github.com/zephyrproject-rtos/zephyr/issues/28876; we
keep support for the second variable around for compatibility with
versions 2.2 and earlier, which didn't have ZEPHYR_BASE in cache.
The cached ZEPHYR_BASE was added in
https://github.com/zephyrproject-rtos/zephyr/pull/23054.)
'''
try:
cache = zcmake.CMakeCache.from_build_dir(path)
except FileNotFoundError:
cache = {}
if 'ZEPHYR_BASE' in cache or 'ZEPHYR_TOOLCHAIN_VARIANT' in cache:
log.dbg(f'{path} is a zephyr build directory',
level=log.VERBOSE_EXTREME)
return True
log.dbg(f'{path} is NOT a valid zephyr build directory',
level=log.VERBOSE_EXTREME)
return False
def flash(self, **kwargs):
self.check_cmd(self.commander)
if self.bin_name is None:
raise ValueError('Cannot flash; bin_name is missing')
lines = ['r'] # Reset and halt the target
if self.erase:
lines.append('erase') # Erase all flash sectors
lines.append('loadfile {} 0x{:x}'.format(self.bin_name,
self.flash_addr))
lines.append('g') # Start the CPU
lines.append('q') # Close the connection and quit
log.dbg('JLink commander script:')
log.dbg('\n'.join(lines))
# Don't use NamedTemporaryFile: the resulting file can't be
# opened again on Windows.
with tempfile.TemporaryDirectory(suffix='jlink') as d:
fname = os.path.join(d, 'runner.jlink')
with open(fname, 'wb') as f:
f.writelines(bytes(line + '\n', 'utf-8') for line in lines)
cmd = ([self.commander] +
['-if', self.iface,
'-speed', self.speed,
'-device', self.device,
'-CommanderScript', fname])
log.inf('Flashing Target Device')
self.check_call(cmd)
def check_output(args, cwd=None):
'''Runs subprocess.check_output(args, cwd=cwd) after
logging the call at VERBOSE_VERY level.'''
cmd_str = quote_sh_list(args)
log.dbg(f"running '{cmd_str}' in {cwd or os.getcwd()}",
level=log.VERBOSE_VERY)
return subprocess.check_output(args, cwd=cwd)
def bootstrap(self, args):
manifest_url = args.manifest_url or MANIFEST_URL_DEFAULT
manifest_rev = args.manifest_rev or MANIFEST_REV_DEFAULT
topdir = util.canon_path(args.directory or os.getcwd())
west_dir = join(topdir, WEST_DIR)
try:
already = util.west_topdir(topdir, fall_back=False)
self.die_already(already)
except util.WestNotFound:
pass
log.banner('Initializing in', topdir)
if not isdir(topdir):
self.create(topdir, exist_ok=False)
# Clone the manifest repository into a temporary directory.
tempdir = join(west_dir, 'manifest-tmp')
if exists(tempdir):
log.dbg('removing existing temporary manifest directory', tempdir)
shutil.rmtree(tempdir)
try:
self.clone_manifest(manifest_url, manifest_rev, tempdir)
except subprocess.CalledProcessError:
shutil.rmtree(tempdir, ignore_errors=True)
raise
# Verify the manifest file exists.
temp_manifest = join(tempdir, 'west.yml')
if not exists(temp_manifest):
log.die(f'can\'t init: no "west.yml" found in {tempdir}\n'
f' Hint: check --manifest-url={manifest_url} and '
'--manifest-rev={manifest_rev}\n'
f' You may need to remove {west_dir} before retrying.')
# Parse the manifest to get the manifest path, if it declares one.
# Otherwise, use the URL. Ignore imports -- all we really
# want to know is if there's a "self: path:" or not.
projects = Manifest.from_file(temp_manifest,
import_flags=ImportFlag.IGNORE,
topdir=topdir).projects
manifest_project = projects[MANIFEST_PROJECT_INDEX]
if manifest_project.path:
manifest_path = manifest_project.path
else:
manifest_path = posixpath.basename(urlparse(manifest_url).path)
manifest_abspath = join(topdir, manifest_path)
log.dbg('moving', tempdir, 'to', manifest_abspath,
level=log.VERBOSE_EXTREME)
try:
shutil.move(tempdir, manifest_abspath)
except shutil.Error as e:
log.die(e)
log.small_banner('setting manifest.path to', manifest_path)
update_config('manifest', 'path', manifest_path, topdir=topdir)
return topdir
def git(self, cmd, extra_args=(), capture_stdout=False,
capture_stderr=False, check=True, cwd=None):
'''Helper for running a git command using metadata from a Project
instance.
:param cmd: git command as a string (or list of strings); all strings
are formatted using self.format() before use.
:param extra_args: sequence of additional arguments to pass to the git
command (useful mostly if cmd is a string).
:param capture_stdout: True if stdout should be captured into the
returned object instead of being printed.
:param capture_stderr: Like capture_stdout, but for stderr. Use with
caution as it prevents error messages from being
shown to the user.
:param check: True if a subprocess.CalledProcessError should be raised
if the git command finishes with a non-zero return code.
:param cwd: directory to run command in (default: self.abspath)
Returns a CompletedProcess (which is back-ported for Python 3.4).'''
_warn_once_if_no_git()
if isinstance(cmd, str):
cmd_list = shlex.split(cmd)
else:
cmd_list = list(cmd)
extra_args = list(extra_args)
if cwd is None:
cwd = self.abspath
args = ['git'] + [self.format(arg) for arg in cmd_list] + extra_args
cmd_str = util.quote_sh_list(args)
log.dbg("running '{}'".format(cmd_str), 'in', cwd,
level=log.VERBOSE_VERY)
popen = subprocess.Popen(
args, cwd=cwd,
stdout=subprocess.PIPE if capture_stdout else None,
stderr=subprocess.PIPE if capture_stderr else None)
stdout, stderr = popen.communicate()
dbg_msg = "'{}' in {} finished with exit status {}".format(
cmd_str, cwd, popen.returncode)
if capture_stdout:
dbg_msg += " and wrote {} to stdout".format(stdout)
if capture_stderr:
dbg_msg += " and wrote {} to stderr".format(stderr)
log.dbg(dbg_msg, level=log.VERBOSE_VERY)
if check and popen.returncode:
raise subprocess.CalledProcessError(popen.returncode, cmd_list,
output=stdout, stderr=stderr)
else:
return CompletedProcess(popen.args, popen.returncode,
stdout, stderr)
def read(self, args):
section, key = self._sk(args)
cfg = configparser.ConfigParser()
read_config(configfile=args.configfile or ALL, config=cfg)
value = cfg.get(section, key, fallback=None)
if value is not None:
log.inf(value)
else:
log.dbg(f'{args.name} is unset')
raise CommandError(returncode=1)
def print_loot(self, name, project, z_project, args):
# Print a list of out of tree outstanding patches in the given
# project.
#
# name: project name
# project: the west.manifest.Project instance in the NCS manifest
# z_project: the Project instance in the upstream manifest
msg = project.format('{name_and_path} outstanding downstream patches:')
# Get the upstream revision of the project. The zephyr project
# has to be treated as a special case.
if name == 'zephyr':
z_rev = self.zephyr_rev
msg += ' NOTE: {} *must* be up to date'.format(z_rev)
else:
z_rev = z_project.revision
log.banner(msg)
try:
nsha = project.sha(project.revision)
project.git('cat-file -e ' + nsha)
except subprocess.CalledProcessError:
log.wrn(
"can't get loot; please run \"west update {}\"".format(
project.name),
'(need revision {})'.format(project.revision))
return
try:
zsha = z_project.sha(z_project.revision)
z_project.git('cat-file -e ' + zsha)
except subprocess.CalledProcessError:
log.wrn("can't get loot; please fetch upstream URL", z_project.url,
'(need revision {})'.format(z_project.revision))
return
try:
analyzer = nwh.RepoAnalyzer(project, z_project, project.revision,
z_rev)
except nwh.InvalidRepositoryError as ire:
log.die(str(ire))
try:
for c in analyzer.downstream_outstanding:
if args.files and not nwh.commit_affects_files(c, args.files):
log.dbg(
'skipping {}; it does not affect file filter'.format(
c.oid),
level=log.VERBOSE_VERY)
continue
if args.sha_only:
log.inf(str(c.oid))
else:
log.inf('- {} {}'.format(c.oid, nwh.commit_shortlog(c)))
except nwh.UnknownCommitsError as uce:
log.die('unknown commits:', str(uce))
def collectTargetDependencies(self, cfgTargets, cfgTarget, pkg):
log.dbg(f" - collecting target dependencies for {pkg.cfg.name}")
# walk through target's dependencies
for dep in cfgTarget.target.dependencies:
# extract dep name from its id
depFragments = dep.id.split(":")
depName = depFragments[0]
log.dbg(f" - adding pending relationship for {depName}")
# create relationship data between dependency packages
rd = RelationshipData()
rd.ownerType = RelationshipDataElementType.TARGETNAME
rd.ownerTargetName = pkg.cfg.name
rd.otherType = RelationshipDataElementType.TARGETNAME
rd.otherTargetName = depName
rd.rlnType = "HAS_PREREQUISITE"
# add it to pending relationships queue
self.pendingRelationships.append(rd)
# if this is a target with any build artifacts (e.g. non-UTILITY),
# also create STATIC_LINK relationship for dependency build files,
# together with this Package's own target build file
if len(cfgTarget.target.artifacts) == 0:
continue
# find the filename for the dependency's build product, using the
# codemodel (since we might not have created this dependency's
# Package or File yet)
depAbspath = ""
for ct in cfgTargets:
if ct.name == depName:
# skip utility targets
if len(ct.target.artifacts) == 0:
continue
# all targets use the same relativeBaseDir, so this works
# even though pkg is the owner package
depAbspath = os.path.join(pkg.cfg.relativeBaseDir,
ct.target.artifacts[0])
break
if depAbspath == "":
continue
# create relationship data between build files
rd = RelationshipData()
rd.ownerType = RelationshipDataElementType.FILENAME
rd.ownerFileAbspath = pkg.targetBuildFile.abspath
rd.otherType = RelationshipDataElementType.FILENAME
rd.otherFileAbspath = depAbspath
rd.rlnType = "STATIC_LINK"
# add it to pending relationships queue
self.pendingRelationships.append(rd)
def set_zephyr_base(args):
'''Ensure ZEPHYR_BASE is set, emitting warnings if that's not
possible, or if the user is pointing it somewhere different than
what the manifest expects.'''
zb_env = os.environ.get('ZEPHYR_BASE')
if args.zephyr_base:
# The command line --zephyr-base takes precedence over
# everything else.
zb = os.path.abspath(args.zephyr_base)
zb_origin = 'command line'
else:
# If the user doesn't specify it concretely, use the project
# with path 'zephyr' if that exists, or the ZEPHYR_BASE value
# in the calling environment.
#
# At some point, we need a more flexible way to set environment
# variables based on manifest contents, but this is good enough
# to get started with and to ask for wider testing.
try:
manifest = Manifest.from_file()
except MalformedConfig as e:
log.die('Parsing of manifest file failed during command',
args.command, ':', *e.args)
for project in manifest.projects:
if project.path == 'zephyr':
zb = project.abspath
zb_origin = 'manifest file {}'.format(manifest.path)
break
else:
if zb_env is None:
log.wrn('no --zephyr-base given, ZEPHYR_BASE is unset,',
'and no manifest project has path "zephyr"')
zb = None
zb_origin = None
else:
zb = zb_env
zb_origin = 'environment'
if zb_env and os.path.abspath(zb) != os.path.abspath(zb_env):
# The environment ZEPHYR_BASE takes precedence over either the
# command line or the manifest, but in normal multi-repo
# operation we shouldn't expect to need to set ZEPHYR_BASE to
# point to some random place. In practice, this is probably
# happening because zephyr-env.sh/cmd was run in some other
# zephyr installation, and the user forgot about that.
log.wrn('ZEPHYR_BASE={}'.format(zb_env),
'in the calling environment, but has been set to', zb,
'instead by the', zb_origin)
os.environ['ZEPHYR_BASE'] = zb
log.dbg('ZEPHYR_BASE={} (origin: {})'.format(zb, zb_origin))
def run_cmake(args, quiet=False):
'''Run cmake to (re)generate a build system'''
cmake = shutil.which('cmake')
if cmake is None:
log.die('CMake is not installed or cannot be found; cannot build.')
cmd = [cmake] + args
kwargs = dict()
if quiet:
kwargs['stdout'] = subprocess.DEVNULL
kwargs['stderr'] = subprocess.STDOUT
log.dbg('Running CMake:', cmd, level=log.VERBOSE_VERY)
log.dbg('As command:', quote_sh_list(cmd), level=log.VERBOSE_VERY)
subprocess.check_call(cmd, **kwargs)
def do_run(self, args, ignored):
self.args = args # for check_force
# Find the build directory and parse .config and DT.
build_dir = find_build_dir(args.build_dir)
self.check_force(os.path.isdir(build_dir),
'no such build directory {}'.format(build_dir))
self.check_force(is_zephyr_build(build_dir),
"build directory {} doesn't look like a Zephyr build "
'directory'.format(build_dir))
bcfg = BuildConfiguration(build_dir)
# Decide on output formats.
formats = []
bin_exists = 'CONFIG_BUILD_OUTPUT_BIN' in bcfg
if args.gen_bin:
self.check_force(bin_exists,
'--bin given but CONFIG_BUILD_OUTPUT_BIN not set '
"in build directory's ({}) .config".
format(build_dir))
formats.append('bin')
elif args.gen_bin is None and bin_exists:
formats.append('bin')
hex_exists = 'CONFIG_BUILD_OUTPUT_HEX' in bcfg
if args.gen_hex:
self.check_force(hex_exists,
'--hex given but CONFIG_BUILD_OUTPUT_HEX not set '
"in build directory's ({}) .config".
format(build_dir))
formats.append('hex')
elif args.gen_hex is None and hex_exists:
formats.append('hex')
if not formats:
if not args.quiet:
log.dbg('nothing to do: no output files')
return
# Delegate to the signer.
if args.tool == 'imgtool':
signer = ImgtoolSigner()
elif args.tool == 'rimage':
signer = RimageSigner()
# (Add support for other signers here in elif blocks)
else:
raise RuntimeError("can't happen")
signer.sign(self, build_dir, bcfg, formats)