def toplevel_projects(self, args):
# Return a list of projects from args.projects, or scream and
# die if any projects are either unknown or not defined in the
# manifest repository.
ids = args.projects
assert ids
mr_projects, mr_unknown = projects_unknown(
Manifest.from_file(import_flags=ImportFlag.IGNORE_PROJECTS), ids)
if not mr_unknown:
return mr_projects
try:
manifest = Manifest.from_file()
except ManifestImportFailed:
log.die('one or more projects are unknown or defined via '
'imports; please run plain "west update".')
projects, unknown = projects_unknown(manifest, ids)
if unknown:
die_unknown(unknown)
else:
# All of the ids are known projects, but some of them
# are not defined in the manifest repository.
mr_unknown_set = set(mr_unknown)
from_projects = [p for p in ids if p in mr_unknown_set]
log.die(f'refusing to update project: ' + " ".join(from_projects) +
'\n' + ' It or they were resolved via project imports.\n'
' Only plain "west update" can currently update them.')
def test_import_project_release_dir(tmpdir):
# Tests for a workspace that imports a directory from a project
# at a fixed release.
remotes = tmpdir / 'remotes'
empty_project = remotes / 'empty_project'
create_repo(empty_project)
add_commit(empty_project, 'empty-project empty commit')
imported = remotes / 'imported'
create_repo(imported)
add_commit(imported, 'add directory of imports',
files={'test.d/1.yml':
f'''\
manifest:
projects:
- name: west.d/1.yml-p1
url: {empty_project}
- name: west.d/1.yml-p2
url: {empty_project}
''',
'test.d/2.yml':
f'''\
manifest:
projects:
- name: west.d/2.yml-p1
url: {empty_project}
'''})
add_tag(imported, 'import-tag')
ws = tmpdir / 'ws'
create_workspace(ws, and_git=True)
manifest_repo = ws / 'mp'
add_commit(manifest_repo, 'manifest repo commit',
files={'west.yml':
f'''
manifest:
projects:
- name: imported
url: {imported}
revision: import-tag
import: test.d
'''})
cmd(f'init -l {manifest_repo}')
with pytest.raises(ManifestImportFailed):
Manifest.from_file(topdir=ws)
cmd('update', cwd=ws)
actual = Manifest.from_file(topdir=ws).projects
expected = [ManifestProject(path='mp', topdir=ws),
Project('imported', imported,
revision='import-tag', topdir=ws),
Project('west.d/1.yml-p1', empty_project, topdir=ws),
Project('west.d/1.yml-p2', empty_project, topdir=ws),
Project('west.d/2.yml-p1', empty_project, topdir=ws)]
for a, e in zip(actual, expected):
check_proj_consistency(a, e)
def test_import_project_rolling(repos_tmpdir):
# Like test_import_project_release, but with a rolling downstream
# that pulls master. We also use west init -l.
remotes = repos_tmpdir / 'repos'
zephyr = remotes / 'zephyr'
ws = repos_tmpdir / 'ws'
create_workspace(ws, and_git=True)
manifest_repo = ws / 'mp'
create_repo(manifest_repo)
add_commit(manifest_repo, 'manifest repo commit',
# zephyr revision is implicitly master:
files={'west.yml':
f'''
manifest:
projects:
- name: zephyr
url: {zephyr}
import: true
'''})
cmd(f'init -l {manifest_repo}')
with pytest.raises(ManifestImportFailed):
Manifest.from_file(topdir=ws)
cmd('update', cwd=ws)
actual = Manifest.from_file(topdir=ws).projects
expected = [ManifestProject(path='mp', topdir=ws),
Project('zephyr', zephyr,
revision='master', topdir=ws),
Project('Kconfiglib', remotes / 'Kconfiglib',
revision='zephyr', path='subdir/Kconfiglib',
topdir=ws),
Project('tagged_repo', remotes / 'tagged_repo',
revision='v1.0', topdir=ws),
Project('net-tools', remotes / 'net-tools',
clone_depth=1, topdir=ws,
west_commands='scripts/west-commands.yml')]
for a, e in zip(actual, expected):
check_proj_consistency(a, e)
# Add a commit in the remote zephyr repository and make sure it
# *does* affect our local workspace, since we're rolling with its
# master branch.
zephyr_ws = ws / 'zephyr'
head_before = rev_parse(zephyr_ws, 'HEAD')
add_commit(zephyr, 'this better show up',
files={'should-clone': ''})
cmd('update', cwd=ws)
assert head_before != rev_parse(zephyr_ws, 'HEAD')
assert (zephyr_ws / 'should-clone').check(file=1)
def do_run(self, args, unknown_args):
env = Manifest.from_file()
try:
kafl_path = env.get_projects(['kafl'])[0].abspath
except ValueError as e:
kafl_path = env.get_projects(['manifest'])[0].abspath
capstone_path = env.get_projects(['capstone'])[0].abspath
libxdc_path = env.get_projects(['libxdc'])[0].abspath
qemu_path = env.get_projects(['qemu'])[0].abspath
kafl_bin_path = os.path.join(kafl_path, 'kafl_fuzz.py')
if not os.path.exists(kafl_bin_path):
log.wrn("Could not find kAFL-Fuzzer in %s" % kafl_path)
kafl_bin_path = ""
qemu_bin_path = os.path.join(qemu_path,
'x86_64-softmmu/qemu-system-x86_64')
if not os.path.exists(qemu_bin_path):
log.wrn("Could not find kAFL Qemu binary in %s" % qemu_path)
qemu_bin_path = ""
# project executables
print("KAFL_BIN_PATH=%s" % kafl_bin_path)
print("KAFL_QEMU_PATH=%s" % qemu_bin_path)
# project libraries/includes
print("C_INCLUDE_PATH=%s/include:%s" % (capstone_path, libxdc_path))
print("LIBRARY_PATH=%s:%s" % (capstone_path, libxdc_path))
print("LD_LIBRARY_PATH=%s:%s" % (capstone_path, libxdc_path))
def main(args) -> None:
"""Entry point.
Args:
args: CLI arguments.
"""
config = yaml.load(open(args.config).read(), Loader=Loader)
manifest = Manifest.from_file(args.manifest)
ids = get_docset_ids(config, manifest, args.only)
output = Path(args.output)
output.mkdir(exist_ok=True)
for docset, id in ids.items():
log.info(f"Creating cache files for {docset}...")
with open(output / f"{docset}-{id}.zip", "wb") as f:
with zipfile.ZipFile(f, "w", zipfile.ZIP_DEFLATED) as zf:
docset_files = chain(
args.build_dir.glob(f"{docset}/**/*"),
args.build_dir.glob(f"html/{docset}/**/*"),
)
for file in docset_files:
if file.name == ENV_CACHE_PICKLE_FILENAME:
continue
if file.name == ENV_PICKLE_FILENAME:
file_cache = file.parent / ENV_CACHE_PICKLE_FILENAME
shutil.copy(file, file_cache)
patch_environment(file_cache, Path(manifest.topdir))
file = file_cache
arcname = file.relative_to(args.build_dir)
zf.write(file, arcname)
def do_run(self, args, user_args):
# manifest.path is not supposed to be set during init, thus clear it
# for the session and update it to correct location when complete.
if config.get('manifest', 'path', fallback=None) is not None:
config.remove_option('manifest', 'path')
manifest_file = os.path.join(args.local or args.cache, 'west.yml')
projects = Manifest.from_file(manifest_file).projects
manifest_project = projects[MANIFEST_PROJECT_INDEX]
if args.local is not None:
rel_manifest = os.path.relpath(args.local, util.west_topdir())
update_config('manifest', 'path', rel_manifest)
else:
if manifest_project.path == '':
url_path = urlparse(args.manifest_url).path
manifest_project.path = posixpath.basename(url_path)
manifest_project.abspath = os.path.realpath(
os.path.join(util.west_topdir(), manifest_project.path))
manifest_project.name = manifest_project.path
shutil.move(args.cache, manifest_project.abspath)
update_config('manifest', 'path', manifest_project.path)
for project in projects:
if project.path == 'zephyr':
update_config('zephyr', 'base', project.path)
def do_run(self, args, extra_args):
env = Manifest.from_file()
for query in extra_args:
try:
prj = env.get_projects([query])
print(prj[0].abspath)
except ValueError as e:
# check if `manifest` is the kAFL repo..
if query != 'kafl':
log.err(
"Could not find %s in west projects. Try `west list`."
% query)
return
try:
# check if manifest repo is kAFL
kafl_path = env.get_projects(['manifest'])[0].abspath
if os.path.exists(kafl_path + '/kafl_fuzz.py'):
log.wrn(
"Returning `manifest` repo path for query `%s`.." %
query)
print(kafl_path)
except ValueError as e:
log.err(
"Could not find %s in west projects. Try `west list`."
% query)
except Exception as e:
log.err(str(e))
def extension_commands(manifest=None):
'''Get descriptions of available extension commands.
The return value is an ordered map from project paths to lists of
WestExtCommandSpec objects, for projects which define extension
commands. The map's iteration order matches the manifest.projects
order.
The return value is empty if configuration option
``commands.allow_extensions`` is false.
:param manifest: a parsed ``west.manifest.Manifest`` object, or None
to reload a new one.
'''
allow_extensions = config.getboolean('commands',
'allow_extensions',
fallback=True)
if not allow_extensions:
return {}
if manifest is None:
manifest = Manifest.from_file()
specs = OrderedDict()
for project in manifest.projects:
if project.west_commands:
specs[project.path] = _ext_specs(project)
return specs
def load_manifest(self):
# Try to parse the manifest. We'll save it if that works, so
# it doesn't have to be re-parsed.
if not self.topdir:
return
try:
self.manifest = Manifest.from_file(topdir=self.topdir)
except (ManifestVersionError, MalformedManifest, MalformedConfig,
FileNotFoundError, ManifestImportFailed) as e:
# Defer exception handling to WestCommand.run(), which uses
# handle_builtin_manifest_load_err() to decide what to do.
#
# Make sure to update that function if you change the
# exceptions caught here. Unexpected exceptions should
# propagate up and fail fast.
#
# This might be OK, e.g. if we're running 'west config
# manifest.path foo' to fix the MalformedConfig error, but
# there's no way to know until we've parsed the command
# line arguments.
if isinstance(e, _ManifestImportDepth):
log.wrn('recursion depth exceeded during manifest resolution; '
'your manifest likely contains an import loop. '
'Run "west -v manifest --resolve" to debug.')
self.mle = e
def test_fs_topdir(fs_topdir):
# The API should be able to find a manifest file based on the file
# system and west configuration. The resulting topdir and abspath
# attributes should work as specified.
content = '''\
manifest:
projects:
- name: project-from-manifest-dir
url: from-manifest-dir
'''
west_yml = str(fs_topdir / 'mp' / 'west.yml')
with open(west_yml, 'w') as f:
f.write(content)
# manifest_path() should discover west_yml.
assert manifest_path() == west_yml
# Manifest.from_file() should as well.
# The project hierarchy should be rooted in the topdir.
manifest = Manifest.from_file()
assert manifest.topdir is not None
assert manifest.topdir == str(fs_topdir)
assert len(manifest.projects) == 2
p = manifest.projects[1]
assert p.name == 'project-from-manifest-dir'
assert p.url == 'from-manifest-dir'
assert p.topdir is not None
assert PurePath(p.topdir) == PurePath(str(fs_topdir))
def update_all(self, args):
# Plain 'west update' is the 'easy' case: since the user just
# wants us to update everything, we don't have to keep track
# of projects appearing or disappearing as a result of fetching
# new revisions from projects with imports.
#
# So we just re-parse the manifest, but force west.manifest to
# call our importer whenever it encounters an import statement
# in a project, allowing us to control the recursion so it
# always uses the latest manifest data.
self.updated = set()
manifest = Manifest.from_file(importer=self.update_importer,
import_flags=ImportFlag.FORCE_PROJECTS)
failed = []
for project in manifest.projects:
if (isinstance(project, ManifestProject)
or project.name in self.updated):
continue
try:
self.update(project)
self.updated.add(project.name)
except subprocess.CalledProcessError:
failed.append(project)
self._handle_failed(args, failed)
def projects(self, manifest_file):
try:
return Manifest.from_file(manifest_file).projects
except MalformedManifest as mm:
log.die(mm.args[0])
except MalformedConfig as mc:
log.die(mc.args[0])
def west_projects():
manifest_file = None
projects = []
# West is imported here, as it is optional
# (and thus maybe not installed)
# if user is providing a specific modules list.
from west.manifest import Manifest
from west.util import WestNotFound
from west.version import __version__ as WestVersion
from packaging import version
try:
manifest = Manifest.from_file()
if version.parse(WestVersion) >= version.parse('0.9.0'):
projects = [
p for p in manifest.get_projects([]) if manifest.is_active(p)
]
else:
projects = manifest.get_projects([])
manifest_file = manifest.path
return {'manifest': manifest_file, 'projects': projects}
except WestNotFound:
# Only accept WestNotFound, meaning we are not in a west
# workspace. Such setup is allowed, as west may be installed
# but the project is not required to use west.
pass
return None
def bootstrap(self, args):
manifest_url = args.manifest_url or MANIFEST_URL_DEFAULT
manifest_rev = args.manifest_rev or MANIFEST_REV_DEFAULT
topdir = util.canon_path(args.directory or os.getcwd())
west_dir = join(topdir, WEST_DIR)
try:
already = util.west_topdir(topdir, fall_back=False)
self.die_already(already)
except util.WestNotFound:
pass
log.banner('Initializing in', topdir)
if not isdir(topdir):
self.create(topdir, exist_ok=False)
# Clone the manifest repository into a temporary directory.
tempdir = join(west_dir, 'manifest-tmp')
if exists(tempdir):
log.dbg('removing existing temporary manifest directory', tempdir)
shutil.rmtree(tempdir)
try:
self.clone_manifest(manifest_url, manifest_rev, tempdir)
except subprocess.CalledProcessError:
shutil.rmtree(tempdir, ignore_errors=True)
raise
# Verify the manifest file exists.
temp_manifest = join(tempdir, 'west.yml')
if not exists(temp_manifest):
log.die(f'can\'t init: no "west.yml" found in {tempdir}\n'
f' Hint: check --manifest-url={manifest_url} and '
'--manifest-rev={manifest_rev}\n'
f' You may need to remove {west_dir} before retrying.')
# Parse the manifest to get the manifest path, if it declares one.
# Otherwise, use the URL. Ignore imports -- all we really
# want to know is if there's a "self: path:" or not.
projects = Manifest.from_file(temp_manifest,
import_flags=ImportFlag.IGNORE,
topdir=topdir).projects
manifest_project = projects[MANIFEST_PROJECT_INDEX]
if manifest_project.path:
manifest_path = manifest_project.path
else:
manifest_path = posixpath.basename(urlparse(manifest_url).path)
manifest_abspath = join(topdir, manifest_path)
log.dbg('moving', tempdir, 'to', manifest_abspath,
level=log.VERBOSE_EXTREME)
try:
shutil.move(tempdir, manifest_abspath)
except shutil.Error as e:
log.die(e)
log.small_banner('setting manifest.path to', manifest_path)
update_config('manifest', 'path', manifest_path, topdir=topdir)
return topdir
def test_fs_topdir_freestanding_manifest(tmpdir, fs_topdir):
# The API should be able to parse a random manifest file
# in a location that has nothing to do with the current topdir.
#
# The resulting Manifest will have projects rooted at topdir.
#
# If it has a self path, that's its path, and the ManifestProject
# is rooted at topdir. Otherwise, its path and abspath are None.
topdir = str(fs_topdir)
# Case 1: self path is present. ManifestProject is rooted
# within the same topdir as the projects.
content = '''\
manifest:
projects:
- name: name
url: url
self:
path: my-path
'''
yml = str(tmpdir / 'random.yml')
with open(yml, 'w') as f:
f.write(content)
manifest = Manifest.from_file(source_file=yml, topdir=topdir)
assert PurePath(manifest.topdir) == PurePath(topdir)
mproj = manifest.projects[0]
assert mproj.path == 'my-path'
assert PurePath(mproj.abspath) == PurePath(str(fs_topdir / 'my-path'))
# Case 1: self path is missing
content = '''\
manifest:
projects:
- name: name
url: url
'''
yml = str(tmpdir / 'random.yml')
with open(yml, 'w') as f:
f.write(content)
manifest = Manifest.from_file(source_file=yml, topdir=topdir)
assert PurePath(manifest.topdir) == PurePath(topdir)
mproj = manifest.projects[0]
assert mproj.path is None
assert mproj.abspath is None
def _all_projects():
# Get a list of project objects from the manifest.
#
# If the manifest is malformed, a fatal error occurs and the
# command aborts.
try:
return list(Manifest.from_file().projects)
except MalformedManifest as m:
log.die(m.args[0])
def set_zephyr_base(args):
'''Ensure ZEPHYR_BASE is set, emitting warnings if that's not
possible, or if the user is pointing it somewhere different than
what the manifest expects.'''
zb_env = os.environ.get('ZEPHYR_BASE')
if args.zephyr_base:
# The command line --zephyr-base takes precedence over
# everything else.
zb = os.path.abspath(args.zephyr_base)
zb_origin = 'command line'
else:
# If the user doesn't specify it concretely, use the project
# with path 'zephyr' if that exists, or the ZEPHYR_BASE value
# in the calling environment.
#
# At some point, we need a more flexible way to set environment
# variables based on manifest contents, but this is good enough
# to get started with and to ask for wider testing.
try:
manifest = Manifest.from_file()
except MalformedConfig as e:
log.die('Parsing of manifest file failed during command',
args.command, ':', *e.args)
for project in manifest.projects:
if project.path == 'zephyr':
zb = project.abspath
zb_origin = 'manifest file {}'.format(manifest.path)
break
else:
if zb_env is None:
log.wrn('no --zephyr-base given, ZEPHYR_BASE is unset,',
'and no manifest project has path "zephyr"')
zb = None
zb_origin = None
else:
zb = zb_env
zb_origin = 'environment'
if zb_env and os.path.abspath(zb) != os.path.abspath(zb_env):
# The environment ZEPHYR_BASE takes precedence over either the
# command line or the manifest, but in normal multi-repo
# operation we shouldn't expect to need to set ZEPHYR_BASE to
# point to some random place. In practice, this is probably
# happening because zephyr-env.sh/cmd was run in some other
# zephyr installation, and the user forgot about that.
log.wrn('ZEPHYR_BASE={}'.format(zb_env),
'in the calling environment, but has been set to', zb,
'instead by the', zb_origin)
os.environ['ZEPHYR_BASE'] = zb
log.dbg('ZEPHYR_BASE={} (origin: {})'.format(zb, zb_origin))
def _special_project(args, name):
# Returns a Project instance for one of the special repositories in west/,
# so that we can reuse the project-related functions for them
if name == 'manifest':
url = config.get(name, 'remote', fallback='origin')
revision = config.get(name, 'revision', fallback='master')
return SpecialProject(name,
revision=revision,
path=os.path.join('west', name),
url=url)
return Manifest.from_file(_manifest_path(args), name).west_project
def do_run(self, args, user_args):
# We assume --freeze here. Future extensions to the group
# --freeze is part of can move this code around.
frozen = Manifest.from_file().as_frozen_dict()
# This is a destructive operation, so it's done here to avoid
# impacting code which doesn't expect this representer to be
# in place.
yaml.SafeDumper.add_representer(collections.OrderedDict, self._rep)
if args.out:
with open(args.out, 'w') as f:
yaml.safe_dump(frozen, default_flow_style=False, stream=f)
else:
yaml.safe_dump(frozen, default_flow_style=False, stream=sys.stdout)
def _freeze(self, args):
try:
frozen = Manifest.from_file().as_frozen_dict()
except RuntimeError as re:
log.die(*(list(re.args) + ['(run "west update" and retry)']))
# This is a destructive operation, so it's done here to avoid
# impacting code which doesn't expect this representer to be
# in place.
yaml.SafeDumper.add_representer(collections.OrderedDict, self._rep)
if args.out:
with open(args.out, 'w') as f:
yaml.safe_dump(frozen, default_flow_style=False, stream=f)
else:
yaml.safe_dump(frozen, default_flow_style=False, stream=sys.stdout)
def do_run(self, args, user_args):
manifest_file = os.path.join(args.local or args.cache, 'west.yml')
project = Manifest.from_file(manifest_file)\
.projects[MANIFEST_PROJECT_INDEX]
if args.local is not None:
rel_manifest = os.path.relpath(args.local, util.west_topdir())
_update_key(config, 'manifest', 'path', rel_manifest)
else:
if project.path is None:
url_path = urlparse(args.manifest_url).path
project.path = posixpath.basename(url_path)
project.name = project.path
_inf(project, 'Creating repository for {name_and_path}')
shutil.move(args.cache, project.abspath)
_update_key(config, 'manifest', 'path', project.path)
def updater(remotes):
# Create a v2.0 tag on the remote tagged_repo repository.
add_commit(remotes.tagged_repo, 'another tagged_repo tagged commit')
add_tag(remotes.tagged_repo, 'v2.0')
# Update the manifest file to point the project's revision at
# the new tag.
manifest = Manifest.from_file(topdir=west_init_tmpdir)
for p in manifest.projects:
if p.name == 'tagged_repo':
p.revision = 'v2.0'
break
else:
assert False, 'no tagged_repo'
with open(west_init_tmpdir / 'zephyr' / 'west.yml', 'w') as f:
f.write(manifest.as_yaml()) # NOT as_frozen_yaml().
# Pull down the v2.0 tag into west_init_tmpdir.
cmd('update tagged_repo')
def external_commands(manifest=None):
'''Get descriptions of available external commands.
The return value is an ordered map from project paths to lists of
WestExtCommandSpec objects, for projects which define external
commands. The map's iteration order matches the manifest.projects
order.
:param manifest: a parsed `west.manifest.Manifest` object, or None
to reload a new one.
'''
if manifest is None:
manifest = Manifest.from_file()
specs = OrderedDict()
for project in manifest.projects:
if project.west_commands:
specs[project.path] = _ext_specs(project)
return specs
def do_run(self, args, user_args):
# All of these commands need the manifest. We are deliberately
# loading it again instead of using self.manifest to emit
# debug logs if enabled, which are turned off when the
# manifest is initially parsed in main.py.
#
# The code in main.py is responsible for handling any errors
# and printing useful messages.
manifest = Manifest.from_file(topdir=self.topdir)
dump_kwargs = {'default_flow_style': False, 'sort_keys': False}
if args.validate:
pass # nothing more to do
elif args.resolve:
self._dump(args, manifest.as_yaml(**dump_kwargs))
elif args.freeze:
self._dump(args, manifest.as_frozen_yaml(**dump_kwargs))
else:
# Can't happen.
raise RuntimeError(f'internal error: unhandled args {args}')
def do_run(self, args, user_args):
# We assume --freeze here. Future extensions to the group
# --freeze is part of can move this code around.
m = Manifest.from_file()
# Build a 'frozen' representation of all projects, except the
# manifest project.
projects = list(m.projects)
del projects[MANIFEST_PROJECT_INDEX]
frozen_projects = []
for project in projects:
sha = _sha(project, '{qual_manifest_rev_branch}')
d = project.as_dict()
d['revision'] = sha
frozen_projects.append(d)
# We include the defaults value here even though all projects
# are fully specified in order to make the resulting manifest
# easy to extend by users who want to reuse the defaults.
o = collections.OrderedDict()
o['west'] = m.west_project.as_dict()
o['west']['revision'] = _sha(m.west_project,
'{qual_manifest_rev_branch}')
o['manifest'] = collections.OrderedDict()
o['manifest']['defaults'] = m.defaults.as_dict()
o['manifest']['remotes'] = [r.as_dict() for r in m.remotes]
o['manifest']['projects'] = frozen_projects
o['manifest']['self'] = m.projects[MANIFEST_PROJECT_INDEX].as_dict()
# This is a destructive operation, so it's done here to avoid
# impacting code which doesn't expect this representer to be
# in place.
yaml.SafeDumper.add_representer(collections.OrderedDict, self._rep)
if args.out:
with open(args.out, 'w') as f:
yaml.safe_dump(o, default_flow_style=False, stream=f)
else:
yaml.safe_dump(o, default_flow_style=False, stream=sys.stdout)
def setup(app):
rev = get_github_rev()
# try to get url from West; this adds compatibility with repos
# located elsewhere
if WestManifest is not None:
baseurl = WestManifest.from_file().get_projects(['zephyr'])[0].url
else:
baseurl = None
# or fallback to default
if baseurl is None:
baseurl = 'https://github.com/zephyrproject-rtos/zephyr'
app.add_role('zephyr_file', autolink('{}/blob/{}/%s'.format(baseurl, rev)))
app.add_role('zephyr_raw', autolink('{}/raw/{}/%s'.format(baseurl, rev)))
# The role just creates new nodes based on information in the
# arguments; its behavior doesn't depend on any other documents.
return {
'parallel_read_safe': True,
'parallel_write_safe': True,
}
def do_run(self, args, unknown_args):
self.setup_upstream_downstream(args)
# Get a dict containing projects that are in the NCS which are
# *not* imported from Zephyr in nrf/west.yml. We will treat
# these specially to make the output easier to understand.
ignored_imports = Manifest.from_file(
import_flags=ImportFlag.IGNORE_PROJECTS)
in_nrf = set(p.name for p in
ignored_imports.projects[MANIFEST_PROJECT_INDEX + 1:])
# This is a dict mapping names of projects which *are* imported
# from zephyr to the Project instances.
self.imported_pmap = {name: project for name, project in
self.ncs_pmap.items() if name not in in_nrf}
log.inf('Comparing your manifest-rev branches with zephyr/west.yml '
f'at {self.zephyr_rev}' +
(', sha: ' + self.zephyr_sha
if self.zephyr_rev != self.zephyr_sha else ''))
log.inf()
present_blacklisted = []
present_allowed = []
missing_blacklisted = []
missing_allowed = []
for zp in self.z_pmap.values():
nn = to_ncs_name(zp)
present = nn in self.ncs_pmap
blacklisted = PurePath(zp.path) in _PROJECT_BLACKLIST
if present:
if blacklisted:
present_blacklisted.append(zp)
else:
present_allowed.append(zp)
else:
if blacklisted:
missing_blacklisted.append(zp)
else:
missing_allowed.append(zp)
def print_lst(projects):
for p in projects:
log.inf(f'{_name_and_path(p)}')
if missing_blacklisted and log.VERBOSE >= log.VERBOSE_NORMAL:
log.banner('blacklisted zephyr projects',
'not in nrf (these are all OK):')
print_lst(missing_blacklisted)
log.banner('blacklisted zephyr projects in NCS:')
if present_blacklisted:
log.wrn(f'these should all be removed from {self.manifest.path}!')
print_lst(present_blacklisted)
else:
log.inf('none (OK)')
log.banner('non-blacklisted zephyr projects missing from NCS:')
if missing_allowed:
west_yml = self.manifest.path
log.wrn(
f'missing projects should be added to NCS or blacklisted\n'
f" To add to NCS:\n"
f" 1. do the zephyr mergeup\n"
f" 2. update zephyr revision in {west_yml}\n"
f" 3. add projects to zephyr's name_whitelist in "
f"{west_yml}\n"
f" 4. run west {self.name} again to check your work\n"
f" To blacklist: edit _PROJECT_BLACKLIST in {__file__}")
for p in missing_allowed:
log.small_banner(f'{_name_and_path(p)}:')
log.inf(f'upstream revision: {p.revision}')
log.inf(f'upstream URL: {p.url}')
else:
log.inf('none (OK)')
if present_allowed:
log.banner('projects in both zephyr and NCS:')
for zp in present_allowed:
# Do some extra checking on unmerged commits.
self.allowed_project(zp)
if log.VERBOSE <= log.VERBOSE_NONE:
log.inf('\nNote: verbose output was omitted,',
'use "west -v ncs-compare" for more details.')
def test_invalid(topdir, invalid):
with pytest.raises(MalformedManifest):
Manifest.from_file(invalid)
def _west_project():
# Returns a Project instance for west.
return Manifest.from_file(sections=['west']).west_project
def set_zephyr_base(args):
'''Ensure ZEPHYR_BASE is set
Order of precedence:
1) Value given as command line argument
2) Value from environment setting: ZEPHYR_BASE
3) Value of zephyr.base setting in west config file
Order of precedence between 2) and 3) can be changed with the setting
zephyr.base-prefer.
zephyr.base-prefer takes the values 'env' and 'configfile'
If 2) and 3) has different values and zephyr.base-prefer is unset a warning
is printed to the user.'''
if args.zephyr_base:
# The command line --zephyr-base takes precedence over
# everything else.
zb = os.path.abspath(args.zephyr_base)
zb_origin = 'command line'
else:
# If the user doesn't specify it concretely, then use ZEPHYR_BASE
# from the environment or zephyr.base from west.configuration.
#
# `west init` will configure zephyr.base to the project that has path
# 'zephyr'.
#
# At some point, we need a more flexible way to set environment
# variables based on manifest contents, but this is good enough
# to get started with and to ask for wider testing.
zb_env = os.environ.get('ZEPHYR_BASE')
zb_prefer = config.config.get('zephyr', 'base-prefer', fallback=None)
zb_config = config.config.get('zephyr', 'base', fallback=None)
if zb_config is not None:
zb_config = os.path.join(west_topdir(), zb_config)
if zb_prefer == 'env' and zb_env is not None:
zb = zb_env
zb_origin = 'env'
elif zb_prefer == 'configfile' and zb_config is not None:
zb = zb_config
zb_origin = 'configfile'
elif zb_env is not None:
zb = zb_env
zb_origin = 'env'
try:
different = (zb_config is not None
and not os.path.samefile(zb_config, zb_env))
except FileNotFoundError:
different = (zb_config is not None and
(os.path.normpath(os.path.abspath(zb_config)) !=
os.path.normpath(os.path.abspath(zb_env))))
if different:
# The environment ZEPHYR_BASE takes precedence over the config
# setting, but in normal multi-repo operation we shouldn't
# expect to need to set ZEPHYR_BASE.
# Therefore issue a warning as it might have happened that
# zephyr-env.sh/cmd was run in some other zephyr installation,
# and the user forgot about that.
log.wrn(
'ZEPHYR_BASE={}'.format(zb_env),
'in the calling environment will be used, but was set '
'to', zb_config, 'in west config.\n To disable this '
'warning, execute '
'\'west config --global zephyr.base-prefer env\'')
elif zb_config is not None:
zb = zb_config
zb_origin = 'configfile'
else:
zb = None
zb_origin = None
# No --zephyr-base, no ZEPHYR_BASE envronment and no zephyr.base
# Fallback to loop over projects, to identify if a project has path
# 'zephyr' for fallback.
try:
manifest = Manifest.from_file()
for project in manifest.projects:
if project.path == 'zephyr':
zb = project.abspath
zb_origin = 'manifest file {}'.format(manifest.path)
break
else:
log.err('no --zephyr-base given, ZEPHYR_BASE is unset,',
'west config contains no zephyr.base setting,',
'and no manifest project has path "zephyr"')
except MalformedConfig as e:
log.wrn("Can't set ZEPHYR_BASE:",
'parsing of manifest file failed during command',
args.command, ':', *e.args)
except WestNotFound:
log.wrn("Can't set ZEPHYR_BASE:",
'not currently in a west installation')
if zb is not None:
os.environ['ZEPHYR_BASE'] = zb
log.dbg('ZEPHYR_BASE={} (origin: {})'.format(zb, zb_origin))
