def do_run(self, args, unknown_args):
self.setup_upstream_downstream(args)
# Get a dict containing projects that are in the NCS which are
# *not* imported from Zephyr in nrf/west.yml. We will treat
# these specially to make the output easier to understand.
ignored_imports = Manifest.from_file(
import_flags=ImportFlag.IGNORE_PROJECTS)
in_nrf = set(p.name
for p in ignored_imports.projects[MANIFEST_PROJECT_INDEX +
1:])
# This is a dict mapping names of projects which *are* imported
# from zephyr to the Project instances.
self.imported_pmap = {
name: project
for name, project in self.ncs_pmap.items() if name not in in_nrf
}
log.inf(
'Comparing your manifest-rev branches with zephyr/west.yml '
f'at {self.zephyr_rev}' +
(', sha: ' +
self.zephyr_sha if self.zephyr_rev != self.zephyr_sha else ''))
log.inf()
present_blacklisted = []
present_allowed = []
missing_blacklisted = []
missing_allowed = []
for zp in self.z_pmap.values():
nn = to_ncs_name(zp)
present = nn in self.ncs_pmap
blacklisted = PurePath(zp.path) in _PROJECT_BLACKLIST
if present:
if blacklisted:
present_blacklisted.append(zp)
else:
present_allowed.append(zp)
else:
if blacklisted:
missing_blacklisted.append(zp)
else:
missing_allowed.append(zp)
def print_lst(projects):
for p in projects:
log.inf(f'{_name_and_path(p)}')
if missing_blacklisted and log.VERBOSE >= log.VERBOSE_NORMAL:
log.banner('blacklisted zephyr projects',
'not in nrf (these are all OK):')
print_lst(missing_blacklisted)
log.banner('blacklisted zephyr projects in NCS:')
if present_blacklisted:
log.wrn(f'these should all be removed from {self.manifest.path}!')
print_lst(present_blacklisted)
else:
log.inf('none (OK)')
log.banner('non-blacklisted zephyr projects missing from NCS:')
if missing_allowed:
west_yml = self.manifest.path
log.wrn(f'missing projects should be added to NCS or blacklisted\n'
f" To add to NCS:\n"
f" 1. do the zephyr mergeup\n"
f" 2. update zephyr revision in {west_yml}\n"
f" 3. add projects to zephyr's name_whitelist in "
f"{west_yml}\n"
f" 4. run west {self.name} again to check your work\n"
f" To blacklist: edit _PROJECT_BLACKLIST in {__file__}")
for p in missing_allowed:
log.small_banner(f'{_name_and_path(p)}:')
log.inf(f'upstream revision: {p.revision}')
log.inf(f'upstream URL: {p.url}')
else:
log.inf('none (OK)')
if present_allowed:
log.banner('projects in both zephyr and NCS:')
for zp in present_allowed:
# Do some extra checking on unmerged commits.
self.allowed_project(zp)
if log.VERBOSE <= log.VERBOSE_NONE:
log.inf('\nNote: verbose output was omitted,',
'use "west -v ncs-compare" for more details.')
def allowed_project(self, zp):
nn = to_ncs_name(zp)
np = self.ncs_pmap[nn]
# is_imported is true if we imported this project from the
# zephyr manifest rather than defining it directly ourselves
# in nrf/west.yml.
is_imported = nn in self.imported_pmap
imported = ', imported from zephyr' if is_imported else ''
banner = f'{nn} ({zp.path}){imported}:'
nrev = 'refs/heads/manifest-rev'
if np.name == 'zephyr':
zrev = self.zephyr_sha
else:
zrev = zp.revision
nsha = self.checked_sha(np, nrev)
zsha = self.checked_sha(zp, zrev)
if not np.is_cloned() or nsha is None or zsha is None:
log.small_banner(banner)
if not np.is_cloned():
log.wrn('project is not cloned; please run "west update"')
elif nsha is None:
log.wrn(f"can't compare; please run \"west update {nn}\" "
f'(need revision {np.revision})')
elif zsha is None:
log.wrn(f"can't compare; please fetch upstream URL {zp.url} "
f'(need revision {zp.revision})')
return
cp = np.git(f'rev-list --left-right --count {zsha}...{nsha}',
capture_stdout=True)
behind, ahead = [int(c) for c in cp.stdout.split()]
if zsha == nsha:
status = 'up to date'
elif ahead and not behind:
status = f'ahead by {ahead} commit' + ("s" if ahead > 1 else "")
elif np.is_ancestor_of(nsha, zsha):
status = f'behind by {behind} commit' + ("s" if behind > 1 else "")
else:
status = f'diverged: {ahead} ahead, {behind} behind'
commits = f'NCS commit: {nsha}, upstream commit: {zsha}'
if 'up to date' in status or 'ahead by' in status:
if log.VERBOSE > log.VERBOSE_NONE:
# Up to date or ahead: only print in verbose mode.
log.small_banner(banner)
log.inf(commits)
log.inf(status)
likely_merged(np, zp, nsha, zsha)
else:
# Behind or diverged: always print.
if is_imported and 'behind by' in status:
status += ' and imported: update by doing zephyr mergeup'
log.small_banner(banner)
log.inf(commits)
log.msg(status, color=log.WRN_COLOR)
likely_merged(np, zp, nsha, zsha)
def update(self, project):
log.banner(f'updating {project.name_and_path}:')
if not project.is_cloned():
_clone(project)
if self.fs == 'always' or _rev_type(project) not in ('tag', 'commit'):
_fetch(project)
else:
log.dbg('skipping unnecessary fetch')
_update_manifest_rev(project, f'{project.revision}^{{commit}}')
# Head of manifest-rev is now pointing to current manifest revision.
# Thus it is safe to unconditionally clear out the refs/west space.
#
# Doing this here instead of in _fetch() ensures that it gets cleaned
# up when users upgrade from older versions of west (like 0.6.x) that
# didn't handle this properly.
# In future, this can be moved into _fetch() after the install base of
# older west versions is expected to be smaller.
_clean_west_refspace(project)
if not _head_ok(project):
# If nothing is checked out (which usually only happens if we
# called _clone(project) above), check out 'manifest-rev' in a
# detached HEAD state.
#
# Otherwise, the initial state would have nothing checked out,
# and HEAD would point to a non-existent refs/heads/master
# branch (that would get created if the user makes an initial
# commit). Among other things, this ensures the rev-parse
# --abbrev-ref HEAD below will always succeed.
#
# The --detach flag is strictly redundant here, because
# the refs/heads/<branch> form already detaches HEAD, but
# it avoids a spammy detached HEAD warning from Git.
project.git('checkout --detach ' + QUAL_MANIFEST_REV)
try:
sha = project.sha(QUAL_MANIFEST_REV)
except subprocess.CalledProcessError:
# This is a sign something's really wrong. Add more help.
log.err(f'no SHA for branch {MANIFEST_REV} '
f'in {project.name_and_path}; was the branch deleted?')
raise
cp = project.git('rev-parse --abbrev-ref HEAD', capture_stdout=True)
current_branch = cp.stdout.decode('utf-8').strip()
if current_branch != 'HEAD':
is_ancestor = project.is_ancestor_of(sha, current_branch)
try_rebase = self.args.rebase
else: # HEAD means no branch is checked out.
# If no branch is checked out, 'rebase' and
# 'keep_descendants' don't matter.
is_ancestor = False
try_rebase = False
if self.args.keep_descendants and is_ancestor:
# A descendant is currently checked out and keep_descendants was
# given, so there's nothing more to do.
log.small_banner(f'{project.name}: left descendant branch '
f'"{current_branch}" checked out')
elif try_rebase:
# Attempt a rebase.
log.small_banner(f'{project.name}: rebasing to {MANIFEST_REV}')
project.git('rebase ' + QUAL_MANIFEST_REV)
else:
# We can't keep a descendant or rebase, so just check
# out the new detached HEAD, then print some helpful context.
project.git('checkout --detach --quiet ' + sha)
log.small_banner(
f'{project.name}: checked out {sha} as detached HEAD')
_post_checkout_help(project, current_branch, sha, is_ancestor)
def bootstrap(self, args):
manifest_url = args.manifest_url or MANIFEST_URL_DEFAULT
manifest_rev = args.manifest_rev or MANIFEST_REV_DEFAULT
topdir = util.canon_path(args.directory or os.getcwd())
west_dir = join(topdir, WEST_DIR)
try:
already = util.west_topdir(topdir, fall_back=False)
self.die_already(already)
except util.WestNotFound:
pass
log.banner('Initializing in', topdir)
if not isdir(topdir):
self.create(topdir, exist_ok=False)
# Clone the manifest repository into a temporary directory.
tempdir = join(west_dir, 'manifest-tmp')
if exists(tempdir):
log.dbg('removing existing temporary manifest directory', tempdir)
shutil.rmtree(tempdir)
try:
self.clone_manifest(manifest_url, manifest_rev, tempdir)
except subprocess.CalledProcessError:
shutil.rmtree(tempdir, ignore_errors=True)
raise
# Verify the manifest file exists.
temp_manifest = join(tempdir, 'west.yml')
if not exists(temp_manifest):
log.die(f'can\'t init: no "west.yml" found in {tempdir}\n'
f' Hint: check --manifest-url={manifest_url} and '
'--manifest-rev={manifest_rev}\n'
f' You may need to remove {west_dir} before retrying.')
# Parse the manifest to get the manifest path, if it declares one.
# Otherwise, use the URL. Ignore imports -- all we really
# want to know is if there's a "self: path:" or not.
projects = Manifest.from_file(temp_manifest,
import_flags=ImportFlag.IGNORE,
topdir=topdir).projects
manifest_project = projects[MANIFEST_PROJECT_INDEX]
if manifest_project.path:
manifest_path = manifest_project.path
else:
manifest_path = posixpath.basename(urlparse(manifest_url).path)
manifest_abspath = join(topdir, manifest_path)
log.dbg('moving',
tempdir,
'to',
manifest_abspath,
level=log.VERBOSE_EXTREME)
try:
shutil.move(tempdir, manifest_abspath)
except shutil.Error as e:
log.die(e)
log.small_banner('setting manifest.path to', manifest_path)
update_config('manifest', 'path', manifest_path, topdir=topdir)
return topdir
def _clone(project):
log.small_banner(project.format('{name}: cloning and initializing'))
project.git('init {abspath}', cwd=util.west_topdir())
# This remote is only added for the user's convenience. We always fetch
# directly from the URL specified in the manifest.
project.git('remote add -- {remote_name} {url}')
def allowed_project(self, zp):
nn = to_ncs_name(zp)
np = self.ncs_pmap[nn]
banner = f'{nn} ({zp.path}):'
if np.name == 'zephyr':
nrev = self.manifest.get_projects(['zephyr'])[0].revision
zrev = self.zephyr_sha
else:
nrev = np.revision
zrev = zp.revision
nsha = self.checked_sha(np, nrev)
zsha = self.checked_sha(zp, zrev)
if not np.is_cloned() or nsha is None or zsha is None:
log.small_banner(banner)
if not np.is_cloned():
log.wrn('project is not cloned; please run "west update"')
elif nsha is None:
log.wrn(f"can't compare; please run \"west update {nn}\" "
f'(need revision {np.revision})')
elif zsha is None:
log.wrn(f"can't compare; please fetch upstream URL {zp.url} "
f'(need revision {zp.revision})')
return
cp = np.git(f'rev-list --left-right --count {zsha}...{nsha}',
capture_stdout=True)
behind, ahead = [int(c) for c in cp.stdout.split()]
if zsha == nsha:
status = 'up to date'
elif ahead and not behind:
status = f'ahead by {ahead} commits'
elif np.is_ancestor_of(nsha, zsha):
status = (f'behind by {behind} commits, '
f'can be fast-forwarded to {zsha}')
else:
status = f'diverged from upstream: {ahead} ahead, {behind} behind'
upstream_rev = f'upstream revision: {zrev}'
if zrev != zsha:
upstream_rev += f' ({zsha})'
downstream_rev = 'NCS revision: ' + nrev
if nrev != nsha:
downstream_rev += f' ({nsha})'
if 'up to date' in status or 'ahead by' in status:
if log.VERBOSE > log.VERBOSE_NONE:
# Up to date or ahead: only print in verbose mode.
log.small_banner(banner)
status += ', ' + downstream_rev
if 'ahead by' in status:
status += ', ' + upstream_rev
log.inf(status)
likely_merged(np, zp, nsha, zsha)
else:
# Behind or diverged: always print.
log.small_banner(banner)
log.msg(status, color=log.WRN_COLOR)
log.inf(upstream_rev)
log.inf(downstream_rev)
likely_merged(np, zp, nsha, zsha)
def _checkout_detach(project, revision):
_git(project, 'checkout --detach --quiet ' + revision)
log.small_banner(
project.format("{name}: checked out {r} as detached HEAD",
r=_sha(project, revision)))
def print_lst(projects):
for p in projects:
log.small_banner(f'{_name_and_path(p)}')
def do_run(self, args, unknown_args):
check_west_version()
self.setup_upstream_downstream(args)
log.inf('Comparing nrf/west.yml with zephyr/west.yml at revision '
f'{self.zephyr_rev}' +
(', sha: ' + self.zephyr_sha
if self.zephyr_rev != self.zephyr_sha else ''))
log.inf()
present_blacklisted = []
present_allowed = []
missing_blacklisted = []
missing_allowed = []
for zp in self.z_pmap.values():
nn = to_ncs_name(zp)
present = nn in self.ncs_pmap
blacklisted = PurePath(zp.path) in _PROJECT_BLACKLIST
if present:
if blacklisted:
present_blacklisted.append(zp)
else:
present_allowed.append(zp)
else:
if blacklisted:
missing_blacklisted.append(zp)
else:
missing_allowed.append(zp)
def print_lst(projects):
for p in projects:
log.small_banner(f'{_name_and_path(p)}')
if missing_blacklisted and log.VERBOSE >= log.VERBOSE_NORMAL:
log.banner('blacklisted zephyr projects',
'not in nrf (these are all OK):')
print_lst(missing_blacklisted)
log.banner('blacklisted zephyr projects in nrf:')
if present_blacklisted:
log.wrn('these should all be removed from nrf')
print_lst(present_blacklisted)
else:
log.inf('none (OK)')
log.banner('non-blacklisted zephyr projects',
'missing from nrf:')
if missing_allowed:
log.wrn('these should be blacklisted or added to nrf')
for p in missing_allowed:
log.small_banner(f'{_name_and_path(p)}:')
log.inf(f'upstream revision: {p.revision}')
log.inf(f'upstream URL: {p.url}')
else:
log.inf('none (OK)')
if present_allowed:
log.banner('projects in both zephyr and nrf:')
for zp in present_allowed:
# Do some extra checking on unmerged commits.
self.allowed_project(zp)
if log.VERBOSE <= log.VERBOSE_NONE:
log.inf('\nNote: verbose output was omitted,',
'use "west -v ncs-compare" for more details.')
def allowed_project(self, zp):
nn = self.to_ncs_name(zp)
np = self.ncs_pmap[nn]
if np.name == 'zephyr':
nrev = self.manifest.get_projects(['zephyr'])[0].revision
zrev = self.zephyr_sha
else:
nrev = np.revision
zrev = zp.revision
log.small_banner(zp.format('{ncs_name} ({path}):', ncs_name=nn))
try:
nsha = np.sha(nrev)
np.git('cat-file -e ' + nsha)
except subprocess.CalledProcessError:
log.wrn("can't compare; please run \"west update {}\"".format(nn),
'(need revision {})'.format(np.revision))
return
try:
zsha = np.sha(zrev)
np.git('cat-file -e ' + zsha)
except subprocess.CalledProcessError:
log.wrn("can't compare; please fetch upstream URL", zp.url,
'(need revision {})'.format(zp.revision))
return
upstream_rev = 'upstream revision: ' + zrev
if zrev != zsha:
upstream_rev += ' ({})'.format(zsha)
downstream_rev = 'NCS revision: ' + nrev
if nrev != nsha:
downstream_rev += ' ({})'.format(nsha)
cp = np.git('rev-list --left-right --count {}...{}'.format(zsha, nsha),
capture_stdout=True)
behind, ahead = [int(c) for c in cp.stdout.split()]
if zsha == nsha:
status = 'up to date'
elif ahead and not behind:
status = 'ahead by {} commits'.format(ahead)
elif np.is_ancestor_of(nsha, zsha):
status = (
'behind by {} commits, can be fast-forwarded to {}'.format(
behind, zsha))
else:
status = ('diverged from upstream: {} ahead, {} behind'.format(
ahead, behind))
if 'behind' in status or 'diverged' in status:
color = log.WRN_COLOR
else:
color = log.INF_COLOR
log.msg(status, color=color)
log.inf(upstream_rev)
log.inf(downstream_rev)
analyzer = nwh.RepoAnalyzer(np, zp, nsha, zsha)
likely_merged = analyzer.likely_merged
if likely_merged:
# likely_merged is a map from downstream commits to
# lists of upstream commits that look similar.
log.msg('downstream patches which are likely merged upstream',
'(revert these if appropriate):',
color=log.WRN_COLOR)
for dc, ucs in likely_merged.items():
log.inf('- {} ({})\n Similar upstream commits:'.format(
dc.oid, nwh.commit_shortlog(dc)))
for uc in ucs:
log.inf(' {} ({})'.format(uc.oid,
nwh.commit_shortlog(uc)))
else:
log.inf('no downstream patches seem to have been merged upstream')
def print_lst(projects):
for p in projects:
log.small_banner(p.format('{name_and_path}'))
def allowed_project(self, zp):
nn = self.to_ncs_name(zp)
np = self.ncs_pmap[nn]
banner = zp.format('{ncs_name} ({path}):', ncs_name=nn)
if np.name == 'zephyr':
nrev = self.manifest.get_projects(['zephyr'])[0].revision
zrev = self.zephyr_sha
else:
nrev = np.revision
zrev = zp.revision
nsha = self.checked_sha(np, nrev)
zsha = self.checked_sha(zp, zrev)
if not np.is_cloned() or nsha is None or zsha is None:
log.small_banner(banner)
if not np.is_cloned():
log.wrn('project is not cloned; please run "west update"')
elif nsha is None:
log.wrn(
"can't compare; please run \"west update {}\"".format(nn),
'(need revision {})'.format(np.revision))
elif zsha is None:
log.wrn("can't compare; please fetch upstream URL", zp.url,
'(need revision {})'.format(zp.revision))
return
cp = np.git('rev-list --left-right --count {}...{}'.format(zsha, nsha),
capture_stdout=True)
behind, ahead = [int(c) for c in cp.stdout.split()]
if zsha == nsha:
status = 'up to date'
elif ahead and not behind:
status = 'ahead by {} commits'.format(ahead)
elif np.is_ancestor_of(nsha, zsha):
status = (
'behind by {} commits, can be fast-forwarded to {}'.format(
behind, zsha))
else:
status = ('diverged from upstream: {} ahead, {} behind'.format(
ahead, behind))
upstream_rev = 'upstream revision: ' + zrev
if zrev != zsha:
upstream_rev += ' ({})'.format(zsha)
downstream_rev = 'NCS revision: ' + nrev
if nrev != nsha:
downstream_rev += ' ({})'.format(nsha)
if 'up to date' in status or 'ahead by' in status:
if log.VERBOSE > log.VERBOSE_NONE:
# Up to date or ahead: only print in verbose mode.
log.small_banner(banner)
status += ', ' + downstream_rev
if 'ahead by' in status:
status += ', ' + upstream_rev
log.inf(status)
self.likely_merged(np, zp, nsha, zsha)
else:
# Behind or diverged: always print.
log.small_banner(banner)
log.msg(status, color=log.WRN_COLOR)
log.inf(upstream_rev)
log.inf(downstream_rev)
self.likely_merged(np, zp, nsha, zsha)
