def do_run(self, args, unknown_args): self.setup_upstream_downstream(args) # Get a dict containing projects that are in the NCS which are # *not* imported from Zephyr in nrf/west.yml. We will treat # these specially to make the output easier to understand. ignored_imports = Manifest.from_file( import_flags=ImportFlag.IGNORE_PROJECTS) in_nrf = set(p.name for p in ignored_imports.projects[MANIFEST_PROJECT_INDEX + 1:]) # This is a dict mapping names of projects which *are* imported # from zephyr to the Project instances. self.imported_pmap = { name: project for name, project in self.ncs_pmap.items() if name not in in_nrf } log.inf( 'Comparing your manifest-rev branches with zephyr/west.yml ' f'at {self.zephyr_rev}' + (', sha: ' + self.zephyr_sha if self.zephyr_rev != self.zephyr_sha else '')) log.inf() present_blacklisted = [] present_allowed = [] missing_blacklisted = [] missing_allowed = [] for zp in self.z_pmap.values(): nn = to_ncs_name(zp) present = nn in self.ncs_pmap blacklisted = PurePath(zp.path) in _PROJECT_BLACKLIST if present: if blacklisted: present_blacklisted.append(zp) else: present_allowed.append(zp) else: if blacklisted: missing_blacklisted.append(zp) else: missing_allowed.append(zp) def print_lst(projects): for p in projects: log.inf(f'{_name_and_path(p)}') if missing_blacklisted and log.VERBOSE >= log.VERBOSE_NORMAL: log.banner('blacklisted zephyr projects', 'not in nrf (these are all OK):') print_lst(missing_blacklisted) log.banner('blacklisted zephyr projects in NCS:') if present_blacklisted: log.wrn(f'these should all be removed from {self.manifest.path}!') print_lst(present_blacklisted) else: log.inf('none (OK)') log.banner('non-blacklisted zephyr projects missing from NCS:') if missing_allowed: west_yml = self.manifest.path log.wrn(f'missing projects should be added to NCS or blacklisted\n' f" To add to NCS:\n" f" 1. do the zephyr mergeup\n" f" 2. update zephyr revision in {west_yml}\n" f" 3. add projects to zephyr's name_whitelist in " f"{west_yml}\n" f" 4. run west {self.name} again to check your work\n" f" To blacklist: edit _PROJECT_BLACKLIST in {__file__}") for p in missing_allowed: log.small_banner(f'{_name_and_path(p)}:') log.inf(f'upstream revision: {p.revision}') log.inf(f'upstream URL: {p.url}') else: log.inf('none (OK)') if present_allowed: log.banner('projects in both zephyr and NCS:') for zp in present_allowed: # Do some extra checking on unmerged commits. self.allowed_project(zp) if log.VERBOSE <= log.VERBOSE_NONE: log.inf('\nNote: verbose output was omitted,', 'use "west -v ncs-compare" for more details.')
def allowed_project(self, zp): nn = to_ncs_name(zp) np = self.ncs_pmap[nn] # is_imported is true if we imported this project from the # zephyr manifest rather than defining it directly ourselves # in nrf/west.yml. is_imported = nn in self.imported_pmap imported = ', imported from zephyr' if is_imported else '' banner = f'{nn} ({zp.path}){imported}:' nrev = 'refs/heads/manifest-rev' if np.name == 'zephyr': zrev = self.zephyr_sha else: zrev = zp.revision nsha = self.checked_sha(np, nrev) zsha = self.checked_sha(zp, zrev) if not np.is_cloned() or nsha is None or zsha is None: log.small_banner(banner) if not np.is_cloned(): log.wrn('project is not cloned; please run "west update"') elif nsha is None: log.wrn(f"can't compare; please run \"west update {nn}\" " f'(need revision {np.revision})') elif zsha is None: log.wrn(f"can't compare; please fetch upstream URL {zp.url} " f'(need revision {zp.revision})') return cp = np.git(f'rev-list --left-right --count {zsha}...{nsha}', capture_stdout=True) behind, ahead = [int(c) for c in cp.stdout.split()] if zsha == nsha: status = 'up to date' elif ahead and not behind: status = f'ahead by {ahead} commit' + ("s" if ahead > 1 else "") elif np.is_ancestor_of(nsha, zsha): status = f'behind by {behind} commit' + ("s" if behind > 1 else "") else: status = f'diverged: {ahead} ahead, {behind} behind' commits = f'NCS commit: {nsha}, upstream commit: {zsha}' if 'up to date' in status or 'ahead by' in status: if log.VERBOSE > log.VERBOSE_NONE: # Up to date or ahead: only print in verbose mode. log.small_banner(banner) log.inf(commits) log.inf(status) likely_merged(np, zp, nsha, zsha) else: # Behind or diverged: always print. if is_imported and 'behind by' in status: status += ' and imported: update by doing zephyr mergeup' log.small_banner(banner) log.inf(commits) log.msg(status, color=log.WRN_COLOR) likely_merged(np, zp, nsha, zsha)
def update(self, project): log.banner(f'updating {project.name_and_path}:') if not project.is_cloned(): _clone(project) if self.fs == 'always' or _rev_type(project) not in ('tag', 'commit'): _fetch(project) else: log.dbg('skipping unnecessary fetch') _update_manifest_rev(project, f'{project.revision}^{{commit}}') # Head of manifest-rev is now pointing to current manifest revision. # Thus it is safe to unconditionally clear out the refs/west space. # # Doing this here instead of in _fetch() ensures that it gets cleaned # up when users upgrade from older versions of west (like 0.6.x) that # didn't handle this properly. # In future, this can be moved into _fetch() after the install base of # older west versions is expected to be smaller. _clean_west_refspace(project) if not _head_ok(project): # If nothing is checked out (which usually only happens if we # called _clone(project) above), check out 'manifest-rev' in a # detached HEAD state. # # Otherwise, the initial state would have nothing checked out, # and HEAD would point to a non-existent refs/heads/master # branch (that would get created if the user makes an initial # commit). Among other things, this ensures the rev-parse # --abbrev-ref HEAD below will always succeed. # # The --detach flag is strictly redundant here, because # the refs/heads/<branch> form already detaches HEAD, but # it avoids a spammy detached HEAD warning from Git. project.git('checkout --detach ' + QUAL_MANIFEST_REV) try: sha = project.sha(QUAL_MANIFEST_REV) except subprocess.CalledProcessError: # This is a sign something's really wrong. Add more help. log.err(f'no SHA for branch {MANIFEST_REV} ' f'in {project.name_and_path}; was the branch deleted?') raise cp = project.git('rev-parse --abbrev-ref HEAD', capture_stdout=True) current_branch = cp.stdout.decode('utf-8').strip() if current_branch != 'HEAD': is_ancestor = project.is_ancestor_of(sha, current_branch) try_rebase = self.args.rebase else: # HEAD means no branch is checked out. # If no branch is checked out, 'rebase' and # 'keep_descendants' don't matter. is_ancestor = False try_rebase = False if self.args.keep_descendants and is_ancestor: # A descendant is currently checked out and keep_descendants was # given, so there's nothing more to do. log.small_banner(f'{project.name}: left descendant branch ' f'"{current_branch}" checked out') elif try_rebase: # Attempt a rebase. log.small_banner(f'{project.name}: rebasing to {MANIFEST_REV}') project.git('rebase ' + QUAL_MANIFEST_REV) else: # We can't keep a descendant or rebase, so just check # out the new detached HEAD, then print some helpful context. project.git('checkout --detach --quiet ' + sha) log.small_banner( f'{project.name}: checked out {sha} as detached HEAD') _post_checkout_help(project, current_branch, sha, is_ancestor)
def bootstrap(self, args): manifest_url = args.manifest_url or MANIFEST_URL_DEFAULT manifest_rev = args.manifest_rev or MANIFEST_REV_DEFAULT topdir = util.canon_path(args.directory or os.getcwd()) west_dir = join(topdir, WEST_DIR) try: already = util.west_topdir(topdir, fall_back=False) self.die_already(already) except util.WestNotFound: pass log.banner('Initializing in', topdir) if not isdir(topdir): self.create(topdir, exist_ok=False) # Clone the manifest repository into a temporary directory. tempdir = join(west_dir, 'manifest-tmp') if exists(tempdir): log.dbg('removing existing temporary manifest directory', tempdir) shutil.rmtree(tempdir) try: self.clone_manifest(manifest_url, manifest_rev, tempdir) except subprocess.CalledProcessError: shutil.rmtree(tempdir, ignore_errors=True) raise # Verify the manifest file exists. temp_manifest = join(tempdir, 'west.yml') if not exists(temp_manifest): log.die(f'can\'t init: no "west.yml" found in {tempdir}\n' f' Hint: check --manifest-url={manifest_url} and ' '--manifest-rev={manifest_rev}\n' f' You may need to remove {west_dir} before retrying.') # Parse the manifest to get the manifest path, if it declares one. # Otherwise, use the URL. Ignore imports -- all we really # want to know is if there's a "self: path:" or not. projects = Manifest.from_file(temp_manifest, import_flags=ImportFlag.IGNORE, topdir=topdir).projects manifest_project = projects[MANIFEST_PROJECT_INDEX] if manifest_project.path: manifest_path = manifest_project.path else: manifest_path = posixpath.basename(urlparse(manifest_url).path) manifest_abspath = join(topdir, manifest_path) log.dbg('moving', tempdir, 'to', manifest_abspath, level=log.VERBOSE_EXTREME) try: shutil.move(tempdir, manifest_abspath) except shutil.Error as e: log.die(e) log.small_banner('setting manifest.path to', manifest_path) update_config('manifest', 'path', manifest_path, topdir=topdir) return topdir
def _clone(project): log.small_banner(project.format('{name}: cloning and initializing')) project.git('init {abspath}', cwd=util.west_topdir()) # This remote is only added for the user's convenience. We always fetch # directly from the URL specified in the manifest. project.git('remote add -- {remote_name} {url}')
def allowed_project(self, zp): nn = to_ncs_name(zp) np = self.ncs_pmap[nn] banner = f'{nn} ({zp.path}):' if np.name == 'zephyr': nrev = self.manifest.get_projects(['zephyr'])[0].revision zrev = self.zephyr_sha else: nrev = np.revision zrev = zp.revision nsha = self.checked_sha(np, nrev) zsha = self.checked_sha(zp, zrev) if not np.is_cloned() or nsha is None or zsha is None: log.small_banner(banner) if not np.is_cloned(): log.wrn('project is not cloned; please run "west update"') elif nsha is None: log.wrn(f"can't compare; please run \"west update {nn}\" " f'(need revision {np.revision})') elif zsha is None: log.wrn(f"can't compare; please fetch upstream URL {zp.url} " f'(need revision {zp.revision})') return cp = np.git(f'rev-list --left-right --count {zsha}...{nsha}', capture_stdout=True) behind, ahead = [int(c) for c in cp.stdout.split()] if zsha == nsha: status = 'up to date' elif ahead and not behind: status = f'ahead by {ahead} commits' elif np.is_ancestor_of(nsha, zsha): status = (f'behind by {behind} commits, ' f'can be fast-forwarded to {zsha}') else: status = f'diverged from upstream: {ahead} ahead, {behind} behind' upstream_rev = f'upstream revision: {zrev}' if zrev != zsha: upstream_rev += f' ({zsha})' downstream_rev = 'NCS revision: ' + nrev if nrev != nsha: downstream_rev += f' ({nsha})' if 'up to date' in status or 'ahead by' in status: if log.VERBOSE > log.VERBOSE_NONE: # Up to date or ahead: only print in verbose mode. log.small_banner(banner) status += ', ' + downstream_rev if 'ahead by' in status: status += ', ' + upstream_rev log.inf(status) likely_merged(np, zp, nsha, zsha) else: # Behind or diverged: always print. log.small_banner(banner) log.msg(status, color=log.WRN_COLOR) log.inf(upstream_rev) log.inf(downstream_rev) likely_merged(np, zp, nsha, zsha)
def _checkout_detach(project, revision): _git(project, 'checkout --detach --quiet ' + revision) log.small_banner( project.format("{name}: checked out {r} as detached HEAD", r=_sha(project, revision)))
def print_lst(projects): for p in projects: log.small_banner(f'{_name_and_path(p)}')
def do_run(self, args, unknown_args): check_west_version() self.setup_upstream_downstream(args) log.inf('Comparing nrf/west.yml with zephyr/west.yml at revision ' f'{self.zephyr_rev}' + (', sha: ' + self.zephyr_sha if self.zephyr_rev != self.zephyr_sha else '')) log.inf() present_blacklisted = [] present_allowed = [] missing_blacklisted = [] missing_allowed = [] for zp in self.z_pmap.values(): nn = to_ncs_name(zp) present = nn in self.ncs_pmap blacklisted = PurePath(zp.path) in _PROJECT_BLACKLIST if present: if blacklisted: present_blacklisted.append(zp) else: present_allowed.append(zp) else: if blacklisted: missing_blacklisted.append(zp) else: missing_allowed.append(zp) def print_lst(projects): for p in projects: log.small_banner(f'{_name_and_path(p)}') if missing_blacklisted and log.VERBOSE >= log.VERBOSE_NORMAL: log.banner('blacklisted zephyr projects', 'not in nrf (these are all OK):') print_lst(missing_blacklisted) log.banner('blacklisted zephyr projects in nrf:') if present_blacklisted: log.wrn('these should all be removed from nrf') print_lst(present_blacklisted) else: log.inf('none (OK)') log.banner('non-blacklisted zephyr projects', 'missing from nrf:') if missing_allowed: log.wrn('these should be blacklisted or added to nrf') for p in missing_allowed: log.small_banner(f'{_name_and_path(p)}:') log.inf(f'upstream revision: {p.revision}') log.inf(f'upstream URL: {p.url}') else: log.inf('none (OK)') if present_allowed: log.banner('projects in both zephyr and nrf:') for zp in present_allowed: # Do some extra checking on unmerged commits. self.allowed_project(zp) if log.VERBOSE <= log.VERBOSE_NONE: log.inf('\nNote: verbose output was omitted,', 'use "west -v ncs-compare" for more details.')
def allowed_project(self, zp): nn = self.to_ncs_name(zp) np = self.ncs_pmap[nn] if np.name == 'zephyr': nrev = self.manifest.get_projects(['zephyr'])[0].revision zrev = self.zephyr_sha else: nrev = np.revision zrev = zp.revision log.small_banner(zp.format('{ncs_name} ({path}):', ncs_name=nn)) try: nsha = np.sha(nrev) np.git('cat-file -e ' + nsha) except subprocess.CalledProcessError: log.wrn("can't compare; please run \"west update {}\"".format(nn), '(need revision {})'.format(np.revision)) return try: zsha = np.sha(zrev) np.git('cat-file -e ' + zsha) except subprocess.CalledProcessError: log.wrn("can't compare; please fetch upstream URL", zp.url, '(need revision {})'.format(zp.revision)) return upstream_rev = 'upstream revision: ' + zrev if zrev != zsha: upstream_rev += ' ({})'.format(zsha) downstream_rev = 'NCS revision: ' + nrev if nrev != nsha: downstream_rev += ' ({})'.format(nsha) cp = np.git('rev-list --left-right --count {}...{}'.format(zsha, nsha), capture_stdout=True) behind, ahead = [int(c) for c in cp.stdout.split()] if zsha == nsha: status = 'up to date' elif ahead and not behind: status = 'ahead by {} commits'.format(ahead) elif np.is_ancestor_of(nsha, zsha): status = ( 'behind by {} commits, can be fast-forwarded to {}'.format( behind, zsha)) else: status = ('diverged from upstream: {} ahead, {} behind'.format( ahead, behind)) if 'behind' in status or 'diverged' in status: color = log.WRN_COLOR else: color = log.INF_COLOR log.msg(status, color=color) log.inf(upstream_rev) log.inf(downstream_rev) analyzer = nwh.RepoAnalyzer(np, zp, nsha, zsha) likely_merged = analyzer.likely_merged if likely_merged: # likely_merged is a map from downstream commits to # lists of upstream commits that look similar. log.msg('downstream patches which are likely merged upstream', '(revert these if appropriate):', color=log.WRN_COLOR) for dc, ucs in likely_merged.items(): log.inf('- {} ({})\n Similar upstream commits:'.format( dc.oid, nwh.commit_shortlog(dc))) for uc in ucs: log.inf(' {} ({})'.format(uc.oid, nwh.commit_shortlog(uc))) else: log.inf('no downstream patches seem to have been merged upstream')
def print_lst(projects): for p in projects: log.small_banner(p.format('{name_and_path}'))
def allowed_project(self, zp): nn = self.to_ncs_name(zp) np = self.ncs_pmap[nn] banner = zp.format('{ncs_name} ({path}):', ncs_name=nn) if np.name == 'zephyr': nrev = self.manifest.get_projects(['zephyr'])[0].revision zrev = self.zephyr_sha else: nrev = np.revision zrev = zp.revision nsha = self.checked_sha(np, nrev) zsha = self.checked_sha(zp, zrev) if not np.is_cloned() or nsha is None or zsha is None: log.small_banner(banner) if not np.is_cloned(): log.wrn('project is not cloned; please run "west update"') elif nsha is None: log.wrn( "can't compare; please run \"west update {}\"".format(nn), '(need revision {})'.format(np.revision)) elif zsha is None: log.wrn("can't compare; please fetch upstream URL", zp.url, '(need revision {})'.format(zp.revision)) return cp = np.git('rev-list --left-right --count {}...{}'.format(zsha, nsha), capture_stdout=True) behind, ahead = [int(c) for c in cp.stdout.split()] if zsha == nsha: status = 'up to date' elif ahead and not behind: status = 'ahead by {} commits'.format(ahead) elif np.is_ancestor_of(nsha, zsha): status = ( 'behind by {} commits, can be fast-forwarded to {}'.format( behind, zsha)) else: status = ('diverged from upstream: {} ahead, {} behind'.format( ahead, behind)) upstream_rev = 'upstream revision: ' + zrev if zrev != zsha: upstream_rev += ' ({})'.format(zsha) downstream_rev = 'NCS revision: ' + nrev if nrev != nsha: downstream_rev += ' ({})'.format(nsha) if 'up to date' in status or 'ahead by' in status: if log.VERBOSE > log.VERBOSE_NONE: # Up to date or ahead: only print in verbose mode. log.small_banner(banner) status += ', ' + downstream_rev if 'ahead by' in status: status += ', ' + upstream_rev log.inf(status) self.likely_merged(np, zp, nsha, zsha) else: # Behind or diverged: always print. log.small_banner(banner) log.msg(status, color=log.WRN_COLOR) log.inf(upstream_rev) log.inf(downstream_rev) self.likely_merged(np, zp, nsha, zsha)