def run_command(self, args, config):
workspace_path = get_workspace_path()
manifest = get_workspace_manifest()
manifest_repo = find_source_manifest_repo(manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo)
cfg, user_cfg, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfg_file'])
if manifest_repo in cfg:
manifest_repo_path = config['cfg_file'].manifest_repo_abs_path(
manifest_repo)
elif manifest_repo in user_cfg:
manifest_repo_path = config[
'user_cfg_file'].manifest_repo_abs_path(manifest_repo)
else:
manifest_repo_path = None
pin_path = self.__get_pin_path(args, workspace_path,
manifest_repo_path, manifest)
pin = ManifestXml(pin_path)
manifest_sources = manifest.get_repo_sources(
manifest.general_config.current_combo)
check_dirty_repos(manifest, workspace_path)
for source in manifest_sources:
local_path = os.path.join(workspace_path, source.root)
repo = Repo(local_path)
origin = repo.remotes.origin
origin.fetch()
self.__pin_matches_project(pin, manifest, workspace_path)
sparse_enabled = sparse_checkout_enabled(workspace_path,
manifest_sources)
if sparse_enabled:
ui_functions.print_info_msg(SPARSE_RESET, header=False)
reset_sparse_checkout(workspace_path, manifest_sources)
submodule_combo = pin.general_config.current_combo
try:
deinit_full(workspace_path, manifest, args.verbose)
except Exception as e:
ui_functions.print_error_msg(SUBMODULE_DEINIT_FAILED, header=False)
if args.verbose:
ui_functions.print_error_msg(e, header=False)
pin_repo_sources = pin.get_repo_sources(
pin.general_config.current_combo)
try:
checkout_repos(args.verbose, args.override, pin_repo_sources,
workspace_path, manifest)
manifest.write_current_combo(humble.PIN_COMBO.format(args.pinfile))
finally:
cache_path = None
cache_obj = get_repo_cache_obj(config)
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(
SUBMODULE_CACHE_REPO_NAME)
maintain_submodules(workspace_path, pin, submodule_combo,
args.verbose, cache_path)
if sparse_enabled:
ui_functions.print_info_msg(SPARSE_CHECKOUT, header=False)
sparse_checkout(workspace_path, pin_repo_sources, manifest)
def maintain_submodules(workspace,
manifest,
combo_name,
verbose=False,
cache_path=None):
"""
Updates the submodules for a specific repo.
workspace - Path to the current workspace.
manifest - The manifest parser object for the project.
combo_name - The combination name to use for submodule maintenance.
verbose - Enable verbose messages.
cache_path - Path to the submodule cache repo. A value of None indicates that no cache repo exists.
"""
# Process each repo that may have submodules enabled
ui_functions.print_info_msg(strings.SUBMOD_INIT_UPDATE)
repo_sources = manifest.get_repo_sources(combo_name)
for source in repo_sources:
# Open the repo and process submodules
try:
repo = git.Repo(os.path.join(workspace, source.root))
except Exception as repo_error:
if args.verbose:
ui_functions.print_error_msg(
strings.SUBMOD_EXCEPTION.format(repo_error))
continue
# Collect the submodule initialization data from manifest as well as if submodules
# should be processed for the repo.
repo_subs = manifest.get_submodule_init_paths(source.remote_name,
combo_name)
repo_subs_enabled = _get_submodule_enable(manifest, source.remote_name,
combo_name)
if not repo_subs_enabled:
continue
# Initialize submodules
if len(repo_subs) > 0:
_init(repo, repo_subs, verbose)
# Perform sync/update
if len(repo_subs) == 0:
_update(repo, None, verbose, cache_path=cache_path)
else:
_update(repo, repo_subs, verbose, cache_path=cache_path)
def run_command(self, args, config):
# Configure git long path support
ui_functions.print_info_msg(humble.LONGPATH_CONFIG, header=False)
set_long_path_support()
print()
# Remove unneeded instead of entries from git global config
ui_functions.print_info_msg(humble.CLEAN_INSTEAD_OFS, header=False)
print()
# If in a valid workspace run the following for each repo:
# git reflog --expire, git gc, git remote prune origin
try:
workspace_path = get_workspace_path()
except EdkrepoWorkspaceInvalidException:
workspace_path = None
ui_functions.print_error_msg(humble.NO_WOKKSPACE, header=False)
print()
if workspace_path:
manifest = get_workspace_manifest()
repos_to_maintain = manifest.get_repo_sources(
manifest.general_config.current_combo)
for repo_to_maintain in repos_to_maintain:
local_repo_path = os.path.join(workspace_path,
repo_to_maintain.root)
repo = Repo(local_repo_path)
ui_functions.print_info_msg(humble.REPO_MAINTENANCE.format(
repo_to_maintain.root),
header=False)
ui_functions.print_info_msg(humble.REFLOG_EXPIRE, header=False)
repo.git.reflog('expire', '--expire=now', '--all')
ui_functions.print_info_msg(humble.GC_AGGRESSIVE, header=False)
repo.git.gc('--aggressive', '--prune=now')
ui_functions.print_info_msg(humble.REMOTE_PRUNE, header=False)
repo.git.remote('prune', 'origin')
print()
def run_command(self, args, config):
workspace_path = get_workspace_path()
initial_manifest = get_workspace_manifest()
current_combo = initial_manifest.general_config.current_combo
initial_sources = initial_manifest.get_repo_sources(current_combo)
initial_hooks = initial_manifest.repo_hooks
initial_combo = current_combo
try:
pull_workspace_manifest_repo(initial_manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo, False)
except:
pull_all_manifest_repos(config['cfg_file'],
config['user_cfg_file'], False)
source_global_manifest_repo = find_source_manifest_repo(
initial_manifest, config['cfg_file'], config['user_cfg_file'],
args.source_manifest_repo)
cfg_manifest_repos, user_cfg_manifest_repos, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfg_file'])
if source_global_manifest_repo in cfg_manifest_repos:
global_manifest_directory = config[
'cfg_file'].manifest_repo_abs_path(source_global_manifest_repo)
verify_single_manifest(config['cfg_file'],
source_global_manifest_repo,
get_workspace_manifest_file(), args.verbose)
elif source_global_manifest_repo in user_cfg_manifest_repos:
global_manifest_directory = config[
'user_cfg_file'].manifest_repo_abs_path(
source_global_manifest_repo)
verify_single_manifest(config['user_cfg_file'],
source_global_manifest_repo,
get_workspace_manifest_file(), args.verbose)
else:
global_manifest_directory = None
if global_manifest_directory is not None:
update_editor_config(config, global_manifest_directory)
if not args.update_local_manifest:
self.__check_for_new_manifest(args, config, initial_manifest,
workspace_path,
global_manifest_directory)
check_dirty_repos(initial_manifest, workspace_path)
# Determine if sparse checkout needs to be disabled for this operation
sparse_settings = initial_manifest.sparse_settings
sparse_enabled = sparse_checkout_enabled(workspace_path,
initial_sources)
sparse_reset_required = False
if sparse_settings is None:
sparse_reset_required = True
elif args.update_local_manifest:
sparse_reset_required = True
if sparse_enabled and sparse_reset_required:
ui_functions.print_info_msg(SPARSE_RESET, header=False)
reset_sparse_checkout(workspace_path, initial_sources)
# Get the latest manifest if requested
if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse
self.__update_local_manifest(args, config, initial_manifest,
workspace_path,
global_manifest_directory)
manifest = get_workspace_manifest()
if args.update_local_manifest:
try:
repo_sources_to_sync = manifest.get_repo_sources(current_combo)
except ValueError:
# The manifest file was updated and the initial combo is no longer present so use the default combo
current_combo = manifest.general_config.default_combo
repo_sources_to_sync = manifest.get_repo_sources(current_combo)
else:
repo_sources_to_sync = manifest.get_repo_sources(current_combo)
manifest.write_current_combo(current_combo)
# At this point both new and old manifest files are ready so we can deinit any
# submodules that are removed due to a manifest update.
if not args.skip_submodule:
deinit_submodules(workspace_path, initial_manifest, initial_combo,
manifest, current_combo, args.verbose)
sync_error = False
# Calculate the hooks which need to be updated, added or removed for the sync
if args.update_local_manifest:
new_hooks = manifest.repo_hooks
hooks_add = set(new_hooks).difference(set(initial_hooks))
hooks_update = set(initial_hooks).intersection(set(new_hooks))
hooks_uninstall = set(initial_hooks).difference(set(new_hooks))
else:
hooks_add = None
hooks_update = initial_hooks
hooks_uninstall = None
# Update submodule configuration
if not args.update_local_manifest: #Performance optimization, __update_local_manifest() will do this
self.__check_submodule_config(workspace_path, manifest,
repo_sources_to_sync)
clean_git_globalconfig()
for repo_to_sync in repo_sources_to_sync:
local_repo_path = os.path.join(workspace_path, repo_to_sync.root)
# Update any hooks
if global_manifest_directory is not None:
update_hooks(hooks_add, hooks_update, hooks_uninstall,
local_repo_path, repo_to_sync, config,
global_manifest_directory)
repo = Repo(local_repo_path)
#Fetch notes
repo.remotes.origin.fetch("refs/notes/*:refs/notes/*")
if repo_to_sync.commit is None and repo_to_sync.tag is None:
local_commits = False
initial_active_branch = repo.active_branch
repo.remotes.origin.fetch(
"refs/heads/{0}:refs/remotes/origin/{0}".format(
repo_to_sync.branch))
#The new branch may not exist in the heads list yet if it is a new branch
repo.git.checkout(repo_to_sync.branch)
if not args.fetch:
ui_functions.print_info_msg(SYNCING.format(
repo_to_sync.root, repo.active_branch),
header=False)
else:
ui_functions.print_info_msg(FETCHING.format(
repo_to_sync.root, repo.active_branch),
header=False)
try:
repo.remotes.origin.fetch()
except GitCommandError as e:
prune_needed = False
prune_needed_heuristic_str = "error: some local refs could not be updated"
if e.stdout.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if e.stderr.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if prune_needed:
time.sleep(1.0)
repo.git.remote('prune', 'origin')
time.sleep(1.0)
repo.remotes.origin.fetch()
else:
raise
if has_primary_repo_remote(repo, args.verbose):
fetch_from_primary_repo(repo, repo_to_sync, args.verbose)
if not args.override and not repo.is_ancestor(
ancestor_rev='HEAD',
rev='origin/{}'.format(repo_to_sync.branch)):
ui_functions.print_info_msg(SYNC_COMMITS_ON_TARGET.format(
repo_to_sync.branch, repo_to_sync.root),
header=False)
local_commits = True
sync_error = True
if not args.fetch and (not local_commits or args.override):
repo.head.reset(commit='origin/{}'.format(
repo_to_sync.branch),
working_tree=True)
# Check to see if mirror is up to date
if not in_sync_with_primary(repo, repo_to_sync, args.verbose):
ui_functions.print_info_msg(MIRROR_BEHIND_PRIMARY_REPO,
header=False)
# Switch back to the initially active branch before exiting
repo.heads[initial_active_branch.name].checkout()
# Warn user if local branch is behind target branch
try:
latest_sha = get_latest_sha(repo, repo_to_sync.branch)
commit_count = int(
repo.git.rev_list('--count',
'{}..HEAD'.format(latest_sha)))
branch_origin = next(
itertools.islice(repo.iter_commits(), commit_count,
commit_count + 1))
behind_count = int(
repo.git.rev_list(
'--count', '{}..{}'.format(branch_origin.hexsha,
latest_sha)))
if behind_count:
ui_functions.print_info_msg(SYNC_NEEDS_REBASE.format(
behind_count=behind_count,
target_remote='origin',
target_branch=repo_to_sync.branch,
local_branch=initial_active_branch.name,
repo_folder=repo_to_sync.root),
header=False)
except:
ui_functions.print_error_msg(SYNC_REBASE_CALC_FAIL,
header=False)
elif args.verbose:
ui_functions.print_warning_msg(NO_SYNC_DETACHED_HEAD.format(
repo_to_sync.root),
header=False)
# Update commit message templates
if global_manifest_directory is not None:
update_repo_commit_template(workspace_path, repo, repo_to_sync,
config, global_manifest_directory)
if sync_error:
ui_functions.print_error_msg(SYNC_ERROR, header=False)
# Initialize submodules
if not args.skip_submodule:
cache_path = None
cache_obj = get_repo_cache_obj(config)
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(
SUBMODULE_CACHE_REPO_NAME)
maintain_submodules(workspace_path, manifest, current_combo,
args.verbose, cache_path)
# Restore sparse checkout state
if sparse_enabled:
ui_functions.print_info_msg(SPARSE_CHECKOUT, header=False)
sparse_checkout(workspace_path, repo_sources_to_sync, manifest)
def __update_local_manifest(self, args, config, initial_manifest,
workspace_path, global_manifest_directory):
#if the manifest repository for the current manifest was not found then there is no project with the manifest
#specified project name in the index file for any of the manifest repositories
if global_manifest_directory is None:
raise EdkrepoManifestNotFoundException(
SOURCE_MANIFEST_REPO_NOT_FOUND.format(
initial_manifest.project_info.codename))
local_manifest_dir = os.path.join(workspace_path, 'repo')
current_combo = initial_manifest.general_config.current_combo
initial_sources = initial_manifest.get_repo_sources(current_combo)
# Do a fetch for each repo in the initial to ensure that newly created upstream branches are available
for initial_repo in initial_sources:
local_repo_path = os.path.join(workspace_path, initial_repo.root)
repo = Repo(local_repo_path)
origin = repo.remotes.origin
try:
origin.fetch()
except GitCommandError as e:
prune_needed = False
prune_needed_heuristic_str = "error: some local refs could not be updated"
if e.stdout.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if e.stderr.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if prune_needed:
# The sleep is to give the operating system time to close all the file handles that Git has open
time.sleep(1.0)
repo.git.remote('prune', 'origin')
time.sleep(1.0)
origin.fetch()
else:
raise
#see if there is an entry in CiIndex.xml that matches the prject name of the current manifest
index_path = os.path.join(global_manifest_directory, 'CiIndex.xml')
ci_index_xml = CiIndexXml(index_path)
if initial_manifest.project_info.codename not in ci_index_xml.project_list:
raise EdkrepoManifestNotFoundException(
SYNC_MANIFEST_NOT_FOUND.format(
initial_manifest.project_info.codename))
initial_manifest_remotes = {
name: url
for name, url in initial_manifest.remotes
}
ci_index_xml_rel_path = os.path.normpath(
ci_index_xml.get_project_xml(
initial_manifest.project_info.codename))
global_manifest_path = os.path.join(global_manifest_directory,
ci_index_xml_rel_path)
new_manifest_to_check = ManifestXml(global_manifest_path)
# Does the current combo exist in the new manifest? If not check to see if you can use the repo sources from
# the default combo
initial_combos = combinations_in_manifest(initial_manifest)
new_combos = combinations_in_manifest(new_manifest_to_check)
if current_combo not in new_combos:
new_sources_for_current_combo = new_manifest_to_check.get_repo_sources(
new_manifest_to_check.general_config.default_combo)
new_sources = new_sources_for_current_combo
else:
new_sources_for_current_combo = new_manifest_to_check.get_repo_sources(
current_combo)
new_sources = new_manifest_to_check.get_repo_sources(current_combo)
remove_included_config(initial_manifest.remotes,
initial_manifest.submodule_alternate_remotes,
local_manifest_dir)
write_included_config(
new_manifest_to_check.remotes,
new_manifest_to_check.submodule_alternate_remotes,
local_manifest_dir)
self.__check_submodule_config(workspace_path, new_manifest_to_check,
new_sources_for_current_combo)
# Check that the repo sources lists are the same. If they are not the same and the override flag is not set, throw an exception.
if not args.override and set(initial_sources) != set(new_sources):
raise EdkrepoManifestChangedException(
SYNC_REPO_CHANGE.format(
initial_manifest.project_info.codename))
elif args.override and set(initial_sources) != set(new_sources):
#get a set of repo source tuples that are not in both the new and old manifest
uncommon_sources = []
initial_common = []
new_common = []
for initial in initial_sources:
common = False
for new in new_sources:
if initial.root == new.root:
if initial.remote_name == new.remote_name:
if initial.remote_url == new.remote_url:
# If the source is unchanged between the old and the new manifest,
# add it to the common lists
common = True
initial_common.append(initial)
new_common.append(new)
break
# If the source is different between the old and the new manifest, add it to the uncommon list
if not common:
uncommon_sources.append(initial)
for new in new_sources:
common = False
for initial in initial_sources:
if new.root == initial.root:
if new.remote_name == initial.remote_name:
if new.remote_url == initial.remote_url:
common = True
break
# If the source is different between the old and the new manifest, add it to the uncommon list
if not common:
uncommon_sources.append(new)
uncommon_sources = set(uncommon_sources)
initial_common = set(initial_common)
new_common = set(new_common)
sources_to_move = []
sources_to_remove = []
sources_to_clone = []
for source in uncommon_sources:
found_source = False
for source_to_check in initial_sources:
if source_to_check.root == source.root:
if source_to_check.remote_name == source.remote_name:
if source_to_check.remote_url == source.remote_url:
found_source = True
break
# If the source that is different came from the old manifest, then it is now outdated and either needs
# to be deleted or moved to an archival location.
if found_source:
roots = [s.root for s in new_sources]
# If there is a source in the new manifest that goes into the same folder name as a source in the
# old manifest, then we need to move that old folder to an archival location.
if source.root in roots:
sources_to_move.append(source)
else:
# If it doesn't exist at all in the new manifest, tell the user it is old and no longer used.
sources_to_remove.append(source)
else:
# If the source that is different came from the new manifest, then we need to clone that new
# Git repository.
sources_to_clone.append(source)
# Move the obsolete Git repositories to archival locations.
for source in sources_to_move:
old_dir = os.path.join(workspace_path, source.root)
new_dir = generate_name_for_obsolete_backup(old_dir)
ui_functions.print_warning_msg(SYNC_SOURCE_MOVE_WARNING.format(
source.root, new_dir),
header=False)
new_dir = os.path.join(workspace_path, new_dir)
try:
shutil.move(old_dir, new_dir)
except:
ui_functions.print_error_msg(SYNC_MOVE_FAILED.format(
initial_dir=source.root, new_dir=new_dir),
header=False)
raise
# Tell the user about any Git repositories that are no longer used.
if len(sources_to_remove) > 0:
ui_functions.print_warning_msg(SYNC_REMOVE_WARNING,
header=False)
for source in sources_to_remove:
path_to_source = os.path.join(workspace_path, source.root)
ui_functions.print_warning_msg(path_to_source, header=False)
if len(sources_to_remove) > 0:
ui_functions.print_warning_msg(SYNC_REMOVE_LIST_END_FORMATTING,
header=False)
# Clone any new Git repositories
clone_repos(args, workspace_path, sources_to_clone,
new_manifest_to_check.repo_hooks, config,
new_manifest_to_check)
# Make a list of and only checkout repos that were newly cloned. Sync keeps repos on their initial active branches
# cloning the entire combo can prevent existing repos from correctly being returned to their proper branch
repos_to_checkout = []
if sources_to_clone:
for new_source in new_sources_for_current_combo:
for source in sources_to_clone:
if source.root == new_source.root:
repos_to_checkout.append(source)
repos_to_checkout.extend(
self.__check_combo_sha_tag_branch(workspace_path,
initial_common, new_common))
if repos_to_checkout:
checkout_repos(args.verbose, args.override, repos_to_checkout,
workspace_path, new_manifest_to_check)
#remove the old manifest file and copy the new one
ui_functions.print_info_msg(UPDATING_MANIFEST, header=False)
local_manifest_path = os.path.join(local_manifest_dir, 'Manifest.xml')
os.remove(local_manifest_path)
shutil.copy(global_manifest_path, local_manifest_path)
# Update the source manifest repository tag in the local copy of the manifest XML
new_manifest = ManifestXml(local_manifest_path)
try:
if 'source_manifest_repo' in vars(args).keys():
find_source_manifest_repo(new_manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo)
else:
find_source_manifest_repo(new_manifest, config['cfg_file'],
config['user_cfg_file'], None)
except EdkrepoManifestNotFoundException:
pass