def __check_for_new_manifest(self, args, config, initial_manifest,
workspace_path, global_manifest_directory):
#if the manifest repository for the current manifest was not found then there is no project with the manifest
#specified project name in the index file for any of the manifest repositories
if global_manifest_directory is None:
if args.override:
return
else:
raise EdkrepoManifestNotFoundException(
SYNC_MANIFEST_NOT_FOUND.format(
initial_manifest.project_info.codename))
#see if there is an entry in CiIndex.xml that matches the prject name of the current manifest
index_path = os.path.join(global_manifest_directory, 'CiIndex.xml')
ci_index_xml = CiIndexXml(index_path)
if initial_manifest.project_info.codename not in ci_index_xml.project_list:
if args.override:
return
else:
raise EdkrepoManifestNotFoundException(
SYNC_MANIFEST_NOT_FOUND.format(
initial_manifest.project_info.codename))
ci_index_xml_rel_path = ci_index_xml.get_project_xml(
initial_manifest.project_info.codename)
global_manifest_path = os.path.join(
global_manifest_directory, os.path.normpath(ci_index_xml_rel_path))
global_manifest = ManifestXml(global_manifest_path)
if not initial_manifest.equals(global_manifest, True):
ui_functions.print_warning_msg(SYNC_MANIFEST_DIFF_WARNING,
header=False)
ui_functions.print_info_msg(SYNC_MANIFEST_UPDATE, header=False)
def run_command(self, args, config):
ui_functions.init_color_console(args.color)
manifest = get_workspace_manifest()
combo_archive = []
combo_list = [c.name for c in manifest.combinations]
if args.archived:
combo_archive = [c.name for c in manifest.archived_combinations]
combo_list.extend(combo_archive)
if manifest.general_config.current_combo not in combo_list:
combo_list.append(manifest.general_config.current_combo)
for combo in sorted(combo_list):
if combo == manifest.general_config.current_combo:
print("* {}{}{}".format(Fore.GREEN, combo, Fore.RESET))
elif combo in combo_archive:
print(" {}{}{}{}".format(Fore.YELLOW, Style.BRIGHT, combo,
Style.RESET_ALL))
else:
ui_functions.print_info_msg(" {}".format(combo), header=False)
if args.verbose:
sources = manifest.get_repo_sources(combo)
length = len(max([source.root for source in sources], key=len))
for source in sources:
ui_functions.print_info_msg(" {} : {}".format(
source.root.ljust(length), source.branch),
header=False)
def run_command(self, args, config):
workspace_path = get_workspace_path()
manifest = get_workspace_manifest()
manifest_repo = find_source_manifest_repo(manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo)
cfg, user_cfg, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfg_file'])
if manifest_repo in cfg:
manifest_repo_path = config['cfg_file'].manifest_repo_abs_path(
manifest_repo)
elif manifest_repo in user_cfg:
manifest_repo_path = config[
'user_cfg_file'].manifest_repo_abs_path(manifest_repo)
else:
manifest_repo_path = None
pin_path = self.__get_pin_path(args, workspace_path,
manifest_repo_path, manifest)
pin = ManifestXml(pin_path)
manifest_sources = manifest.get_repo_sources(
manifest.general_config.current_combo)
check_dirty_repos(manifest, workspace_path)
for source in manifest_sources:
local_path = os.path.join(workspace_path, source.root)
repo = Repo(local_path)
origin = repo.remotes.origin
origin.fetch()
self.__pin_matches_project(pin, manifest, workspace_path)
sparse_enabled = sparse_checkout_enabled(workspace_path,
manifest_sources)
if sparse_enabled:
ui_functions.print_info_msg(SPARSE_RESET, header=False)
reset_sparse_checkout(workspace_path, manifest_sources)
submodule_combo = pin.general_config.current_combo
try:
deinit_full(workspace_path, manifest, args.verbose)
except Exception as e:
ui_functions.print_error_msg(SUBMODULE_DEINIT_FAILED, header=False)
if args.verbose:
ui_functions.print_error_msg(e, header=False)
pin_repo_sources = pin.get_repo_sources(
pin.general_config.current_combo)
try:
checkout_repos(args.verbose, args.override, pin_repo_sources,
workspace_path, manifest)
manifest.write_current_combo(humble.PIN_COMBO.format(args.pinfile))
finally:
cache_path = None
cache_obj = get_repo_cache_obj(config)
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(
SUBMODULE_CACHE_REPO_NAME)
maintain_submodules(workspace_path, pin, submodule_combo,
args.verbose, cache_path)
if sparse_enabled:
ui_functions.print_info_msg(SPARSE_CHECKOUT, header=False)
sparse_checkout(workspace_path, pin_repo_sources, manifest)
def run_command(self, args, config):
cfg_repos, user_cfg_repos, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfg_file'])
if args.action == 'list':
for repo in cfg_repos:
ui_functions.print_info_msg(humble.CFG_LIST_ENTRY.format(repo),
header=False)
for repo in user_cfg_repos:
ui_functions.print_info_msg(
humble.USER_CFG_LIST_ENTRY.format(repo), header=False)
elif (args.action == ('add' or 'remove')) and not args.name:
raise EdkrepoInvalidParametersException(humble.NAME_REQUIRED)
elif args.action == 'add' and (not args.branch or not args.url
or not args.path):
raise EdkrepoInvalidParametersException(humble.ADD_REQUIRED)
elif args.action == 'remove' and args.name and args.name in cfg_repos:
raise EdkrepoInvalidParametersException(humble.CANNOT_REMOVE_CFG)
elif args.action == 'remove' and args.name not in config[
'user_cfg_file'].manifest_repo_list:
raise EdkrepoInvalidParametersException(humble.REMOVE_NOT_EXIST)
elif args.action == 'add' and (args.name in cfg_repos
or args.name in user_cfg_repos):
raise EdkrepoInvalidParametersException(
humble.ALREADY_EXISTS.format(args.name))
user_cfg_file_path = config['user_cfg_file'].cfg_filename
if args.action == 'add' or 'remove':
user_cfg_file = configparser.ConfigParser(allow_no_value=True)
user_cfg_file.read(user_cfg_file_path)
if args.action == 'add':
if not user_cfg_file.has_section('manifest-repos'):
user_cfg_file.add_section('manifest-repos')
user_cfg_file.set('manifest-repos', args.name, None)
user_cfg_file.add_section(args.name)
user_cfg_file.set(args.name, 'URL', args.url)
user_cfg_file.set(args.name, 'Branch', args.branch)
user_cfg_file.set(args.name, 'LocalPath', args.path)
if args.action == 'remove':
if user_cfg_file.has_section('manifest-repos'):
if user_cfg_file.has_option('manifest-repos', args.name):
user_cfg_file.remove_option('manifest-repos',
args.name)
else:
raise EdkrepoInvalidParametersException(
humble.REMOVE_NOT_EXIST)
else:
raise EdkrepoInvalidParametersException(
humble.REMOVE_NOT_EXIST)
if user_cfg_file.has_section(args.name):
user_cfg_file.remove_section(args.name)
else:
raise EdkrepoInvalidParametersException(
humble.REMOVE_NOT_EXIST)
with open(user_cfg_file_path, 'w') as cfg_stream:
user_cfg_file.write(cfg_stream)
def run_command(self, args, config):
workspace_path = get_workspace_path()
manifest = get_workspace_manifest()
manifest_config = manifest.general_config
repo_sources_to_reset = manifest.get_repo_sources(manifest_config.current_combo)
for repo_to_reset in repo_sources_to_reset:
local_repo_path = os.path.join(workspace_path, repo_to_reset.root)
repo = Repo(local_repo_path)
if args.verbose:
ui_functions.print_info_msg("{}Resetting {}".format("Hard " if args.hard else "", repo_to_reset.root))
repo.head.reset(working_tree=args.hard)
def run_command(self, args, config):
workspace_path = get_workspace_path()
initial_manifest = get_workspace_manifest()
current_combo = initial_manifest.general_config.current_combo
current_sources = initial_manifest.get_repo_sources(current_combo)
ui_functions.print_info_msg(
humble.STATUS_CURRENT_COMBO.format(current_combo), header=False)
for current_repo in current_sources:
local_repo_path = os.path.join(workspace_path, current_repo.root)
repo = Repo(local_repo_path)
ui_functions.print_info_msg("{}: {}\n".format(
current_repo.root, repo.git.status()),
header=False)
def run_command(self, args, config):
workspace_path = get_workspace_path()
manifest = get_workspace_manifest()
# Set the pinname/path == to the file name provided.
# If a relative paths is provided save the file relative to the current working directory.
if os.path.isabs(os.path.normpath(args.PinFileName)):
pin_file_name = os.path.normpath(args.PinFileName)
else:
pin_file_name = os.path.abspath(os.path.normpath(args.PinFileName))
# If the directory that the pin file is saved in does not exist create it and ensure pin file name uniqueness.
if os.path.isfile(pin_file_name):
raise EdkrepoInvalidParametersException(PIN_FILE_ALREADY_EXISTS)
if not os.path.exists(os.path.dirname(pin_file_name)):
os.mkdir(os.path.dirname(pin_file_name))
repo_sources = manifest.get_repo_sources(
manifest.general_config.current_combo)
# get the repo sources and commit ids for the pin
ui_functions.print_info_msg(GENERATING_PIN_DATA.format(
manifest.project_info.codename,
manifest.general_config.current_combo),
header=False)
updated_repo_sources = []
for repo_source in repo_sources:
local_repo_path = os.path.join(workspace_path, repo_source.root)
if not os.path.exists(local_repo_path):
raise EdkrepoWorkspaceCorruptException(
MISSING_REPO.format(repo_source.root))
repo = Repo(local_repo_path)
commit_id = repo.head.commit.hexsha
if args.verbose:
ui_functions.print_info_msg(GENERATING_REPO_DATA.format(
repo_source.root),
header=False)
ui_functions.print_info_msg(BRANCH.format(repo_source.branch),
header=False)
ui_functions.print_info_msg(COMMIT.format(commit_id),
header=False)
updated_repo_source = repo_source._replace(commit=commit_id)
updated_repo_sources.append(updated_repo_source)
# create the pin
ui_functions.print_info_msg(WRITING_PIN_FILE.format(pin_file_name),
header=False)
manifest.generate_pin_xml(args.Description,
manifest.general_config.current_combo,
updated_repo_sources,
filename=pin_file_name)
def run_command(self, args, config):
workspace_path = get_workspace_path()
manifest = get_workspace_manifest()
manifest_config = manifest.general_config
repo_sources_to_clean = manifest.get_repo_sources(
manifest_config.current_combo)
for repo_to_clean in repo_sources_to_clean:
local_repo_path = os.path.join(workspace_path, repo_to_clean.root)
repo = Repo(local_repo_path)
result = repo.git.clean(f=args.force,
d=args.dirs,
n=(not args.force),
q=(args.quiet and args.force))
if result:
ui_functions.print_info_msg(result, header=False)
def maintain_submodules(workspace,
manifest,
combo_name,
verbose=False,
cache_path=None):
"""
Updates the submodules for a specific repo.
workspace - Path to the current workspace.
manifest - The manifest parser object for the project.
combo_name - The combination name to use for submodule maintenance.
verbose - Enable verbose messages.
cache_path - Path to the submodule cache repo. A value of None indicates that no cache repo exists.
"""
# Process each repo that may have submodules enabled
ui_functions.print_info_msg(strings.SUBMOD_INIT_UPDATE)
repo_sources = manifest.get_repo_sources(combo_name)
for source in repo_sources:
# Open the repo and process submodules
try:
repo = git.Repo(os.path.join(workspace, source.root))
except Exception as repo_error:
if args.verbose:
ui_functions.print_error_msg(
strings.SUBMOD_EXCEPTION.format(repo_error))
continue
# Collect the submodule initialization data from manifest as well as if submodules
# should be processed for the repo.
repo_subs = manifest.get_submodule_init_paths(source.remote_name,
combo_name)
repo_subs_enabled = _get_submodule_enable(manifest, source.remote_name,
combo_name)
if not repo_subs_enabled:
continue
# Initialize submodules
if len(repo_subs) > 0:
_init(repo, repo_subs, verbose)
# Perform sync/update
if len(repo_subs) == 0:
_update(repo, None, verbose, cache_path=cache_path)
else:
_update(repo, repo_subs, verbose, cache_path=cache_path)
def run_command(self, args, config):
# Process enable disable requests
if args.disable:
config['user_cfg_file'].set_caching_state(False)
elif args.enable:
config['user_cfg_file'].set_caching_state(True)
# Get the current state now that we have processed enable/disable
cache_state = config['user_cfg_file'].caching_state
ui_functions.print_info_msg(CACHE_ENABLED.format(cache_state))
if not cache_state:
return
# State is enabled so make sure cache directory exists
cache_obj = get_repo_cache_obj(config)
pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'])
# Check to see if a manifest was provided and add any missing remotes
manifest = None
if args.project is not None:
manifest = _get_manifest(args.project, config, args.source_manifest_repo)
else:
try:
manifest = get_workspace_manifest()
except Exception:
pass
# If manifest is provided attempt to add any remotes that do not exist
if manifest is not None:
add_missing_cache_repos(cache_obj, manifest, True)
# Display all the cache information
if args.info:
ui_functions.print_info_msg(CACHE_INFO)
info = cache_obj.get_cache_info(args.verbose)
for item in info:
ui_functions.print_info_msg(CACHE_INFO_LINE.format(item.path, item.remote, item.url))
# Do an update if requested
if args.update:
ui_functions.print_info_msg(CACHE_FETCH)
cache_obj.update_cache(verbose=True)
# Close the cache repos
cache_obj.close(args.verbose)
def __check_combo_sha_tag_branch(self, workspace_path, initial_sources,
new_sources):
# Checks for changes in the defined SHAs, Tags or branches in the checked out combo. Returns
# a list of repos to checkout. Checks to see if user is on appropriate SHA, tag or branch and
# throws and exception if not.
repos_to_checkout = []
for initial_source in initial_sources:
for new_source in new_sources:
if initial_source.root == new_source.root and initial_source.remote_name == new_source.remote_name and initial_source.remote_url == new_source.remote_url:
local_repo_path = os.path.join(workspace_path,
initial_source.root)
repo = Repo(local_repo_path)
if initial_source.commit and initial_source.commit != new_source.commit:
if repo.head.object.hexsha != initial_source.commit:
ui_functions.print_info_msg(
SYNC_BRANCH_CHANGE_ON_LOCAL.format(
initial_source.branch, new_source.branch,
initial_source.root),
header=False)
repos_to_checkout.append(new_source)
break
elif initial_source.tag and initial_source.tag != new_source.tag:
tag_sha = repo.git.rev_list(
'-n 1', initial_source.tag
) #according to gitpython docs must change - to _
if tag_sha != repo.head.object.hexsha:
ui_functions.print_info_msg(
SYNC_BRANCH_CHANGE_ON_LOCAL.format(
initial_source.branch, new_source.branch,
initial_source.root),
header=False)
repos_to_checkout.append(new_source)
break
elif initial_source.branch and initial_source.branch != new_source.branch:
if repo.active_branch.name != initial_source.branch:
ui_functions.print_info_msg(
SYNC_BRANCH_CHANGE_ON_LOCAL.format(
initial_source.branch, new_source.branch,
initial_source.root),
header=False)
repos_to_checkout.append(new_source)
break
return repos_to_checkout
def __update_local_manifest(self, args, config, initial_manifest,
workspace_path, global_manifest_directory):
#if the manifest repository for the current manifest was not found then there is no project with the manifest
#specified project name in the index file for any of the manifest repositories
if global_manifest_directory is None:
raise EdkrepoManifestNotFoundException(
SOURCE_MANIFEST_REPO_NOT_FOUND.format(
initial_manifest.project_info.codename))
local_manifest_dir = os.path.join(workspace_path, 'repo')
current_combo = initial_manifest.general_config.current_combo
initial_sources = initial_manifest.get_repo_sources(current_combo)
# Do a fetch for each repo in the initial to ensure that newly created upstream branches are available
for initial_repo in initial_sources:
local_repo_path = os.path.join(workspace_path, initial_repo.root)
repo = Repo(local_repo_path)
origin = repo.remotes.origin
try:
origin.fetch()
except GitCommandError as e:
prune_needed = False
prune_needed_heuristic_str = "error: some local refs could not be updated"
if e.stdout.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if e.stderr.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if prune_needed:
# The sleep is to give the operating system time to close all the file handles that Git has open
time.sleep(1.0)
repo.git.remote('prune', 'origin')
time.sleep(1.0)
origin.fetch()
else:
raise
#see if there is an entry in CiIndex.xml that matches the prject name of the current manifest
index_path = os.path.join(global_manifest_directory, 'CiIndex.xml')
ci_index_xml = CiIndexXml(index_path)
if initial_manifest.project_info.codename not in ci_index_xml.project_list:
raise EdkrepoManifestNotFoundException(
SYNC_MANIFEST_NOT_FOUND.format(
initial_manifest.project_info.codename))
initial_manifest_remotes = {
name: url
for name, url in initial_manifest.remotes
}
ci_index_xml_rel_path = os.path.normpath(
ci_index_xml.get_project_xml(
initial_manifest.project_info.codename))
global_manifest_path = os.path.join(global_manifest_directory,
ci_index_xml_rel_path)
new_manifest_to_check = ManifestXml(global_manifest_path)
# Does the current combo exist in the new manifest? If not check to see if you can use the repo sources from
# the default combo
initial_combos = combinations_in_manifest(initial_manifest)
new_combos = combinations_in_manifest(new_manifest_to_check)
if current_combo not in new_combos:
new_sources_for_current_combo = new_manifest_to_check.get_repo_sources(
new_manifest_to_check.general_config.default_combo)
new_sources = new_sources_for_current_combo
else:
new_sources_for_current_combo = new_manifest_to_check.get_repo_sources(
current_combo)
new_sources = new_manifest_to_check.get_repo_sources(current_combo)
remove_included_config(initial_manifest.remotes,
initial_manifest.submodule_alternate_remotes,
local_manifest_dir)
write_included_config(
new_manifest_to_check.remotes,
new_manifest_to_check.submodule_alternate_remotes,
local_manifest_dir)
self.__check_submodule_config(workspace_path, new_manifest_to_check,
new_sources_for_current_combo)
# Check that the repo sources lists are the same. If they are not the same and the override flag is not set, throw an exception.
if not args.override and set(initial_sources) != set(new_sources):
raise EdkrepoManifestChangedException(
SYNC_REPO_CHANGE.format(
initial_manifest.project_info.codename))
elif args.override and set(initial_sources) != set(new_sources):
#get a set of repo source tuples that are not in both the new and old manifest
uncommon_sources = []
initial_common = []
new_common = []
for initial in initial_sources:
common = False
for new in new_sources:
if initial.root == new.root:
if initial.remote_name == new.remote_name:
if initial.remote_url == new.remote_url:
# If the source is unchanged between the old and the new manifest,
# add it to the common lists
common = True
initial_common.append(initial)
new_common.append(new)
break
# If the source is different between the old and the new manifest, add it to the uncommon list
if not common:
uncommon_sources.append(initial)
for new in new_sources:
common = False
for initial in initial_sources:
if new.root == initial.root:
if new.remote_name == initial.remote_name:
if new.remote_url == initial.remote_url:
common = True
break
# If the source is different between the old and the new manifest, add it to the uncommon list
if not common:
uncommon_sources.append(new)
uncommon_sources = set(uncommon_sources)
initial_common = set(initial_common)
new_common = set(new_common)
sources_to_move = []
sources_to_remove = []
sources_to_clone = []
for source in uncommon_sources:
found_source = False
for source_to_check in initial_sources:
if source_to_check.root == source.root:
if source_to_check.remote_name == source.remote_name:
if source_to_check.remote_url == source.remote_url:
found_source = True
break
# If the source that is different came from the old manifest, then it is now outdated and either needs
# to be deleted or moved to an archival location.
if found_source:
roots = [s.root for s in new_sources]
# If there is a source in the new manifest that goes into the same folder name as a source in the
# old manifest, then we need to move that old folder to an archival location.
if source.root in roots:
sources_to_move.append(source)
else:
# If it doesn't exist at all in the new manifest, tell the user it is old and no longer used.
sources_to_remove.append(source)
else:
# If the source that is different came from the new manifest, then we need to clone that new
# Git repository.
sources_to_clone.append(source)
# Move the obsolete Git repositories to archival locations.
for source in sources_to_move:
old_dir = os.path.join(workspace_path, source.root)
new_dir = generate_name_for_obsolete_backup(old_dir)
ui_functions.print_warning_msg(SYNC_SOURCE_MOVE_WARNING.format(
source.root, new_dir),
header=False)
new_dir = os.path.join(workspace_path, new_dir)
try:
shutil.move(old_dir, new_dir)
except:
ui_functions.print_error_msg(SYNC_MOVE_FAILED.format(
initial_dir=source.root, new_dir=new_dir),
header=False)
raise
# Tell the user about any Git repositories that are no longer used.
if len(sources_to_remove) > 0:
ui_functions.print_warning_msg(SYNC_REMOVE_WARNING,
header=False)
for source in sources_to_remove:
path_to_source = os.path.join(workspace_path, source.root)
ui_functions.print_warning_msg(path_to_source, header=False)
if len(sources_to_remove) > 0:
ui_functions.print_warning_msg(SYNC_REMOVE_LIST_END_FORMATTING,
header=False)
# Clone any new Git repositories
clone_repos(args, workspace_path, sources_to_clone,
new_manifest_to_check.repo_hooks, config,
new_manifest_to_check)
# Make a list of and only checkout repos that were newly cloned. Sync keeps repos on their initial active branches
# cloning the entire combo can prevent existing repos from correctly being returned to their proper branch
repos_to_checkout = []
if sources_to_clone:
for new_source in new_sources_for_current_combo:
for source in sources_to_clone:
if source.root == new_source.root:
repos_to_checkout.append(source)
repos_to_checkout.extend(
self.__check_combo_sha_tag_branch(workspace_path,
initial_common, new_common))
if repos_to_checkout:
checkout_repos(args.verbose, args.override, repos_to_checkout,
workspace_path, new_manifest_to_check)
#remove the old manifest file and copy the new one
ui_functions.print_info_msg(UPDATING_MANIFEST, header=False)
local_manifest_path = os.path.join(local_manifest_dir, 'Manifest.xml')
os.remove(local_manifest_path)
shutil.copy(global_manifest_path, local_manifest_path)
# Update the source manifest repository tag in the local copy of the manifest XML
new_manifest = ManifestXml(local_manifest_path)
try:
if 'source_manifest_repo' in vars(args).keys():
find_source_manifest_repo(new_manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo)
else:
find_source_manifest_repo(new_manifest, config['cfg_file'],
config['user_cfg_file'], None)
except EdkrepoManifestNotFoundException:
pass
def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest, cache_obj=None):
for repo_to_clone in repos_to_clone:
local_repo_path = os.path.join(workspace_dir, repo_to_clone.root)
local_repo_url = repo_to_clone.remote_url
cache_path = None
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(local_repo_url)
ui_functions.print_info_msg("Cloning from: " + str(local_repo_url), header = False)
if cache_path is not None:
ui_functions.print_info_msg('+ Using cache at {}'.format(cache_path))
repo = Repo.clone_from(local_repo_url, local_repo_path,
progress=GitProgressHandler(),
reference_if_able=cache_path,
no_checkout=True)
else:
repo = Repo.clone_from(local_repo_url, local_repo_path,
progress=GitProgressHandler(),
no_checkout=True)
# Fetch notes
repo.remotes.origin.fetch("refs/notes/*:refs/notes/*")
# Add the primary remote so that a reference to the latest code is available when
# using a mirror.
if add_primary_repo_remote(repo, repo_to_clone, args.verbose):
fetch_from_primary_repo(repo, repo_to_clone, args.verbose)
# Handle branch/commit/tag checkout if needed. If a combination of these are specified the
# order of importance is 1)commit 2)tag 3)branch with only the higest priority being checked
# out
if repo_to_clone.commit:
if args.verbose and (repo_to_clone.branch or repo_to_clone.tag):
ui_functions.print_info_msg(MULTIPLE_SOURCE_ATTRIBUTES_SPECIFIED.format(repo_to_clone.root))
repo.git.checkout(repo_to_clone.commit)
elif repo_to_clone.tag and repo_to_clone.commit is None:
if args.verbose and repo_to_clone.branch:
ui_functions.print_info_msg(TAG_AND_BRANCH_SPECIFIED.format(repo_to_clone.root))
repo.git.checkout(repo_to_clone.tag)
elif repo_to_clone.branch and (repo_to_clone.commit is None and repo_to_clone.tag is None):
if repo_to_clone.branch not in repo.remotes['origin'].refs:
raise EdkrepoManifestInvalidException('The specified remote branch does not exist')
branch_name = repo_to_clone.branch
local_branch = repo.create_head(branch_name, repo.remotes['origin'].refs[branch_name])
repo.heads[local_branch.name].set_tracking_branch(repo.remotes['origin'].refs[branch_name])
repo.heads[local_branch.name].checkout()
else:
raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT)
try:
if 'source_manifest_repo' in vars(args).keys():
src_manifest_repo = find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo, False)
else:
src_manifest_repo = find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], None, False)
except EdkrepoManifestNotFoundException:
src_manifest_repo = None
if src_manifest_repo:
cfg, user_cfg, conflicts = list_available_manifest_repos(config['cfg_file'], config['user_cfg_file'])
if src_manifest_repo in cfg:
global_manifest_directory = config['cfg_file'].manifest_repo_abs_path(src_manifest_repo)
elif src_manifest_repo in user_cfg:
global_manifest_directory = config['user_cfg_file'].manifest_repo_abs_path(src_manifest_repo)
else:
global_manifest_directory = None
else:
global_manifest_directory = None
if global_manifest_directory:
# Install git hooks if there is a manifest repo associated with the manifest being cloned
install_hooks(project_client_side_hooks, local_repo_path, repo_to_clone, config, global_manifest_directory)
# Add the commit template if it exists.
update_repo_commit_template(workspace_dir, repo, repo_to_clone, config, global_manifest_directory)
# Check to see if mirror is in sync with primary repo
if not in_sync_with_primary(repo, repo_to_clone, args.verbose):
ui_functions.print_warning_msg(MIRROR_BEHIND_PRIMARY_REPO)
def run_command(self, args, config):
print()
cfg_file = config['cfg_file']
user_cfg = config['user_cfg_file']
cfg_man_repos, user_cfg_man_repos, conflicts = list_available_manifest_repos(
cfg_file, user_cfg)
man_repos = {}
pull_all_manifest_repos(cfg_file, user_cfg, False)
# Get paths to the global manifest dirs and their index files
for repo in cfg_man_repos:
global_manifest_directory = cfg_file.manifest_repo_abs_path(repo)
index_path = os.path.join(global_manifest_directory, 'CiIndex.xml')
man_repos[repo] = (global_manifest_directory, index_path)
for repo in user_cfg_man_repos:
global_manifest_directory = user_cfg.manifest_repo_abs_path(repo)
index_path = os.path.join(global_manifest_directory, 'CiIndex.xml')
man_repos[repo] = (global_manifest_directory, index_path)
try:
wkspc_manifest = get_workspace_manifest()
current_project = wkspc_manifest.project_info.codename
src_man_repo = find_source_manifest_repo(wkspc_manifest, cfg_file,
user_cfg, None)
except EdkrepoWorkspaceInvalidException:
current_project = None
src_man_repo = None
except EdkrepoManifestNotFoundException:
src_man_repo = None
for repo in man_repos.keys():
print()
ui_functions.print_info_msg("Manifest directory:", header=False)
ui_functions.print_info_msg(repo, header=False)
if args.verbose:
ui_functions.print_info_msg('Manifest directory path:',
header=False)
ui_functions.print_info_msg(man_repos[repo][0], header=False)
print()
ci_index_xml = CiIndexXml(man_repos[repo][1])
# Attempt to make sure the manifest data is good
try:
validate_manifest_repo(man_repos[repo][0], args.verbose,
args.archived)
except:
print()
ui_functions.print_info_msg("Projects:", header=False)
for project in sorted(ci_index_xml.project_list):
if (project == current_project and src_man_repo == repo) or (
not src_man_repo and project == current_project):
ui_functions.print_info_msg(project, header=False)
else:
ui_functions.print_warning_msg(project, header=False)
if args.verbose:
ui_functions.print_info_msg(" -> {}".format(
ci_index_xml.get_project_xml(project)),
header=False)
proj_manifest = ManifestXml(
find_project_in_single_index(project, ci_index_xml,
man_repos[repo][0])[1])
ui_functions.print_info_msg(" -> DevLead: {}".format(
' '.join(
x for x in proj_manifest.project_info.dev_leads)),
header=False)
if args.archived:
print()
ui_functions.print_info_msg("Archived Projects:", header=False)
for project in sorted(ci_index_xml.archived_project_list):
if project == current_project:
ui_functions.print_info_msg(project, header=False)
else:
ui_functions.print_warning_msg(project, header=False)
if args.verbose:
ui_functions.print_info_msg(" -> {}".format(
ci_index_xml.get_project_xml(project)),
header=False)
proj_manifest = ManifestXml(
find_project_in_single_index(
project, ci_index_xml, man_repos[repo][0])[1])
ui_functions.print_info_msg(" -> DevLead: {}".format(
' '.join(x for x in
proj_manifest.project_info.dev_leads)),
header=False)
def run_command(self, args, config):
# Collect workspace/repo data
workspace_path = get_workspace_path()
manifest = get_workspace_manifest()
current_combo = manifest.general_config.current_combo
repo_list = manifest.get_repo_sources(current_combo)
sparse_settings = manifest.sparse_settings
sparse_enabled = sparse_checkout_enabled(workspace_path, repo_list)
# Determine if settings are being chaged or just status display
if args.enable or args.disable:
# Handle sparse checkout changes
if args.enable and args.disable:
raise EdkrepoSparseException(SPARSE_ENABLE_DISABLE)
elif (args.enable and sparse_enabled) or (args.disable
and not sparse_enabled):
raise EdkrepoSparseException(SPARSE_NO_CHANGE)
check_dirty_repos(manifest, workspace_path)
if args.enable and not sparse_enabled:
ui_functions.print_info_msg(SPARSE_ENABLE, header=False)
sparse_checkout(workspace_path, repo_list, manifest)
elif args.disable and sparse_enabled:
ui_functions.print_info_msg(SPARSE_DISABLE, header=False)
reset_sparse_checkout(workspace_path, repo_list, True)
else:
# Display the current status of the project
ui_functions.print_info_msg(SPARSE_STATUS, header=False)
ui_functions.print_info_msg(
SPARSE_CHECKOUT_STATUS.format(sparse_enabled), header=False)
if sparse_settings is not None:
ui_functions.print_info_msg(SPARSE_BY_DEFAULT_STATUS.format(
sparse_settings.sparse_by_default),
header=False)
ui_functions.print_info_msg(
SPARSE_ENABLED_REPOS.format(current_combo), header=False)
for repo in [x for x in repo_list if x.sparse]:
ui_functions.print_info_msg('- {}: {}'.format(
repo.root, repo.remote_url),
header=False)
def run_command(self, args, config):
# Configure git long path support
ui_functions.print_info_msg(humble.LONGPATH_CONFIG, header=False)
set_long_path_support()
print()
# Remove unneeded instead of entries from git global config
ui_functions.print_info_msg(humble.CLEAN_INSTEAD_OFS, header=False)
print()
# If in a valid workspace run the following for each repo:
# git reflog --expire, git gc, git remote prune origin
try:
workspace_path = get_workspace_path()
except EdkrepoWorkspaceInvalidException:
workspace_path = None
ui_functions.print_error_msg(humble.NO_WOKKSPACE, header=False)
print()
if workspace_path:
manifest = get_workspace_manifest()
repos_to_maintain = manifest.get_repo_sources(
manifest.general_config.current_combo)
for repo_to_maintain in repos_to_maintain:
local_repo_path = os.path.join(workspace_path,
repo_to_maintain.root)
repo = Repo(local_repo_path)
ui_functions.print_info_msg(humble.REPO_MAINTENANCE.format(
repo_to_maintain.root),
header=False)
ui_functions.print_info_msg(humble.REFLOG_EXPIRE, header=False)
repo.git.reflog('expire', '--expire=now', '--all')
ui_functions.print_info_msg(humble.GC_AGGRESSIVE, header=False)
repo.git.gc('--aggressive', '--prune=now')
ui_functions.print_info_msg(humble.REMOTE_PRUNE, header=False)
repo.git.remote('prune', 'origin')
print()
def run_command(self, args, config):
workspace_path = get_workspace_path()
initial_manifest = get_workspace_manifest()
current_combo = initial_manifest.general_config.current_combo
initial_sources = initial_manifest.get_repo_sources(current_combo)
initial_hooks = initial_manifest.repo_hooks
initial_combo = current_combo
try:
pull_workspace_manifest_repo(initial_manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo, False)
except:
pull_all_manifest_repos(config['cfg_file'],
config['user_cfg_file'], False)
source_global_manifest_repo = find_source_manifest_repo(
initial_manifest, config['cfg_file'], config['user_cfg_file'],
args.source_manifest_repo)
cfg_manifest_repos, user_cfg_manifest_repos, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfg_file'])
if source_global_manifest_repo in cfg_manifest_repos:
global_manifest_directory = config[
'cfg_file'].manifest_repo_abs_path(source_global_manifest_repo)
verify_single_manifest(config['cfg_file'],
source_global_manifest_repo,
get_workspace_manifest_file(), args.verbose)
elif source_global_manifest_repo in user_cfg_manifest_repos:
global_manifest_directory = config[
'user_cfg_file'].manifest_repo_abs_path(
source_global_manifest_repo)
verify_single_manifest(config['user_cfg_file'],
source_global_manifest_repo,
get_workspace_manifest_file(), args.verbose)
else:
global_manifest_directory = None
if global_manifest_directory is not None:
update_editor_config(config, global_manifest_directory)
if not args.update_local_manifest:
self.__check_for_new_manifest(args, config, initial_manifest,
workspace_path,
global_manifest_directory)
check_dirty_repos(initial_manifest, workspace_path)
# Determine if sparse checkout needs to be disabled for this operation
sparse_settings = initial_manifest.sparse_settings
sparse_enabled = sparse_checkout_enabled(workspace_path,
initial_sources)
sparse_reset_required = False
if sparse_settings is None:
sparse_reset_required = True
elif args.update_local_manifest:
sparse_reset_required = True
if sparse_enabled and sparse_reset_required:
ui_functions.print_info_msg(SPARSE_RESET, header=False)
reset_sparse_checkout(workspace_path, initial_sources)
# Get the latest manifest if requested
if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse
self.__update_local_manifest(args, config, initial_manifest,
workspace_path,
global_manifest_directory)
manifest = get_workspace_manifest()
if args.update_local_manifest:
try:
repo_sources_to_sync = manifest.get_repo_sources(current_combo)
except ValueError:
# The manifest file was updated and the initial combo is no longer present so use the default combo
current_combo = manifest.general_config.default_combo
repo_sources_to_sync = manifest.get_repo_sources(current_combo)
else:
repo_sources_to_sync = manifest.get_repo_sources(current_combo)
manifest.write_current_combo(current_combo)
# At this point both new and old manifest files are ready so we can deinit any
# submodules that are removed due to a manifest update.
if not args.skip_submodule:
deinit_submodules(workspace_path, initial_manifest, initial_combo,
manifest, current_combo, args.verbose)
sync_error = False
# Calculate the hooks which need to be updated, added or removed for the sync
if args.update_local_manifest:
new_hooks = manifest.repo_hooks
hooks_add = set(new_hooks).difference(set(initial_hooks))
hooks_update = set(initial_hooks).intersection(set(new_hooks))
hooks_uninstall = set(initial_hooks).difference(set(new_hooks))
else:
hooks_add = None
hooks_update = initial_hooks
hooks_uninstall = None
# Update submodule configuration
if not args.update_local_manifest: #Performance optimization, __update_local_manifest() will do this
self.__check_submodule_config(workspace_path, manifest,
repo_sources_to_sync)
clean_git_globalconfig()
for repo_to_sync in repo_sources_to_sync:
local_repo_path = os.path.join(workspace_path, repo_to_sync.root)
# Update any hooks
if global_manifest_directory is not None:
update_hooks(hooks_add, hooks_update, hooks_uninstall,
local_repo_path, repo_to_sync, config,
global_manifest_directory)
repo = Repo(local_repo_path)
#Fetch notes
repo.remotes.origin.fetch("refs/notes/*:refs/notes/*")
if repo_to_sync.commit is None and repo_to_sync.tag is None:
local_commits = False
initial_active_branch = repo.active_branch
repo.remotes.origin.fetch(
"refs/heads/{0}:refs/remotes/origin/{0}".format(
repo_to_sync.branch))
#The new branch may not exist in the heads list yet if it is a new branch
repo.git.checkout(repo_to_sync.branch)
if not args.fetch:
ui_functions.print_info_msg(SYNCING.format(
repo_to_sync.root, repo.active_branch),
header=False)
else:
ui_functions.print_info_msg(FETCHING.format(
repo_to_sync.root, repo.active_branch),
header=False)
try:
repo.remotes.origin.fetch()
except GitCommandError as e:
prune_needed = False
prune_needed_heuristic_str = "error: some local refs could not be updated"
if e.stdout.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if e.stderr.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if prune_needed:
time.sleep(1.0)
repo.git.remote('prune', 'origin')
time.sleep(1.0)
repo.remotes.origin.fetch()
else:
raise
if has_primary_repo_remote(repo, args.verbose):
fetch_from_primary_repo(repo, repo_to_sync, args.verbose)
if not args.override and not repo.is_ancestor(
ancestor_rev='HEAD',
rev='origin/{}'.format(repo_to_sync.branch)):
ui_functions.print_info_msg(SYNC_COMMITS_ON_TARGET.format(
repo_to_sync.branch, repo_to_sync.root),
header=False)
local_commits = True
sync_error = True
if not args.fetch and (not local_commits or args.override):
repo.head.reset(commit='origin/{}'.format(
repo_to_sync.branch),
working_tree=True)
# Check to see if mirror is up to date
if not in_sync_with_primary(repo, repo_to_sync, args.verbose):
ui_functions.print_info_msg(MIRROR_BEHIND_PRIMARY_REPO,
header=False)
# Switch back to the initially active branch before exiting
repo.heads[initial_active_branch.name].checkout()
# Warn user if local branch is behind target branch
try:
latest_sha = get_latest_sha(repo, repo_to_sync.branch)
commit_count = int(
repo.git.rev_list('--count',
'{}..HEAD'.format(latest_sha)))
branch_origin = next(
itertools.islice(repo.iter_commits(), commit_count,
commit_count + 1))
behind_count = int(
repo.git.rev_list(
'--count', '{}..{}'.format(branch_origin.hexsha,
latest_sha)))
if behind_count:
ui_functions.print_info_msg(SYNC_NEEDS_REBASE.format(
behind_count=behind_count,
target_remote='origin',
target_branch=repo_to_sync.branch,
local_branch=initial_active_branch.name,
repo_folder=repo_to_sync.root),
header=False)
except:
ui_functions.print_error_msg(SYNC_REBASE_CALC_FAIL,
header=False)
elif args.verbose:
ui_functions.print_warning_msg(NO_SYNC_DETACHED_HEAD.format(
repo_to_sync.root),
header=False)
# Update commit message templates
if global_manifest_directory is not None:
update_repo_commit_template(workspace_path, repo, repo_to_sync,
config, global_manifest_directory)
if sync_error:
ui_functions.print_error_msg(SYNC_ERROR, header=False)
# Initialize submodules
if not args.skip_submodule:
cache_path = None
cache_obj = get_repo_cache_obj(config)
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(
SUBMODULE_CACHE_REPO_NAME)
maintain_submodules(workspace_path, manifest, current_combo,
args.verbose, cache_path)
# Restore sparse checkout state
if sparse_enabled:
ui_functions.print_info_msg(SPARSE_CHECKOUT, header=False)
sparse_checkout(workspace_path, repo_sources_to_sync, manifest)
def run_command(self, args, config):
pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'],
False)
name_or_manifest = args.ProjectNameOrManifestFile
workspace_dir = args.Workspace
# Check to see if requested workspace exists. If not create it. If so check for empty
if workspace_dir == '.':
# User has selected the directory they are running edkrepo from
workspace_dir = os.getcwd()
else:
workspace_dir = os.path.abspath(workspace_dir)
if sys.platform == "win32":
subst = get_subst_drive_dict()
drive = os.path.splitdrive(workspace_dir)[0][0].upper()
if drive in subst:
workspace_dir = os.path.join(
subst[drive],
os.path.splitdrive(workspace_dir)[1][1:])
workspace_dir = os.path.normpath(workspace_dir)
if os.path.isdir(workspace_dir) and os.listdir(workspace_dir):
raise EdkrepoInvalidParametersException(CLONE_INVALID_WORKSPACE)
if not os.path.isdir(workspace_dir):
os.makedirs(workspace_dir)
cfg, user_cfg, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfg_file'])
try:
manifest_repo, source_cfg, global_manifest_path = find_project_in_all_indices(
args.ProjectNameOrManifestFile, config['cfg_file'],
config['user_cfg_file'],
PROJ_NOT_IN_REPO.format(args.ProjectNameOrManifestFile),
SOURCE_MANIFEST_REPO_NOT_FOUND.format(
args.ProjectNameOrManifestFile), args.source_manifest_repo)
except EdkrepoManifestNotFoundException:
raise EdkrepoInvalidParametersException(CLONE_INVALID_PROJECT_ARG)
# If this manifest is in a defined manifest repository validate the manifest within the manifest repo
if manifest_repo in cfg:
verify_single_manifest(config['cfg_file'], manifest_repo,
global_manifest_path)
update_editor_config(
config,
config['cfg_file'].manifest_repo_abs_path(manifest_repo))
elif manifest_repo in user_cfg:
verify_single_manifest(config['user_cfg_file'], manifest_repo,
global_manifest_path)
update_editor_config(
config,
config['user_cfg_file'].manifest_repo_abs_path(manifest_repo))
# Copy project manifest to local manifest dir and rename it Manifest.xml.
local_manifest_dir = os.path.join(workspace_dir, "repo")
os.makedirs(local_manifest_dir)
local_manifest_path = os.path.join(local_manifest_dir, "Manifest.xml")
shutil.copy(global_manifest_path, local_manifest_path)
manifest = ManifestXml(local_manifest_path)
# Update the source manifest repository tag in the local copy of the manifest XML
try:
if 'source_manifest_repo' in vars(args).keys():
find_source_manifest_repo(manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo)
else:
find_source_manifest_repo(manifest, config['cfg_file'],
config['user_cfg_file'], None)
except EdkrepoManifestNotFoundException:
pass
# Process the combination name and make sure it can be found in the manifest
if args.Combination is not None:
try:
combo_name = case_insensitive_single_match(
args.Combination, combinations_in_manifest(manifest))
except:
#remove the repo directory and Manifest.xml from the workspace so the next time the user trys to clone
#they will have an empty workspace and then raise an exception
shutil.rmtree(local_manifest_dir)
raise EdkrepoInvalidParametersException(
CLONE_INVALID_COMBO_ARG)
manifest.write_current_combo(combo_name)
elif manifest.is_pin_file():
# Since pin files are subset of manifest files they do not have a "default combo" it is set to None. In this
# case use the current_combo instead.
combo_name = manifest.general_config.current_combo
else:
# If a combo was not specified or a pin file used the default combo should be cloned. Also ensure that the
# current combo is updated to match.
combo_name = manifest.general_config.default_combo
manifest.write_current_combo(combo_name)
# Get the list of repos to clone and clone them
repo_sources_to_clone = manifest.get_repo_sources(combo_name)
#check that the repo sources do not contain duplicated local roots
local_roots = [r.root for r in repo_sources_to_clone]
for root in local_roots:
if local_roots.count(root) > 1:
#remove the repo dir and manifest.xml so the next time the user trys to clone they will have an empty
#workspace
shutil.rmtree(local_manifest_dir)
raise EdkrepoManifestInvalidException(
CLONE_INVALID_LOCAL_ROOTS)
project_client_side_hooks = manifest.repo_hooks
# Set up submodule alt url config settings prior to cloning any repos
submodule_included_configs = write_included_config(
manifest.remotes, manifest.submodule_alternate_remotes,
local_manifest_dir)
write_conditional_include(workspace_dir, repo_sources_to_clone,
submodule_included_configs)
# Determine if caching is going to be used and then clone
cache_obj = get_repo_cache_obj(config)
if cache_obj is not None:
add_missing_cache_repos(cache_obj, manifest, args.verbose)
clone_repos(args, workspace_dir, repo_sources_to_clone,
project_client_side_hooks, config, manifest, cache_obj)
# Init submodules
if not args.skip_submodule:
cache_path = None
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(
SUBMODULE_CACHE_REPO_NAME)
maintain_submodules(workspace_dir, manifest, combo_name,
args.verbose, cache_path)
# Perform a sparse checkout if requested.
use_sparse = args.sparse
sparse_settings = manifest.sparse_settings
if sparse_settings is None:
# No SparseCheckout information in manifest so skip sparse checkout
use_sparse = False
elif sparse_settings.sparse_by_default:
# Sparse settings enabled by default for the project
use_sparse = True
if args.nosparse:
# Command line disables sparse checkout
use_sparse = False
if use_sparse:
ui_functions.print_info_msg(SPARSE_CHECKOUT)
sparse_checkout(workspace_dir, repo_sources_to_clone, manifest)
