def run_command(self, args, config):
# Process enable disable requests
if args.disable:
config['user_cfg_file'].set_caching_state(False)
elif args.enable:
config['user_cfg_file'].set_caching_state(True)
# Get the current state now that we have processed enable/disable
cache_state = config['user_cfg_file'].caching_state
ui_functions.print_info_msg(CACHE_ENABLED.format(cache_state))
if not cache_state:
return
# State is enabled so make sure cache directory exists
cache_obj = get_repo_cache_obj(config)
pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'])
# Check to see if a manifest was provided and add any missing remotes
manifest = None
if args.project is not None:
manifest = _get_manifest(args.project, config, args.source_manifest_repo)
else:
try:
manifest = get_workspace_manifest()
except Exception:
pass
# If manifest is provided attempt to add any remotes that do not exist
if manifest is not None:
add_missing_cache_repos(cache_obj, manifest, True)
# Display all the cache information
if args.info:
ui_functions.print_info_msg(CACHE_INFO)
info = cache_obj.get_cache_info(args.verbose)
for item in info:
ui_functions.print_info_msg(CACHE_INFO_LINE.format(item.path, item.remote, item.url))
# Do an update if requested
if args.update:
ui_functions.print_info_msg(CACHE_FETCH)
cache_obj.update_cache(verbose=True)
# Close the cache repos
cache_obj.close(args.verbose)
def run_command(self, args, config):
workspace_path = get_workspace_path()
initial_manifest = get_workspace_manifest()
current_combo = initial_manifest.general_config.current_combo
initial_sources = initial_manifest.get_repo_sources(current_combo)
initial_hooks = initial_manifest.repo_hooks
initial_combo = current_combo
try:
pull_workspace_manifest_repo(initial_manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo, False)
except:
pull_all_manifest_repos(config['cfg_file'],
config['user_cfg_file'], False)
source_global_manifest_repo = find_source_manifest_repo(
initial_manifest, config['cfg_file'], config['user_cfg_file'],
args.source_manifest_repo)
cfg_manifest_repos, user_cfg_manifest_repos, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfg_file'])
if source_global_manifest_repo in cfg_manifest_repos:
global_manifest_directory = config[
'cfg_file'].manifest_repo_abs_path(source_global_manifest_repo)
verify_single_manifest(config['cfg_file'],
source_global_manifest_repo,
get_workspace_manifest_file(), args.verbose)
elif source_global_manifest_repo in user_cfg_manifest_repos:
global_manifest_directory = config[
'user_cfg_file'].manifest_repo_abs_path(
source_global_manifest_repo)
verify_single_manifest(config['user_cfg_file'],
source_global_manifest_repo,
get_workspace_manifest_file(), args.verbose)
else:
global_manifest_directory = None
if global_manifest_directory is not None:
update_editor_config(config, global_manifest_directory)
if not args.update_local_manifest:
self.__check_for_new_manifest(args, config, initial_manifest,
workspace_path,
global_manifest_directory)
check_dirty_repos(initial_manifest, workspace_path)
# Determine if sparse checkout needs to be disabled for this operation
sparse_settings = initial_manifest.sparse_settings
sparse_enabled = sparse_checkout_enabled(workspace_path,
initial_sources)
sparse_reset_required = False
if sparse_settings is None:
sparse_reset_required = True
elif args.update_local_manifest:
sparse_reset_required = True
if sparse_enabled and sparse_reset_required:
ui_functions.print_info_msg(SPARSE_RESET, header=False)
reset_sparse_checkout(workspace_path, initial_sources)
# Get the latest manifest if requested
if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse
self.__update_local_manifest(args, config, initial_manifest,
workspace_path,
global_manifest_directory)
manifest = get_workspace_manifest()
if args.update_local_manifest:
try:
repo_sources_to_sync = manifest.get_repo_sources(current_combo)
except ValueError:
# The manifest file was updated and the initial combo is no longer present so use the default combo
current_combo = manifest.general_config.default_combo
repo_sources_to_sync = manifest.get_repo_sources(current_combo)
else:
repo_sources_to_sync = manifest.get_repo_sources(current_combo)
manifest.write_current_combo(current_combo)
# At this point both new and old manifest files are ready so we can deinit any
# submodules that are removed due to a manifest update.
if not args.skip_submodule:
deinit_submodules(workspace_path, initial_manifest, initial_combo,
manifest, current_combo, args.verbose)
sync_error = False
# Calculate the hooks which need to be updated, added or removed for the sync
if args.update_local_manifest:
new_hooks = manifest.repo_hooks
hooks_add = set(new_hooks).difference(set(initial_hooks))
hooks_update = set(initial_hooks).intersection(set(new_hooks))
hooks_uninstall = set(initial_hooks).difference(set(new_hooks))
else:
hooks_add = None
hooks_update = initial_hooks
hooks_uninstall = None
# Update submodule configuration
if not args.update_local_manifest: #Performance optimization, __update_local_manifest() will do this
self.__check_submodule_config(workspace_path, manifest,
repo_sources_to_sync)
clean_git_globalconfig()
for repo_to_sync in repo_sources_to_sync:
local_repo_path = os.path.join(workspace_path, repo_to_sync.root)
# Update any hooks
if global_manifest_directory is not None:
update_hooks(hooks_add, hooks_update, hooks_uninstall,
local_repo_path, repo_to_sync, config,
global_manifest_directory)
repo = Repo(local_repo_path)
#Fetch notes
repo.remotes.origin.fetch("refs/notes/*:refs/notes/*")
if repo_to_sync.commit is None and repo_to_sync.tag is None:
local_commits = False
initial_active_branch = repo.active_branch
repo.remotes.origin.fetch(
"refs/heads/{0}:refs/remotes/origin/{0}".format(
repo_to_sync.branch))
#The new branch may not exist in the heads list yet if it is a new branch
repo.git.checkout(repo_to_sync.branch)
if not args.fetch:
ui_functions.print_info_msg(SYNCING.format(
repo_to_sync.root, repo.active_branch),
header=False)
else:
ui_functions.print_info_msg(FETCHING.format(
repo_to_sync.root, repo.active_branch),
header=False)
try:
repo.remotes.origin.fetch()
except GitCommandError as e:
prune_needed = False
prune_needed_heuristic_str = "error: some local refs could not be updated"
if e.stdout.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if e.stderr.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if prune_needed:
time.sleep(1.0)
repo.git.remote('prune', 'origin')
time.sleep(1.0)
repo.remotes.origin.fetch()
else:
raise
if has_primary_repo_remote(repo, args.verbose):
fetch_from_primary_repo(repo, repo_to_sync, args.verbose)
if not args.override and not repo.is_ancestor(
ancestor_rev='HEAD',
rev='origin/{}'.format(repo_to_sync.branch)):
ui_functions.print_info_msg(SYNC_COMMITS_ON_TARGET.format(
repo_to_sync.branch, repo_to_sync.root),
header=False)
local_commits = True
sync_error = True
if not args.fetch and (not local_commits or args.override):
repo.head.reset(commit='origin/{}'.format(
repo_to_sync.branch),
working_tree=True)
# Check to see if mirror is up to date
if not in_sync_with_primary(repo, repo_to_sync, args.verbose):
ui_functions.print_info_msg(MIRROR_BEHIND_PRIMARY_REPO,
header=False)
# Switch back to the initially active branch before exiting
repo.heads[initial_active_branch.name].checkout()
# Warn user if local branch is behind target branch
try:
latest_sha = get_latest_sha(repo, repo_to_sync.branch)
commit_count = int(
repo.git.rev_list('--count',
'{}..HEAD'.format(latest_sha)))
branch_origin = next(
itertools.islice(repo.iter_commits(), commit_count,
commit_count + 1))
behind_count = int(
repo.git.rev_list(
'--count', '{}..{}'.format(branch_origin.hexsha,
latest_sha)))
if behind_count:
ui_functions.print_info_msg(SYNC_NEEDS_REBASE.format(
behind_count=behind_count,
target_remote='origin',
target_branch=repo_to_sync.branch,
local_branch=initial_active_branch.name,
repo_folder=repo_to_sync.root),
header=False)
except:
ui_functions.print_error_msg(SYNC_REBASE_CALC_FAIL,
header=False)
elif args.verbose:
ui_functions.print_warning_msg(NO_SYNC_DETACHED_HEAD.format(
repo_to_sync.root),
header=False)
# Update commit message templates
if global_manifest_directory is not None:
update_repo_commit_template(workspace_path, repo, repo_to_sync,
config, global_manifest_directory)
if sync_error:
ui_functions.print_error_msg(SYNC_ERROR, header=False)
# Initialize submodules
if not args.skip_submodule:
cache_path = None
cache_obj = get_repo_cache_obj(config)
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(
SUBMODULE_CACHE_REPO_NAME)
maintain_submodules(workspace_path, manifest, current_combo,
args.verbose, cache_path)
# Restore sparse checkout state
if sparse_enabled:
ui_functions.print_info_msg(SPARSE_CHECKOUT, header=False)
sparse_checkout(workspace_path, repo_sources_to_sync, manifest)
def run_command(self, args, config):
print()
init_color_console(args.color)
pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'])
print()
cfg_manifest_repos, user_config_manifest_repos, conflicts = list_available_manifest_repos(config['cfg_file'], config['user_cfg_file'])
found_manifests = {}
manifests = {}
repo_urls = set()
config_manifest_repos_project_list = []
user_config_manifest_repos_project_list = []
for manifest_repo in cfg_manifest_repos:
# Get path to global manifest file
global_manifest_directory = config['cfg_file'].manifest_repo_abs_path(manifest_repo)
if args.verbose:
print(humble.MANIFEST_DIRECTORY)
print(global_manifest_directory)
print()
#Create a dictionary containing all the manifests listed in the CiIndex.xml file
index_path = os.path.join(global_manifest_directory, CI_INDEX_FILE_NAME)
print(index_path)
ci_index_xml = CiIndexXml(index_path)
config_manifest_repos_project_list = ci_index_xml.project_list
if args.archived:
config_manifest_repos_project_list.extend(ci_index_xml.archived_project_list)
for project in config_manifest_repos_project_list:
xml_file = ci_index_xml.get_project_xml(project)
manifest = ManifestXml(os.path.normpath(os.path.join(global_manifest_directory, xml_file)))
found_manifests['{}:{}'.format(manifest_repo, project)] = manifest
combo_list = [c.name for c in manifest.combinations]
if args.archived:
combo_list.extend([c.name for c in manifest.archived_combinations])
for combo in combo_list:
sources = manifest.get_repo_sources(combo)
for source in sources:
repo_urls.add(self.get_repo_url(source.remote_url))
for manifest_repo in user_config_manifest_repos:
# Get path to global manifest file
global_manifest_directory = config['user_cfg_file'].manifest_repo_abs_path(manifest_repo)
if args.verbose:
print(humble.MANIFEST_DIRECTORY)
print(global_manifest_directory)
print()
#Create a dictionary containing all the manifests listed in the CiIndex.xml file
index_path = os.path.join(global_manifest_directory, CI_INDEX_FILE_NAME)
ci_index_xml = CiIndexXml(index_path)
user_config_manifest_repos_project_list = ci_index_xml.project_list
if args.archived:
user_config_manifest_repos_project_list.extend(ci_index_xml.archived_project_list)
for project in user_config_manifest_repos_project_list:
xml_file = ci_index_xml.get_project_xml(project)
manifest = ManifestXml(os.path.normpath(os.path.join(global_manifest_directory, xml_file)))
found_manifests['{}:{}'.format(manifest_repo, project)] = manifest
combo_list = [c.name for c in manifest.combinations]
if args.archived:
combo_list.extend([c.name for c in manifest.archived_combinations])
for combo in combo_list:
sources = manifest.get_repo_sources(combo)
for source in sources:
repo_urls.add(self.get_repo_url(source.remote_url))
#Remove the manifest repo portion of the key is there is not a duplicate project name
key_list = list(found_manifests)
for entry in key_list:
new_key = entry.split(':')[1]
value = found_manifests[entry]
del found_manifests[entry]
for found_manifest in list(found_manifests):
if found_manifest.split(':')[1] == new_key:
new_key = 'Manifest Repository: {} Project: {}'.format(entry.split(':')[0], entry.split(':')[1])
break
if new_key in manifests.keys():
new_key = 'Manifest Repository: {} Project: {}'.format(entry.split(':'[0]), entry.split(':')[1])
manifests[new_key] = value
#Sort the manifests so projects will be displayed alphabetically
manifests = collections.OrderedDict(sorted(manifests.items()))
project_justify = len(max(manifests.keys(), key=len))
#Determine the names of the repositories
self.generate_repo_names(repo_urls, manifests, args.archived)
print(humble.REPOSITORIES)
#If the user provided a list of repositories to view, check to make sure
#at least one repository will be shown, if not provide an error
if args.repos and len([x for x in self.repo_names if x in args.repos]) <= 0:
raise EdkrepoInvalidParametersException(humble.REPO_NOT_FOUND_IN_MANIFEST.format(','.join(args.repos)))
#For each each git repository...
for repo_name in self.repo_names:
if args.repos and repo_name not in args.repos:
continue
repo = self.repo_names[repo_name][0]
print(humble.REPO_NAME_AND_URL.format(repo_name, repo))
print(humble.BRANCHES)
#Determine the list of branches that used by any branch combination in any manifest
branches = set()
for project_name in manifests:
combo_list = [c.name for c in manifests[project_name].combinations]
if args.archived:
combo_list.extend([c.name for c in manifests[project_name].archived_combinations])
for combo in combo_list:
sources = manifests[project_name].get_repo_sources(combo)
for source in sources:
if self.get_repo_url(source.remote_url) == repo:
branches.add(source.branch)
#Sort the branch names so they will be displayed alphabetically
#with the exception that if a branch named "master" exists, then it
#will be displayed first
branches = sorted(branches, key=str.casefold)
if 'master' in branches:
branches.remove('master')
branches.insert(0, 'master')
#For each interesting branch in the current git repository...
for branch in branches:
print(humble.BRANCH_FORMAT_STRING.format(branch))
#Determine the branch combinations that use that branch
for project_name in manifests:
combos = []
combo_list = [c.name for c in manifests[project_name].combinations]
if args.archived:
combo_list.extend([c.name for c in manifests[project_name].archived_combinations])
for combo in combo_list:
sources = manifests[project_name].get_repo_sources(combo)
for source in sources:
if self.get_repo_url(source.remote_url) == repo and source.branch == branch:
combos.append(combo)
break
if len(combos) > 0:
#Sort the branch combinations so they will be displayed alphabetically
#with the exception that the default branch combination for the manifest
#file will be displayed first
combos = sorted(combos, key=str.casefold)
default_combo = manifests[project_name].general_config.default_combo
if default_combo in combos:
combos.remove(default_combo)
combos.insert(0, default_combo)
first_combo = True
for combo in combos:
#Print the project name
if first_combo:
project_name_print = humble.PROJECT_NAME_FORMAT_STRING.format(project_name.ljust(project_justify))
first_combo = False
else:
project_name_print = '{} '.format((' ' * len(project_name)).ljust(project_justify))
#Print the branch combination name, if this is the default branch combination,
#then print it in green color with *'s around it
if default_combo == combo:
print(humble.DEFAULT_COMBO_FORMAT_STRING.format(project_name_print, combo))
else:
print(humble.COMBO_FORMAT_STRING.format(project_name_print, combo))
def run_command(self, args, config):
print()
cfg_file = config['cfg_file']
user_cfg = config['user_cfg_file']
cfg_man_repos, user_cfg_man_repos, conflicts = list_available_manifest_repos(
cfg_file, user_cfg)
man_repos = {}
pull_all_manifest_repos(cfg_file, user_cfg, False)
# Get paths to the global manifest dirs and their index files
for repo in cfg_man_repos:
global_manifest_directory = cfg_file.manifest_repo_abs_path(repo)
index_path = os.path.join(global_manifest_directory, 'CiIndex.xml')
man_repos[repo] = (global_manifest_directory, index_path)
for repo in user_cfg_man_repos:
global_manifest_directory = user_cfg.manifest_repo_abs_path(repo)
index_path = os.path.join(global_manifest_directory, 'CiIndex.xml')
man_repos[repo] = (global_manifest_directory, index_path)
try:
wkspc_manifest = get_workspace_manifest()
current_project = wkspc_manifest.project_info.codename
src_man_repo = find_source_manifest_repo(wkspc_manifest, cfg_file,
user_cfg, None)
except EdkrepoWorkspaceInvalidException:
current_project = None
src_man_repo = None
except EdkrepoManifestNotFoundException:
src_man_repo = None
for repo in man_repos.keys():
print()
ui_functions.print_info_msg("Manifest directory:", header=False)
ui_functions.print_info_msg(repo, header=False)
if args.verbose:
ui_functions.print_info_msg('Manifest directory path:',
header=False)
ui_functions.print_info_msg(man_repos[repo][0], header=False)
print()
ci_index_xml = CiIndexXml(man_repos[repo][1])
# Attempt to make sure the manifest data is good
try:
validate_manifest_repo(man_repos[repo][0], args.verbose,
args.archived)
except:
print()
ui_functions.print_info_msg("Projects:", header=False)
for project in sorted(ci_index_xml.project_list):
if (project == current_project and src_man_repo == repo) or (
not src_man_repo and project == current_project):
ui_functions.print_info_msg(project, header=False)
else:
ui_functions.print_warning_msg(project, header=False)
if args.verbose:
ui_functions.print_info_msg(" -> {}".format(
ci_index_xml.get_project_xml(project)),
header=False)
proj_manifest = ManifestXml(
find_project_in_single_index(project, ci_index_xml,
man_repos[repo][0])[1])
ui_functions.print_info_msg(" -> DevLead: {}".format(
' '.join(
x for x in proj_manifest.project_info.dev_leads)),
header=False)
if args.archived:
print()
ui_functions.print_info_msg("Archived Projects:", header=False)
for project in sorted(ci_index_xml.archived_project_list):
if project == current_project:
ui_functions.print_info_msg(project, header=False)
else:
ui_functions.print_warning_msg(project, header=False)
if args.verbose:
ui_functions.print_info_msg(" -> {}".format(
ci_index_xml.get_project_xml(project)),
header=False)
proj_manifest = ManifestXml(
find_project_in_single_index(
project, ci_index_xml, man_repos[repo][0])[1])
ui_functions.print_info_msg(" -> DevLead: {}".format(
' '.join(x for x in
proj_manifest.project_info.dev_leads)),
header=False)
def run_command(self, args, config):
pull_all_manifest_repos(config['cfg_file'],
config['user_cfg_file'],
reset_hard=args.hard)
def run_command(self, args, config):
pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'],
False)
name_or_manifest = args.ProjectNameOrManifestFile
workspace_dir = args.Workspace
# Check to see if requested workspace exists. If not create it. If so check for empty
if workspace_dir == '.':
# User has selected the directory they are running edkrepo from
workspace_dir = os.getcwd()
else:
workspace_dir = os.path.abspath(workspace_dir)
if sys.platform == "win32":
subst = get_subst_drive_dict()
drive = os.path.splitdrive(workspace_dir)[0][0].upper()
if drive in subst:
workspace_dir = os.path.join(
subst[drive],
os.path.splitdrive(workspace_dir)[1][1:])
workspace_dir = os.path.normpath(workspace_dir)
if os.path.isdir(workspace_dir) and os.listdir(workspace_dir):
raise EdkrepoInvalidParametersException(CLONE_INVALID_WORKSPACE)
if not os.path.isdir(workspace_dir):
os.makedirs(workspace_dir)
cfg, user_cfg, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfg_file'])
try:
manifest_repo, source_cfg, global_manifest_path = find_project_in_all_indices(
args.ProjectNameOrManifestFile, config['cfg_file'],
config['user_cfg_file'],
PROJ_NOT_IN_REPO.format(args.ProjectNameOrManifestFile),
SOURCE_MANIFEST_REPO_NOT_FOUND.format(
args.ProjectNameOrManifestFile), args.source_manifest_repo)
except EdkrepoManifestNotFoundException:
raise EdkrepoInvalidParametersException(CLONE_INVALID_PROJECT_ARG)
# If this manifest is in a defined manifest repository validate the manifest within the manifest repo
if manifest_repo in cfg:
verify_single_manifest(config['cfg_file'], manifest_repo,
global_manifest_path)
update_editor_config(
config,
config['cfg_file'].manifest_repo_abs_path(manifest_repo))
elif manifest_repo in user_cfg:
verify_single_manifest(config['user_cfg_file'], manifest_repo,
global_manifest_path)
update_editor_config(
config,
config['user_cfg_file'].manifest_repo_abs_path(manifest_repo))
# Copy project manifest to local manifest dir and rename it Manifest.xml.
local_manifest_dir = os.path.join(workspace_dir, "repo")
os.makedirs(local_manifest_dir)
local_manifest_path = os.path.join(local_manifest_dir, "Manifest.xml")
shutil.copy(global_manifest_path, local_manifest_path)
manifest = ManifestXml(local_manifest_path)
# Update the source manifest repository tag in the local copy of the manifest XML
try:
if 'source_manifest_repo' in vars(args).keys():
find_source_manifest_repo(manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo)
else:
find_source_manifest_repo(manifest, config['cfg_file'],
config['user_cfg_file'], None)
except EdkrepoManifestNotFoundException:
pass
# Process the combination name and make sure it can be found in the manifest
if args.Combination is not None:
try:
combo_name = case_insensitive_single_match(
args.Combination, combinations_in_manifest(manifest))
except:
#remove the repo directory and Manifest.xml from the workspace so the next time the user trys to clone
#they will have an empty workspace and then raise an exception
shutil.rmtree(local_manifest_dir)
raise EdkrepoInvalidParametersException(
CLONE_INVALID_COMBO_ARG)
manifest.write_current_combo(combo_name)
elif manifest.is_pin_file():
# Since pin files are subset of manifest files they do not have a "default combo" it is set to None. In this
# case use the current_combo instead.
combo_name = manifest.general_config.current_combo
else:
# If a combo was not specified or a pin file used the default combo should be cloned. Also ensure that the
# current combo is updated to match.
combo_name = manifest.general_config.default_combo
manifest.write_current_combo(combo_name)
# Get the list of repos to clone and clone them
repo_sources_to_clone = manifest.get_repo_sources(combo_name)
#check that the repo sources do not contain duplicated local roots
local_roots = [r.root for r in repo_sources_to_clone]
for root in local_roots:
if local_roots.count(root) > 1:
#remove the repo dir and manifest.xml so the next time the user trys to clone they will have an empty
#workspace
shutil.rmtree(local_manifest_dir)
raise EdkrepoManifestInvalidException(
CLONE_INVALID_LOCAL_ROOTS)
project_client_side_hooks = manifest.repo_hooks
# Set up submodule alt url config settings prior to cloning any repos
submodule_included_configs = write_included_config(
manifest.remotes, manifest.submodule_alternate_remotes,
local_manifest_dir)
write_conditional_include(workspace_dir, repo_sources_to_clone,
submodule_included_configs)
# Determine if caching is going to be used and then clone
cache_obj = get_repo_cache_obj(config)
if cache_obj is not None:
add_missing_cache_repos(cache_obj, manifest, args.verbose)
clone_repos(args, workspace_dir, repo_sources_to_clone,
project_client_side_hooks, config, manifest, cache_obj)
# Init submodules
if not args.skip_submodule:
cache_path = None
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(
SUBMODULE_CACHE_REPO_NAME)
maintain_submodules(workspace_dir, manifest, combo_name,
args.verbose, cache_path)
# Perform a sparse checkout if requested.
use_sparse = args.sparse
sparse_settings = manifest.sparse_settings
if sparse_settings is None:
# No SparseCheckout information in manifest so skip sparse checkout
use_sparse = False
elif sparse_settings.sparse_by_default:
# Sparse settings enabled by default for the project
use_sparse = True
if args.nosparse:
# Command line disables sparse checkout
use_sparse = False
if use_sparse:
ui_functions.print_info_msg(SPARSE_CHECKOUT)
sparse_checkout(workspace_dir, repo_sources_to_clone, manifest)