def run_command(self, args, config):
workspace_path = get_workspace_path()
manifest = get_workspace_manifest()
manifest_repo = find_source_manifest_repo(manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo)
cfg, user_cfg, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfg_file'])
if manifest_repo in cfg:
manifest_repo_path = config['cfg_file'].manifest_repo_abs_path(
manifest_repo)
elif manifest_repo in user_cfg:
manifest_repo_path = config[
'user_cfg_file'].manifest_repo_abs_path(manifest_repo)
else:
manifest_repo_path = None
pin_path = self.__get_pin_path(args, workspace_path,
manifest_repo_path, manifest)
pin = ManifestXml(pin_path)
manifest_sources = manifest.get_repo_sources(
manifest.general_config.current_combo)
check_dirty_repos(manifest, workspace_path)
for source in manifest_sources:
local_path = os.path.join(workspace_path, source.root)
repo = Repo(local_path)
origin = repo.remotes.origin
origin.fetch()
self.__pin_matches_project(pin, manifest, workspace_path)
sparse_enabled = sparse_checkout_enabled(workspace_path,
manifest_sources)
if sparse_enabled:
ui_functions.print_info_msg(SPARSE_RESET, header=False)
reset_sparse_checkout(workspace_path, manifest_sources)
submodule_combo = pin.general_config.current_combo
try:
deinit_full(workspace_path, manifest, args.verbose)
except Exception as e:
ui_functions.print_error_msg(SUBMODULE_DEINIT_FAILED, header=False)
if args.verbose:
ui_functions.print_error_msg(e, header=False)
pin_repo_sources = pin.get_repo_sources(
pin.general_config.current_combo)
try:
checkout_repos(args.verbose, args.override, pin_repo_sources,
workspace_path, manifest)
manifest.write_current_combo(humble.PIN_COMBO.format(args.pinfile))
finally:
cache_path = None
cache_obj = get_repo_cache_obj(config)
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(
SUBMODULE_CACHE_REPO_NAME)
maintain_submodules(workspace_path, pin, submodule_combo,
args.verbose, cache_path)
if sparse_enabled:
ui_functions.print_info_msg(SPARSE_CHECKOUT, header=False)
sparse_checkout(workspace_path, pin_repo_sources, manifest)
def __check_for_new_manifest(self, args, config, initial_manifest,
workspace_path, global_manifest_directory):
#if the manifest repository for the current manifest was not found then there is no project with the manifest
#specified project name in the index file for any of the manifest repositories
if global_manifest_directory is None:
if args.override:
return
else:
raise EdkrepoManifestNotFoundException(
SYNC_MANIFEST_NOT_FOUND.format(
initial_manifest.project_info.codename))
#see if there is an entry in CiIndex.xml that matches the prject name of the current manifest
index_path = os.path.join(global_manifest_directory, 'CiIndex.xml')
ci_index_xml = CiIndexXml(index_path)
if initial_manifest.project_info.codename not in ci_index_xml.project_list:
if args.override:
return
else:
raise EdkrepoManifestNotFoundException(
SYNC_MANIFEST_NOT_FOUND.format(
initial_manifest.project_info.codename))
ci_index_xml_rel_path = ci_index_xml.get_project_xml(
initial_manifest.project_info.codename)
global_manifest_path = os.path.join(
global_manifest_directory, os.path.normpath(ci_index_xml_rel_path))
global_manifest = ManifestXml(global_manifest_path)
if not initial_manifest.equals(global_manifest, True):
ui_functions.print_warning_msg(SYNC_MANIFEST_DIFF_WARNING,
header=False)
ui_functions.print_info_msg(SYNC_MANIFEST_UPDATE, header=False)
def validate_parsing(self):
try:
self._manifest_xmldata = ManifestXml(self._manifestfile)
return ("PARSING", True, None
) # (validation type, status, message)
except Exception as e_message:
self._manifest_xmldata = None
return ("PARSING", False, e_message)
def verify_single_manifest(cfg_file, manifest_repo, manifest_path, verbose=False):
manifest = ManifestXml(manifest_path)
print(VERIFY_PROJ.format(manifest.project_info.codename))
index_path = os.path.join(cfg_file.manifest_repo_abs_path(manifest_repo), CI_INDEX_FILE_NAME)
proj_val_data = validate_manifestfiles([manifest_path])
proj_val_error = get_manifest_validation_status(proj_val_data)
if proj_val_error:
if verbose:
print_manifest_errors(proj_val_data)
raise EdkrepoManifestInvalidException(VERIFY_PROJ_FAIL.format(manifest.project_info.codename))
def main(args):
# Optional data init
new_manifest = None
all_new_combos = None
# Extract basic manifest data
manifest = ManifestXml(args.manifest_file)
init_manifest = manifest
all_combos = manifest.combinations
all_combos.extend(manifest.archived_combinations)
if args.new_manifest is not None:
new_manifest = ManifestXml(args.new_manifest)
all_new_combos = new_manifest.combinations
all_new_combos.extend(new_manifest.archived_combinations)
init_manifest = new_manifest
# Determine current and new combo information
current_combo = manifest.general_config.current_combo
new_combo = args.new_combo
for combo in all_combos:
if args.combo is not None:
if args.combo.lower() == combo.name.lower():
current_combo = combo.name
if args.new_combo is not None:
if args.new_combo.lower() == combo.name.lower():
new_combo = combo.name
init_combo = current_combo
if new_combo is not None:
init_combo = new_combo
if args.deinit_full:
deinit_full(args.workspace, manifest, args.verbose)
elif args.init_full:
init_full(args.workspace, manifest, args.verbose)
else:
if args.deinit:
deinit_submodules(args.workspace, manifest, current_combo,
new_manifest, new_combo, args.verbose)
maintain_submodules(args.workspace, init_manifest, init_combo,
args.verbose)
return 0
def _get_manifest(project, config, source_manifest_repo=None):
try:
manifest_repo, source_cfg, manifest_path = find_project_in_all_indices(
project,
config['cfg_file'],
config['user_cfg_file'],
PROJECT_NOT_FOUND.format(project),
NO_INSTANCE.format(project),
source_manifest_repo)
except Exception:
raise EdkrepoCacheException(UNABLE_TO_LOAD_MANIFEST)
try:
manifest = ManifestXml(manifest_path)
except Exception:
raise EdkrepoCacheException(UNABLE_TO_PARSE_MANIFEST)
return manifest
def checkout_pin(parsed_args, config):
pins = []
manifest = get_workspace_manifest()
manifest_directory = None
try:
source_manifest_repo = find_source_manifest_repo(
manifest,
config['cfg_file'],
config['user_cfg_file'],
)
if source_manifest_repo:
cfg, user_cfg, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfgFile'])
if source_manifest_repo in cfg:
manifest_directory = config['cfg_file'].manifest_repo_abs_path(
source_manifest_repo)
elif source_manifest_repo in user_cfg:
manifest_directory = config[
'user_cfg_file'].manifest_repo_abs_path(
source_manifest_repo)
else:
manifest_directory = None
except EdkrepoManifestNotFoundException:
manifest_directory = None
if manifest_directory:
pin_folder = os.path.normpath(
os.path.join(manifest_directory, manifest.general_config.pin_path))
for dirpath, _, filenames in os.walk(pin_folder):
for file in filenames:
pin_file = os.path.join(dirpath, file)
# Capture error output from manifest parser stdout so it is hidden unless verbose is enabled
stdout = sys.stdout
sys.stdout = io.StringIO()
pin = ManifestXml(pin_file)
parse_output = sys.stdout.getvalue()
sys.stdout = stdout
if parsed_args.verbose and parse_output.strip() != '':
print('Pin {} Parsing Errors: {}\n'.format(
file, parse_output.strip()))
if pin.project_info.codename == manifest.project_info.codename:
pins.append(file)
print(' '.join(pins))
def __update_local_manifest(self, args, config, initial_manifest,
workspace_path, global_manifest_directory):
#if the manifest repository for the current manifest was not found then there is no project with the manifest
#specified project name in the index file for any of the manifest repositories
if global_manifest_directory is None:
raise EdkrepoManifestNotFoundException(
SOURCE_MANIFEST_REPO_NOT_FOUND.format(
initial_manifest.project_info.codename))
local_manifest_dir = os.path.join(workspace_path, 'repo')
current_combo = initial_manifest.general_config.current_combo
initial_sources = initial_manifest.get_repo_sources(current_combo)
# Do a fetch for each repo in the initial to ensure that newly created upstream branches are available
for initial_repo in initial_sources:
local_repo_path = os.path.join(workspace_path, initial_repo.root)
repo = Repo(local_repo_path)
origin = repo.remotes.origin
try:
origin.fetch()
except GitCommandError as e:
prune_needed = False
prune_needed_heuristic_str = "error: some local refs could not be updated"
if e.stdout.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if e.stderr.strip().find(prune_needed_heuristic_str) != -1:
prune_needed = True
if prune_needed:
# The sleep is to give the operating system time to close all the file handles that Git has open
time.sleep(1.0)
repo.git.remote('prune', 'origin')
time.sleep(1.0)
origin.fetch()
else:
raise
#see if there is an entry in CiIndex.xml that matches the prject name of the current manifest
index_path = os.path.join(global_manifest_directory, 'CiIndex.xml')
ci_index_xml = CiIndexXml(index_path)
if initial_manifest.project_info.codename not in ci_index_xml.project_list:
raise EdkrepoManifestNotFoundException(
SYNC_MANIFEST_NOT_FOUND.format(
initial_manifest.project_info.codename))
initial_manifest_remotes = {
name: url
for name, url in initial_manifest.remotes
}
ci_index_xml_rel_path = os.path.normpath(
ci_index_xml.get_project_xml(
initial_manifest.project_info.codename))
global_manifest_path = os.path.join(global_manifest_directory,
ci_index_xml_rel_path)
new_manifest_to_check = ManifestXml(global_manifest_path)
# Does the current combo exist in the new manifest? If not check to see if you can use the repo sources from
# the default combo
initial_combos = combinations_in_manifest(initial_manifest)
new_combos = combinations_in_manifest(new_manifest_to_check)
if current_combo not in new_combos:
new_sources_for_current_combo = new_manifest_to_check.get_repo_sources(
new_manifest_to_check.general_config.default_combo)
new_sources = new_sources_for_current_combo
else:
new_sources_for_current_combo = new_manifest_to_check.get_repo_sources(
current_combo)
new_sources = new_manifest_to_check.get_repo_sources(current_combo)
remove_included_config(initial_manifest.remotes,
initial_manifest.submodule_alternate_remotes,
local_manifest_dir)
write_included_config(
new_manifest_to_check.remotes,
new_manifest_to_check.submodule_alternate_remotes,
local_manifest_dir)
self.__check_submodule_config(workspace_path, new_manifest_to_check,
new_sources_for_current_combo)
# Check that the repo sources lists are the same. If they are not the same and the override flag is not set, throw an exception.
if not args.override and set(initial_sources) != set(new_sources):
raise EdkrepoManifestChangedException(
SYNC_REPO_CHANGE.format(
initial_manifest.project_info.codename))
elif args.override and set(initial_sources) != set(new_sources):
#get a set of repo source tuples that are not in both the new and old manifest
uncommon_sources = []
initial_common = []
new_common = []
for initial in initial_sources:
common = False
for new in new_sources:
if initial.root == new.root:
if initial.remote_name == new.remote_name:
if initial.remote_url == new.remote_url:
# If the source is unchanged between the old and the new manifest,
# add it to the common lists
common = True
initial_common.append(initial)
new_common.append(new)
break
# If the source is different between the old and the new manifest, add it to the uncommon list
if not common:
uncommon_sources.append(initial)
for new in new_sources:
common = False
for initial in initial_sources:
if new.root == initial.root:
if new.remote_name == initial.remote_name:
if new.remote_url == initial.remote_url:
common = True
break
# If the source is different between the old and the new manifest, add it to the uncommon list
if not common:
uncommon_sources.append(new)
uncommon_sources = set(uncommon_sources)
initial_common = set(initial_common)
new_common = set(new_common)
sources_to_move = []
sources_to_remove = []
sources_to_clone = []
for source in uncommon_sources:
found_source = False
for source_to_check in initial_sources:
if source_to_check.root == source.root:
if source_to_check.remote_name == source.remote_name:
if source_to_check.remote_url == source.remote_url:
found_source = True
break
# If the source that is different came from the old manifest, then it is now outdated and either needs
# to be deleted or moved to an archival location.
if found_source:
roots = [s.root for s in new_sources]
# If there is a source in the new manifest that goes into the same folder name as a source in the
# old manifest, then we need to move that old folder to an archival location.
if source.root in roots:
sources_to_move.append(source)
else:
# If it doesn't exist at all in the new manifest, tell the user it is old and no longer used.
sources_to_remove.append(source)
else:
# If the source that is different came from the new manifest, then we need to clone that new
# Git repository.
sources_to_clone.append(source)
# Move the obsolete Git repositories to archival locations.
for source in sources_to_move:
old_dir = os.path.join(workspace_path, source.root)
new_dir = generate_name_for_obsolete_backup(old_dir)
ui_functions.print_warning_msg(SYNC_SOURCE_MOVE_WARNING.format(
source.root, new_dir),
header=False)
new_dir = os.path.join(workspace_path, new_dir)
try:
shutil.move(old_dir, new_dir)
except:
ui_functions.print_error_msg(SYNC_MOVE_FAILED.format(
initial_dir=source.root, new_dir=new_dir),
header=False)
raise
# Tell the user about any Git repositories that are no longer used.
if len(sources_to_remove) > 0:
ui_functions.print_warning_msg(SYNC_REMOVE_WARNING,
header=False)
for source in sources_to_remove:
path_to_source = os.path.join(workspace_path, source.root)
ui_functions.print_warning_msg(path_to_source, header=False)
if len(sources_to_remove) > 0:
ui_functions.print_warning_msg(SYNC_REMOVE_LIST_END_FORMATTING,
header=False)
# Clone any new Git repositories
clone_repos(args, workspace_path, sources_to_clone,
new_manifest_to_check.repo_hooks, config,
new_manifest_to_check)
# Make a list of and only checkout repos that were newly cloned. Sync keeps repos on their initial active branches
# cloning the entire combo can prevent existing repos from correctly being returned to their proper branch
repos_to_checkout = []
if sources_to_clone:
for new_source in new_sources_for_current_combo:
for source in sources_to_clone:
if source.root == new_source.root:
repos_to_checkout.append(source)
repos_to_checkout.extend(
self.__check_combo_sha_tag_branch(workspace_path,
initial_common, new_common))
if repos_to_checkout:
checkout_repos(args.verbose, args.override, repos_to_checkout,
workspace_path, new_manifest_to_check)
#remove the old manifest file and copy the new one
ui_functions.print_info_msg(UPDATING_MANIFEST, header=False)
local_manifest_path = os.path.join(local_manifest_dir, 'Manifest.xml')
os.remove(local_manifest_path)
shutil.copy(global_manifest_path, local_manifest_path)
# Update the source manifest repository tag in the local copy of the manifest XML
new_manifest = ManifestXml(local_manifest_path)
try:
if 'source_manifest_repo' in vars(args).keys():
find_source_manifest_repo(new_manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo)
else:
find_source_manifest_repo(new_manifest, config['cfg_file'],
config['user_cfg_file'], None)
except EdkrepoManifestNotFoundException:
pass
def run_command(self, args, config):
print()
init_color_console(args.color)
pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'])
print()
cfg_manifest_repos, user_config_manifest_repos, conflicts = list_available_manifest_repos(config['cfg_file'], config['user_cfg_file'])
found_manifests = {}
manifests = {}
repo_urls = set()
config_manifest_repos_project_list = []
user_config_manifest_repos_project_list = []
for manifest_repo in cfg_manifest_repos:
# Get path to global manifest file
global_manifest_directory = config['cfg_file'].manifest_repo_abs_path(manifest_repo)
if args.verbose:
print(humble.MANIFEST_DIRECTORY)
print(global_manifest_directory)
print()
#Create a dictionary containing all the manifests listed in the CiIndex.xml file
index_path = os.path.join(global_manifest_directory, CI_INDEX_FILE_NAME)
print(index_path)
ci_index_xml = CiIndexXml(index_path)
config_manifest_repos_project_list = ci_index_xml.project_list
if args.archived:
config_manifest_repos_project_list.extend(ci_index_xml.archived_project_list)
for project in config_manifest_repos_project_list:
xml_file = ci_index_xml.get_project_xml(project)
manifest = ManifestXml(os.path.normpath(os.path.join(global_manifest_directory, xml_file)))
found_manifests['{}:{}'.format(manifest_repo, project)] = manifest
combo_list = [c.name for c in manifest.combinations]
if args.archived:
combo_list.extend([c.name for c in manifest.archived_combinations])
for combo in combo_list:
sources = manifest.get_repo_sources(combo)
for source in sources:
repo_urls.add(self.get_repo_url(source.remote_url))
for manifest_repo in user_config_manifest_repos:
# Get path to global manifest file
global_manifest_directory = config['user_cfg_file'].manifest_repo_abs_path(manifest_repo)
if args.verbose:
print(humble.MANIFEST_DIRECTORY)
print(global_manifest_directory)
print()
#Create a dictionary containing all the manifests listed in the CiIndex.xml file
index_path = os.path.join(global_manifest_directory, CI_INDEX_FILE_NAME)
ci_index_xml = CiIndexXml(index_path)
user_config_manifest_repos_project_list = ci_index_xml.project_list
if args.archived:
user_config_manifest_repos_project_list.extend(ci_index_xml.archived_project_list)
for project in user_config_manifest_repos_project_list:
xml_file = ci_index_xml.get_project_xml(project)
manifest = ManifestXml(os.path.normpath(os.path.join(global_manifest_directory, xml_file)))
found_manifests['{}:{}'.format(manifest_repo, project)] = manifest
combo_list = [c.name for c in manifest.combinations]
if args.archived:
combo_list.extend([c.name for c in manifest.archived_combinations])
for combo in combo_list:
sources = manifest.get_repo_sources(combo)
for source in sources:
repo_urls.add(self.get_repo_url(source.remote_url))
#Remove the manifest repo portion of the key is there is not a duplicate project name
key_list = list(found_manifests)
for entry in key_list:
new_key = entry.split(':')[1]
value = found_manifests[entry]
del found_manifests[entry]
for found_manifest in list(found_manifests):
if found_manifest.split(':')[1] == new_key:
new_key = 'Manifest Repository: {} Project: {}'.format(entry.split(':')[0], entry.split(':')[1])
break
if new_key in manifests.keys():
new_key = 'Manifest Repository: {} Project: {}'.format(entry.split(':'[0]), entry.split(':')[1])
manifests[new_key] = value
#Sort the manifests so projects will be displayed alphabetically
manifests = collections.OrderedDict(sorted(manifests.items()))
project_justify = len(max(manifests.keys(), key=len))
#Determine the names of the repositories
self.generate_repo_names(repo_urls, manifests, args.archived)
print(humble.REPOSITORIES)
#If the user provided a list of repositories to view, check to make sure
#at least one repository will be shown, if not provide an error
if args.repos and len([x for x in self.repo_names if x in args.repos]) <= 0:
raise EdkrepoInvalidParametersException(humble.REPO_NOT_FOUND_IN_MANIFEST.format(','.join(args.repos)))
#For each each git repository...
for repo_name in self.repo_names:
if args.repos and repo_name not in args.repos:
continue
repo = self.repo_names[repo_name][0]
print(humble.REPO_NAME_AND_URL.format(repo_name, repo))
print(humble.BRANCHES)
#Determine the list of branches that used by any branch combination in any manifest
branches = set()
for project_name in manifests:
combo_list = [c.name for c in manifests[project_name].combinations]
if args.archived:
combo_list.extend([c.name for c in manifests[project_name].archived_combinations])
for combo in combo_list:
sources = manifests[project_name].get_repo_sources(combo)
for source in sources:
if self.get_repo_url(source.remote_url) == repo:
branches.add(source.branch)
#Sort the branch names so they will be displayed alphabetically
#with the exception that if a branch named "master" exists, then it
#will be displayed first
branches = sorted(branches, key=str.casefold)
if 'master' in branches:
branches.remove('master')
branches.insert(0, 'master')
#For each interesting branch in the current git repository...
for branch in branches:
print(humble.BRANCH_FORMAT_STRING.format(branch))
#Determine the branch combinations that use that branch
for project_name in manifests:
combos = []
combo_list = [c.name for c in manifests[project_name].combinations]
if args.archived:
combo_list.extend([c.name for c in manifests[project_name].archived_combinations])
for combo in combo_list:
sources = manifests[project_name].get_repo_sources(combo)
for source in sources:
if self.get_repo_url(source.remote_url) == repo and source.branch == branch:
combos.append(combo)
break
if len(combos) > 0:
#Sort the branch combinations so they will be displayed alphabetically
#with the exception that the default branch combination for the manifest
#file will be displayed first
combos = sorted(combos, key=str.casefold)
default_combo = manifests[project_name].general_config.default_combo
if default_combo in combos:
combos.remove(default_combo)
combos.insert(0, default_combo)
first_combo = True
for combo in combos:
#Print the project name
if first_combo:
project_name_print = humble.PROJECT_NAME_FORMAT_STRING.format(project_name.ljust(project_justify))
first_combo = False
else:
project_name_print = '{} '.format((' ' * len(project_name)).ljust(project_justify))
#Print the branch combination name, if this is the default branch combination,
#then print it in green color with *'s around it
if default_combo == combo:
print(humble.DEFAULT_COMBO_FORMAT_STRING.format(project_name_print, combo))
else:
print(humble.COMBO_FORMAT_STRING.format(project_name_print, combo))
def run_command(self, args, config):
print()
cfg_file = config['cfg_file']
user_cfg = config['user_cfg_file']
cfg_man_repos, user_cfg_man_repos, conflicts = list_available_manifest_repos(
cfg_file, user_cfg)
man_repos = {}
pull_all_manifest_repos(cfg_file, user_cfg, False)
# Get paths to the global manifest dirs and their index files
for repo in cfg_man_repos:
global_manifest_directory = cfg_file.manifest_repo_abs_path(repo)
index_path = os.path.join(global_manifest_directory, 'CiIndex.xml')
man_repos[repo] = (global_manifest_directory, index_path)
for repo in user_cfg_man_repos:
global_manifest_directory = user_cfg.manifest_repo_abs_path(repo)
index_path = os.path.join(global_manifest_directory, 'CiIndex.xml')
man_repos[repo] = (global_manifest_directory, index_path)
try:
wkspc_manifest = get_workspace_manifest()
current_project = wkspc_manifest.project_info.codename
src_man_repo = find_source_manifest_repo(wkspc_manifest, cfg_file,
user_cfg, None)
except EdkrepoWorkspaceInvalidException:
current_project = None
src_man_repo = None
except EdkrepoManifestNotFoundException:
src_man_repo = None
for repo in man_repos.keys():
print()
ui_functions.print_info_msg("Manifest directory:", header=False)
ui_functions.print_info_msg(repo, header=False)
if args.verbose:
ui_functions.print_info_msg('Manifest directory path:',
header=False)
ui_functions.print_info_msg(man_repos[repo][0], header=False)
print()
ci_index_xml = CiIndexXml(man_repos[repo][1])
# Attempt to make sure the manifest data is good
try:
validate_manifest_repo(man_repos[repo][0], args.verbose,
args.archived)
except:
print()
ui_functions.print_info_msg("Projects:", header=False)
for project in sorted(ci_index_xml.project_list):
if (project == current_project and src_man_repo == repo) or (
not src_man_repo and project == current_project):
ui_functions.print_info_msg(project, header=False)
else:
ui_functions.print_warning_msg(project, header=False)
if args.verbose:
ui_functions.print_info_msg(" -> {}".format(
ci_index_xml.get_project_xml(project)),
header=False)
proj_manifest = ManifestXml(
find_project_in_single_index(project, ci_index_xml,
man_repos[repo][0])[1])
ui_functions.print_info_msg(" -> DevLead: {}".format(
' '.join(
x for x in proj_manifest.project_info.dev_leads)),
header=False)
if args.archived:
print()
ui_functions.print_info_msg("Archived Projects:", header=False)
for project in sorted(ci_index_xml.archived_project_list):
if project == current_project:
ui_functions.print_info_msg(project, header=False)
else:
ui_functions.print_warning_msg(project, header=False)
if args.verbose:
ui_functions.print_info_msg(" -> {}".format(
ci_index_xml.get_project_xml(project)),
header=False)
proj_manifest = ManifestXml(
find_project_in_single_index(
project, ci_index_xml, man_repos[repo][0])[1])
ui_functions.print_info_msg(" -> DevLead: {}".format(
' '.join(x for x in
proj_manifest.project_info.dev_leads)),
header=False)
def run_command(self, args, config):
less_path, use_less = find_less()
if use_less:
output_string = ''
separator = '\n'
cfg, user_cfg, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfg_file'])
try:
manifest = get_workspace_manifest()
pull_workspace_manifest_repo(manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo, False)
src_manifest_repo = find_source_manifest_repo(
manifest, config['cfg_file'], config['user_cfg_file'],
args.source_manifest_repo)
if src_manifest_repo in cfg:
manifest_directory = config['cfg_file'].manifest_repo_abs_path(
src_manifest_repo)
elif src_manifest_repo in user_cfg:
manifest_directory = config[
'user_cfg_file'].manifest_repo_abs_path(src_manifest_repo)
else:
raise EdkrepoManifestNotFoundException(
SOURCE_MANIFEST_REPO_NOT_FOUND.format(
manifest.project_info.codename))
except EdkrepoWorkspaceInvalidException:
if not args.project:
raise EdkrepoInvalidParametersException(humble.NOT_IN_WKSPCE)
else:
#arg parse provides a list so only use the first item since we are limiting users to one project
manifest_repo, src_cfg, manifest_path = find_project_in_all_indices(
args.project[0], config['cfg_file'],
config['user_cfg_file'],
PROJ_NOT_IN_REPO.format(args.project[0]),
SOURCE_MANIFEST_REPO_NOT_FOUND.format(args.project[0]))
if manifest_repo in cfg:
manifest_directory = config[
'cfg_file'].manifest_repo_abs_path(manifest_repo)
elif manifest_repo in user_cfg:
manifest_directory = config[
'user_cfg_file'].manifest_repo_abs_path(manifest_repo)
manifest = ManifestXml(manifest_path)
if manifest.general_config.pin_path is None:
print(humble.NO_PIN_FOLDER)
return
pin_folder = os.path.normpath(
os.path.join(manifest_directory, manifest.general_config.pin_path))
if args.verbose:
if not use_less:
print(humble.PIN_FOLDER.format(pin_folder))
else:
output_string = (humble.PIN_FOLDER.format(pin_folder))
for dirpath, _, filenames in os.walk(pin_folder):
for file in filenames:
pin_file = os.path.join(dirpath, file)
# Capture error output from manifest parser stdout to properly interleave with other pin data
stdout = sys.stdout
sys.stdout = io.StringIO()
try:
pin = ManifestXml(pin_file)
except TypeError:
continue
parse_output = sys.stdout.getvalue()
sys.stdout = stdout
if pin.project_info.codename == manifest.project_info.codename:
if not use_less:
print('Pin File: {}'.format(file))
if args.verbose and not args.description:
print('Parsing Errors: {}\n'.format(
parse_output.strip()))
elif args.verbose and args.description:
print('Parsing Errors: {}'.format(
parse_output.strip()))
if args.description:
print('Description: {}\n'.format(
pin.project_info.description))
elif use_less:
output_string = separator.join(
(output_string, 'Pin File: {}'.format(file)))
if args.verbose and not args.description:
output_string = separator.join(
(output_string, 'Parsing Errors: {}\n'.format(
parse_output.strip())))
elif args.verbose and args.description:
output_string = separator.join(
(output_string, 'Parsing Errors: {}'.format(
parse_output.strip())))
if args.description:
output_string = separator.join(
(output_string, 'Description: {}\n'.format(
pin.project_info.description)))
if less_path:
less_output = subprocess.Popen(
[str(less_path), '-F', '-R', '-S', '-X', '-K'],
stdin=subprocess.PIPE,
stdout=sys.stdout,
universal_newlines=True)
less_output.communicate(input=output_string)
def find_project_in_all_indices(project,
edkrepo_cfg,
edkrepo_user_cfg,
except_msg_man_repo,
except_msg_not_found,
man_repo=None):
'''
Finds the project in all manifest repositories listed in the edkrepo.efg and
edkrepo_user.cfg. If a project with the same name is found uses man_repo to select
the correct entry
'''
cfg_man_repos, user_cfg_man_repos, conflicts = list_available_manifest_repos(
edkrepo_cfg, edkrepo_user_cfg)
projects = {}
for repo in cfg_man_repos:
manifest_dir = edkrepo_cfg.manifest_repo_abs_path(repo)
# If the manifest directory does not exist clone it.
if not os.path.exists(manifest_dir):
pull_single_manifest_repo(
edkrepo_cfg.get_manifest_repo_url(repo),
edkrepo_cfg.get_manifest_repo_branch(repo),
edkrepo_cfg.get_manifest_repo_local_path(repo),
reset_hard=False)
index_file = CiIndexXml(os.path.join(manifest_dir, CI_INDEX_FILE_NAME))
found, man_path = find_project_in_single_index(project, index_file,
manifest_dir)
if found:
projects[repo] = ('edkrepo_cfg', man_path)
for repo in user_cfg_man_repos:
manifest_dir = edkrepo_user_cfg.manifest_repo_abs_path(repo)
if not os.path.exists(manifest_dir):
pull_single_manifest_repo(
edkrepo_user_cfg.get_manifest_repo_url(repo),
edkrepo_user_cfg.get_manifest_repo_branch(repo),
edkrepo_user_cfg.get_manifest_repo_local_path(repo),
reset_hard=False)
index_file = CiIndexXml(os.path.join(manifest_dir, CI_INDEX_FILE_NAME))
found, man_path = find_project_in_single_index(project, index_file,
manifest_dir)
if found:
projects[repo] = ('edkrepo_user_cfg', man_path)
if len(projects.keys()) == 1:
repo = list(projects.keys())[0]
return repo, projects[repo][0], projects[repo][1]
elif len(projects.keys()) > 1 and man_repo:
try:
return man_repo, projects[man_repo][0], projects[man_repo][1]
except KeyError:
raise EdkrepoInvalidParametersException(except_msg_man_repo)
elif os.path.isabs(project):
manifest = ManifestXml(project)
try:
found_manifest_repo, found_cfg, found_project = find_project_in_all_indices(
manifest.project_info.codename, edkrepo_cfg, edkrepo_user_cfg,
except_msg_man_repo, except_msg_not_found, man_repo)
return found_manifest_repo, found_cfg, project
except EdkrepoManifestNotFoundException:
return None, None, project
elif os.path.isfile(os.path.join(os.getcwd(), project)):
manifest = ManifestXml(os.path.join(os.getcwd(), project))
try:
found_manifest_repo, found_cfg, found_project = find_project_in_all_indices(
manifest.project_info.codename, edkrepo_cfg, edkrepo_user_cfg,
except_msg_man_repo, except_msg_not_found, man_repo)
return found_manifest_repo, found_cfg, project
except EdkrepoManifestNotFoundException:
return None, None, os.path.join(os.getcwd(), project)
elif not os.path.dirname(project):
for repo in cfg_man_repos:
if (man_repo and (repo == man_repo)) or not man_repo:
for dirpath, dirname, filenames in os.walk(
edkrepo_cfg.manifest_repo_abs_path(repo)):
if project in filenames:
return repo, 'edkrepo_cfg', os.path.join(
dirpath, project)
for repo in user_cfg_man_repos:
if (man_repo and (repo == man_repo)) or not man_repo:
for dirpath, dirname, filenames in os.walk(
edkrepo_user_cfg.manifest_repo_abs_path(repo)):
if project in filenames:
return repo, 'edkrepo_user_cfg', os.path.join(
dirpath, project)
raise EdkrepoManifestNotFoundException(
humble.PROJ_NOT_IN_REPO.format(project))
else:
raise EdkrepoManifestNotFoundException(
humble.PROJ_NOT_IN_REPO.format(project))
def run_command(self, args, config):
pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'],
False)
name_or_manifest = args.ProjectNameOrManifestFile
workspace_dir = args.Workspace
# Check to see if requested workspace exists. If not create it. If so check for empty
if workspace_dir == '.':
# User has selected the directory they are running edkrepo from
workspace_dir = os.getcwd()
else:
workspace_dir = os.path.abspath(workspace_dir)
if sys.platform == "win32":
subst = get_subst_drive_dict()
drive = os.path.splitdrive(workspace_dir)[0][0].upper()
if drive in subst:
workspace_dir = os.path.join(
subst[drive],
os.path.splitdrive(workspace_dir)[1][1:])
workspace_dir = os.path.normpath(workspace_dir)
if os.path.isdir(workspace_dir) and os.listdir(workspace_dir):
raise EdkrepoInvalidParametersException(CLONE_INVALID_WORKSPACE)
if not os.path.isdir(workspace_dir):
os.makedirs(workspace_dir)
cfg, user_cfg, conflicts = list_available_manifest_repos(
config['cfg_file'], config['user_cfg_file'])
try:
manifest_repo, source_cfg, global_manifest_path = find_project_in_all_indices(
args.ProjectNameOrManifestFile, config['cfg_file'],
config['user_cfg_file'],
PROJ_NOT_IN_REPO.format(args.ProjectNameOrManifestFile),
SOURCE_MANIFEST_REPO_NOT_FOUND.format(
args.ProjectNameOrManifestFile), args.source_manifest_repo)
except EdkrepoManifestNotFoundException:
raise EdkrepoInvalidParametersException(CLONE_INVALID_PROJECT_ARG)
# If this manifest is in a defined manifest repository validate the manifest within the manifest repo
if manifest_repo in cfg:
verify_single_manifest(config['cfg_file'], manifest_repo,
global_manifest_path)
update_editor_config(
config,
config['cfg_file'].manifest_repo_abs_path(manifest_repo))
elif manifest_repo in user_cfg:
verify_single_manifest(config['user_cfg_file'], manifest_repo,
global_manifest_path)
update_editor_config(
config,
config['user_cfg_file'].manifest_repo_abs_path(manifest_repo))
# Copy project manifest to local manifest dir and rename it Manifest.xml.
local_manifest_dir = os.path.join(workspace_dir, "repo")
os.makedirs(local_manifest_dir)
local_manifest_path = os.path.join(local_manifest_dir, "Manifest.xml")
shutil.copy(global_manifest_path, local_manifest_path)
manifest = ManifestXml(local_manifest_path)
# Update the source manifest repository tag in the local copy of the manifest XML
try:
if 'source_manifest_repo' in vars(args).keys():
find_source_manifest_repo(manifest, config['cfg_file'],
config['user_cfg_file'],
args.source_manifest_repo)
else:
find_source_manifest_repo(manifest, config['cfg_file'],
config['user_cfg_file'], None)
except EdkrepoManifestNotFoundException:
pass
# Process the combination name and make sure it can be found in the manifest
if args.Combination is not None:
try:
combo_name = case_insensitive_single_match(
args.Combination, combinations_in_manifest(manifest))
except:
#remove the repo directory and Manifest.xml from the workspace so the next time the user trys to clone
#they will have an empty workspace and then raise an exception
shutil.rmtree(local_manifest_dir)
raise EdkrepoInvalidParametersException(
CLONE_INVALID_COMBO_ARG)
manifest.write_current_combo(combo_name)
elif manifest.is_pin_file():
# Since pin files are subset of manifest files they do not have a "default combo" it is set to None. In this
# case use the current_combo instead.
combo_name = manifest.general_config.current_combo
else:
# If a combo was not specified or a pin file used the default combo should be cloned. Also ensure that the
# current combo is updated to match.
combo_name = manifest.general_config.default_combo
manifest.write_current_combo(combo_name)
# Get the list of repos to clone and clone them
repo_sources_to_clone = manifest.get_repo_sources(combo_name)
#check that the repo sources do not contain duplicated local roots
local_roots = [r.root for r in repo_sources_to_clone]
for root in local_roots:
if local_roots.count(root) > 1:
#remove the repo dir and manifest.xml so the next time the user trys to clone they will have an empty
#workspace
shutil.rmtree(local_manifest_dir)
raise EdkrepoManifestInvalidException(
CLONE_INVALID_LOCAL_ROOTS)
project_client_side_hooks = manifest.repo_hooks
# Set up submodule alt url config settings prior to cloning any repos
submodule_included_configs = write_included_config(
manifest.remotes, manifest.submodule_alternate_remotes,
local_manifest_dir)
write_conditional_include(workspace_dir, repo_sources_to_clone,
submodule_included_configs)
# Determine if caching is going to be used and then clone
cache_obj = get_repo_cache_obj(config)
if cache_obj is not None:
add_missing_cache_repos(cache_obj, manifest, args.verbose)
clone_repos(args, workspace_dir, repo_sources_to_clone,
project_client_side_hooks, config, manifest, cache_obj)
# Init submodules
if not args.skip_submodule:
cache_path = None
if cache_obj is not None:
cache_path = cache_obj.get_cache_path(
SUBMODULE_CACHE_REPO_NAME)
maintain_submodules(workspace_dir, manifest, combo_name,
args.verbose, cache_path)
# Perform a sparse checkout if requested.
use_sparse = args.sparse
sparse_settings = manifest.sparse_settings
if sparse_settings is None:
# No SparseCheckout information in manifest so skip sparse checkout
use_sparse = False
elif sparse_settings.sparse_by_default:
# Sparse settings enabled by default for the project
use_sparse = True
if args.nosparse:
# Command line disables sparse checkout
use_sparse = False
if use_sparse:
ui_functions.print_info_msg(SPARSE_CHECKOUT)
sparse_checkout(workspace_dir, repo_sources_to_clone, manifest)