def __check_for_new_manifest(self, args, config, initial_manifest, workspace_path, global_manifest_directory): #if the manifest repository for the current manifest was not found then there is no project with the manifest #specified project name in the index file for any of the manifest repositories if global_manifest_directory is None: if args.override: return else: raise EdkrepoManifestNotFoundException( SYNC_MANIFEST_NOT_FOUND.format( initial_manifest.project_info.codename)) #see if there is an entry in CiIndex.xml that matches the prject name of the current manifest index_path = os.path.join(global_manifest_directory, 'CiIndex.xml') ci_index_xml = CiIndexXml(index_path) if initial_manifest.project_info.codename not in ci_index_xml.project_list: if args.override: return else: raise EdkrepoManifestNotFoundException( SYNC_MANIFEST_NOT_FOUND.format( initial_manifest.project_info.codename)) ci_index_xml_rel_path = ci_index_xml.get_project_xml( initial_manifest.project_info.codename) global_manifest_path = os.path.join( global_manifest_directory, os.path.normpath(ci_index_xml_rel_path)) global_manifest = ManifestXml(global_manifest_path) if not initial_manifest.equals(global_manifest, True): ui_functions.print_warning_msg(SYNC_MANIFEST_DIFF_WARNING, header=False) ui_functions.print_info_msg(SYNC_MANIFEST_UPDATE, header=False)
def validate_manifestrepo(global_manifest_directory, verify_archived=False): manifestfile_validation = {} # Open CiIndex.xml and parse the data ci_index_filename = os.path.join(global_manifest_directory, 'CiIndex.xml') ci_index_xml = CiIndexXml(ci_index_filename) # Check every entry in the CiIndex.xml file to verify consistency project_list = ci_index_xml.project_list if verify_archived: project_list.extend(ci_index_xml.archived_project_list) for project in project_list: manifest_filepath = None manifest_obj = None validate_parsing = None val_name_duplication = None results = [] # Get project XML file path project_path = os.path.normpath(ci_index_xml.get_project_xml(project)) # Validate parsing and add results manifest_filepath = os.path.join(global_manifest_directory, project_path) manifest_obj = ValidateManifest(manifest_filepath) validate_parsing = manifest_obj.validate_parsing() results.append(validate_parsing) # Validate Code name and add results val_codename = manifest_obj.validate_codename(project) results.append(val_codename) # Verify that name not already used and add results. The name is not case sensitive. val_name_duplication = manifest_obj.validate_case_insensitive_single_match( project, project_list, ci_index_filename) results.append(val_name_duplication) # Add all results to a dictionary with file path as key manifestfile_validation[manifest_filepath] = results return manifestfile_validation
def find_source_manifest_repo(project_manifest, edkrepo_cfg, edkrepo_user_cfg, man_repo=None, update_source_manifest_repo=True): ''' Finds the source manifest repo for a given project. ''' if project_manifest.general_config.source_manifest_repo: source_manifest_repo = project_manifest.general_config.source_manifest_repo cfg_manifest_repos, user_cfg_manifest_repos, _ = list_available_manifest_repos( edkrepo_cfg, edkrepo_user_cfg) manifest_dir = None if source_manifest_repo in cfg_manifest_repos: manifest_dir = edkrepo_cfg.manifest_repo_abs_path( source_manifest_repo) elif source_manifest_repo in user_cfg_manifest_repos: manifest_dir = edkrepo_user_cfg.manifest_repo_abs_path( source_manifest_repo) if manifest_dir is not None: index_file_path = os.path.join(manifest_dir, CI_INDEX_FILE_NAME) if os.path.isfile(index_file_path): index_file = CiIndexXml(index_file_path) found, _ = find_project_in_single_index( project_manifest.project_info.codename, index_file, manifest_dir) if found: return source_manifest_repo try: src_man_repo, _, _ = find_project_in_all_indices( project_manifest.project_info.codename, edkrepo_cfg, edkrepo_user_cfg, humble.PROJ_NOT_IN_REPO.format( project_manifest.project_info.codename), humble.SOURCE_MANIFEST_REPO_NOT_FOUND.format( project_manifest.project_info.codename), man_repo) except EdkrepoManifestNotFoundException: src_man_repo = None if src_man_repo is not None and update_source_manifest_repo: project_manifest.write_source_manifest_repo(src_man_repo) return src_man_repo
def __update_local_manifest(self, args, config, initial_manifest, workspace_path, global_manifest_directory): #if the manifest repository for the current manifest was not found then there is no project with the manifest #specified project name in the index file for any of the manifest repositories if global_manifest_directory is None: raise EdkrepoManifestNotFoundException( SOURCE_MANIFEST_REPO_NOT_FOUND.format( initial_manifest.project_info.codename)) local_manifest_dir = os.path.join(workspace_path, 'repo') current_combo = initial_manifest.general_config.current_combo initial_sources = initial_manifest.get_repo_sources(current_combo) # Do a fetch for each repo in the initial to ensure that newly created upstream branches are available for initial_repo in initial_sources: local_repo_path = os.path.join(workspace_path, initial_repo.root) repo = Repo(local_repo_path) origin = repo.remotes.origin try: origin.fetch() except GitCommandError as e: prune_needed = False prune_needed_heuristic_str = "error: some local refs could not be updated" if e.stdout.strip().find(prune_needed_heuristic_str) != -1: prune_needed = True if e.stderr.strip().find(prune_needed_heuristic_str) != -1: prune_needed = True if prune_needed: # The sleep is to give the operating system time to close all the file handles that Git has open time.sleep(1.0) repo.git.remote('prune', 'origin') time.sleep(1.0) origin.fetch() else: raise #see if there is an entry in CiIndex.xml that matches the prject name of the current manifest index_path = os.path.join(global_manifest_directory, 'CiIndex.xml') ci_index_xml = CiIndexXml(index_path) if initial_manifest.project_info.codename not in ci_index_xml.project_list: raise EdkrepoManifestNotFoundException( SYNC_MANIFEST_NOT_FOUND.format( initial_manifest.project_info.codename)) initial_manifest_remotes = { name: url for name, url in initial_manifest.remotes } ci_index_xml_rel_path = os.path.normpath( ci_index_xml.get_project_xml( initial_manifest.project_info.codename)) global_manifest_path = os.path.join(global_manifest_directory, ci_index_xml_rel_path) new_manifest_to_check = ManifestXml(global_manifest_path) # Does the current combo exist in the new manifest? If not check to see if you can use the repo sources from # the default combo initial_combos = combinations_in_manifest(initial_manifest) new_combos = combinations_in_manifest(new_manifest_to_check) if current_combo not in new_combos: new_sources_for_current_combo = new_manifest_to_check.get_repo_sources( new_manifest_to_check.general_config.default_combo) new_sources = new_sources_for_current_combo else: new_sources_for_current_combo = new_manifest_to_check.get_repo_sources( current_combo) new_sources = new_manifest_to_check.get_repo_sources(current_combo) remove_included_config(initial_manifest.remotes, initial_manifest.submodule_alternate_remotes, local_manifest_dir) write_included_config( new_manifest_to_check.remotes, new_manifest_to_check.submodule_alternate_remotes, local_manifest_dir) self.__check_submodule_config(workspace_path, new_manifest_to_check, new_sources_for_current_combo) # Check that the repo sources lists are the same. If they are not the same and the override flag is not set, throw an exception. if not args.override and set(initial_sources) != set(new_sources): raise EdkrepoManifestChangedException( SYNC_REPO_CHANGE.format( initial_manifest.project_info.codename)) elif args.override and set(initial_sources) != set(new_sources): #get a set of repo source tuples that are not in both the new and old manifest uncommon_sources = [] initial_common = [] new_common = [] for initial in initial_sources: common = False for new in new_sources: if initial.root == new.root: if initial.remote_name == new.remote_name: if initial.remote_url == new.remote_url: # If the source is unchanged between the old and the new manifest, # add it to the common lists common = True initial_common.append(initial) new_common.append(new) break # If the source is different between the old and the new manifest, add it to the uncommon list if not common: uncommon_sources.append(initial) for new in new_sources: common = False for initial in initial_sources: if new.root == initial.root: if new.remote_name == initial.remote_name: if new.remote_url == initial.remote_url: common = True break # If the source is different between the old and the new manifest, add it to the uncommon list if not common: uncommon_sources.append(new) uncommon_sources = set(uncommon_sources) initial_common = set(initial_common) new_common = set(new_common) sources_to_move = [] sources_to_remove = [] sources_to_clone = [] for source in uncommon_sources: found_source = False for source_to_check in initial_sources: if source_to_check.root == source.root: if source_to_check.remote_name == source.remote_name: if source_to_check.remote_url == source.remote_url: found_source = True break # If the source that is different came from the old manifest, then it is now outdated and either needs # to be deleted or moved to an archival location. if found_source: roots = [s.root for s in new_sources] # If there is a source in the new manifest that goes into the same folder name as a source in the # old manifest, then we need to move that old folder to an archival location. if source.root in roots: sources_to_move.append(source) else: # If it doesn't exist at all in the new manifest, tell the user it is old and no longer used. sources_to_remove.append(source) else: # If the source that is different came from the new manifest, then we need to clone that new # Git repository. sources_to_clone.append(source) # Move the obsolete Git repositories to archival locations. for source in sources_to_move: old_dir = os.path.join(workspace_path, source.root) new_dir = generate_name_for_obsolete_backup(old_dir) ui_functions.print_warning_msg(SYNC_SOURCE_MOVE_WARNING.format( source.root, new_dir), header=False) new_dir = os.path.join(workspace_path, new_dir) try: shutil.move(old_dir, new_dir) except: ui_functions.print_error_msg(SYNC_MOVE_FAILED.format( initial_dir=source.root, new_dir=new_dir), header=False) raise # Tell the user about any Git repositories that are no longer used. if len(sources_to_remove) > 0: ui_functions.print_warning_msg(SYNC_REMOVE_WARNING, header=False) for source in sources_to_remove: path_to_source = os.path.join(workspace_path, source.root) ui_functions.print_warning_msg(path_to_source, header=False) if len(sources_to_remove) > 0: ui_functions.print_warning_msg(SYNC_REMOVE_LIST_END_FORMATTING, header=False) # Clone any new Git repositories clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, new_manifest_to_check) # Make a list of and only checkout repos that were newly cloned. Sync keeps repos on their initial active branches # cloning the entire combo can prevent existing repos from correctly being returned to their proper branch repos_to_checkout = [] if sources_to_clone: for new_source in new_sources_for_current_combo: for source in sources_to_clone: if source.root == new_source.root: repos_to_checkout.append(source) repos_to_checkout.extend( self.__check_combo_sha_tag_branch(workspace_path, initial_common, new_common)) if repos_to_checkout: checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check) #remove the old manifest file and copy the new one ui_functions.print_info_msg(UPDATING_MANIFEST, header=False) local_manifest_path = os.path.join(local_manifest_dir, 'Manifest.xml') os.remove(local_manifest_path) shutil.copy(global_manifest_path, local_manifest_path) # Update the source manifest repository tag in the local copy of the manifest XML new_manifest = ManifestXml(local_manifest_path) try: if 'source_manifest_repo' in vars(args).keys(): find_source_manifest_repo(new_manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) else: find_source_manifest_repo(new_manifest, config['cfg_file'], config['user_cfg_file'], None) except EdkrepoManifestNotFoundException: pass
def run_command(self, args, config): print() init_color_console(args.color) pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file']) print() cfg_manifest_repos, user_config_manifest_repos, conflicts = list_available_manifest_repos(config['cfg_file'], config['user_cfg_file']) found_manifests = {} manifests = {} repo_urls = set() config_manifest_repos_project_list = [] user_config_manifest_repos_project_list = [] for manifest_repo in cfg_manifest_repos: # Get path to global manifest file global_manifest_directory = config['cfg_file'].manifest_repo_abs_path(manifest_repo) if args.verbose: print(humble.MANIFEST_DIRECTORY) print(global_manifest_directory) print() #Create a dictionary containing all the manifests listed in the CiIndex.xml file index_path = os.path.join(global_manifest_directory, CI_INDEX_FILE_NAME) print(index_path) ci_index_xml = CiIndexXml(index_path) config_manifest_repos_project_list = ci_index_xml.project_list if args.archived: config_manifest_repos_project_list.extend(ci_index_xml.archived_project_list) for project in config_manifest_repos_project_list: xml_file = ci_index_xml.get_project_xml(project) manifest = ManifestXml(os.path.normpath(os.path.join(global_manifest_directory, xml_file))) found_manifests['{}:{}'.format(manifest_repo, project)] = manifest combo_list = [c.name for c in manifest.combinations] if args.archived: combo_list.extend([c.name for c in manifest.archived_combinations]) for combo in combo_list: sources = manifest.get_repo_sources(combo) for source in sources: repo_urls.add(self.get_repo_url(source.remote_url)) for manifest_repo in user_config_manifest_repos: # Get path to global manifest file global_manifest_directory = config['user_cfg_file'].manifest_repo_abs_path(manifest_repo) if args.verbose: print(humble.MANIFEST_DIRECTORY) print(global_manifest_directory) print() #Create a dictionary containing all the manifests listed in the CiIndex.xml file index_path = os.path.join(global_manifest_directory, CI_INDEX_FILE_NAME) ci_index_xml = CiIndexXml(index_path) user_config_manifest_repos_project_list = ci_index_xml.project_list if args.archived: user_config_manifest_repos_project_list.extend(ci_index_xml.archived_project_list) for project in user_config_manifest_repos_project_list: xml_file = ci_index_xml.get_project_xml(project) manifest = ManifestXml(os.path.normpath(os.path.join(global_manifest_directory, xml_file))) found_manifests['{}:{}'.format(manifest_repo, project)] = manifest combo_list = [c.name for c in manifest.combinations] if args.archived: combo_list.extend([c.name for c in manifest.archived_combinations]) for combo in combo_list: sources = manifest.get_repo_sources(combo) for source in sources: repo_urls.add(self.get_repo_url(source.remote_url)) #Remove the manifest repo portion of the key is there is not a duplicate project name key_list = list(found_manifests) for entry in key_list: new_key = entry.split(':')[1] value = found_manifests[entry] del found_manifests[entry] for found_manifest in list(found_manifests): if found_manifest.split(':')[1] == new_key: new_key = 'Manifest Repository: {} Project: {}'.format(entry.split(':')[0], entry.split(':')[1]) break if new_key in manifests.keys(): new_key = 'Manifest Repository: {} Project: {}'.format(entry.split(':'[0]), entry.split(':')[1]) manifests[new_key] = value #Sort the manifests so projects will be displayed alphabetically manifests = collections.OrderedDict(sorted(manifests.items())) project_justify = len(max(manifests.keys(), key=len)) #Determine the names of the repositories self.generate_repo_names(repo_urls, manifests, args.archived) print(humble.REPOSITORIES) #If the user provided a list of repositories to view, check to make sure #at least one repository will be shown, if not provide an error if args.repos and len([x for x in self.repo_names if x in args.repos]) <= 0: raise EdkrepoInvalidParametersException(humble.REPO_NOT_FOUND_IN_MANIFEST.format(','.join(args.repos))) #For each each git repository... for repo_name in self.repo_names: if args.repos and repo_name not in args.repos: continue repo = self.repo_names[repo_name][0] print(humble.REPO_NAME_AND_URL.format(repo_name, repo)) print(humble.BRANCHES) #Determine the list of branches that used by any branch combination in any manifest branches = set() for project_name in manifests: combo_list = [c.name for c in manifests[project_name].combinations] if args.archived: combo_list.extend([c.name for c in manifests[project_name].archived_combinations]) for combo in combo_list: sources = manifests[project_name].get_repo_sources(combo) for source in sources: if self.get_repo_url(source.remote_url) == repo: branches.add(source.branch) #Sort the branch names so they will be displayed alphabetically #with the exception that if a branch named "master" exists, then it #will be displayed first branches = sorted(branches, key=str.casefold) if 'master' in branches: branches.remove('master') branches.insert(0, 'master') #For each interesting branch in the current git repository... for branch in branches: print(humble.BRANCH_FORMAT_STRING.format(branch)) #Determine the branch combinations that use that branch for project_name in manifests: combos = [] combo_list = [c.name for c in manifests[project_name].combinations] if args.archived: combo_list.extend([c.name for c in manifests[project_name].archived_combinations]) for combo in combo_list: sources = manifests[project_name].get_repo_sources(combo) for source in sources: if self.get_repo_url(source.remote_url) == repo and source.branch == branch: combos.append(combo) break if len(combos) > 0: #Sort the branch combinations so they will be displayed alphabetically #with the exception that the default branch combination for the manifest #file will be displayed first combos = sorted(combos, key=str.casefold) default_combo = manifests[project_name].general_config.default_combo if default_combo in combos: combos.remove(default_combo) combos.insert(0, default_combo) first_combo = True for combo in combos: #Print the project name if first_combo: project_name_print = humble.PROJECT_NAME_FORMAT_STRING.format(project_name.ljust(project_justify)) first_combo = False else: project_name_print = '{} '.format((' ' * len(project_name)).ljust(project_justify)) #Print the branch combination name, if this is the default branch combination, #then print it in green color with *'s around it if default_combo == combo: print(humble.DEFAULT_COMBO_FORMAT_STRING.format(project_name_print, combo)) else: print(humble.COMBO_FORMAT_STRING.format(project_name_print, combo))
def run_command(self, args, config): print() cfg_file = config['cfg_file'] user_cfg = config['user_cfg_file'] cfg_man_repos, user_cfg_man_repos, conflicts = list_available_manifest_repos( cfg_file, user_cfg) man_repos = {} pull_all_manifest_repos(cfg_file, user_cfg, False) # Get paths to the global manifest dirs and their index files for repo in cfg_man_repos: global_manifest_directory = cfg_file.manifest_repo_abs_path(repo) index_path = os.path.join(global_manifest_directory, 'CiIndex.xml') man_repos[repo] = (global_manifest_directory, index_path) for repo in user_cfg_man_repos: global_manifest_directory = user_cfg.manifest_repo_abs_path(repo) index_path = os.path.join(global_manifest_directory, 'CiIndex.xml') man_repos[repo] = (global_manifest_directory, index_path) try: wkspc_manifest = get_workspace_manifest() current_project = wkspc_manifest.project_info.codename src_man_repo = find_source_manifest_repo(wkspc_manifest, cfg_file, user_cfg, None) except EdkrepoWorkspaceInvalidException: current_project = None src_man_repo = None except EdkrepoManifestNotFoundException: src_man_repo = None for repo in man_repos.keys(): print() ui_functions.print_info_msg("Manifest directory:", header=False) ui_functions.print_info_msg(repo, header=False) if args.verbose: ui_functions.print_info_msg('Manifest directory path:', header=False) ui_functions.print_info_msg(man_repos[repo][0], header=False) print() ci_index_xml = CiIndexXml(man_repos[repo][1]) # Attempt to make sure the manifest data is good try: validate_manifest_repo(man_repos[repo][0], args.verbose, args.archived) except: print() ui_functions.print_info_msg("Projects:", header=False) for project in sorted(ci_index_xml.project_list): if (project == current_project and src_man_repo == repo) or ( not src_man_repo and project == current_project): ui_functions.print_info_msg(project, header=False) else: ui_functions.print_warning_msg(project, header=False) if args.verbose: ui_functions.print_info_msg(" -> {}".format( ci_index_xml.get_project_xml(project)), header=False) proj_manifest = ManifestXml( find_project_in_single_index(project, ci_index_xml, man_repos[repo][0])[1]) ui_functions.print_info_msg(" -> DevLead: {}".format( ' '.join( x for x in proj_manifest.project_info.dev_leads)), header=False) if args.archived: print() ui_functions.print_info_msg("Archived Projects:", header=False) for project in sorted(ci_index_xml.archived_project_list): if project == current_project: ui_functions.print_info_msg(project, header=False) else: ui_functions.print_warning_msg(project, header=False) if args.verbose: ui_functions.print_info_msg(" -> {}".format( ci_index_xml.get_project_xml(project)), header=False) proj_manifest = ManifestXml( find_project_in_single_index( project, ci_index_xml, man_repos[repo][0])[1]) ui_functions.print_info_msg(" -> DevLead: {}".format( ' '.join(x for x in proj_manifest.project_info.dev_leads)), header=False)
def find_project_in_all_indices(project, edkrepo_cfg, edkrepo_user_cfg, except_msg_man_repo, except_msg_not_found, man_repo=None): ''' Finds the project in all manifest repositories listed in the edkrepo.efg and edkrepo_user.cfg. If a project with the same name is found uses man_repo to select the correct entry ''' cfg_man_repos, user_cfg_man_repos, conflicts = list_available_manifest_repos( edkrepo_cfg, edkrepo_user_cfg) projects = {} for repo in cfg_man_repos: manifest_dir = edkrepo_cfg.manifest_repo_abs_path(repo) # If the manifest directory does not exist clone it. if not os.path.exists(manifest_dir): pull_single_manifest_repo( edkrepo_cfg.get_manifest_repo_url(repo), edkrepo_cfg.get_manifest_repo_branch(repo), edkrepo_cfg.get_manifest_repo_local_path(repo), reset_hard=False) index_file = CiIndexXml(os.path.join(manifest_dir, CI_INDEX_FILE_NAME)) found, man_path = find_project_in_single_index(project, index_file, manifest_dir) if found: projects[repo] = ('edkrepo_cfg', man_path) for repo in user_cfg_man_repos: manifest_dir = edkrepo_user_cfg.manifest_repo_abs_path(repo) if not os.path.exists(manifest_dir): pull_single_manifest_repo( edkrepo_user_cfg.get_manifest_repo_url(repo), edkrepo_user_cfg.get_manifest_repo_branch(repo), edkrepo_user_cfg.get_manifest_repo_local_path(repo), reset_hard=False) index_file = CiIndexXml(os.path.join(manifest_dir, CI_INDEX_FILE_NAME)) found, man_path = find_project_in_single_index(project, index_file, manifest_dir) if found: projects[repo] = ('edkrepo_user_cfg', man_path) if len(projects.keys()) == 1: repo = list(projects.keys())[0] return repo, projects[repo][0], projects[repo][1] elif len(projects.keys()) > 1 and man_repo: try: return man_repo, projects[man_repo][0], projects[man_repo][1] except KeyError: raise EdkrepoInvalidParametersException(except_msg_man_repo) elif os.path.isabs(project): manifest = ManifestXml(project) try: found_manifest_repo, found_cfg, found_project = find_project_in_all_indices( manifest.project_info.codename, edkrepo_cfg, edkrepo_user_cfg, except_msg_man_repo, except_msg_not_found, man_repo) return found_manifest_repo, found_cfg, project except EdkrepoManifestNotFoundException: return None, None, project elif os.path.isfile(os.path.join(os.getcwd(), project)): manifest = ManifestXml(os.path.join(os.getcwd(), project)) try: found_manifest_repo, found_cfg, found_project = find_project_in_all_indices( manifest.project_info.codename, edkrepo_cfg, edkrepo_user_cfg, except_msg_man_repo, except_msg_not_found, man_repo) return found_manifest_repo, found_cfg, project except EdkrepoManifestNotFoundException: return None, None, os.path.join(os.getcwd(), project) elif not os.path.dirname(project): for repo in cfg_man_repos: if (man_repo and (repo == man_repo)) or not man_repo: for dirpath, dirname, filenames in os.walk( edkrepo_cfg.manifest_repo_abs_path(repo)): if project in filenames: return repo, 'edkrepo_cfg', os.path.join( dirpath, project) for repo in user_cfg_man_repos: if (man_repo and (repo == man_repo)) or not man_repo: for dirpath, dirname, filenames in os.walk( edkrepo_user_cfg.manifest_repo_abs_path(repo)): if project in filenames: return repo, 'edkrepo_user_cfg', os.path.join( dirpath, project) raise EdkrepoManifestNotFoundException( humble.PROJ_NOT_IN_REPO.format(project)) else: raise EdkrepoManifestNotFoundException( humble.PROJ_NOT_IN_REPO.format(project))