def __check_for_new_manifest(self, args, config, initial_manifest, workspace_path, global_manifest_directory): #if the manifest repository for the current manifest was not found then there is no project with the manifest #specified project name in the index file for any of the manifest repositories if global_manifest_directory is None: if args.override: return else: raise EdkrepoManifestNotFoundException( SYNC_MANIFEST_NOT_FOUND.format( initial_manifest.project_info.codename)) #see if there is an entry in CiIndex.xml that matches the prject name of the current manifest index_path = os.path.join(global_manifest_directory, 'CiIndex.xml') ci_index_xml = CiIndexXml(index_path) if initial_manifest.project_info.codename not in ci_index_xml.project_list: if args.override: return else: raise EdkrepoManifestNotFoundException( SYNC_MANIFEST_NOT_FOUND.format( initial_manifest.project_info.codename)) ci_index_xml_rel_path = ci_index_xml.get_project_xml( initial_manifest.project_info.codename) global_manifest_path = os.path.join( global_manifest_directory, os.path.normpath(ci_index_xml_rel_path)) global_manifest = ManifestXml(global_manifest_path) if not initial_manifest.equals(global_manifest, True): ui_functions.print_warning_msg(SYNC_MANIFEST_DIFF_WARNING, header=False) ui_functions.print_info_msg(SYNC_MANIFEST_UPDATE, header=False)
def run_command(self, args, config): ui_functions.init_color_console(args.color) manifest = get_workspace_manifest() combo_archive = [] combo_list = [c.name for c in manifest.combinations] if args.archived: combo_archive = [c.name for c in manifest.archived_combinations] combo_list.extend(combo_archive) if manifest.general_config.current_combo not in combo_list: combo_list.append(manifest.general_config.current_combo) for combo in sorted(combo_list): if combo == manifest.general_config.current_combo: print("* {}{}{}".format(Fore.GREEN, combo, Fore.RESET)) elif combo in combo_archive: print(" {}{}{}{}".format(Fore.YELLOW, Style.BRIGHT, combo, Style.RESET_ALL)) else: ui_functions.print_info_msg(" {}".format(combo), header=False) if args.verbose: sources = manifest.get_repo_sources(combo) length = len(max([source.root for source in sources], key=len)) for source in sources: ui_functions.print_info_msg(" {} : {}".format( source.root.ljust(length), source.branch), header=False)
def run_command(self, args, config): workspace_path = get_workspace_path() manifest = get_workspace_manifest() manifest_repo = find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) cfg, user_cfg, conflicts = list_available_manifest_repos( config['cfg_file'], config['user_cfg_file']) if manifest_repo in cfg: manifest_repo_path = config['cfg_file'].manifest_repo_abs_path( manifest_repo) elif manifest_repo in user_cfg: manifest_repo_path = config[ 'user_cfg_file'].manifest_repo_abs_path(manifest_repo) else: manifest_repo_path = None pin_path = self.__get_pin_path(args, workspace_path, manifest_repo_path, manifest) pin = ManifestXml(pin_path) manifest_sources = manifest.get_repo_sources( manifest.general_config.current_combo) check_dirty_repos(manifest, workspace_path) for source in manifest_sources: local_path = os.path.join(workspace_path, source.root) repo = Repo(local_path) origin = repo.remotes.origin origin.fetch() self.__pin_matches_project(pin, manifest, workspace_path) sparse_enabled = sparse_checkout_enabled(workspace_path, manifest_sources) if sparse_enabled: ui_functions.print_info_msg(SPARSE_RESET, header=False) reset_sparse_checkout(workspace_path, manifest_sources) submodule_combo = pin.general_config.current_combo try: deinit_full(workspace_path, manifest, args.verbose) except Exception as e: ui_functions.print_error_msg(SUBMODULE_DEINIT_FAILED, header=False) if args.verbose: ui_functions.print_error_msg(e, header=False) pin_repo_sources = pin.get_repo_sources( pin.general_config.current_combo) try: checkout_repos(args.verbose, args.override, pin_repo_sources, workspace_path, manifest) manifest.write_current_combo(humble.PIN_COMBO.format(args.pinfile)) finally: cache_path = None cache_obj = get_repo_cache_obj(config) if cache_obj is not None: cache_path = cache_obj.get_cache_path( SUBMODULE_CACHE_REPO_NAME) maintain_submodules(workspace_path, pin, submodule_combo, args.verbose, cache_path) if sparse_enabled: ui_functions.print_info_msg(SPARSE_CHECKOUT, header=False) sparse_checkout(workspace_path, pin_repo_sources, manifest)
def run_command(self, args, config): cfg_repos, user_cfg_repos, conflicts = list_available_manifest_repos( config['cfg_file'], config['user_cfg_file']) if args.action == 'list': for repo in cfg_repos: ui_functions.print_info_msg(humble.CFG_LIST_ENTRY.format(repo), header=False) for repo in user_cfg_repos: ui_functions.print_info_msg( humble.USER_CFG_LIST_ENTRY.format(repo), header=False) elif (args.action == ('add' or 'remove')) and not args.name: raise EdkrepoInvalidParametersException(humble.NAME_REQUIRED) elif args.action == 'add' and (not args.branch or not args.url or not args.path): raise EdkrepoInvalidParametersException(humble.ADD_REQUIRED) elif args.action == 'remove' and args.name and args.name in cfg_repos: raise EdkrepoInvalidParametersException(humble.CANNOT_REMOVE_CFG) elif args.action == 'remove' and args.name not in config[ 'user_cfg_file'].manifest_repo_list: raise EdkrepoInvalidParametersException(humble.REMOVE_NOT_EXIST) elif args.action == 'add' and (args.name in cfg_repos or args.name in user_cfg_repos): raise EdkrepoInvalidParametersException( humble.ALREADY_EXISTS.format(args.name)) user_cfg_file_path = config['user_cfg_file'].cfg_filename if args.action == 'add' or 'remove': user_cfg_file = configparser.ConfigParser(allow_no_value=True) user_cfg_file.read(user_cfg_file_path) if args.action == 'add': if not user_cfg_file.has_section('manifest-repos'): user_cfg_file.add_section('manifest-repos') user_cfg_file.set('manifest-repos', args.name, None) user_cfg_file.add_section(args.name) user_cfg_file.set(args.name, 'URL', args.url) user_cfg_file.set(args.name, 'Branch', args.branch) user_cfg_file.set(args.name, 'LocalPath', args.path) if args.action == 'remove': if user_cfg_file.has_section('manifest-repos'): if user_cfg_file.has_option('manifest-repos', args.name): user_cfg_file.remove_option('manifest-repos', args.name) else: raise EdkrepoInvalidParametersException( humble.REMOVE_NOT_EXIST) else: raise EdkrepoInvalidParametersException( humble.REMOVE_NOT_EXIST) if user_cfg_file.has_section(args.name): user_cfg_file.remove_section(args.name) else: raise EdkrepoInvalidParametersException( humble.REMOVE_NOT_EXIST) with open(user_cfg_file_path, 'w') as cfg_stream: user_cfg_file.write(cfg_stream)
def run_command(self, args, config): workspace_path = get_workspace_path() manifest = get_workspace_manifest() manifest_config = manifest.general_config repo_sources_to_reset = manifest.get_repo_sources(manifest_config.current_combo) for repo_to_reset in repo_sources_to_reset: local_repo_path = os.path.join(workspace_path, repo_to_reset.root) repo = Repo(local_repo_path) if args.verbose: ui_functions.print_info_msg("{}Resetting {}".format("Hard " if args.hard else "", repo_to_reset.root)) repo.head.reset(working_tree=args.hard)
def run_command(self, args, config): workspace_path = get_workspace_path() initial_manifest = get_workspace_manifest() current_combo = initial_manifest.general_config.current_combo current_sources = initial_manifest.get_repo_sources(current_combo) ui_functions.print_info_msg( humble.STATUS_CURRENT_COMBO.format(current_combo), header=False) for current_repo in current_sources: local_repo_path = os.path.join(workspace_path, current_repo.root) repo = Repo(local_repo_path) ui_functions.print_info_msg("{}: {}\n".format( current_repo.root, repo.git.status()), header=False)
def run_command(self, args, config): workspace_path = get_workspace_path() manifest = get_workspace_manifest() # Set the pinname/path == to the file name provided. # If a relative paths is provided save the file relative to the current working directory. if os.path.isabs(os.path.normpath(args.PinFileName)): pin_file_name = os.path.normpath(args.PinFileName) else: pin_file_name = os.path.abspath(os.path.normpath(args.PinFileName)) # If the directory that the pin file is saved in does not exist create it and ensure pin file name uniqueness. if os.path.isfile(pin_file_name): raise EdkrepoInvalidParametersException(PIN_FILE_ALREADY_EXISTS) if not os.path.exists(os.path.dirname(pin_file_name)): os.mkdir(os.path.dirname(pin_file_name)) repo_sources = manifest.get_repo_sources( manifest.general_config.current_combo) # get the repo sources and commit ids for the pin ui_functions.print_info_msg(GENERATING_PIN_DATA.format( manifest.project_info.codename, manifest.general_config.current_combo), header=False) updated_repo_sources = [] for repo_source in repo_sources: local_repo_path = os.path.join(workspace_path, repo_source.root) if not os.path.exists(local_repo_path): raise EdkrepoWorkspaceCorruptException( MISSING_REPO.format(repo_source.root)) repo = Repo(local_repo_path) commit_id = repo.head.commit.hexsha if args.verbose: ui_functions.print_info_msg(GENERATING_REPO_DATA.format( repo_source.root), header=False) ui_functions.print_info_msg(BRANCH.format(repo_source.branch), header=False) ui_functions.print_info_msg(COMMIT.format(commit_id), header=False) updated_repo_source = repo_source._replace(commit=commit_id) updated_repo_sources.append(updated_repo_source) # create the pin ui_functions.print_info_msg(WRITING_PIN_FILE.format(pin_file_name), header=False) manifest.generate_pin_xml(args.Description, manifest.general_config.current_combo, updated_repo_sources, filename=pin_file_name)
def run_command(self, args, config): workspace_path = get_workspace_path() manifest = get_workspace_manifest() manifest_config = manifest.general_config repo_sources_to_clean = manifest.get_repo_sources( manifest_config.current_combo) for repo_to_clean in repo_sources_to_clean: local_repo_path = os.path.join(workspace_path, repo_to_clean.root) repo = Repo(local_repo_path) result = repo.git.clean(f=args.force, d=args.dirs, n=(not args.force), q=(args.quiet and args.force)) if result: ui_functions.print_info_msg(result, header=False)
def maintain_submodules(workspace, manifest, combo_name, verbose=False, cache_path=None): """ Updates the submodules for a specific repo. workspace - Path to the current workspace. manifest - The manifest parser object for the project. combo_name - The combination name to use for submodule maintenance. verbose - Enable verbose messages. cache_path - Path to the submodule cache repo. A value of None indicates that no cache repo exists. """ # Process each repo that may have submodules enabled ui_functions.print_info_msg(strings.SUBMOD_INIT_UPDATE) repo_sources = manifest.get_repo_sources(combo_name) for source in repo_sources: # Open the repo and process submodules try: repo = git.Repo(os.path.join(workspace, source.root)) except Exception as repo_error: if args.verbose: ui_functions.print_error_msg( strings.SUBMOD_EXCEPTION.format(repo_error)) continue # Collect the submodule initialization data from manifest as well as if submodules # should be processed for the repo. repo_subs = manifest.get_submodule_init_paths(source.remote_name, combo_name) repo_subs_enabled = _get_submodule_enable(manifest, source.remote_name, combo_name) if not repo_subs_enabled: continue # Initialize submodules if len(repo_subs) > 0: _init(repo, repo_subs, verbose) # Perform sync/update if len(repo_subs) == 0: _update(repo, None, verbose, cache_path=cache_path) else: _update(repo, repo_subs, verbose, cache_path=cache_path)
def run_command(self, args, config): # Process enable disable requests if args.disable: config['user_cfg_file'].set_caching_state(False) elif args.enable: config['user_cfg_file'].set_caching_state(True) # Get the current state now that we have processed enable/disable cache_state = config['user_cfg_file'].caching_state ui_functions.print_info_msg(CACHE_ENABLED.format(cache_state)) if not cache_state: return # State is enabled so make sure cache directory exists cache_obj = get_repo_cache_obj(config) pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file']) # Check to see if a manifest was provided and add any missing remotes manifest = None if args.project is not None: manifest = _get_manifest(args.project, config, args.source_manifest_repo) else: try: manifest = get_workspace_manifest() except Exception: pass # If manifest is provided attempt to add any remotes that do not exist if manifest is not None: add_missing_cache_repos(cache_obj, manifest, True) # Display all the cache information if args.info: ui_functions.print_info_msg(CACHE_INFO) info = cache_obj.get_cache_info(args.verbose) for item in info: ui_functions.print_info_msg(CACHE_INFO_LINE.format(item.path, item.remote, item.url)) # Do an update if requested if args.update: ui_functions.print_info_msg(CACHE_FETCH) cache_obj.update_cache(verbose=True) # Close the cache repos cache_obj.close(args.verbose)
def __check_combo_sha_tag_branch(self, workspace_path, initial_sources, new_sources): # Checks for changes in the defined SHAs, Tags or branches in the checked out combo. Returns # a list of repos to checkout. Checks to see if user is on appropriate SHA, tag or branch and # throws and exception if not. repos_to_checkout = [] for initial_source in initial_sources: for new_source in new_sources: if initial_source.root == new_source.root and initial_source.remote_name == new_source.remote_name and initial_source.remote_url == new_source.remote_url: local_repo_path = os.path.join(workspace_path, initial_source.root) repo = Repo(local_repo_path) if initial_source.commit and initial_source.commit != new_source.commit: if repo.head.object.hexsha != initial_source.commit: ui_functions.print_info_msg( SYNC_BRANCH_CHANGE_ON_LOCAL.format( initial_source.branch, new_source.branch, initial_source.root), header=False) repos_to_checkout.append(new_source) break elif initial_source.tag and initial_source.tag != new_source.tag: tag_sha = repo.git.rev_list( '-n 1', initial_source.tag ) #according to gitpython docs must change - to _ if tag_sha != repo.head.object.hexsha: ui_functions.print_info_msg( SYNC_BRANCH_CHANGE_ON_LOCAL.format( initial_source.branch, new_source.branch, initial_source.root), header=False) repos_to_checkout.append(new_source) break elif initial_source.branch and initial_source.branch != new_source.branch: if repo.active_branch.name != initial_source.branch: ui_functions.print_info_msg( SYNC_BRANCH_CHANGE_ON_LOCAL.format( initial_source.branch, new_source.branch, initial_source.root), header=False) repos_to_checkout.append(new_source) break return repos_to_checkout
def __update_local_manifest(self, args, config, initial_manifest, workspace_path, global_manifest_directory): #if the manifest repository for the current manifest was not found then there is no project with the manifest #specified project name in the index file for any of the manifest repositories if global_manifest_directory is None: raise EdkrepoManifestNotFoundException( SOURCE_MANIFEST_REPO_NOT_FOUND.format( initial_manifest.project_info.codename)) local_manifest_dir = os.path.join(workspace_path, 'repo') current_combo = initial_manifest.general_config.current_combo initial_sources = initial_manifest.get_repo_sources(current_combo) # Do a fetch for each repo in the initial to ensure that newly created upstream branches are available for initial_repo in initial_sources: local_repo_path = os.path.join(workspace_path, initial_repo.root) repo = Repo(local_repo_path) origin = repo.remotes.origin try: origin.fetch() except GitCommandError as e: prune_needed = False prune_needed_heuristic_str = "error: some local refs could not be updated" if e.stdout.strip().find(prune_needed_heuristic_str) != -1: prune_needed = True if e.stderr.strip().find(prune_needed_heuristic_str) != -1: prune_needed = True if prune_needed: # The sleep is to give the operating system time to close all the file handles that Git has open time.sleep(1.0) repo.git.remote('prune', 'origin') time.sleep(1.0) origin.fetch() else: raise #see if there is an entry in CiIndex.xml that matches the prject name of the current manifest index_path = os.path.join(global_manifest_directory, 'CiIndex.xml') ci_index_xml = CiIndexXml(index_path) if initial_manifest.project_info.codename not in ci_index_xml.project_list: raise EdkrepoManifestNotFoundException( SYNC_MANIFEST_NOT_FOUND.format( initial_manifest.project_info.codename)) initial_manifest_remotes = { name: url for name, url in initial_manifest.remotes } ci_index_xml_rel_path = os.path.normpath( ci_index_xml.get_project_xml( initial_manifest.project_info.codename)) global_manifest_path = os.path.join(global_manifest_directory, ci_index_xml_rel_path) new_manifest_to_check = ManifestXml(global_manifest_path) # Does the current combo exist in the new manifest? If not check to see if you can use the repo sources from # the default combo initial_combos = combinations_in_manifest(initial_manifest) new_combos = combinations_in_manifest(new_manifest_to_check) if current_combo not in new_combos: new_sources_for_current_combo = new_manifest_to_check.get_repo_sources( new_manifest_to_check.general_config.default_combo) new_sources = new_sources_for_current_combo else: new_sources_for_current_combo = new_manifest_to_check.get_repo_sources( current_combo) new_sources = new_manifest_to_check.get_repo_sources(current_combo) remove_included_config(initial_manifest.remotes, initial_manifest.submodule_alternate_remotes, local_manifest_dir) write_included_config( new_manifest_to_check.remotes, new_manifest_to_check.submodule_alternate_remotes, local_manifest_dir) self.__check_submodule_config(workspace_path, new_manifest_to_check, new_sources_for_current_combo) # Check that the repo sources lists are the same. If they are not the same and the override flag is not set, throw an exception. if not args.override and set(initial_sources) != set(new_sources): raise EdkrepoManifestChangedException( SYNC_REPO_CHANGE.format( initial_manifest.project_info.codename)) elif args.override and set(initial_sources) != set(new_sources): #get a set of repo source tuples that are not in both the new and old manifest uncommon_sources = [] initial_common = [] new_common = [] for initial in initial_sources: common = False for new in new_sources: if initial.root == new.root: if initial.remote_name == new.remote_name: if initial.remote_url == new.remote_url: # If the source is unchanged between the old and the new manifest, # add it to the common lists common = True initial_common.append(initial) new_common.append(new) break # If the source is different between the old and the new manifest, add it to the uncommon list if not common: uncommon_sources.append(initial) for new in new_sources: common = False for initial in initial_sources: if new.root == initial.root: if new.remote_name == initial.remote_name: if new.remote_url == initial.remote_url: common = True break # If the source is different between the old and the new manifest, add it to the uncommon list if not common: uncommon_sources.append(new) uncommon_sources = set(uncommon_sources) initial_common = set(initial_common) new_common = set(new_common) sources_to_move = [] sources_to_remove = [] sources_to_clone = [] for source in uncommon_sources: found_source = False for source_to_check in initial_sources: if source_to_check.root == source.root: if source_to_check.remote_name == source.remote_name: if source_to_check.remote_url == source.remote_url: found_source = True break # If the source that is different came from the old manifest, then it is now outdated and either needs # to be deleted or moved to an archival location. if found_source: roots = [s.root for s in new_sources] # If there is a source in the new manifest that goes into the same folder name as a source in the # old manifest, then we need to move that old folder to an archival location. if source.root in roots: sources_to_move.append(source) else: # If it doesn't exist at all in the new manifest, tell the user it is old and no longer used. sources_to_remove.append(source) else: # If the source that is different came from the new manifest, then we need to clone that new # Git repository. sources_to_clone.append(source) # Move the obsolete Git repositories to archival locations. for source in sources_to_move: old_dir = os.path.join(workspace_path, source.root) new_dir = generate_name_for_obsolete_backup(old_dir) ui_functions.print_warning_msg(SYNC_SOURCE_MOVE_WARNING.format( source.root, new_dir), header=False) new_dir = os.path.join(workspace_path, new_dir) try: shutil.move(old_dir, new_dir) except: ui_functions.print_error_msg(SYNC_MOVE_FAILED.format( initial_dir=source.root, new_dir=new_dir), header=False) raise # Tell the user about any Git repositories that are no longer used. if len(sources_to_remove) > 0: ui_functions.print_warning_msg(SYNC_REMOVE_WARNING, header=False) for source in sources_to_remove: path_to_source = os.path.join(workspace_path, source.root) ui_functions.print_warning_msg(path_to_source, header=False) if len(sources_to_remove) > 0: ui_functions.print_warning_msg(SYNC_REMOVE_LIST_END_FORMATTING, header=False) # Clone any new Git repositories clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, new_manifest_to_check) # Make a list of and only checkout repos that were newly cloned. Sync keeps repos on their initial active branches # cloning the entire combo can prevent existing repos from correctly being returned to their proper branch repos_to_checkout = [] if sources_to_clone: for new_source in new_sources_for_current_combo: for source in sources_to_clone: if source.root == new_source.root: repos_to_checkout.append(source) repos_to_checkout.extend( self.__check_combo_sha_tag_branch(workspace_path, initial_common, new_common)) if repos_to_checkout: checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check) #remove the old manifest file and copy the new one ui_functions.print_info_msg(UPDATING_MANIFEST, header=False) local_manifest_path = os.path.join(local_manifest_dir, 'Manifest.xml') os.remove(local_manifest_path) shutil.copy(global_manifest_path, local_manifest_path) # Update the source manifest repository tag in the local copy of the manifest XML new_manifest = ManifestXml(local_manifest_path) try: if 'source_manifest_repo' in vars(args).keys(): find_source_manifest_repo(new_manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) else: find_source_manifest_repo(new_manifest, config['cfg_file'], config['user_cfg_file'], None) except EdkrepoManifestNotFoundException: pass
def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest, cache_obj=None): for repo_to_clone in repos_to_clone: local_repo_path = os.path.join(workspace_dir, repo_to_clone.root) local_repo_url = repo_to_clone.remote_url cache_path = None if cache_obj is not None: cache_path = cache_obj.get_cache_path(local_repo_url) ui_functions.print_info_msg("Cloning from: " + str(local_repo_url), header = False) if cache_path is not None: ui_functions.print_info_msg('+ Using cache at {}'.format(cache_path)) repo = Repo.clone_from(local_repo_url, local_repo_path, progress=GitProgressHandler(), reference_if_able=cache_path, no_checkout=True) else: repo = Repo.clone_from(local_repo_url, local_repo_path, progress=GitProgressHandler(), no_checkout=True) # Fetch notes repo.remotes.origin.fetch("refs/notes/*:refs/notes/*") # Add the primary remote so that a reference to the latest code is available when # using a mirror. if add_primary_repo_remote(repo, repo_to_clone, args.verbose): fetch_from_primary_repo(repo, repo_to_clone, args.verbose) # Handle branch/commit/tag checkout if needed. If a combination of these are specified the # order of importance is 1)commit 2)tag 3)branch with only the higest priority being checked # out if repo_to_clone.commit: if args.verbose and (repo_to_clone.branch or repo_to_clone.tag): ui_functions.print_info_msg(MULTIPLE_SOURCE_ATTRIBUTES_SPECIFIED.format(repo_to_clone.root)) repo.git.checkout(repo_to_clone.commit) elif repo_to_clone.tag and repo_to_clone.commit is None: if args.verbose and repo_to_clone.branch: ui_functions.print_info_msg(TAG_AND_BRANCH_SPECIFIED.format(repo_to_clone.root)) repo.git.checkout(repo_to_clone.tag) elif repo_to_clone.branch and (repo_to_clone.commit is None and repo_to_clone.tag is None): if repo_to_clone.branch not in repo.remotes['origin'].refs: raise EdkrepoManifestInvalidException('The specified remote branch does not exist') branch_name = repo_to_clone.branch local_branch = repo.create_head(branch_name, repo.remotes['origin'].refs[branch_name]) repo.heads[local_branch.name].set_tracking_branch(repo.remotes['origin'].refs[branch_name]) repo.heads[local_branch.name].checkout() else: raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT) try: if 'source_manifest_repo' in vars(args).keys(): src_manifest_repo = find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo, False) else: src_manifest_repo = find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], None, False) except EdkrepoManifestNotFoundException: src_manifest_repo = None if src_manifest_repo: cfg, user_cfg, conflicts = list_available_manifest_repos(config['cfg_file'], config['user_cfg_file']) if src_manifest_repo in cfg: global_manifest_directory = config['cfg_file'].manifest_repo_abs_path(src_manifest_repo) elif src_manifest_repo in user_cfg: global_manifest_directory = config['user_cfg_file'].manifest_repo_abs_path(src_manifest_repo) else: global_manifest_directory = None else: global_manifest_directory = None if global_manifest_directory: # Install git hooks if there is a manifest repo associated with the manifest being cloned install_hooks(project_client_side_hooks, local_repo_path, repo_to_clone, config, global_manifest_directory) # Add the commit template if it exists. update_repo_commit_template(workspace_dir, repo, repo_to_clone, config, global_manifest_directory) # Check to see if mirror is in sync with primary repo if not in_sync_with_primary(repo, repo_to_clone, args.verbose): ui_functions.print_warning_msg(MIRROR_BEHIND_PRIMARY_REPO)
def run_command(self, args, config): print() cfg_file = config['cfg_file'] user_cfg = config['user_cfg_file'] cfg_man_repos, user_cfg_man_repos, conflicts = list_available_manifest_repos( cfg_file, user_cfg) man_repos = {} pull_all_manifest_repos(cfg_file, user_cfg, False) # Get paths to the global manifest dirs and their index files for repo in cfg_man_repos: global_manifest_directory = cfg_file.manifest_repo_abs_path(repo) index_path = os.path.join(global_manifest_directory, 'CiIndex.xml') man_repos[repo] = (global_manifest_directory, index_path) for repo in user_cfg_man_repos: global_manifest_directory = user_cfg.manifest_repo_abs_path(repo) index_path = os.path.join(global_manifest_directory, 'CiIndex.xml') man_repos[repo] = (global_manifest_directory, index_path) try: wkspc_manifest = get_workspace_manifest() current_project = wkspc_manifest.project_info.codename src_man_repo = find_source_manifest_repo(wkspc_manifest, cfg_file, user_cfg, None) except EdkrepoWorkspaceInvalidException: current_project = None src_man_repo = None except EdkrepoManifestNotFoundException: src_man_repo = None for repo in man_repos.keys(): print() ui_functions.print_info_msg("Manifest directory:", header=False) ui_functions.print_info_msg(repo, header=False) if args.verbose: ui_functions.print_info_msg('Manifest directory path:', header=False) ui_functions.print_info_msg(man_repos[repo][0], header=False) print() ci_index_xml = CiIndexXml(man_repos[repo][1]) # Attempt to make sure the manifest data is good try: validate_manifest_repo(man_repos[repo][0], args.verbose, args.archived) except: print() ui_functions.print_info_msg("Projects:", header=False) for project in sorted(ci_index_xml.project_list): if (project == current_project and src_man_repo == repo) or ( not src_man_repo and project == current_project): ui_functions.print_info_msg(project, header=False) else: ui_functions.print_warning_msg(project, header=False) if args.verbose: ui_functions.print_info_msg(" -> {}".format( ci_index_xml.get_project_xml(project)), header=False) proj_manifest = ManifestXml( find_project_in_single_index(project, ci_index_xml, man_repos[repo][0])[1]) ui_functions.print_info_msg(" -> DevLead: {}".format( ' '.join( x for x in proj_manifest.project_info.dev_leads)), header=False) if args.archived: print() ui_functions.print_info_msg("Archived Projects:", header=False) for project in sorted(ci_index_xml.archived_project_list): if project == current_project: ui_functions.print_info_msg(project, header=False) else: ui_functions.print_warning_msg(project, header=False) if args.verbose: ui_functions.print_info_msg(" -> {}".format( ci_index_xml.get_project_xml(project)), header=False) proj_manifest = ManifestXml( find_project_in_single_index( project, ci_index_xml, man_repos[repo][0])[1]) ui_functions.print_info_msg(" -> DevLead: {}".format( ' '.join(x for x in proj_manifest.project_info.dev_leads)), header=False)
def run_command(self, args, config): # Collect workspace/repo data workspace_path = get_workspace_path() manifest = get_workspace_manifest() current_combo = manifest.general_config.current_combo repo_list = manifest.get_repo_sources(current_combo) sparse_settings = manifest.sparse_settings sparse_enabled = sparse_checkout_enabled(workspace_path, repo_list) # Determine if settings are being chaged or just status display if args.enable or args.disable: # Handle sparse checkout changes if args.enable and args.disable: raise EdkrepoSparseException(SPARSE_ENABLE_DISABLE) elif (args.enable and sparse_enabled) or (args.disable and not sparse_enabled): raise EdkrepoSparseException(SPARSE_NO_CHANGE) check_dirty_repos(manifest, workspace_path) if args.enable and not sparse_enabled: ui_functions.print_info_msg(SPARSE_ENABLE, header=False) sparse_checkout(workspace_path, repo_list, manifest) elif args.disable and sparse_enabled: ui_functions.print_info_msg(SPARSE_DISABLE, header=False) reset_sparse_checkout(workspace_path, repo_list, True) else: # Display the current status of the project ui_functions.print_info_msg(SPARSE_STATUS, header=False) ui_functions.print_info_msg( SPARSE_CHECKOUT_STATUS.format(sparse_enabled), header=False) if sparse_settings is not None: ui_functions.print_info_msg(SPARSE_BY_DEFAULT_STATUS.format( sparse_settings.sparse_by_default), header=False) ui_functions.print_info_msg( SPARSE_ENABLED_REPOS.format(current_combo), header=False) for repo in [x for x in repo_list if x.sparse]: ui_functions.print_info_msg('- {}: {}'.format( repo.root, repo.remote_url), header=False)
def run_command(self, args, config): # Configure git long path support ui_functions.print_info_msg(humble.LONGPATH_CONFIG, header=False) set_long_path_support() print() # Remove unneeded instead of entries from git global config ui_functions.print_info_msg(humble.CLEAN_INSTEAD_OFS, header=False) print() # If in a valid workspace run the following for each repo: # git reflog --expire, git gc, git remote prune origin try: workspace_path = get_workspace_path() except EdkrepoWorkspaceInvalidException: workspace_path = None ui_functions.print_error_msg(humble.NO_WOKKSPACE, header=False) print() if workspace_path: manifest = get_workspace_manifest() repos_to_maintain = manifest.get_repo_sources( manifest.general_config.current_combo) for repo_to_maintain in repos_to_maintain: local_repo_path = os.path.join(workspace_path, repo_to_maintain.root) repo = Repo(local_repo_path) ui_functions.print_info_msg(humble.REPO_MAINTENANCE.format( repo_to_maintain.root), header=False) ui_functions.print_info_msg(humble.REFLOG_EXPIRE, header=False) repo.git.reflog('expire', '--expire=now', '--all') ui_functions.print_info_msg(humble.GC_AGGRESSIVE, header=False) repo.git.gc('--aggressive', '--prune=now') ui_functions.print_info_msg(humble.REMOTE_PRUNE, header=False) repo.git.remote('prune', 'origin') print()
def run_command(self, args, config): workspace_path = get_workspace_path() initial_manifest = get_workspace_manifest() current_combo = initial_manifest.general_config.current_combo initial_sources = initial_manifest.get_repo_sources(current_combo) initial_hooks = initial_manifest.repo_hooks initial_combo = current_combo try: pull_workspace_manifest_repo(initial_manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo, False) except: pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'], False) source_global_manifest_repo = find_source_manifest_repo( initial_manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) cfg_manifest_repos, user_cfg_manifest_repos, conflicts = list_available_manifest_repos( config['cfg_file'], config['user_cfg_file']) if source_global_manifest_repo in cfg_manifest_repos: global_manifest_directory = config[ 'cfg_file'].manifest_repo_abs_path(source_global_manifest_repo) verify_single_manifest(config['cfg_file'], source_global_manifest_repo, get_workspace_manifest_file(), args.verbose) elif source_global_manifest_repo in user_cfg_manifest_repos: global_manifest_directory = config[ 'user_cfg_file'].manifest_repo_abs_path( source_global_manifest_repo) verify_single_manifest(config['user_cfg_file'], source_global_manifest_repo, get_workspace_manifest_file(), args.verbose) else: global_manifest_directory = None if global_manifest_directory is not None: update_editor_config(config, global_manifest_directory) if not args.update_local_manifest: self.__check_for_new_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) check_dirty_repos(initial_manifest, workspace_path) # Determine if sparse checkout needs to be disabled for this operation sparse_settings = initial_manifest.sparse_settings sparse_enabled = sparse_checkout_enabled(workspace_path, initial_sources) sparse_reset_required = False if sparse_settings is None: sparse_reset_required = True elif args.update_local_manifest: sparse_reset_required = True if sparse_enabled and sparse_reset_required: ui_functions.print_info_msg(SPARSE_RESET, header=False) reset_sparse_checkout(workspace_path, initial_sources) # Get the latest manifest if requested if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) manifest = get_workspace_manifest() if args.update_local_manifest: try: repo_sources_to_sync = manifest.get_repo_sources(current_combo) except ValueError: # The manifest file was updated and the initial combo is no longer present so use the default combo current_combo = manifest.general_config.default_combo repo_sources_to_sync = manifest.get_repo_sources(current_combo) else: repo_sources_to_sync = manifest.get_repo_sources(current_combo) manifest.write_current_combo(current_combo) # At this point both new and old manifest files are ready so we can deinit any # submodules that are removed due to a manifest update. if not args.skip_submodule: deinit_submodules(workspace_path, initial_manifest, initial_combo, manifest, current_combo, args.verbose) sync_error = False # Calculate the hooks which need to be updated, added or removed for the sync if args.update_local_manifest: new_hooks = manifest.repo_hooks hooks_add = set(new_hooks).difference(set(initial_hooks)) hooks_update = set(initial_hooks).intersection(set(new_hooks)) hooks_uninstall = set(initial_hooks).difference(set(new_hooks)) else: hooks_add = None hooks_update = initial_hooks hooks_uninstall = None # Update submodule configuration if not args.update_local_manifest: #Performance optimization, __update_local_manifest() will do this self.__check_submodule_config(workspace_path, manifest, repo_sources_to_sync) clean_git_globalconfig() for repo_to_sync in repo_sources_to_sync: local_repo_path = os.path.join(workspace_path, repo_to_sync.root) # Update any hooks if global_manifest_directory is not None: update_hooks(hooks_add, hooks_update, hooks_uninstall, local_repo_path, repo_to_sync, config, global_manifest_directory) repo = Repo(local_repo_path) #Fetch notes repo.remotes.origin.fetch("refs/notes/*:refs/notes/*") if repo_to_sync.commit is None and repo_to_sync.tag is None: local_commits = False initial_active_branch = repo.active_branch repo.remotes.origin.fetch( "refs/heads/{0}:refs/remotes/origin/{0}".format( repo_to_sync.branch)) #The new branch may not exist in the heads list yet if it is a new branch repo.git.checkout(repo_to_sync.branch) if not args.fetch: ui_functions.print_info_msg(SYNCING.format( repo_to_sync.root, repo.active_branch), header=False) else: ui_functions.print_info_msg(FETCHING.format( repo_to_sync.root, repo.active_branch), header=False) try: repo.remotes.origin.fetch() except GitCommandError as e: prune_needed = False prune_needed_heuristic_str = "error: some local refs could not be updated" if e.stdout.strip().find(prune_needed_heuristic_str) != -1: prune_needed = True if e.stderr.strip().find(prune_needed_heuristic_str) != -1: prune_needed = True if prune_needed: time.sleep(1.0) repo.git.remote('prune', 'origin') time.sleep(1.0) repo.remotes.origin.fetch() else: raise if has_primary_repo_remote(repo, args.verbose): fetch_from_primary_repo(repo, repo_to_sync, args.verbose) if not args.override and not repo.is_ancestor( ancestor_rev='HEAD', rev='origin/{}'.format(repo_to_sync.branch)): ui_functions.print_info_msg(SYNC_COMMITS_ON_TARGET.format( repo_to_sync.branch, repo_to_sync.root), header=False) local_commits = True sync_error = True if not args.fetch and (not local_commits or args.override): repo.head.reset(commit='origin/{}'.format( repo_to_sync.branch), working_tree=True) # Check to see if mirror is up to date if not in_sync_with_primary(repo, repo_to_sync, args.verbose): ui_functions.print_info_msg(MIRROR_BEHIND_PRIMARY_REPO, header=False) # Switch back to the initially active branch before exiting repo.heads[initial_active_branch.name].checkout() # Warn user if local branch is behind target branch try: latest_sha = get_latest_sha(repo, repo_to_sync.branch) commit_count = int( repo.git.rev_list('--count', '{}..HEAD'.format(latest_sha))) branch_origin = next( itertools.islice(repo.iter_commits(), commit_count, commit_count + 1)) behind_count = int( repo.git.rev_list( '--count', '{}..{}'.format(branch_origin.hexsha, latest_sha))) if behind_count: ui_functions.print_info_msg(SYNC_NEEDS_REBASE.format( behind_count=behind_count, target_remote='origin', target_branch=repo_to_sync.branch, local_branch=initial_active_branch.name, repo_folder=repo_to_sync.root), header=False) except: ui_functions.print_error_msg(SYNC_REBASE_CALC_FAIL, header=False) elif args.verbose: ui_functions.print_warning_msg(NO_SYNC_DETACHED_HEAD.format( repo_to_sync.root), header=False) # Update commit message templates if global_manifest_directory is not None: update_repo_commit_template(workspace_path, repo, repo_to_sync, config, global_manifest_directory) if sync_error: ui_functions.print_error_msg(SYNC_ERROR, header=False) # Initialize submodules if not args.skip_submodule: cache_path = None cache_obj = get_repo_cache_obj(config) if cache_obj is not None: cache_path = cache_obj.get_cache_path( SUBMODULE_CACHE_REPO_NAME) maintain_submodules(workspace_path, manifest, current_combo, args.verbose, cache_path) # Restore sparse checkout state if sparse_enabled: ui_functions.print_info_msg(SPARSE_CHECKOUT, header=False) sparse_checkout(workspace_path, repo_sources_to_sync, manifest)
def run_command(self, args, config): pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'], False) name_or_manifest = args.ProjectNameOrManifestFile workspace_dir = args.Workspace # Check to see if requested workspace exists. If not create it. If so check for empty if workspace_dir == '.': # User has selected the directory they are running edkrepo from workspace_dir = os.getcwd() else: workspace_dir = os.path.abspath(workspace_dir) if sys.platform == "win32": subst = get_subst_drive_dict() drive = os.path.splitdrive(workspace_dir)[0][0].upper() if drive in subst: workspace_dir = os.path.join( subst[drive], os.path.splitdrive(workspace_dir)[1][1:]) workspace_dir = os.path.normpath(workspace_dir) if os.path.isdir(workspace_dir) and os.listdir(workspace_dir): raise EdkrepoInvalidParametersException(CLONE_INVALID_WORKSPACE) if not os.path.isdir(workspace_dir): os.makedirs(workspace_dir) cfg, user_cfg, conflicts = list_available_manifest_repos( config['cfg_file'], config['user_cfg_file']) try: manifest_repo, source_cfg, global_manifest_path = find_project_in_all_indices( args.ProjectNameOrManifestFile, config['cfg_file'], config['user_cfg_file'], PROJ_NOT_IN_REPO.format(args.ProjectNameOrManifestFile), SOURCE_MANIFEST_REPO_NOT_FOUND.format( args.ProjectNameOrManifestFile), args.source_manifest_repo) except EdkrepoManifestNotFoundException: raise EdkrepoInvalidParametersException(CLONE_INVALID_PROJECT_ARG) # If this manifest is in a defined manifest repository validate the manifest within the manifest repo if manifest_repo in cfg: verify_single_manifest(config['cfg_file'], manifest_repo, global_manifest_path) update_editor_config( config, config['cfg_file'].manifest_repo_abs_path(manifest_repo)) elif manifest_repo in user_cfg: verify_single_manifest(config['user_cfg_file'], manifest_repo, global_manifest_path) update_editor_config( config, config['user_cfg_file'].manifest_repo_abs_path(manifest_repo)) # Copy project manifest to local manifest dir and rename it Manifest.xml. local_manifest_dir = os.path.join(workspace_dir, "repo") os.makedirs(local_manifest_dir) local_manifest_path = os.path.join(local_manifest_dir, "Manifest.xml") shutil.copy(global_manifest_path, local_manifest_path) manifest = ManifestXml(local_manifest_path) # Update the source manifest repository tag in the local copy of the manifest XML try: if 'source_manifest_repo' in vars(args).keys(): find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) else: find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], None) except EdkrepoManifestNotFoundException: pass # Process the combination name and make sure it can be found in the manifest if args.Combination is not None: try: combo_name = case_insensitive_single_match( args.Combination, combinations_in_manifest(manifest)) except: #remove the repo directory and Manifest.xml from the workspace so the next time the user trys to clone #they will have an empty workspace and then raise an exception shutil.rmtree(local_manifest_dir) raise EdkrepoInvalidParametersException( CLONE_INVALID_COMBO_ARG) manifest.write_current_combo(combo_name) elif manifest.is_pin_file(): # Since pin files are subset of manifest files they do not have a "default combo" it is set to None. In this # case use the current_combo instead. combo_name = manifest.general_config.current_combo else: # If a combo was not specified or a pin file used the default combo should be cloned. Also ensure that the # current combo is updated to match. combo_name = manifest.general_config.default_combo manifest.write_current_combo(combo_name) # Get the list of repos to clone and clone them repo_sources_to_clone = manifest.get_repo_sources(combo_name) #check that the repo sources do not contain duplicated local roots local_roots = [r.root for r in repo_sources_to_clone] for root in local_roots: if local_roots.count(root) > 1: #remove the repo dir and manifest.xml so the next time the user trys to clone they will have an empty #workspace shutil.rmtree(local_manifest_dir) raise EdkrepoManifestInvalidException( CLONE_INVALID_LOCAL_ROOTS) project_client_side_hooks = manifest.repo_hooks # Set up submodule alt url config settings prior to cloning any repos submodule_included_configs = write_included_config( manifest.remotes, manifest.submodule_alternate_remotes, local_manifest_dir) write_conditional_include(workspace_dir, repo_sources_to_clone, submodule_included_configs) # Determine if caching is going to be used and then clone cache_obj = get_repo_cache_obj(config) if cache_obj is not None: add_missing_cache_repos(cache_obj, manifest, args.verbose) clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest, cache_obj) # Init submodules if not args.skip_submodule: cache_path = None if cache_obj is not None: cache_path = cache_obj.get_cache_path( SUBMODULE_CACHE_REPO_NAME) maintain_submodules(workspace_dir, manifest, combo_name, args.verbose, cache_path) # Perform a sparse checkout if requested. use_sparse = args.sparse sparse_settings = manifest.sparse_settings if sparse_settings is None: # No SparseCheckout information in manifest so skip sparse checkout use_sparse = False elif sparse_settings.sparse_by_default: # Sparse settings enabled by default for the project use_sparse = True if args.nosparse: # Command line disables sparse checkout use_sparse = False if use_sparse: ui_functions.print_info_msg(SPARSE_CHECKOUT) sparse_checkout(workspace_dir, repo_sources_to_clone, manifest)