def run_command(self, args, config): # Process enable disable requests if args.disable: config['user_cfg_file'].set_caching_state(False) elif args.enable: config['user_cfg_file'].set_caching_state(True) # Get the current state now that we have processed enable/disable cache_state = config['user_cfg_file'].caching_state ui_functions.print_info_msg(CACHE_ENABLED.format(cache_state)) if not cache_state: return # State is enabled so make sure cache directory exists cache_obj = get_repo_cache_obj(config) pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file']) # Check to see if a manifest was provided and add any missing remotes manifest = None if args.project is not None: manifest = _get_manifest(args.project, config, args.source_manifest_repo) else: try: manifest = get_workspace_manifest() except Exception: pass # If manifest is provided attempt to add any remotes that do not exist if manifest is not None: add_missing_cache_repos(cache_obj, manifest, True) # Display all the cache information if args.info: ui_functions.print_info_msg(CACHE_INFO) info = cache_obj.get_cache_info(args.verbose) for item in info: ui_functions.print_info_msg(CACHE_INFO_LINE.format(item.path, item.remote, item.url)) # Do an update if requested if args.update: ui_functions.print_info_msg(CACHE_FETCH) cache_obj.update_cache(verbose=True) # Close the cache repos cache_obj.close(args.verbose)
def run_command(self, args, config): workspace_path = get_workspace_path() initial_manifest = get_workspace_manifest() current_combo = initial_manifest.general_config.current_combo initial_sources = initial_manifest.get_repo_sources(current_combo) initial_hooks = initial_manifest.repo_hooks initial_combo = current_combo try: pull_workspace_manifest_repo(initial_manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo, False) except: pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'], False) source_global_manifest_repo = find_source_manifest_repo( initial_manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) cfg_manifest_repos, user_cfg_manifest_repos, conflicts = list_available_manifest_repos( config['cfg_file'], config['user_cfg_file']) if source_global_manifest_repo in cfg_manifest_repos: global_manifest_directory = config[ 'cfg_file'].manifest_repo_abs_path(source_global_manifest_repo) verify_single_manifest(config['cfg_file'], source_global_manifest_repo, get_workspace_manifest_file(), args.verbose) elif source_global_manifest_repo in user_cfg_manifest_repos: global_manifest_directory = config[ 'user_cfg_file'].manifest_repo_abs_path( source_global_manifest_repo) verify_single_manifest(config['user_cfg_file'], source_global_manifest_repo, get_workspace_manifest_file(), args.verbose) else: global_manifest_directory = None if global_manifest_directory is not None: update_editor_config(config, global_manifest_directory) if not args.update_local_manifest: self.__check_for_new_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) check_dirty_repos(initial_manifest, workspace_path) # Determine if sparse checkout needs to be disabled for this operation sparse_settings = initial_manifest.sparse_settings sparse_enabled = sparse_checkout_enabled(workspace_path, initial_sources) sparse_reset_required = False if sparse_settings is None: sparse_reset_required = True elif args.update_local_manifest: sparse_reset_required = True if sparse_enabled and sparse_reset_required: ui_functions.print_info_msg(SPARSE_RESET, header=False) reset_sparse_checkout(workspace_path, initial_sources) # Get the latest manifest if requested if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) manifest = get_workspace_manifest() if args.update_local_manifest: try: repo_sources_to_sync = manifest.get_repo_sources(current_combo) except ValueError: # The manifest file was updated and the initial combo is no longer present so use the default combo current_combo = manifest.general_config.default_combo repo_sources_to_sync = manifest.get_repo_sources(current_combo) else: repo_sources_to_sync = manifest.get_repo_sources(current_combo) manifest.write_current_combo(current_combo) # At this point both new and old manifest files are ready so we can deinit any # submodules that are removed due to a manifest update. if not args.skip_submodule: deinit_submodules(workspace_path, initial_manifest, initial_combo, manifest, current_combo, args.verbose) sync_error = False # Calculate the hooks which need to be updated, added or removed for the sync if args.update_local_manifest: new_hooks = manifest.repo_hooks hooks_add = set(new_hooks).difference(set(initial_hooks)) hooks_update = set(initial_hooks).intersection(set(new_hooks)) hooks_uninstall = set(initial_hooks).difference(set(new_hooks)) else: hooks_add = None hooks_update = initial_hooks hooks_uninstall = None # Update submodule configuration if not args.update_local_manifest: #Performance optimization, __update_local_manifest() will do this self.__check_submodule_config(workspace_path, manifest, repo_sources_to_sync) clean_git_globalconfig() for repo_to_sync in repo_sources_to_sync: local_repo_path = os.path.join(workspace_path, repo_to_sync.root) # Update any hooks if global_manifest_directory is not None: update_hooks(hooks_add, hooks_update, hooks_uninstall, local_repo_path, repo_to_sync, config, global_manifest_directory) repo = Repo(local_repo_path) #Fetch notes repo.remotes.origin.fetch("refs/notes/*:refs/notes/*") if repo_to_sync.commit is None and repo_to_sync.tag is None: local_commits = False initial_active_branch = repo.active_branch repo.remotes.origin.fetch( "refs/heads/{0}:refs/remotes/origin/{0}".format( repo_to_sync.branch)) #The new branch may not exist in the heads list yet if it is a new branch repo.git.checkout(repo_to_sync.branch) if not args.fetch: ui_functions.print_info_msg(SYNCING.format( repo_to_sync.root, repo.active_branch), header=False) else: ui_functions.print_info_msg(FETCHING.format( repo_to_sync.root, repo.active_branch), header=False) try: repo.remotes.origin.fetch() except GitCommandError as e: prune_needed = False prune_needed_heuristic_str = "error: some local refs could not be updated" if e.stdout.strip().find(prune_needed_heuristic_str) != -1: prune_needed = True if e.stderr.strip().find(prune_needed_heuristic_str) != -1: prune_needed = True if prune_needed: time.sleep(1.0) repo.git.remote('prune', 'origin') time.sleep(1.0) repo.remotes.origin.fetch() else: raise if has_primary_repo_remote(repo, args.verbose): fetch_from_primary_repo(repo, repo_to_sync, args.verbose) if not args.override and not repo.is_ancestor( ancestor_rev='HEAD', rev='origin/{}'.format(repo_to_sync.branch)): ui_functions.print_info_msg(SYNC_COMMITS_ON_TARGET.format( repo_to_sync.branch, repo_to_sync.root), header=False) local_commits = True sync_error = True if not args.fetch and (not local_commits or args.override): repo.head.reset(commit='origin/{}'.format( repo_to_sync.branch), working_tree=True) # Check to see if mirror is up to date if not in_sync_with_primary(repo, repo_to_sync, args.verbose): ui_functions.print_info_msg(MIRROR_BEHIND_PRIMARY_REPO, header=False) # Switch back to the initially active branch before exiting repo.heads[initial_active_branch.name].checkout() # Warn user if local branch is behind target branch try: latest_sha = get_latest_sha(repo, repo_to_sync.branch) commit_count = int( repo.git.rev_list('--count', '{}..HEAD'.format(latest_sha))) branch_origin = next( itertools.islice(repo.iter_commits(), commit_count, commit_count + 1)) behind_count = int( repo.git.rev_list( '--count', '{}..{}'.format(branch_origin.hexsha, latest_sha))) if behind_count: ui_functions.print_info_msg(SYNC_NEEDS_REBASE.format( behind_count=behind_count, target_remote='origin', target_branch=repo_to_sync.branch, local_branch=initial_active_branch.name, repo_folder=repo_to_sync.root), header=False) except: ui_functions.print_error_msg(SYNC_REBASE_CALC_FAIL, header=False) elif args.verbose: ui_functions.print_warning_msg(NO_SYNC_DETACHED_HEAD.format( repo_to_sync.root), header=False) # Update commit message templates if global_manifest_directory is not None: update_repo_commit_template(workspace_path, repo, repo_to_sync, config, global_manifest_directory) if sync_error: ui_functions.print_error_msg(SYNC_ERROR, header=False) # Initialize submodules if not args.skip_submodule: cache_path = None cache_obj = get_repo_cache_obj(config) if cache_obj is not None: cache_path = cache_obj.get_cache_path( SUBMODULE_CACHE_REPO_NAME) maintain_submodules(workspace_path, manifest, current_combo, args.verbose, cache_path) # Restore sparse checkout state if sparse_enabled: ui_functions.print_info_msg(SPARSE_CHECKOUT, header=False) sparse_checkout(workspace_path, repo_sources_to_sync, manifest)
def run_command(self, args, config): print() init_color_console(args.color) pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file']) print() cfg_manifest_repos, user_config_manifest_repos, conflicts = list_available_manifest_repos(config['cfg_file'], config['user_cfg_file']) found_manifests = {} manifests = {} repo_urls = set() config_manifest_repos_project_list = [] user_config_manifest_repos_project_list = [] for manifest_repo in cfg_manifest_repos: # Get path to global manifest file global_manifest_directory = config['cfg_file'].manifest_repo_abs_path(manifest_repo) if args.verbose: print(humble.MANIFEST_DIRECTORY) print(global_manifest_directory) print() #Create a dictionary containing all the manifests listed in the CiIndex.xml file index_path = os.path.join(global_manifest_directory, CI_INDEX_FILE_NAME) print(index_path) ci_index_xml = CiIndexXml(index_path) config_manifest_repos_project_list = ci_index_xml.project_list if args.archived: config_manifest_repos_project_list.extend(ci_index_xml.archived_project_list) for project in config_manifest_repos_project_list: xml_file = ci_index_xml.get_project_xml(project) manifest = ManifestXml(os.path.normpath(os.path.join(global_manifest_directory, xml_file))) found_manifests['{}:{}'.format(manifest_repo, project)] = manifest combo_list = [c.name for c in manifest.combinations] if args.archived: combo_list.extend([c.name for c in manifest.archived_combinations]) for combo in combo_list: sources = manifest.get_repo_sources(combo) for source in sources: repo_urls.add(self.get_repo_url(source.remote_url)) for manifest_repo in user_config_manifest_repos: # Get path to global manifest file global_manifest_directory = config['user_cfg_file'].manifest_repo_abs_path(manifest_repo) if args.verbose: print(humble.MANIFEST_DIRECTORY) print(global_manifest_directory) print() #Create a dictionary containing all the manifests listed in the CiIndex.xml file index_path = os.path.join(global_manifest_directory, CI_INDEX_FILE_NAME) ci_index_xml = CiIndexXml(index_path) user_config_manifest_repos_project_list = ci_index_xml.project_list if args.archived: user_config_manifest_repos_project_list.extend(ci_index_xml.archived_project_list) for project in user_config_manifest_repos_project_list: xml_file = ci_index_xml.get_project_xml(project) manifest = ManifestXml(os.path.normpath(os.path.join(global_manifest_directory, xml_file))) found_manifests['{}:{}'.format(manifest_repo, project)] = manifest combo_list = [c.name for c in manifest.combinations] if args.archived: combo_list.extend([c.name for c in manifest.archived_combinations]) for combo in combo_list: sources = manifest.get_repo_sources(combo) for source in sources: repo_urls.add(self.get_repo_url(source.remote_url)) #Remove the manifest repo portion of the key is there is not a duplicate project name key_list = list(found_manifests) for entry in key_list: new_key = entry.split(':')[1] value = found_manifests[entry] del found_manifests[entry] for found_manifest in list(found_manifests): if found_manifest.split(':')[1] == new_key: new_key = 'Manifest Repository: {} Project: {}'.format(entry.split(':')[0], entry.split(':')[1]) break if new_key in manifests.keys(): new_key = 'Manifest Repository: {} Project: {}'.format(entry.split(':'[0]), entry.split(':')[1]) manifests[new_key] = value #Sort the manifests so projects will be displayed alphabetically manifests = collections.OrderedDict(sorted(manifests.items())) project_justify = len(max(manifests.keys(), key=len)) #Determine the names of the repositories self.generate_repo_names(repo_urls, manifests, args.archived) print(humble.REPOSITORIES) #If the user provided a list of repositories to view, check to make sure #at least one repository will be shown, if not provide an error if args.repos and len([x for x in self.repo_names if x in args.repos]) <= 0: raise EdkrepoInvalidParametersException(humble.REPO_NOT_FOUND_IN_MANIFEST.format(','.join(args.repos))) #For each each git repository... for repo_name in self.repo_names: if args.repos and repo_name not in args.repos: continue repo = self.repo_names[repo_name][0] print(humble.REPO_NAME_AND_URL.format(repo_name, repo)) print(humble.BRANCHES) #Determine the list of branches that used by any branch combination in any manifest branches = set() for project_name in manifests: combo_list = [c.name for c in manifests[project_name].combinations] if args.archived: combo_list.extend([c.name for c in manifests[project_name].archived_combinations]) for combo in combo_list: sources = manifests[project_name].get_repo_sources(combo) for source in sources: if self.get_repo_url(source.remote_url) == repo: branches.add(source.branch) #Sort the branch names so they will be displayed alphabetically #with the exception that if a branch named "master" exists, then it #will be displayed first branches = sorted(branches, key=str.casefold) if 'master' in branches: branches.remove('master') branches.insert(0, 'master') #For each interesting branch in the current git repository... for branch in branches: print(humble.BRANCH_FORMAT_STRING.format(branch)) #Determine the branch combinations that use that branch for project_name in manifests: combos = [] combo_list = [c.name for c in manifests[project_name].combinations] if args.archived: combo_list.extend([c.name for c in manifests[project_name].archived_combinations]) for combo in combo_list: sources = manifests[project_name].get_repo_sources(combo) for source in sources: if self.get_repo_url(source.remote_url) == repo and source.branch == branch: combos.append(combo) break if len(combos) > 0: #Sort the branch combinations so they will be displayed alphabetically #with the exception that the default branch combination for the manifest #file will be displayed first combos = sorted(combos, key=str.casefold) default_combo = manifests[project_name].general_config.default_combo if default_combo in combos: combos.remove(default_combo) combos.insert(0, default_combo) first_combo = True for combo in combos: #Print the project name if first_combo: project_name_print = humble.PROJECT_NAME_FORMAT_STRING.format(project_name.ljust(project_justify)) first_combo = False else: project_name_print = '{} '.format((' ' * len(project_name)).ljust(project_justify)) #Print the branch combination name, if this is the default branch combination, #then print it in green color with *'s around it if default_combo == combo: print(humble.DEFAULT_COMBO_FORMAT_STRING.format(project_name_print, combo)) else: print(humble.COMBO_FORMAT_STRING.format(project_name_print, combo))
def run_command(self, args, config): print() cfg_file = config['cfg_file'] user_cfg = config['user_cfg_file'] cfg_man_repos, user_cfg_man_repos, conflicts = list_available_manifest_repos( cfg_file, user_cfg) man_repos = {} pull_all_manifest_repos(cfg_file, user_cfg, False) # Get paths to the global manifest dirs and their index files for repo in cfg_man_repos: global_manifest_directory = cfg_file.manifest_repo_abs_path(repo) index_path = os.path.join(global_manifest_directory, 'CiIndex.xml') man_repos[repo] = (global_manifest_directory, index_path) for repo in user_cfg_man_repos: global_manifest_directory = user_cfg.manifest_repo_abs_path(repo) index_path = os.path.join(global_manifest_directory, 'CiIndex.xml') man_repos[repo] = (global_manifest_directory, index_path) try: wkspc_manifest = get_workspace_manifest() current_project = wkspc_manifest.project_info.codename src_man_repo = find_source_manifest_repo(wkspc_manifest, cfg_file, user_cfg, None) except EdkrepoWorkspaceInvalidException: current_project = None src_man_repo = None except EdkrepoManifestNotFoundException: src_man_repo = None for repo in man_repos.keys(): print() ui_functions.print_info_msg("Manifest directory:", header=False) ui_functions.print_info_msg(repo, header=False) if args.verbose: ui_functions.print_info_msg('Manifest directory path:', header=False) ui_functions.print_info_msg(man_repos[repo][0], header=False) print() ci_index_xml = CiIndexXml(man_repos[repo][1]) # Attempt to make sure the manifest data is good try: validate_manifest_repo(man_repos[repo][0], args.verbose, args.archived) except: print() ui_functions.print_info_msg("Projects:", header=False) for project in sorted(ci_index_xml.project_list): if (project == current_project and src_man_repo == repo) or ( not src_man_repo and project == current_project): ui_functions.print_info_msg(project, header=False) else: ui_functions.print_warning_msg(project, header=False) if args.verbose: ui_functions.print_info_msg(" -> {}".format( ci_index_xml.get_project_xml(project)), header=False) proj_manifest = ManifestXml( find_project_in_single_index(project, ci_index_xml, man_repos[repo][0])[1]) ui_functions.print_info_msg(" -> DevLead: {}".format( ' '.join( x for x in proj_manifest.project_info.dev_leads)), header=False) if args.archived: print() ui_functions.print_info_msg("Archived Projects:", header=False) for project in sorted(ci_index_xml.archived_project_list): if project == current_project: ui_functions.print_info_msg(project, header=False) else: ui_functions.print_warning_msg(project, header=False) if args.verbose: ui_functions.print_info_msg(" -> {}".format( ci_index_xml.get_project_xml(project)), header=False) proj_manifest = ManifestXml( find_project_in_single_index( project, ci_index_xml, man_repos[repo][0])[1]) ui_functions.print_info_msg(" -> DevLead: {}".format( ' '.join(x for x in proj_manifest.project_info.dev_leads)), header=False)
def run_command(self, args, config): pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'], reset_hard=args.hard)
def run_command(self, args, config): pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'], False) name_or_manifest = args.ProjectNameOrManifestFile workspace_dir = args.Workspace # Check to see if requested workspace exists. If not create it. If so check for empty if workspace_dir == '.': # User has selected the directory they are running edkrepo from workspace_dir = os.getcwd() else: workspace_dir = os.path.abspath(workspace_dir) if sys.platform == "win32": subst = get_subst_drive_dict() drive = os.path.splitdrive(workspace_dir)[0][0].upper() if drive in subst: workspace_dir = os.path.join( subst[drive], os.path.splitdrive(workspace_dir)[1][1:]) workspace_dir = os.path.normpath(workspace_dir) if os.path.isdir(workspace_dir) and os.listdir(workspace_dir): raise EdkrepoInvalidParametersException(CLONE_INVALID_WORKSPACE) if not os.path.isdir(workspace_dir): os.makedirs(workspace_dir) cfg, user_cfg, conflicts = list_available_manifest_repos( config['cfg_file'], config['user_cfg_file']) try: manifest_repo, source_cfg, global_manifest_path = find_project_in_all_indices( args.ProjectNameOrManifestFile, config['cfg_file'], config['user_cfg_file'], PROJ_NOT_IN_REPO.format(args.ProjectNameOrManifestFile), SOURCE_MANIFEST_REPO_NOT_FOUND.format( args.ProjectNameOrManifestFile), args.source_manifest_repo) except EdkrepoManifestNotFoundException: raise EdkrepoInvalidParametersException(CLONE_INVALID_PROJECT_ARG) # If this manifest is in a defined manifest repository validate the manifest within the manifest repo if manifest_repo in cfg: verify_single_manifest(config['cfg_file'], manifest_repo, global_manifest_path) update_editor_config( config, config['cfg_file'].manifest_repo_abs_path(manifest_repo)) elif manifest_repo in user_cfg: verify_single_manifest(config['user_cfg_file'], manifest_repo, global_manifest_path) update_editor_config( config, config['user_cfg_file'].manifest_repo_abs_path(manifest_repo)) # Copy project manifest to local manifest dir and rename it Manifest.xml. local_manifest_dir = os.path.join(workspace_dir, "repo") os.makedirs(local_manifest_dir) local_manifest_path = os.path.join(local_manifest_dir, "Manifest.xml") shutil.copy(global_manifest_path, local_manifest_path) manifest = ManifestXml(local_manifest_path) # Update the source manifest repository tag in the local copy of the manifest XML try: if 'source_manifest_repo' in vars(args).keys(): find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) else: find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], None) except EdkrepoManifestNotFoundException: pass # Process the combination name and make sure it can be found in the manifest if args.Combination is not None: try: combo_name = case_insensitive_single_match( args.Combination, combinations_in_manifest(manifest)) except: #remove the repo directory and Manifest.xml from the workspace so the next time the user trys to clone #they will have an empty workspace and then raise an exception shutil.rmtree(local_manifest_dir) raise EdkrepoInvalidParametersException( CLONE_INVALID_COMBO_ARG) manifest.write_current_combo(combo_name) elif manifest.is_pin_file(): # Since pin files are subset of manifest files they do not have a "default combo" it is set to None. In this # case use the current_combo instead. combo_name = manifest.general_config.current_combo else: # If a combo was not specified or a pin file used the default combo should be cloned. Also ensure that the # current combo is updated to match. combo_name = manifest.general_config.default_combo manifest.write_current_combo(combo_name) # Get the list of repos to clone and clone them repo_sources_to_clone = manifest.get_repo_sources(combo_name) #check that the repo sources do not contain duplicated local roots local_roots = [r.root for r in repo_sources_to_clone] for root in local_roots: if local_roots.count(root) > 1: #remove the repo dir and manifest.xml so the next time the user trys to clone they will have an empty #workspace shutil.rmtree(local_manifest_dir) raise EdkrepoManifestInvalidException( CLONE_INVALID_LOCAL_ROOTS) project_client_side_hooks = manifest.repo_hooks # Set up submodule alt url config settings prior to cloning any repos submodule_included_configs = write_included_config( manifest.remotes, manifest.submodule_alternate_remotes, local_manifest_dir) write_conditional_include(workspace_dir, repo_sources_to_clone, submodule_included_configs) # Determine if caching is going to be used and then clone cache_obj = get_repo_cache_obj(config) if cache_obj is not None: add_missing_cache_repos(cache_obj, manifest, args.verbose) clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest, cache_obj) # Init submodules if not args.skip_submodule: cache_path = None if cache_obj is not None: cache_path = cache_obj.get_cache_path( SUBMODULE_CACHE_REPO_NAME) maintain_submodules(workspace_dir, manifest, combo_name, args.verbose, cache_path) # Perform a sparse checkout if requested. use_sparse = args.sparse sparse_settings = manifest.sparse_settings if sparse_settings is None: # No SparseCheckout information in manifest so skip sparse checkout use_sparse = False elif sparse_settings.sparse_by_default: # Sparse settings enabled by default for the project use_sparse = True if args.nosparse: # Command line disables sparse checkout use_sparse = False if use_sparse: ui_functions.print_info_msg(SPARSE_CHECKOUT) sparse_checkout(workspace_dir, repo_sources_to_clone, manifest)