def confirm_missing_deps(deps: Collection[Tuple[str, str]], watcher: ProcessWatcher, i18n: I18n) -> bool: opts = [] total_isize, total_dsize = None, None pkgs_data = pacman.map_updates_data(pkgs=tuple(d[0] for d in deps if d[1] != 'aur'), description=True) or dict() for dep in deps: ver, desc, isize, dsize = None, None, None, None data = pkgs_data.get(dep[0]) if data: desc, isize, dsize = (data.get(f) for f in ('des', 's', 'ds')) if isize is not None: if total_isize is None: total_isize = 0 total_isize += isize if dsize is not None: if total_dsize is None: total_dsize = 0 total_dsize += dsize label = f"{dep[0]} | " \ f"{i18n['size'].capitalize()}: {get_human_size_str(isize) if isize is not None else '?'}" \ f"{' ({}: {})'.format(i18n['download'].capitalize(), get_human_size_str(dsize)) if dsize else ''}" op = InputOption(label=label, value=dep[0], tooltip=desc) op.read_only = True op.icon_path = _get_repo_icon(dep[1]) opts.append(op) comp = MultipleSelectComponent(label='', options=opts, default_options=set(opts)) body = StringIO() body.write('<p>') body.write(i18n['arch.missing_deps.body'].format(deps=bold(str(len(deps))))) if total_isize is not None or total_dsize is not None: body.write(' (') if total_isize is not None: body.write(f"{i18n['size'].capitalize()}: {bold(get_human_size_str(total_isize))} | ") if total_dsize is not None: body.write(f"{i18n['download'].capitalize()}: {bold(get_human_size_str(total_dsize))}") body.write(')') body.write(':</p>') return watcher.request_confirmation(title=i18n['arch.missing_deps.title'], body=body.getvalue(), components=[comp], confirmation_label=i18n['continue'].capitalize(), deny_label=i18n['cancel'].capitalize(), min_width=625)
def _fill_to_install(self, context: UpdateRequirementsContext) -> bool: ti = time.time() self.logger.info("Discovering updates missing packages") deps_data, deps_checked = {}, set() deps = self.deps_analyser.map_missing_deps(pkgs_data=context.pkgs_data, provided_map=context.provided_map, aur_index=context.aur_index, deps_checked=deps_checked, sort=True, deps_data=deps_data, remote_provided_map=context.remote_provided_map, remote_repo_map=context.remote_repo_map, watcher=self.watcher, automatch_providers=context.arch_config['automatch_providers']) if deps is None: tf = time.time() self.logger.info("It took {0:.2f} seconds to retrieve required upgrade packages".format(tf - ti)) return False # the user called the process off if deps: # filtering selected packages selected_names = {p for p in context.to_update} deps = [dep for dep in deps if dep[0] not in selected_names] if deps: sorted_pkgs = {} aur_to_install_data = {} all_to_install_data = {} for idx, dep in enumerate(deps): data = deps_data[dep[0]] pkg = ArchPackage(name=dep[0], version=data['v'], latest_version=data['v'], repository=dep[1], i18n=self.i18n) sorted_pkgs[idx] = pkg context.to_install[dep[0]] = pkg if pkg.repository == 'aur': context.aur_to_install[pkg.name] = pkg aur_to_install_data[pkg.name] = data else: context.repo_to_install[pkg.name] = pkg if context.repo_to_install: all_to_install_data.update(pacman.map_updates_data(context.repo_to_install.keys())) if aur_to_install_data: all_to_install_data.update(aur_to_install_data) if all_to_install_data: context.pkgs_data.update(all_to_install_data) self._fill_conflicts(context, context.to_remove.keys()) tf = time.time() self.logger.info("It took {0:.2f} seconds to retrieve required upgrade packages".format(tf - ti)) return True
def _fill_single_providers_data(self, all_missing_deps: Iterable[Tuple[str, str]], repo_missing_deps: Iterable[str], aur_missing_deps: Iterable[str], deps_data: Dict[str, dict]): """ fills the missing data of the single dependency providers since they are already considered dependencies (when several providers are available for given a dependency, the user must choose first) """ repo_providers_no_data, aur_providers_no_data = None, None for dep_name, dep_repo in all_missing_deps: if dep_repo == '__several__': deps_data[dep_name] = { 'd': None, 'p': {dep_name}, 'r': dep_repo } elif dep_name not in deps_data: if repo_missing_deps and dep_name in repo_missing_deps: if repo_providers_no_data is None: repo_providers_no_data = set() repo_providers_no_data.add(dep_name) elif aur_missing_deps and dep_name in aur_missing_deps: if aur_providers_no_data is None: aur_providers_no_data = set() aur_providers_no_data.add(dep_name) aur_data_filler, aur_providers_data = None, None if aur_providers_no_data: aur_providers_data = dict() aur_data_filler = Thread(target=self._fill_aur_updates_data, args=(aur_providers_no_data, aur_providers_data)) aur_data_filler.start() if repo_providers_no_data: repo_providers_data = pacman.map_updates_data( repo_providers_no_data) if repo_providers_data: deps_data.update(repo_providers_data) if aur_data_filler: aur_data_filler.join() if aur_providers_data: deps_data.update(aur_providers_data)
def summarize(self, pkgs: List[ArchPackage], root_password: str, arch_config: dict) -> UpgradeRequirements: res = UpgradeRequirements([], [], [], []) remote_provided_map = pacman.map_provided(remote=True) remote_repo_map = pacman.map_repositories() context = UpdateRequirementsContext(to_update={}, repo_to_update={}, aur_to_update={}, repo_to_install={}, aur_to_install={}, to_install={}, pkgs_data={}, cannot_upgrade={}, to_remove={}, installed_names=set(), provided_map={}, aur_index=set(), arch_config=arch_config, root_password=root_password, remote_provided_map=remote_provided_map, remote_repo_map=remote_repo_map) self.__fill_aur_index(context) aur_data = {} aur_srcinfo_threads = [] for p in pkgs: context.to_update[p.name] = p if p.repository == 'aur': context.aur_to_update[p.name] = p t = Thread(target=self._fill_aur_pkg_update_data, args=(p, aur_data), daemon=True) t.start() aur_srcinfo_threads.append(t) else: context.repo_to_update[p.name] = p if context.aur_to_update: for t in aur_srcinfo_threads: t.join() self.logger.info("Filling updates data") if context.repo_to_update: context.pkgs_data.update(pacman.map_updates_data(context.repo_to_update.keys())) if aur_data: context.pkgs_data.update(aur_data) self.__fill_provided_map(context) if context.pkgs_data: self._fill_conflicts(context) try: if not self._fill_to_install(context): self.logger.info("The operation was cancelled by the user") return except PackageNotFoundException as e: self.logger.error("Package '{}' not found".format(e.name)) return if context.to_update: installed_sizes = pacman.get_installed_size(list(context.to_update.keys())) sorted_pkgs = [] if context.repo_to_update: # only sorting by name ( pacman already knows the best order to perform the upgrade ) sorted_pkgs.extend(context.repo_to_update.values()) sorted_pkgs.sort(key=lambda pkg: pkg.name) if context.aur_to_update: # adding AUR packages in the end sorted_aur = sorting.sort(context.aur_to_update.keys(), context.pkgs_data, context.provided_map) for aur_pkg in sorted_aur: sorted_pkgs.append(context.aur_to_update[aur_pkg[0]]) res.to_upgrade = [self._map_requirement(pkg, context, installed_sizes) for pkg in sorted_pkgs] if context.to_remove: res.to_remove = [p for p in context.to_remove.values()] if context.cannot_upgrade: res.cannot_upgrade = [d for d in context.cannot_upgrade.values()] if context.to_install: res.to_install = [self._map_requirement(p, context) for p in context.to_install.values()] return res
def fill_providers_deps( self, missing_deps: List[Tuple[str, str]], provided_map: Dict[str, Set[str]], remote_repo_map: Dict[str, str], already_checked: Set[str], remote_provided_map: Dict[str, Set[str]], deps_data: Dict[str, dict], aur_idx: Iterable[str], sort: bool, watcher: ProcessWatcher, automatch_providers: bool) -> List[Tuple[str, str]]: """ :param missing_deps: :param provided_map: :param remote_repo_map: :param already_checked: :param remote_provided_map: :param deps_data: :param aur_idx: :param sort: :param watcher: :param automatch_providers :return: all deps sorted or None if the user declined the providers options """ deps_providers = map_providers( {data[0] for data in missing_deps if data[1] == '__several__'}, remote_provided_map) if deps_providers: all_providers = set() for providers in deps_providers.values(): all_providers.update(providers) providers_repos = pacman.map_repositories(all_providers) selected_providers = confirmation.request_providers( deps_providers, providers_repos, watcher, self.i18n) if not selected_providers: return else: providers_data = pacman.map_updates_data( selected_providers ) # adding the chosen providers to re-check the missing deps provided_map.update( pacman.map_provided(remote=True, pkgs=selected_providers) ) # adding the providers as "installed" packages providers_deps = self.map_missing_deps( pkgs_data=providers_data, provided_map=provided_map, aur_index=aur_idx, deps_checked=already_checked, deps_data=deps_data, sort=False, remote_provided_map=remote_provided_map, remote_repo_map=remote_repo_map, watcher=watcher, choose_providers=True, automatch_providers=automatch_providers) # cleaning the already mapped providers deps: to_remove = [] for idx, dep in enumerate(missing_deps): if dep[1] == '__several__': to_remove.append(idx) for idx, to_remove in enumerate(to_remove): del missing_deps[to_remove - idx] missing_deps.extend(((p, providers_repos.get(p, 'aur')) for p in selected_providers)) for dep in providers_deps: if dep not in missing_deps and dep[1] != '__several__': missing_deps.append(dep) deps_data.update(providers_data) if not self.fill_providers_deps( missing_deps=missing_deps, provided_map=provided_map, remote_repo_map=remote_repo_map, already_checked=already_checked, aur_idx=aur_idx, remote_provided_map=remote_provided_map, deps_data=deps_data, sort=False, watcher=watcher, automatch_providers=automatch_providers): return if sort: missing_to_sort = { d[0] for d in missing_deps if d[1] != '__several__' } return sorting.sort(missing_to_sort, deps_data, provided_map) return missing_deps
def map_missing_deps( self, pkgs_data: Dict[str, dict], provided_map: Dict[str, Set[str]], remote_provided_map: Dict[str, Set[str]], remote_repo_map: Dict[str, str], aur_index: Iterable[str], deps_checked: Set[str], deps_data: Dict[str, dict], sort: bool, watcher: ProcessWatcher, choose_providers: bool = True, automatch_providers: bool = False) -> List[Tuple[str, str]]: sorted_deps = [ ] # it will hold the proper order to install the missing dependencies missing_deps, repo_missing, aur_missing = set(), set(), set() deps_checked.update(pkgs_data.keys()) for p, data in pkgs_data.items(): if data['d']: for dep in data['d']: if dep in pkgs_data: continue if dep not in provided_map: dep_split = self.re_dep_operator.split(dep) dep_name = dep_split[0].strip() if dep_name not in deps_checked: deps_checked.add(dep_name) if dep_name not in provided_map: self._fill_missing_dep( dep_name=dep_name, dep_exp=dep, aur_index=aur_index, missing_deps=missing_deps, remote_provided_map=remote_provided_map, remote_repo_map=remote_repo_map, repo_deps=repo_missing, aur_deps=aur_missing, watcher=watcher, deps_data=deps_data, automatch_providers=automatch_providers) else: version_pattern = '{}='.format(dep_name) version_found = [ p for p in provided_map if p.startswith(version_pattern) ] if version_found: version_found = version_found[0].split( '=')[1] version_informed = dep_split[2].strip() if ':' not in version_informed: version_found = version_found.split( ':')[-1] if '-' not in version_informed: version_found = version_found.split( '-')[0] try: version_found = parse_version( version_found) version_informed = parse_version( version_informed) op = dep_split[ 1] if dep_split[1] != '=' else '==' match = eval( 'version_found {} version_informed' .format(op)) except: match = False traceback.print_exc() if not match: self._fill_missing_dep( dep_name=dep_name, dep_exp=dep, aur_index=aur_index, missing_deps=missing_deps, remote_provided_map= remote_provided_map, remote_repo_map=remote_repo_map, repo_deps=repo_missing, aur_deps=aur_missing, watcher=watcher, deps_data=deps_data, automatch_providers= automatch_providers) else: self._fill_missing_dep( dep_name=dep_name, dep_exp=dep, aur_index=aur_index, missing_deps=missing_deps, remote_provided_map=remote_provided_map, remote_repo_map=remote_repo_map, repo_deps=repo_missing, aur_deps=aur_missing, watcher=watcher, deps_data=deps_data, automatch_providers=automatch_providers ) if missing_deps: if repo_missing: with_single_providers = [] for d in missing_deps: if d[0] in repo_missing and d[0] not in deps_data: if d[1] == '__several__': deps_data[d[0]] = {'d': None, 'p': d[0], 'r': d[1]} else: with_single_providers.append(d[0]) if with_single_providers: data = pacman.map_updates_data(with_single_providers) if data: deps_data.update(data) if aur_missing: aur_threads = [] for pkgname in aur_missing: t = Thread(target=self.__fill_aur_update_data, args=(pkgname, deps_data), daemon=True) t.start() aur_threads.append(t) for t in aur_threads: t.join() missing_subdeps = self.map_missing_deps( pkgs_data=deps_data, provided_map=provided_map, aur_index=aur_index, deps_checked=deps_checked, sort=False, deps_data=deps_data, watcher=watcher, remote_provided_map=remote_provided_map, remote_repo_map=remote_repo_map, automatch_providers=automatch_providers, choose_providers=False) if missing_subdeps: missing_deps.update(missing_subdeps) if sort: sorted_deps.extend(sorting.sort(deps_data.keys(), deps_data)) else: sorted_deps.extend(((dep[0], dep[1]) for dep in missing_deps)) if sorted_deps and choose_providers: return self.fill_providers_deps( missing_deps=sorted_deps, provided_map=provided_map, remote_provided_map=remote_provided_map, remote_repo_map=remote_repo_map, watcher=watcher, sort=sort, already_checked=deps_checked, aur_idx=aur_index, deps_data=deps_data, automatch_providers=automatch_providers) return sorted_deps
def _fill_missing_dep(self, dep_name: str, dep_exp: str, aur_index: Iterable[str], missing_deps: Set[Tuple[str, str]], remote_provided_map: Dict[str, Set[str]], remote_repo_map: Dict[str, str], repo_deps: Set[str], aur_deps: Set[str], deps_data: Dict[str, dict], watcher: ProcessWatcher, automatch_providers: bool): if dep_name == dep_exp: providers = remote_provided_map.get(dep_name) if not providers: # try to find the package through the pacman's search mechanism match = pacman.find_one_match(dep_name) if match: providers = {match} else: # handling cases when the dep has an expression ( e.g: xpto>=0.12 ) providers = remote_provided_map.get(dep_exp) if providers is None: providers = remote_provided_map.get(dep_name) if not providers: # try to find the package through the pacman's search mechanism match = pacman.find_one_match(dep_name) if match: providers = {match} if providers and len(providers) > 1: no_mapped_data = { p for p in providers if p not in deps_data } # checking providers with no mapped data if no_mapped_data: providers_data = pacman.map_updates_data( no_mapped_data) if not providers_data: raise Exception( "Could not retrieve the info from providers: {}" .format(no_mapped_data)) deps_data.update( providers_data) # adding missing providers data matched_providers = set() split_informed_dep = self.re_dep_operator.split(dep_exp) try: version_informed = parse_version(split_informed_dep[2]) exp_op = split_informed_dep[ 1] if split_informed_dep[1] != '=' else '==' for p in providers: provided = deps_data[p]['p'] for provided_exp in provided: split_dep = self.re_dep_operator.split( provided_exp) if len(split_dep ) == 3 and split_dep[0] == dep_name: provided_version = parse_version( split_dep[2]) if eval('provided_version {} version_informed' .format(exp_op)): matched_providers.add(p) break providers = matched_providers except: traceback.print_exc() if providers: if len(providers) > 1: dep_data = None if automatch_providers: exact_matches = [p for p in providers if p == dep_name] if exact_matches: dep_data = (exact_matches[0], remote_repo_map.get(exact_matches[0])) if not dep_data: dep_data = (dep_name, '__several__') else: real_name = providers.pop() dep_data = (real_name, remote_repo_map.get(real_name)) repo_deps.add(dep_data[0]) missing_deps.add(dep_data) elif aur_index and dep_name in aur_index: aur_deps.add(dep_name) missing_deps.add((dep_name, 'aur')) else: if watcher: message.show_dep_not_found(dep_exp, self.i18n, watcher) raise PackageNotFoundException(dep_exp) else: raise PackageNotFoundException(dep_exp)
def fill_providers_deps( self, missing_deps: List[Tuple[str, str]], provided_map: Dict[str, Set[str]], remote_repo_map: Dict[str, str], already_checked: Set[str], remote_provided_map: Dict[str, Set[str]], deps_data: Dict[str, dict], aur_idx: Iterable[str], sort: bool, watcher: ProcessWatcher, automatch_providers: bool, prefer_repository_provider: bool ) -> Optional[List[Tuple[str, str]]]: """ :param missing_deps: :param provided_map: :param remote_repo_map: :param already_checked: :param remote_provided_map: :param deps_data: :param aur_idx: :param sort: :param watcher: :param automatch_providers :param prefer_repository_provider :return: all deps sorted or None if the user declined the providers options """ deps_providers = map_providers( {data[0] for data in missing_deps if data[1] == '__several__'}, remote_provided_map) if deps_providers: providers_repos = {} repos_providers = set() for providers in deps_providers.values(): for provider in providers: if remote_repo_map.get(provider) == 'aur': providers_repos[provider] = 'aur' else: repos_providers.add(provider) providers_repos.update(pacman.map_repositories(repos_providers)) selected_providers = confirmation.request_providers( deps_providers, providers_repos, watcher, self.i18n) if not selected_providers: return else: # adding the chosen providers for re-checking the missing dependencies repo_selected, aur_selected = set(), set() for provider in selected_providers: if provider in repos_providers: repo_selected.add(provider) else: aur_selected.add(provider) providers_data = dict() if repo_selected: providers_data.update( pacman.map_updates_data(repo_selected)) # adding the providers as "installed" packages provided_map.update( pacman.map_provided(remote=True, pkgs=repo_selected)) if aur_selected: for pkgname, pkgdata in self.aur_client.gen_updates_data( aur_selected): providers_data[pkgname] = pkgdata for provider in pkgdata[ 'p']: # adding the providers as "installed" packages currently_provided = provided_map.get( provider, set()) provided_map[provider] = currently_provided currently_provided.add(pkgname) providers_deps = self.map_missing_deps( pkgs_data=providers_data, provided_map=provided_map, aur_index=aur_idx, deps_checked=already_checked, deps_data=deps_data, sort=False, remote_provided_map=remote_provided_map, remote_repo_map=remote_repo_map, watcher=watcher, choose_providers=True, automatch_providers=automatch_providers, prefer_repository_provider=prefer_repository_provider) if providers_deps is None: # it means the user called off the installation process return # cleaning the already mapped providers deps: to_remove = [] for idx, dep in enumerate(missing_deps): if dep[1] == '__several__': to_remove.append(idx) for idx, to_remove in enumerate(to_remove): del missing_deps[to_remove - idx] missing_deps.extend(((p, providers_repos.get(p, 'aur')) for p in selected_providers)) for dep in providers_deps: if dep not in missing_deps and dep[1] != '__several__': missing_deps.append(dep) deps_data.update(providers_data) if not self.fill_providers_deps( missing_deps=missing_deps, provided_map=provided_map, remote_repo_map=remote_repo_map, already_checked=already_checked, aur_idx=aur_idx, remote_provided_map=remote_provided_map, deps_data=deps_data, sort=False, watcher=watcher, automatch_providers=automatch_providers, prefer_repository_provider=prefer_repository_provider): return if sort: missing_to_sort = { d[0] for d in missing_deps if d[1] != '__several__' } return sorting.sort(missing_to_sort, deps_data, provided_map) return missing_deps
def _find_repo_providers( self, dep_name: str, dep_exp: str, remote_provided_map: Dict[str, Set[str]], deps_data: Dict[str, dict], remote_repo_map: Dict[str, str] ) -> Generator[Tuple[str, str, Optional[dict]], None, None]: if dep_name == dep_exp: providers = remote_provided_map.get(dep_name) if providers: for pkgname in providers: yield pkgname, remote_repo_map.get(pkgname), None else: # try to find the package through the pacman's search mechanism match = pacman.find_one_match(dep_name) if match: yield match, remote_repo_map.get(match), None else: # handling cases when the dep has an expression ( e.g: xpto>=0.12 ) exact_exp_providers = remote_provided_map.get(dep_exp) if exact_exp_providers: for p in exact_exp_providers: yield p, remote_repo_map.get(p), None else: providers = remote_provided_map.get(dep_name) if not providers: # try to find the package through the pacman's search mechanism match = pacman.find_one_match(dep_name) if match: providers = {match} if providers: providers_no_provided_data = { p for p in providers if p not in deps_data } missing_providers_data = None if providers_no_provided_data: missing_providers_data = pacman.map_updates_data( providers_no_provided_data) if not missing_providers_data: raise Exception( f"Could not retrieve information from providers: " f"{', '.join(providers_no_provided_data)}") data_not_found = { p for p in providers if p not in missing_providers_data } if data_not_found: raise Exception( f"Could not retrieve information from providers: " f"{', '.join(data_not_found)}") split_informed_dep = self.re_dep_operator.split(dep_exp) version_required = split_informed_dep[2] exp_op = split_informed_dep[ 1] if split_informed_dep[1] != '=' else '==' for p in providers: info = deps_data.get(p) if not info and missing_providers_data: info = missing_providers_data[p] for provided_exp in info['p']: split_dep = self.re_dep_operator.split( provided_exp) if len(split_dep ) == 3 and split_dep[0] == dep_name: version_provided = split_dep[2] if match_required_version( version_provided, exp_op, version_required): yield p, remote_repo_map.get(p), info break