def confirm_missing_deps(deps: Collection[Tuple[str, str]], watcher: ProcessWatcher, i18n: I18n) -> bool:
opts = []
total_isize, total_dsize = None, None
pkgs_data = pacman.map_updates_data(pkgs=tuple(d[0] for d in deps if d[1] != 'aur'), description=True) or dict()
for dep in deps:
ver, desc, isize, dsize = None, None, None, None
data = pkgs_data.get(dep[0])
if data:
desc, isize, dsize = (data.get(f) for f in ('des', 's', 'ds'))
if isize is not None:
if total_isize is None:
total_isize = 0
total_isize += isize
if dsize is not None:
if total_dsize is None:
total_dsize = 0
total_dsize += dsize
label = f"{dep[0]} | " \
f"{i18n['size'].capitalize()}: {get_human_size_str(isize) if isize is not None else '?'}" \
f"{' ({}: {})'.format(i18n['download'].capitalize(), get_human_size_str(dsize)) if dsize else ''}"
op = InputOption(label=label, value=dep[0], tooltip=desc)
op.read_only = True
op.icon_path = _get_repo_icon(dep[1])
opts.append(op)
comp = MultipleSelectComponent(label='', options=opts, default_options=set(opts))
body = StringIO()
body.write('<p>')
body.write(i18n['arch.missing_deps.body'].format(deps=bold(str(len(deps)))))
if total_isize is not None or total_dsize is not None:
body.write(' (')
if total_isize is not None:
body.write(f"{i18n['size'].capitalize()}: {bold(get_human_size_str(total_isize))} | ")
if total_dsize is not None:
body.write(f"{i18n['download'].capitalize()}: {bold(get_human_size_str(total_dsize))}")
body.write(')')
body.write(':</p>')
return watcher.request_confirmation(title=i18n['arch.missing_deps.title'],
body=body.getvalue(),
components=[comp],
confirmation_label=i18n['continue'].capitalize(),
deny_label=i18n['cancel'].capitalize(),
min_width=625)
def _fill_to_install(self, context: UpdateRequirementsContext) -> bool:
ti = time.time()
self.logger.info("Discovering updates missing packages")
deps_data, deps_checked = {}, set()
deps = self.deps_analyser.map_missing_deps(pkgs_data=context.pkgs_data,
provided_map=context.provided_map,
aur_index=context.aur_index,
deps_checked=deps_checked,
sort=True,
deps_data=deps_data,
remote_provided_map=context.remote_provided_map,
remote_repo_map=context.remote_repo_map,
watcher=self.watcher,
automatch_providers=context.arch_config['automatch_providers'])
if deps is None:
tf = time.time()
self.logger.info("It took {0:.2f} seconds to retrieve required upgrade packages".format(tf - ti))
return False # the user called the process off
if deps: # filtering selected packages
selected_names = {p for p in context.to_update}
deps = [dep for dep in deps if dep[0] not in selected_names]
if deps:
sorted_pkgs = {}
aur_to_install_data = {}
all_to_install_data = {}
for idx, dep in enumerate(deps):
data = deps_data[dep[0]]
pkg = ArchPackage(name=dep[0], version=data['v'], latest_version=data['v'], repository=dep[1], i18n=self.i18n)
sorted_pkgs[idx] = pkg
context.to_install[dep[0]] = pkg
if pkg.repository == 'aur':
context.aur_to_install[pkg.name] = pkg
aur_to_install_data[pkg.name] = data
else:
context.repo_to_install[pkg.name] = pkg
if context.repo_to_install:
all_to_install_data.update(pacman.map_updates_data(context.repo_to_install.keys()))
if aur_to_install_data:
all_to_install_data.update(aur_to_install_data)
if all_to_install_data:
context.pkgs_data.update(all_to_install_data)
self._fill_conflicts(context, context.to_remove.keys())
tf = time.time()
self.logger.info("It took {0:.2f} seconds to retrieve required upgrade packages".format(tf - ti))
return True
def _fill_single_providers_data(self,
all_missing_deps: Iterable[Tuple[str,
str]],
repo_missing_deps: Iterable[str],
aur_missing_deps: Iterable[str],
deps_data: Dict[str, dict]):
"""
fills the missing data of the single dependency providers since they are already considered dependencies
(when several providers are available for given a dependency, the user must choose first)
"""
repo_providers_no_data, aur_providers_no_data = None, None
for dep_name, dep_repo in all_missing_deps:
if dep_repo == '__several__':
deps_data[dep_name] = {
'd': None,
'p': {dep_name},
'r': dep_repo
}
elif dep_name not in deps_data:
if repo_missing_deps and dep_name in repo_missing_deps:
if repo_providers_no_data is None:
repo_providers_no_data = set()
repo_providers_no_data.add(dep_name)
elif aur_missing_deps and dep_name in aur_missing_deps:
if aur_providers_no_data is None:
aur_providers_no_data = set()
aur_providers_no_data.add(dep_name)
aur_data_filler, aur_providers_data = None, None
if aur_providers_no_data:
aur_providers_data = dict()
aur_data_filler = Thread(target=self._fill_aur_updates_data,
args=(aur_providers_no_data,
aur_providers_data))
aur_data_filler.start()
if repo_providers_no_data:
repo_providers_data = pacman.map_updates_data(
repo_providers_no_data)
if repo_providers_data:
deps_data.update(repo_providers_data)
if aur_data_filler:
aur_data_filler.join()
if aur_providers_data:
deps_data.update(aur_providers_data)
def summarize(self, pkgs: List[ArchPackage], root_password: str, arch_config: dict) -> UpgradeRequirements:
res = UpgradeRequirements([], [], [], [])
remote_provided_map = pacman.map_provided(remote=True)
remote_repo_map = pacman.map_repositories()
context = UpdateRequirementsContext(to_update={}, repo_to_update={}, aur_to_update={}, repo_to_install={},
aur_to_install={}, to_install={}, pkgs_data={}, cannot_upgrade={},
to_remove={}, installed_names=set(), provided_map={}, aur_index=set(),
arch_config=arch_config, root_password=root_password,
remote_provided_map=remote_provided_map, remote_repo_map=remote_repo_map)
self.__fill_aur_index(context)
aur_data = {}
aur_srcinfo_threads = []
for p in pkgs:
context.to_update[p.name] = p
if p.repository == 'aur':
context.aur_to_update[p.name] = p
t = Thread(target=self._fill_aur_pkg_update_data, args=(p, aur_data), daemon=True)
t.start()
aur_srcinfo_threads.append(t)
else:
context.repo_to_update[p.name] = p
if context.aur_to_update:
for t in aur_srcinfo_threads:
t.join()
self.logger.info("Filling updates data")
if context.repo_to_update:
context.pkgs_data.update(pacman.map_updates_data(context.repo_to_update.keys()))
if aur_data:
context.pkgs_data.update(aur_data)
self.__fill_provided_map(context)
if context.pkgs_data:
self._fill_conflicts(context)
try:
if not self._fill_to_install(context):
self.logger.info("The operation was cancelled by the user")
return
except PackageNotFoundException as e:
self.logger.error("Package '{}' not found".format(e.name))
return
if context.to_update:
installed_sizes = pacman.get_installed_size(list(context.to_update.keys()))
sorted_pkgs = []
if context.repo_to_update: # only sorting by name ( pacman already knows the best order to perform the upgrade )
sorted_pkgs.extend(context.repo_to_update.values())
sorted_pkgs.sort(key=lambda pkg: pkg.name)
if context.aur_to_update: # adding AUR packages in the end
sorted_aur = sorting.sort(context.aur_to_update.keys(), context.pkgs_data, context.provided_map)
for aur_pkg in sorted_aur:
sorted_pkgs.append(context.aur_to_update[aur_pkg[0]])
res.to_upgrade = [self._map_requirement(pkg, context, installed_sizes) for pkg in sorted_pkgs]
if context.to_remove:
res.to_remove = [p for p in context.to_remove.values()]
if context.cannot_upgrade:
res.cannot_upgrade = [d for d in context.cannot_upgrade.values()]
if context.to_install:
res.to_install = [self._map_requirement(p, context) for p in context.to_install.values()]
return res
def fill_providers_deps(
self, missing_deps: List[Tuple[str,
str]], provided_map: Dict[str,
Set[str]],
remote_repo_map: Dict[str, str], already_checked: Set[str],
remote_provided_map: Dict[str, Set[str]], deps_data: Dict[str,
dict],
aur_idx: Iterable[str], sort: bool, watcher: ProcessWatcher,
automatch_providers: bool) -> List[Tuple[str, str]]:
"""
:param missing_deps:
:param provided_map:
:param remote_repo_map:
:param already_checked:
:param remote_provided_map:
:param deps_data:
:param aur_idx:
:param sort:
:param watcher:
:param automatch_providers
:return: all deps sorted or None if the user declined the providers options
"""
deps_providers = map_providers(
{data[0]
for data in missing_deps if data[1] == '__several__'},
remote_provided_map)
if deps_providers:
all_providers = set()
for providers in deps_providers.values():
all_providers.update(providers)
providers_repos = pacman.map_repositories(all_providers)
selected_providers = confirmation.request_providers(
deps_providers, providers_repos, watcher, self.i18n)
if not selected_providers:
return
else:
providers_data = pacman.map_updates_data(
selected_providers
) # adding the chosen providers to re-check the missing deps
provided_map.update(
pacman.map_provided(remote=True, pkgs=selected_providers)
) # adding the providers as "installed" packages
providers_deps = self.map_missing_deps(
pkgs_data=providers_data,
provided_map=provided_map,
aur_index=aur_idx,
deps_checked=already_checked,
deps_data=deps_data,
sort=False,
remote_provided_map=remote_provided_map,
remote_repo_map=remote_repo_map,
watcher=watcher,
choose_providers=True,
automatch_providers=automatch_providers)
# cleaning the already mapped providers deps:
to_remove = []
for idx, dep in enumerate(missing_deps):
if dep[1] == '__several__':
to_remove.append(idx)
for idx, to_remove in enumerate(to_remove):
del missing_deps[to_remove - idx]
missing_deps.extend(((p, providers_repos.get(p, 'aur'))
for p in selected_providers))
for dep in providers_deps:
if dep not in missing_deps and dep[1] != '__several__':
missing_deps.append(dep)
deps_data.update(providers_data)
if not self.fill_providers_deps(
missing_deps=missing_deps,
provided_map=provided_map,
remote_repo_map=remote_repo_map,
already_checked=already_checked,
aur_idx=aur_idx,
remote_provided_map=remote_provided_map,
deps_data=deps_data,
sort=False,
watcher=watcher,
automatch_providers=automatch_providers):
return
if sort:
missing_to_sort = {
d[0]
for d in missing_deps if d[1] != '__several__'
}
return sorting.sort(missing_to_sort, deps_data,
provided_map)
return missing_deps
def map_missing_deps(
self,
pkgs_data: Dict[str, dict],
provided_map: Dict[str, Set[str]],
remote_provided_map: Dict[str, Set[str]],
remote_repo_map: Dict[str, str],
aur_index: Iterable[str],
deps_checked: Set[str],
deps_data: Dict[str, dict],
sort: bool,
watcher: ProcessWatcher,
choose_providers: bool = True,
automatch_providers: bool = False) -> List[Tuple[str, str]]:
sorted_deps = [
] # it will hold the proper order to install the missing dependencies
missing_deps, repo_missing, aur_missing = set(), set(), set()
deps_checked.update(pkgs_data.keys())
for p, data in pkgs_data.items():
if data['d']:
for dep in data['d']:
if dep in pkgs_data:
continue
if dep not in provided_map:
dep_split = self.re_dep_operator.split(dep)
dep_name = dep_split[0].strip()
if dep_name not in deps_checked:
deps_checked.add(dep_name)
if dep_name not in provided_map:
self._fill_missing_dep(
dep_name=dep_name,
dep_exp=dep,
aur_index=aur_index,
missing_deps=missing_deps,
remote_provided_map=remote_provided_map,
remote_repo_map=remote_repo_map,
repo_deps=repo_missing,
aur_deps=aur_missing,
watcher=watcher,
deps_data=deps_data,
automatch_providers=automatch_providers)
else:
version_pattern = '{}='.format(dep_name)
version_found = [
p for p in provided_map
if p.startswith(version_pattern)
]
if version_found:
version_found = version_found[0].split(
'=')[1]
version_informed = dep_split[2].strip()
if ':' not in version_informed:
version_found = version_found.split(
':')[-1]
if '-' not in version_informed:
version_found = version_found.split(
'-')[0]
try:
version_found = parse_version(
version_found)
version_informed = parse_version(
version_informed)
op = dep_split[
1] if dep_split[1] != '=' else '=='
match = eval(
'version_found {} version_informed'
.format(op))
except:
match = False
traceback.print_exc()
if not match:
self._fill_missing_dep(
dep_name=dep_name,
dep_exp=dep,
aur_index=aur_index,
missing_deps=missing_deps,
remote_provided_map=
remote_provided_map,
remote_repo_map=remote_repo_map,
repo_deps=repo_missing,
aur_deps=aur_missing,
watcher=watcher,
deps_data=deps_data,
automatch_providers=
automatch_providers)
else:
self._fill_missing_dep(
dep_name=dep_name,
dep_exp=dep,
aur_index=aur_index,
missing_deps=missing_deps,
remote_provided_map=remote_provided_map,
remote_repo_map=remote_repo_map,
repo_deps=repo_missing,
aur_deps=aur_missing,
watcher=watcher,
deps_data=deps_data,
automatch_providers=automatch_providers
)
if missing_deps:
if repo_missing:
with_single_providers = []
for d in missing_deps:
if d[0] in repo_missing and d[0] not in deps_data:
if d[1] == '__several__':
deps_data[d[0]] = {'d': None, 'p': d[0], 'r': d[1]}
else:
with_single_providers.append(d[0])
if with_single_providers:
data = pacman.map_updates_data(with_single_providers)
if data:
deps_data.update(data)
if aur_missing:
aur_threads = []
for pkgname in aur_missing:
t = Thread(target=self.__fill_aur_update_data,
args=(pkgname, deps_data),
daemon=True)
t.start()
aur_threads.append(t)
for t in aur_threads:
t.join()
missing_subdeps = self.map_missing_deps(
pkgs_data=deps_data,
provided_map=provided_map,
aur_index=aur_index,
deps_checked=deps_checked,
sort=False,
deps_data=deps_data,
watcher=watcher,
remote_provided_map=remote_provided_map,
remote_repo_map=remote_repo_map,
automatch_providers=automatch_providers,
choose_providers=False)
if missing_subdeps:
missing_deps.update(missing_subdeps)
if sort:
sorted_deps.extend(sorting.sort(deps_data.keys(), deps_data))
else:
sorted_deps.extend(((dep[0], dep[1]) for dep in missing_deps))
if sorted_deps and choose_providers:
return self.fill_providers_deps(
missing_deps=sorted_deps,
provided_map=provided_map,
remote_provided_map=remote_provided_map,
remote_repo_map=remote_repo_map,
watcher=watcher,
sort=sort,
already_checked=deps_checked,
aur_idx=aur_index,
deps_data=deps_data,
automatch_providers=automatch_providers)
return sorted_deps
def _fill_missing_dep(self, dep_name: str, dep_exp: str,
aur_index: Iterable[str],
missing_deps: Set[Tuple[str, str]],
remote_provided_map: Dict[str, Set[str]],
remote_repo_map: Dict[str, str], repo_deps: Set[str],
aur_deps: Set[str], deps_data: Dict[str, dict],
watcher: ProcessWatcher, automatch_providers: bool):
if dep_name == dep_exp:
providers = remote_provided_map.get(dep_name)
if not providers: # try to find the package through the pacman's search mechanism
match = pacman.find_one_match(dep_name)
if match:
providers = {match}
else: # handling cases when the dep has an expression ( e.g: xpto>=0.12 )
providers = remote_provided_map.get(dep_exp)
if providers is None:
providers = remote_provided_map.get(dep_name)
if not providers: # try to find the package through the pacman's search mechanism
match = pacman.find_one_match(dep_name)
if match:
providers = {match}
if providers and len(providers) > 1:
no_mapped_data = {
p
for p in providers if p not in deps_data
} # checking providers with no mapped data
if no_mapped_data:
providers_data = pacman.map_updates_data(
no_mapped_data)
if not providers_data:
raise Exception(
"Could not retrieve the info from providers: {}"
.format(no_mapped_data))
deps_data.update(
providers_data) # adding missing providers data
matched_providers = set()
split_informed_dep = self.re_dep_operator.split(dep_exp)
try:
version_informed = parse_version(split_informed_dep[2])
exp_op = split_informed_dep[
1] if split_informed_dep[1] != '=' else '=='
for p in providers:
provided = deps_data[p]['p']
for provided_exp in provided:
split_dep = self.re_dep_operator.split(
provided_exp)
if len(split_dep
) == 3 and split_dep[0] == dep_name:
provided_version = parse_version(
split_dep[2])
if eval('provided_version {} version_informed'
.format(exp_op)):
matched_providers.add(p)
break
providers = matched_providers
except:
traceback.print_exc()
if providers:
if len(providers) > 1:
dep_data = None
if automatch_providers:
exact_matches = [p for p in providers if p == dep_name]
if exact_matches:
dep_data = (exact_matches[0],
remote_repo_map.get(exact_matches[0]))
if not dep_data:
dep_data = (dep_name, '__several__')
else:
real_name = providers.pop()
dep_data = (real_name, remote_repo_map.get(real_name))
repo_deps.add(dep_data[0])
missing_deps.add(dep_data)
elif aur_index and dep_name in aur_index:
aur_deps.add(dep_name)
missing_deps.add((dep_name, 'aur'))
else:
if watcher:
message.show_dep_not_found(dep_exp, self.i18n, watcher)
raise PackageNotFoundException(dep_exp)
else:
raise PackageNotFoundException(dep_exp)
def fill_providers_deps(
self, missing_deps: List[Tuple[str,
str]], provided_map: Dict[str,
Set[str]],
remote_repo_map: Dict[str, str], already_checked: Set[str],
remote_provided_map: Dict[str, Set[str]], deps_data: Dict[str,
dict],
aur_idx: Iterable[str], sort: bool, watcher: ProcessWatcher,
automatch_providers: bool, prefer_repository_provider: bool
) -> Optional[List[Tuple[str, str]]]:
"""
:param missing_deps:
:param provided_map:
:param remote_repo_map:
:param already_checked:
:param remote_provided_map:
:param deps_data:
:param aur_idx:
:param sort:
:param watcher:
:param automatch_providers
:param prefer_repository_provider
:return: all deps sorted or None if the user declined the providers options
"""
deps_providers = map_providers(
{data[0]
for data in missing_deps if data[1] == '__several__'},
remote_provided_map)
if deps_providers:
providers_repos = {}
repos_providers = set()
for providers in deps_providers.values():
for provider in providers:
if remote_repo_map.get(provider) == 'aur':
providers_repos[provider] = 'aur'
else:
repos_providers.add(provider)
providers_repos.update(pacman.map_repositories(repos_providers))
selected_providers = confirmation.request_providers(
deps_providers, providers_repos, watcher, self.i18n)
if not selected_providers:
return
else:
# adding the chosen providers for re-checking the missing dependencies
repo_selected, aur_selected = set(), set()
for provider in selected_providers:
if provider in repos_providers:
repo_selected.add(provider)
else:
aur_selected.add(provider)
providers_data = dict()
if repo_selected:
providers_data.update(
pacman.map_updates_data(repo_selected))
# adding the providers as "installed" packages
provided_map.update(
pacman.map_provided(remote=True, pkgs=repo_selected))
if aur_selected:
for pkgname, pkgdata in self.aur_client.gen_updates_data(
aur_selected):
providers_data[pkgname] = pkgdata
for provider in pkgdata[
'p']: # adding the providers as "installed" packages
currently_provided = provided_map.get(
provider, set())
provided_map[provider] = currently_provided
currently_provided.add(pkgname)
providers_deps = self.map_missing_deps(
pkgs_data=providers_data,
provided_map=provided_map,
aur_index=aur_idx,
deps_checked=already_checked,
deps_data=deps_data,
sort=False,
remote_provided_map=remote_provided_map,
remote_repo_map=remote_repo_map,
watcher=watcher,
choose_providers=True,
automatch_providers=automatch_providers,
prefer_repository_provider=prefer_repository_provider)
if providers_deps is None: # it means the user called off the installation process
return
# cleaning the already mapped providers deps:
to_remove = []
for idx, dep in enumerate(missing_deps):
if dep[1] == '__several__':
to_remove.append(idx)
for idx, to_remove in enumerate(to_remove):
del missing_deps[to_remove - idx]
missing_deps.extend(((p, providers_repos.get(p, 'aur'))
for p in selected_providers))
for dep in providers_deps:
if dep not in missing_deps and dep[1] != '__several__':
missing_deps.append(dep)
deps_data.update(providers_data)
if not self.fill_providers_deps(
missing_deps=missing_deps,
provided_map=provided_map,
remote_repo_map=remote_repo_map,
already_checked=already_checked,
aur_idx=aur_idx,
remote_provided_map=remote_provided_map,
deps_data=deps_data,
sort=False,
watcher=watcher,
automatch_providers=automatch_providers,
prefer_repository_provider=prefer_repository_provider):
return
if sort:
missing_to_sort = {
d[0]
for d in missing_deps if d[1] != '__several__'
}
return sorting.sort(missing_to_sort, deps_data,
provided_map)
return missing_deps
def _find_repo_providers(
self, dep_name: str, dep_exp: str, remote_provided_map: Dict[str,
Set[str]],
deps_data: Dict[str, dict], remote_repo_map: Dict[str, str]
) -> Generator[Tuple[str, str, Optional[dict]], None, None]:
if dep_name == dep_exp:
providers = remote_provided_map.get(dep_name)
if providers:
for pkgname in providers:
yield pkgname, remote_repo_map.get(pkgname), None
else: # try to find the package through the pacman's search mechanism
match = pacman.find_one_match(dep_name)
if match:
yield match, remote_repo_map.get(match), None
else: # handling cases when the dep has an expression ( e.g: xpto>=0.12 )
exact_exp_providers = remote_provided_map.get(dep_exp)
if exact_exp_providers:
for p in exact_exp_providers:
yield p, remote_repo_map.get(p), None
else:
providers = remote_provided_map.get(dep_name)
if not providers: # try to find the package through the pacman's search mechanism
match = pacman.find_one_match(dep_name)
if match:
providers = {match}
if providers:
providers_no_provided_data = {
p
for p in providers if p not in deps_data
}
missing_providers_data = None
if providers_no_provided_data:
missing_providers_data = pacman.map_updates_data(
providers_no_provided_data)
if not missing_providers_data:
raise Exception(
f"Could not retrieve information from providers: "
f"{', '.join(providers_no_provided_data)}")
data_not_found = {
p
for p in providers
if p not in missing_providers_data
}
if data_not_found:
raise Exception(
f"Could not retrieve information from providers: "
f"{', '.join(data_not_found)}")
split_informed_dep = self.re_dep_operator.split(dep_exp)
version_required = split_informed_dep[2]
exp_op = split_informed_dep[
1] if split_informed_dep[1] != '=' else '=='
for p in providers:
info = deps_data.get(p)
if not info and missing_providers_data:
info = missing_providers_data[p]
for provided_exp in info['p']:
split_dep = self.re_dep_operator.split(
provided_exp)
if len(split_dep
) == 3 and split_dep[0] == dep_name:
version_provided = split_dep[2]
if match_required_version(
version_provided, exp_op,
version_required):
yield p, remote_repo_map.get(p), info
break
