def __getpackageurl(package):
packagedb = inary.db.packagedb.PackageDB()
repodb = inary.db.repodb.RepoDB()
pkg = util.parse_package_name_get_name(package)
reponame = None
try:
reponame = packagedb.which_repo(pkg)
except Exception:
# Maybe this package is obsoluted from repository
for repo in repodb.get_binary_repos():
if pkg in packagedb.get_obsoletes(repo):
reponame = repo
if not reponame:
raise PackageNotFound
repourl = repodb.get_repo_url(reponame)
ctx.ui.info(
_("Package \"{0}\" found in repository \"{1}\".").format(
pkg, reponame))
# return _possible_ url for this package
return os.path.join(os.path.dirname(repourl),
util.parse_package_dir_path(package),
package)
def index(self, repo_uri, skip_sources=False):
self.repo_dir = repo_uri
packages = []
specs = []
deltas = {}
ctx.ui.info(_("* Generating index tree...\n"), color="cyan")
pkgs_sorted = False
for fn in os.walk(repo_uri).__next__()[2]:
if fn.endswith(ctx.const.delta_package_suffix) or fn.endswith(
ctx.const.package_suffix):
pkgpath = os.path.join(repo_uri,
util.parse_package_dir_path(fn))
if not os.path.isdir(pkgpath):
os.makedirs(pkgpath)
ctx.ui.info(
"{:80.80}\r".format(_(' -> Sorting: \"{}\"').format(fn)),
noln=False if ctx.config.get_option("verbose") else True)
shutil.copy2(os.path.join(repo_uri, fn), pkgpath)
os.remove(os.path.join(repo_uri, fn))
pkgs_sorted = True
if pkgs_sorted:
ctx.ui.info("{:80.80}\r".format(
util.colorize(_(' * Sorted: \"{}\"').format(fn),
color="green")))
for root, dirs, files in os.walk(repo_uri):
# Filter hidden directories
# TODO: Add --exclude-dirs parameter to CLI and filter according
# directories here
dirs[:] = [d for d in dirs if not d.startswith(".")]
for fn in files:
if fn.endswith(ctx.const.delta_package_suffix):
name = util.parse_package_name_get_name(fn)
deltas.setdefault(name, []).append(os.path.join(root, fn))
elif fn.endswith(ctx.const.package_suffix):
packages.append(os.path.join(root, fn))
if fn == 'components.xml':
self.components.extend(
add_components(os.path.join(root, fn)))
if fn == 'pspec.xml' and not skip_sources:
specs.append((os.path.join(root, fn), repo_uri))
if fn == 'distribution.xml':
self.distribution = add_distro(os.path.join(root, fn))
if fn == 'groups.xml':
self.groups.extend(add_groups(os.path.join(root, fn)))
ctx.ui.info("")
# Create a process pool, as many processes as the number of CPUs we
# have
try:
obsoletes_list = list(map(str, self.distribution.obsoletes))
except AttributeError:
obsoletes_list = []
if obsoletes_list:
ctx.ui.info(_(
" * Added obsoleted packages: [ {} ]".format(obsoletes_list)),
color="blue",
noln=False)
pool = multiprocessing.Pool()
# Before calling pool.map check if list is empty or not: python#12157
if specs:
ctx.ui.info(_(" * Adding source packages: "),
color="blue",
noln=False)
try:
# Add source packages to index using a process pool
self.specs = pool.map(add_spec, specs)
ctx.ui.info("\n")
except BaseException:
# If an exception occurs (like a keyboard interrupt),
# immediately terminate worker processes and propagate
# exception. (CLI honors KeyboardInterrupt exception, if you're
# not using CLI, you must handle KeyboardException yourself)
pool.terminate()
pool.join()
ctx.ui.info("")
raise
latest_packages = []
for pkg in util.filter_latest_packages(packages):
pkg_name = util.parse_package_name(os.path.basename(pkg))[0]
if pkg_name.endswith(ctx.const.debug_name_suffix):
pkg_name = util.remove_suffix(ctx.const.debug_name_suffix,
pkg_name)
if pkg_name not in obsoletes_list:
# Currently, multiprocessing.Pool.map method accepts methods
# with single parameters only. So we have to send our
# parameters as a tuple to workaround that
latest_packages.append((pkg, deltas, repo_uri))
# Before calling pool.map check if list is empty or not: python#12157
if latest_packages:
sorted_pkgs = {}
for pkg in latest_packages:
key = re.search(r"\/((lib)?[\d\w])\/", pkg[0])
key = key.group(1) if key else os.path.dirname(pkg[0])
try:
sorted_pkgs[key].append(pkg)
except KeyError:
sorted_pkgs[key] = [pkg]
self.packages = []
ctx.ui.info(_(" * Adding binary packages: "),
color="blue",
noln=False)
for key, pkgs in sorted(sorted_pkgs.items()):
ctx.ui.info("{:80.80}\r".format(
_(" -> Adding packages from directory \"{}\"... ".format(
key))),
noln=True)
try:
# Add binary packages to index using a process pool
self.packages.extend(pool.map(add_package, pkgs))
except BaseException:
pool.terminate()
pool.join()
ctx.ui.info("")
raise
ctx.ui.info("{:80.80}\r".format(
_(" * Adding packages from directory \"{}\"... done.".
format(key))),
color="green",
noln=False)
ctx.ui.info(_("* Writing index file."), color="blue")
pool.close()
pool.join()