Esempio n. 1
0
    def index(self, repo_uri, skip_sources=False):
        self.repo_dir = repo_uri

        packages = []
        deltas = {}
        for root, dirs, files in os.walk(repo_uri):
            for fn in files:

                if fn.endswith(ctx.const.delta_package_suffix):
                    name, version = util.parse_package_name(fn)
                    deltas.setdefault(name, []).append(os.path.join(root, fn))
                elif fn.endswith(ctx.const.package_suffix):
                    packages.append(os.path.join(root, fn))

                if fn == "components.xml":
                    self.add_components(os.path.join(root, fn))
                if fn == "pspec.xml" and not skip_sources:
                    self.add_spec(os.path.join(root, fn), repo_uri)
                if fn == "distribution.xml":
                    self.add_distro(os.path.join(root, fn))
                if fn == "groups.xml":
                    self.add_groups(os.path.join(root, fn))

        try:
            obsoletes_list = map(str, self.distribution.obsoletes)
        except AttributeError:
            obsoletes_list = []

        for pkg in util.filter_latest_packages(packages):
            pkg_name = util.parse_package_name(os.path.basename(pkg))[0]
            if pkg_name not in obsoletes_list:
                ctx.ui.info(_("Adding %s to package index") % pkg)
                self.add_package(pkg, deltas, repo_uri)
Esempio n. 2
0
    def index(self, repo_uri, skip_sources=False):
        self.repo_dir = repo_uri

        packages = []
        deltas = {}
        for root, dirs, files in os.walk(repo_uri):
            for fn in files:

                if fn.endswith(ctx.const.delta_package_suffix):
                    name, version = util.parse_package_name(fn)
                    deltas.setdefault(name, []).append(os.path.join(root, fn))
                elif fn.endswith(ctx.const.package_suffix):
                    packages.append(os.path.join(root, fn))

                if fn == 'component.xml':
                    ctx.ui.info(_('Adding %s to component index') % fn)
                    self.add_component(os.path.join(root, fn))
                if fn == 'pspec.xml' and not skip_sources:
                    self.add_spec(os.path.join(root, fn), repo_uri)
                if fn == 'distribution.xml':
                    self.add_distro(os.path.join(root, fn))

        try:
            obsoletes_list = map(str, self.distribution.obsoletes)
        except AttributeError:
            obsoletes_list = []

        for pkg in util.filter_latest_packages(packages):
            pkg_name = util.parse_package_name(os.path.basename(pkg))[0]
            if pkg_name not in obsoletes_list:
                ctx.ui.info(_('Adding %s to package index') % pkg)
                self.add_package(pkg, deltas, repo_uri)
Esempio n. 3
0
    def index(self, repo_uri, skip_sources=False):
        self.repo_dir = repo_uri

        packages = []
        specs = []
        deltas = {}

        for root, dirs, files in os.walk(repo_uri):
            # Filter hidden directories
            # TODO: Add --exclude-dirs parameter to CLI and filter according
            # directories here
            dirs[:] = [d for d in dirs if not d.startswith(".")]

            for fn in files:

                if fn.endswith(ctx.const.delta_package_suffix):
                    name, version = util.parse_package_name(fn)
                    deltas.setdefault(name, []).append(os.path.join(root, fn))
                elif fn.endswith(ctx.const.package_suffix):
                    packages.append(os.path.join(root, fn))

                if fn == "components.xml":
                    self.components.extend(add_components(os.path.join(root, fn)))
                if fn == "pspec.xml" and not skip_sources:
                    specs.append((os.path.join(root, fn), repo_uri))
                if fn == "distribution.xml":
                    self.distribution = add_distro(os.path.join(root, fn))
                if fn == "groups.xml":
                    self.groups.extend(add_groups(os.path.join(root, fn)))

        ctx.ui.info("")

        # Create a process pool, as many processes as the number of CPUs we
        # have
        pool = multiprocessing.Pool()

        # Before calling pool.map check if list is empty or not: python#12157
        if specs:
            try:
                # Add source packages to index using a process pool
                self.specs = pool.map(add_spec, specs)
            except:
                # If an exception occurs (like a keyboard interrupt),
                # immediately terminate worker processes and propagate
                # exception. (CLI honors KeyboardInterrupt exception, if you're
                # not using CLI, you must handle KeyboardException yourself)
                pool.terminate()
                pool.join()
                ctx.ui.info("")
                raise

        try:
            obsoletes_list = map(str, self.distribution.obsoletes)
        except AttributeError:
            obsoletes_list = []

        latest_packages = []

        for pkg in util.filter_latest_packages(packages):
            pkg_name = util.parse_package_name(os.path.basename(pkg))[0]
            if pkg_name.endswith(ctx.const.debug_name_suffix):
                pkg_name = util.remove_suffix(ctx.const.debug_name_suffix, pkg_name)
            if pkg_name not in obsoletes_list:
                # Currently, multiprocessing.Pool.map method accepts methods
                # with single parameters only. So we have to send our
                # parameters as a tuple to workaround that

                latest_packages.append((pkg, deltas, repo_uri))

        # Before calling pool.map check if list is empty or not: python#12157
        if latest_packages:
            try:
                # Add binary packages to index using a process pool
                self.packages = pool.map(add_package, latest_packages)
            except:
                pool.terminate()
                pool.join()
                ctx.ui.info("")
                raise

        ctx.ui.info("")
        pool.close()
        pool.join()
Esempio n. 4
0
    def index(self, repo_uri, skip_sources=False):
        self.repo_dir = repo_uri

        packages = []
        specs = []
        deltas = {}

        for root, dirs, files in os.walk(repo_uri):
            # Filter hidden directories
            # TODO: Add --exclude-dirs parameter to CLI and filter according
            # directories here
            dirs[:] = [d for d in dirs if not d.startswith(".")]

            for fn in files:

                if fn.endswith(ctx.const.delta_package_suffix):
                    name, version = util.parse_package_name(fn)
                    deltas.setdefault(name, []).append(os.path.join(root, fn))
                elif fn.endswith(ctx.const.package_suffix):
                    packages.append(os.path.join(root, fn))

                if fn == 'components.xml':
                    self.components.extend(
                        add_components(os.path.join(root, fn)))
                if fn == 'pspec.xml' and not skip_sources:
                    specs.append((os.path.join(root, fn), repo_uri))
                if fn == 'distribution.xml':
                    self.distribution = add_distro(os.path.join(root, fn))
                if fn == 'groups.xml':
                    self.groups.extend(add_groups(os.path.join(root, fn)))

        ctx.ui.info("")

        # Create a process pool, as many processes as the number of CPUs we
        # have
        pool = multiprocessing.Pool()

        # Before calling pool.map check if list is empty or not: python#12157
        if specs:
            try:
                # Add source packages to index using a process pool
                self.specs = pool.map(add_spec, specs)
            except:
                # If an exception occurs (like a keyboard interrupt),
                # immediately terminate worker processes and propagate
                # exception. (CLI honors KeyboardInterrupt exception, if you're
                # not using CLI, you must handle KeyboardException yourself)
                pool.terminate()
                pool.join()
                ctx.ui.info("")
                raise

        try:
            obsoletes_list = map(str, self.distribution.obsoletes)
        except AttributeError:
            obsoletes_list = []

        latest_packages = []

        for pkg in util.filter_latest_packages(packages):
            pkg_name = util.parse_package_name(os.path.basename(pkg))[0]
            if pkg_name.endswith(ctx.const.debug_name_suffix):
                pkg_name = util.remove_suffix(ctx.const.debug_name_suffix,
                                              pkg_name)
            if pkg_name not in obsoletes_list:
                # Currently, multiprocessing.Pool.map method accepts methods
                # with single parameters only. So we have to send our
                # parameters as a tuple to workaround that

                latest_packages.append((pkg, deltas, repo_uri))

        # Before calling pool.map check if list is empty or not: python#12157
        if latest_packages:
            try:
                # Add binary packages to index using a process pool
                self.packages = pool.map(add_package, latest_packages)
            except:
                pool.terminate()
                pool.join()
                ctx.ui.info("")
                raise

        ctx.ui.info("")
        pool.close()
        pool.join()
Esempio n. 5
0
 def scan_bins(self, binpath):
     binpaks = filter_latest_packages(glob.glob(binpath + "/*.pisi"))
     for pak in binpaks:
         self.processPisi(pak)
Esempio n. 6
0
    def index(self, repo_uri, skip_sources=False):
        self.repo_dir = repo_uri

        packages = []
        specs = []
        deltas = {}

        pkgs_sorted = False

        for fn in next(os.walk(repo_uri))[2]:
            if fn.endswith(ctx.const.delta_package_suffix) or fn.endswith(ctx.const.package_suffix):
                name, version = util.parse_package_name(fn)
                if name.split("-").pop() in ["devel", "32bit", "doc", "docs", "userspace"]:
                    name = name[:-1 - len(name.split("-").pop())]
                pkgpath = os.path.join(repo_uri,
                                       name[0:4].lower() if name.startswith("lib") and len(name) > 3 else name.lower()[0],
                                       name.lower())
                if not os.path.isdir(pkgpath):
                    os.makedirs(pkgpath)
                ctx.ui.info("%-80.80s\r" % (_('Sorting: %s ') % fn),
                            noln=False if ctx.config.get_option("verbose") else True)
                shutil.copy2(os.path.join(repo_uri, fn), pkgpath)
                os.remove(os.path.join(repo_uri, fn))
                pkgs_sorted = True

        if pkgs_sorted:
            ctx.ui.info("%-80.80s\r" % '')

        for root, dirs, files in os.walk(repo_uri):
            # Filter hidden directories
            # TODO: Add --exclude-dirs parameter to CLI and filter according
            # directories here
            dirs[:] = [d for d in dirs if not d.startswith(".")]

            for fn in files:

                if fn.endswith(ctx.const.delta_package_suffix):
                    name, version = util.parse_package_name(fn)
                    deltas.setdefault(name, []).append(os.path.join(root, fn))
                elif fn.endswith(ctx.const.package_suffix):
                    packages.append(os.path.join(root, fn))

                if fn == 'components.xml':
                    self.components.extend(add_components(os.path.join(root, fn)))
                if fn == 'pspec.xml' and not skip_sources:
                    specs.append((os.path.join(root, fn), repo_uri))
                if fn == 'distribution.xml':
                    self.distribution = add_distro(os.path.join(root, fn))
                if fn == 'groups.xml':
                    self.groups.extend(add_groups(os.path.join(root, fn)))

        ctx.ui.info("")

        # Create a process pool, as many processes as the number of CPUs we
        # have
        pool = multiprocessing.Pool()

        # Before calling pool.map check if list is empty or not: python#12157
        if specs:
            try:
                # Add source packages to index using a process pool
                self.specs = pool.map(add_spec, specs)
            except:
                # If an exception occurs (like a keyboard interrupt),
                # immediately terminate worker processes and propagate
                # exception. (CLI honors KeyboardInterrupt exception, if you're
                # not using CLI, you must handle KeyboardException yourself)
                pool.terminate()
                pool.join()
                ctx.ui.info("")
                raise

        try:
            obsoletes_list = list(map(str, self.distribution.obsoletes))
        except AttributeError:
            obsoletes_list = []

        latest_packages = []

        for pkg in util.filter_latest_packages(packages):
            pkg_name = util.parse_package_name(os.path.basename(pkg))[0]
            if pkg_name.endswith(ctx.const.debug_name_suffix):
                pkg_name = util.remove_suffix(ctx.const.debug_name_suffix,
                                              pkg_name)
            if pkg_name not in obsoletes_list:
                # Currently, multiprocessing.Pool.map method accepts methods
                # with single parameters only. So we have to send our
                # parameters as a tuple to workaround that

                latest_packages.append((pkg, deltas, repo_uri))

        # Before calling pool.map check if list is empty or not: python#12157
        if latest_packages:
            sorted_pkgs = {}
            for pkg in latest_packages:
                key = re.search("\/((lib)?[\d\w])\/", pkg[0])
                key = key.group(1) if key else os.path.dirname(pkg[0])
                try:
                    sorted_pkgs[key].append(pkg)
                except KeyError:
                    sorted_pkgs[key] = [pkg]
            self.packages = []
            for key, pkgs in sorted(sorted_pkgs.items()):
                ctx.ui.info("%-80.80s\r" % (_("Adding packages from directory %s... " % key)), noln=True)
                try:
                    # Add binary packages to index using a process pool
                    self.packages.extend(pool.map(add_package, pkgs))
                except:
                    pool.terminate()
                    pool.join()
                    ctx.ui.info("")
                    raise
                ctx.ui.info("%-80.80s\r" % (_("Adding packages from directory %s... done." % key)))

        ctx.ui.info("")
        pool.close()
        pool.join()
Esempio n. 7
0
    def index(self, repo_uri, skip_sources=False):
        self.repo_dir = repo_uri

        packages = []
        specs = []
        deltas = {}

        pkgs_sorted = False
        for fn in os.walk(repo_uri).next()[2]:
            if fn.endswith(ctx.const.delta_package_suffix) or fn.endswith(ctx.const.package_suffix):
                pkgpath = os.path.join(repo_uri,
                                       util.parse_package_dir_path(fn))
                if not os.path.isdir(pkgpath): os.makedirs(pkgpath)
                ctx.ui.info("%-80.80s\r" % (_('Sorting: %s ') %
                    fn), noln = False if ctx.config.get_option("verbose") else True)
                shutil.copy2(os.path.join(repo_uri, fn), pkgpath)
                os.remove(os.path.join(repo_uri, fn))
                pkgs_sorted = True
        if pkgs_sorted:
            ctx.ui.info("%-80.80s\r" % '')

        for root, dirs, files in os.walk(repo_uri):
            # Filter hidden directories
            # TODO: Add --exclude-dirs parameter to CLI and filter according
            # directories here
            dirs[:] = [d for d in dirs if not d.startswith(".")]

            for fn in files:

                if fn.endswith(ctx.const.delta_package_suffix):
                    name, version = util.parse_package_name(fn)
                    deltas.setdefault(name, []).append(os.path.join(root, fn))
                elif fn.endswith(ctx.const.package_suffix):
                    packages.append(os.path.join(root, fn))

                if fn == 'components.xml':
                    self.components.extend(add_components(os.path.join(root, fn)))
                if fn == 'pspec.xml' and not skip_sources:
                    specs.append((os.path.join(root, fn), repo_uri))
                if fn == 'distribution.xml':
                    self.distribution = add_distro(os.path.join(root, fn))
                if fn == 'groups.xml':
                    self.groups.extend(add_groups(os.path.join(root, fn)))

        ctx.ui.info("")

        # Create a process pool, as many processes as the number of CPUs we
        # have
        pool = multiprocessing.Pool()

        # Before calling pool.map check if list is empty or not: python#12157
        if specs:
            try:
                # Add source packages to index using a process pool
                self.specs = pool.map(add_spec, specs)
            except:
                # If an exception occurs (like a keyboard interrupt),
                # immediately terminate worker processes and propagate
                # exception. (CLI honors KeyboardInterrupt exception, if you're
                # not using CLI, you must handle KeyboardException yourself)
                pool.terminate()
                pool.join()
                ctx.ui.info("")
                raise

        try:
            obsoletes_list = map(str, self.distribution.obsoletes)
        except AttributeError:
            obsoletes_list = []

        latest_packages = []

        for pkg in util.filter_latest_packages(packages):
            pkg_name = util.parse_package_name(os.path.basename(pkg))[0]
            if pkg_name.endswith(ctx.const.debug_name_suffix):
                pkg_name = util.remove_suffix(ctx.const.debug_name_suffix,
                                              pkg_name)
            if pkg_name not in obsoletes_list:
                # Currently, multiprocessing.Pool.map method accepts methods
                # with single parameters only. So we have to send our
                # parameters as a tuple to workaround that

                latest_packages.append((pkg, deltas, repo_uri))

        # Before calling pool.map check if list is empty or not: python#12157
        if latest_packages:
            sorted_pkgs = {}
            for pkg in latest_packages:
                key = re.search("\/((lib)?[\d\w])\/", pkg[0])
                key = key.group(1) if key else os.path.dirname(pkg[0]) 
                try:
                    sorted_pkgs[key].append(pkg)
                except KeyError:
                    sorted_pkgs[key] = [pkg]
            self.packages = []
            for key, pkgs in sorted(sorted_pkgs.items()):
                ctx.ui.info("%-80.80s\r" % (_("Adding packages from directory %s... " % key)), noln=True)
                try:
                    # Add binary packages to index using a process pool
                    self.packages.extend(pool.map(add_package, pkgs))
                except:
                    pool.terminate()
                    pool.join()
                    ctx.ui.info("")
                    raise
                ctx.ui.info("%-80.80s\r" % (_("Adding packages from directory %s... done." % key)))

        ctx.ui.info("")
        pool.close()
        pool.join()
Esempio n. 8
0
 def scan_bins(self, binpath):
     binpaks = filter_latest_packages(glob.glob(binpath + "/*.pisi"))
     for pak in binpaks:
         self.processPisi(pak)