Exemplo n.º 1
0
def attempt_source_package_import(distro, source, package_root,
                                  importer_handler):
    """Attempt to import a source package, and handle typical errors."""
    package_name = source.get("Package", "unknown")
    try:
        try:
            do_one_sourcepackage(distro, source, package_root,
                                 importer_handler)
        except psycopg2.Error:
            log.exception(
                "Database error: unable to create SourcePackage for %s. "
                "Retrying once..", package_name)
            importer_handler.abort()
            time.sleep(15)
            do_one_sourcepackage(distro, source, package_root,
                                 importer_handler)
    except (InvalidVersionError, MissingRequiredArguments,
            DisplayNameDecodingError):
        log.exception("Unable to create SourcePackageData for %s",
                      package_name)
    except (PoolFileNotFound, ExecutionError):
        # Problems with katie db stuff of opening files
        log.exception("Error processing package files for %s", package_name)
    except psycopg2.Error:
        log.exception(
            "Database errors made me give up: unable to create "
            "SourcePackage for %s", package_name)
        importer_handler.abort()
    except MultiplePackageReleaseError:
        log.exception("Database duplication processing %s", package_name)
Exemplo n.º 2
0
def attempt_source_package_import(distro, source, package_root, importer_handler):
    """Attempt to import a source package, and handle typical errors."""
    package_name = source.get("Package", "unknown")
    try:
        try:
            do_one_sourcepackage(distro, source, package_root, importer_handler)
        except psycopg2.Error:
            log.exception("Database error: unable to create SourcePackage for %s. " "Retrying once..", package_name)
            importer_handler.abort()
            time.sleep(15)
            do_one_sourcepackage(distro, source, package_root, importer_handler)
    except (InvalidVersionError, MissingRequiredArguments, DisplayNameDecodingError):
        log.exception("Unable to create SourcePackageData for %s", package_name)
    except (PoolFileNotFound, ExecutionError):
        # Problems with katie db stuff of opening files
        log.exception("Error processing package files for %s", package_name)
    except psycopg2.Error:
        log.exception("Database errors made me give up: unable to create " "SourcePackage for %s", package_name)
        importer_handler.abort()
    except MultiplePackageReleaseError:
        log.exception("Database duplication processing %s", package_name)
Exemplo n.º 3
0
 def listChannels(cls):
     """See `ISnapStoreClient`."""
     if config.snappy.store_search_url is None:
         return _default_store_channels
     channels = None
     memcache_client = getUtility(IMemcacheClient)
     search_host = urlsplit(config.snappy.store_search_url).hostname
     memcache_key = ("%s:channels" % search_host).encode("UTF-8")
     cached_channels = memcache_client.get(memcache_key)
     if cached_channels is not None:
         try:
             channels = json.loads(cached_channels)
         except JSONDecodeError:
             log.exception(
                 "Cannot load cached channels for %s; deleting" %
                 search_host)
             memcache_client.delete(memcache_key)
     if (channels is None and
             not getFeatureFlag(u"snap.disable_channel_search")):
         path = "api/v1/channels"
         timeline = cls._getTimeline()
         if timeline is not None:
             action = timeline.start("store-search-get", "/" + path)
         channels_url = urlappend(config.snappy.store_search_url, path)
         try:
             response = urlfetch(
                 channels_url, headers={"Accept": "application/hal+json"})
         except requests.HTTPError as e:
             raise cls._makeSnapStoreError(BadSearchResponse, e)
         finally:
             if timeline is not None:
                 action.finish()
         channels = response.json().get("_embedded", {}).get(
             "clickindex:channel", [])
         expire_time = time.time() + 60 * 60 * 24
         memcache_client.set(
             memcache_key, json.dumps(channels), expire_time)
     if channels is None:
         channels = _default_store_channels
     return channels
Exemplo n.º 4
0
    def create_maps(self, arch_component_items):
        # Create the maps
        self.src_map = defaultdict(list)
        self.bin_map = {}

        # Iterate over ArchComponentItems instance to cover
        # all components in all architectures.
        for info_set in arch_component_items:

            # Run over the source stanzas and store info in src_map. We
            # make just one source map (instead of one per architecture)
            # because most of them are the same for all architectures,
            # but we go over it to also cover source packages that only
            # compile for one architecture.
            sources = apt_pkg.TagFile(info_set.srcfile)
            for section in sources:
                try:
                    src_tmp = dict(section)
                    src_tmp['Component'] = info_set.component
                    src_name = src_tmp['Package']
                except KeyError:
                    log.exception(
                        "Invalid Sources stanza in %s",
                        info_set.sources_tagfile)
                    continue
                self.src_map[src_name].append(src_tmp)

            # Check if it's in source-only mode.  If so, skip binary index
            # mapping.
            if info_set.source_only:
                continue

            # Create a tmp map for binaries for one arch/component pair.
            self.bin_map.setdefault(info_set.arch, {})

            tmpbin_map = self.bin_map[info_set.arch]

            binaries = apt_pkg.TagFile(info_set.binfile)
            for section in binaries:
                try:
                    bin_tmp = dict(section)
                    # The component isn't listed in the tagfile.
                    bin_tmp['Component'] = info_set.component
                    bin_name = bin_tmp['Package']
                except KeyError:
                    log.exception(
                        "Invalid Releases stanza in %s",
                        info_set.binaries_tagfile)
                    continue
                tmpbin_map[bin_name] = bin_tmp

            # Run over the D-I stanzas and store info in tmp_bin_map.
            dibinaries = apt_pkg.TagFile(info_set.difile)
            for section in dibinaries:
                try:
                    dibin_tmp = dict(section)
                    dibin_tmp['Component'] = info_set.component
                    dibin_name = dibin_tmp['Package']
                except KeyError:
                    log.exception("Invalid D-I Releases stanza in %s" %
                                  info_set.difile)
                    continue
                tmpbin_map[dibin_name] = dibin_tmp
Exemplo n.º 5
0
def run_gina(options, ztm, target_section):
    # Avoid circular imports.
    from lp.registry.interfaces.pocket import PackagePublishingPocket

    package_root = target_section.root
    distro = target_section.distro
    pocket_distroseries = target_section.pocketrelease
    distroseries = target_section.distroseries
    components = [c.strip() for c in target_section.components.split(",")]
    archs = [a.strip() for a in target_section.architectures.split(",")]
    pocket = target_section.pocket
    component_override = target_section.componentoverride
    source_only = target_section.source_only
    spnames_only = target_section.sourcepackagenames_only

    LIBRHOST = config.librarian.upload_host
    LIBRPORT = config.librarian.upload_port

    log.info("")
    log.info("=== Processing %s/%s/%s ===", distro, distroseries, pocket)
    log.debug("Packages read from: %s", package_root)
    log.info("Components to import: %s", ", ".join(components))
    if component_override is not None:
        log.info("Override components to: %s", component_override)
    log.info("Architectures to import: %s", ", ".join(archs))
    log.debug("Launchpad database: %s", config.database.rw_main_master)
    log.info("SourcePackage Only: %s", source_only)
    log.info("SourcePackageName Only: %s", spnames_only)
    log.debug("Librarian: %s:%s", LIBRHOST, LIBRPORT)
    log.info("")

    if not hasattr(PackagePublishingPocket, pocket.upper()):
        log.error("Could not find a pocket schema for %s", pocket)
        sys.exit(1)

    pocket = getattr(PackagePublishingPocket, pocket.upper())

    if component_override:
        valid_components = [component.name for component in getUtility(IComponentSet)]
        if component_override not in valid_components:
            log.error("Could not find component %s", component_override)
            sys.exit(1)

    try:
        arch_component_items = ArchiveComponentItems(package_root, pocket_distroseries, components, archs, source_only)
    except MangledArchiveError:
        log.exception("Failed to analyze archive for %s", pocket_distroseries)
        sys.exit(1)

    packages_map = PackagesMap(arch_component_items)
    importer_handler = ImporterHandler(ztm, distro, distroseries, package_root, pocket, component_override)

    import_sourcepackages(distro, packages_map, package_root, importer_handler)
    importer_handler.commit()

    # XXX JeroenVermeulen 2011-09-07 bug=843728: Dominate binaries as well.
    dominate_imported_source_packages(ztm, log, distro, distroseries, pocket, packages_map)
    ztm.commit()

    if source_only:
        log.info("Source only mode... done")
        return

    for archtag in archs:
        try:
            importer_handler.ensure_archinfo(archtag)
        except DataSetupError:
            log.exception("Database setup required for run on %s", archtag)
            sys.exit(1)

    import_binarypackages(distro, packages_map, package_root, importer_handler)
    importer_handler.commit()
Exemplo n.º 6
0
def import_binarypackages(distro, packages_map, package_root, importer_handler):
    nosource = []

    # Run over all the architectures we have
    for archtag in packages_map.bin_map.keys():
        npacks = len(packages_map.bin_map[archtag])
        log.info("%i Binary Packages to be imported for %s", npacks, archtag)
        # Go over binarypackages importing them for this architecture
        for package_name in sorted(packages_map.bin_map[archtag].iterkeys()):
            binary = packages_map.bin_map[archtag][package_name]
            try:
                try:
                    do_one_binarypackage(distro, binary, archtag, package_root, importer_handler)
                except psycopg2.Error:
                    log.exception(
                        "Database errors when importing a BinaryPackage " "for %s. Retrying once..", package_name
                    )
                    importer_handler.abort()
                    time.sleep(15)
                    do_one_binarypackage(distro, binary, archtag, package_root, importer_handler)
            except (InvalidVersionError, MissingRequiredArguments):
                log.exception("Unable to create BinaryPackageData for %s", package_name)
                continue
            except (PoolFileNotFound, ExecutionError):
                # Problems with katie db stuff of opening files
                log.exception("Error processing package files for %s", package_name)
                continue
            except MultiplePackageReleaseError:
                log.exception("Database duplication processing %s", package_name)
                continue
            except psycopg2.Error:
                log.exception("Database errors made me give up: unable to create " "BinaryPackage for %s", package_name)
                importer_handler.abort()
                continue
            except NoSourcePackageError:
                log.exception("Failed to create Binary Package for %s", package_name)
                nosource.append(binary)
                continue

        if nosource:
            # XXX kiko 2005-10-23: untested
            log.warn("%i source packages not found", len(nosource))
            for pkg in nosource:
                log.warn(pkg)
Exemplo n.º 7
0
    def create_maps(self, arch_component_items):
        # Create the maps
        self.src_map = defaultdict(list)
        self.bin_map = {}

        # Iterate over ArchComponentItems instance to cover
        # all components in all architectures.
        for info_set in arch_component_items:

            # Run over the source stanzas and store info in src_map. We
            # make just one source map (instead of one per architecture)
            # because most of them are the same for all architectures,
            # but we go over it to also cover source packages that only
            # compile for one architecture.
            sources = apt_pkg.TagFile(info_set.srcfile)
            try:
                for section in sources:
                    try:
                        src_tmp = dict(section)
                        src_tmp['Component'] = info_set.component
                        src_name = src_tmp['Package']
                    except KeyError:
                        log.exception("Invalid Sources stanza in %s",
                                      info_set.sources_tagfile)
                        continue
                    self.src_map[src_name].append(src_tmp)
            except SystemError:
                log.exception("Invalid Sources stanza in %s",
                              info_set.sources_tagfile)

            # Check if it's in source-only mode.  If so, skip binary index
            # mapping.
            if info_set.source_only:
                continue

            # Create a tmp map for binaries for one arch/component pair.
            self.bin_map.setdefault(info_set.arch, {})

            tmpbin_map = self.bin_map[info_set.arch]

            binaries = apt_pkg.TagFile(info_set.binfile)
            for section in binaries:
                try:
                    bin_tmp = dict(section)
                    # The component isn't listed in the tagfile.
                    bin_tmp['Component'] = info_set.component
                    bin_name = bin_tmp['Package']
                except KeyError:
                    log.exception("Invalid Releases stanza in %s",
                                  info_set.binaries_tagfile)
                    continue
                tmpbin_map[bin_name] = bin_tmp

            # Run over the D-I stanzas and store info in tmp_bin_map.
            dibinaries = apt_pkg.TagFile(info_set.difile)
            for section in dibinaries:
                try:
                    dibin_tmp = dict(section)
                    dibin_tmp['Component'] = info_set.component
                    dibin_name = dibin_tmp['Package']
                except KeyError:
                    log.exception("Invalid D-I Releases stanza in %s" %
                                  info_set.difile)
                    continue
                tmpbin_map[dibin_name] = dibin_tmp
Exemplo n.º 8
0
def run_gina(options, ztm, target_section):
    package_root = target_section.root
    distro = target_section.distro
    pocket_distroseries = target_section.pocketrelease
    distroseries = target_section.distroseries
    components = [c.strip() for c in target_section.components.split(",")]
    archs = [a.strip() for a in target_section.architectures.split(",")]
    pocket = target_section.pocket
    component_override = target_section.componentoverride
    source_only = target_section.source_only
    spnames_only = target_section.sourcepackagenames_only

    LIBRHOST = config.librarian.upload_host
    LIBRPORT = config.librarian.upload_port

    log.info("")
    log.info("=== Processing %s/%s/%s ===", distro, distroseries, pocket)
    log.debug("Packages read from: %s", package_root)
    log.info("Components to import: %s", ", ".join(components))
    if component_override is not None:
        log.info("Override components to: %s", component_override)
    log.info("Architectures to import: %s", ", ".join(archs))
    log.debug("Launchpad database: %s", config.database.rw_main_master)
    log.info("SourcePackage Only: %s", source_only)
    log.info("SourcePackageName Only: %s", spnames_only)
    log.debug("Librarian: %s:%s", LIBRHOST, LIBRPORT)
    log.info("")

    if not hasattr(PackagePublishingPocket, pocket.upper()):
        log.error("Could not find a pocket schema for %s", pocket)
        sys.exit(1)

    pocket = getattr(PackagePublishingPocket, pocket.upper())

    if component_override:
        valid_components = [
            component.name for component in getUtility(IComponentSet)
        ]
        if component_override not in valid_components:
            log.error("Could not find component %s", component_override)
            sys.exit(1)

    try:
        arch_component_items = ArchiveComponentItems(package_root,
                                                     pocket_distroseries,
                                                     components, archs,
                                                     source_only)
    except MangledArchiveError:
        log.exception("Failed to analyze archive for %s", pocket_distroseries)
        sys.exit(1)

    packages_map = PackagesMap(arch_component_items)
    importer_handler = ImporterHandler(ztm, distro, distroseries, package_root,
                                       pocket, component_override)

    import_sourcepackages(distro, packages_map, package_root, importer_handler)
    importer_handler.commit()

    # XXX JeroenVermeulen 2011-09-07 bug=843728: Dominate binaries as well.
    dominate_imported_source_packages(ztm, log, distro, distroseries, pocket,
                                      packages_map)
    ztm.commit()

    if source_only:
        log.info('Source only mode... done')
        return

    for archtag in archs:
        try:
            importer_handler.ensure_arch(archtag)
        except DataSetupError:
            log.exception("Database setup required for run on %s", archtag)
            sys.exit(1)

    import_binarypackages(distro, packages_map, package_root, importer_handler)
    importer_handler.commit()
Exemplo n.º 9
0
def import_binarypackages(distro, packages_map, package_root,
                          importer_handler):
    nosource = []

    # Run over all the architectures we have
    for archtag in packages_map.bin_map.keys():
        npacks = len(packages_map.bin_map[archtag])
        log.info('%i Binary Packages to be imported for %s', npacks, archtag)
        # Go over binarypackages importing them for this architecture
        for package_name in sorted(packages_map.bin_map[archtag].iterkeys()):
            binary = packages_map.bin_map[archtag][package_name]
            try:
                try:
                    do_one_binarypackage(distro, binary, archtag, package_root,
                                         importer_handler)
                except psycopg2.Error:
                    log.exception(
                        "Database errors when importing a BinaryPackage "
                        "for %s. Retrying once..", package_name)
                    importer_handler.abort()
                    time.sleep(15)
                    do_one_binarypackage(distro, binary, archtag, package_root,
                                         importer_handler)
            except (InvalidVersionError, MissingRequiredArguments):
                log.exception("Unable to create BinaryPackageData for %s",
                              package_name)
                continue
            except (PoolFileNotFound, ExecutionError):
                # Problems with katie db stuff of opening files
                log.exception("Error processing package files for %s",
                              package_name)
                continue
            except MultiplePackageReleaseError:
                log.exception("Database duplication processing %s",
                              package_name)
                continue
            except psycopg2.Error:
                log.exception(
                    "Database errors made me give up: unable to create "
                    "BinaryPackage for %s", package_name)
                importer_handler.abort()
                continue
            except NoSourcePackageError:
                log.exception("Failed to create Binary Package for %s",
                              package_name)
                nosource.append(binary)
                continue

        if nosource:
            # XXX kiko 2005-10-23: untested
            log.warn('%i source packages not found', len(nosource))
            for pkg in nosource:
                log.warn(pkg)