Exemple #1
0
    def import_packages(self, plug, source_id, url):
        if (not self.filters) and source_id:
            h = rhnSQL.prepare("""
                    select flag, filter
                      from rhnContentSourceFilter
                     where source_id = :source_id
                     order by sort_order """)
            h.execute(source_id=source_id)
            filter_data = h.fetchall_dict() or []
            filters = [(row['flag'], re.split(r'[,\s]+', row['filter']))
                       for row in filter_data]
        else:
            filters = self.filters

        packages = plug.list_packages(filters, self.latest)
        self.all_packages.extend(packages)
        to_process = []
        num_passed = len(packages)
        log(0, "Packages in repo:             %5d" % plug.num_packages)
        if plug.num_excluded:
            log(0, "Packages passed filter rules: %5d" % num_passed)
        channel_id = int(self.channel['id'])
        if self.channel['org_id']:
            self.channel['org_id'] = int(self.channel['org_id'])
        else:
            self.channel['org_id'] = None
        for pack in packages:
            db_pack = rhnPackage.get_info_for_package(
                [pack.name, pack.version, pack.release, pack.epoch, pack.arch],
                channel_id, self.channel['org_id'])

            to_download = True
            to_link = True
            # Package exists in DB
            if db_pack:
                # Path in filesystem is defined
                if db_pack['path']:
                    pack.path = os.path.join(CFG.MOUNT_POINT, db_pack['path'])
                else:
                    pack.path = ""

                if self.metadata_only or self.match_package_checksum(pack.path,
                                                                     pack.checksum_type, pack.checksum):
                    # package is already on disk or not required
                    to_download = False
                    if db_pack['channel_id'] == channel_id:
                        # package is already in the channel
                        to_link = False

                elif db_pack['channel_id'] == channel_id:
                    # different package with SAME NVREA
                    self.disassociate_package(db_pack)

                # just pass data from DB, they will be used if there is no RPM available
                pack.checksum = db_pack['checksum']
                pack.checksum_type = db_pack['checksum_type']
                pack.epoch = db_pack['epoch']

            if to_download or to_link:
                to_process.append((pack, to_download, to_link))

        num_to_process = len(to_process)
        if num_to_process == 0:
            log(0, "No new packages to sync.")
            # If we are just appending, we can exit
            if not self.strict:
                return
        else:
            log(0, "Packages already synced:      %5d" % (num_passed - num_to_process))
            log(0, "Packages to sync:             %5d" % num_to_process)

        self.regen = True
        is_non_local_repo = (url.find("file:/") < 0)

        for (index, what) in enumerate(to_process):
            pack, to_download, to_link = what
            localpath = None
            # pylint: disable=W0703
            try:
                log(0, "%d/%d : %s" % (index + 1, num_to_process, pack.getNVREA()))
                if to_download:
                    pack.path = localpath = plug.get_package(pack, metadata_only=self.metadata_only)
                    pack.load_checksum_from_header()
                    pack.upload_package(self.channel, metadata_only=self.metadata_only)
            except KeyboardInterrupt:
                raise
            except Exception:
                e = sys.exc_info()[1]
                log2stderr(0, e)
                if self.fail:
                    raise
                to_process[index] = (pack, False, False)
                continue
            finally:
                if is_non_local_repo and localpath and os.path.exists(localpath):
                    os.remove(localpath)

        log(0, "Linking packages to channel.")
        if self.strict:
            import_batch = [self.associate_package(pack)
                            for pack in self.all_packages]
        else:
            import_batch = [self.associate_package(pack)
                            for (pack, to_download, to_link) in to_process
                            if to_link]
        backend = SQLBackend()
        caller = "server.app.yumreposync"
        importer = ChannelPackageSubscription(import_batch,
                                              backend, caller=caller, repogen=False,
                                              strict=self.strict)
        importer.run()
        backend.commit()
Exemple #2
0
    def sync(self, update_repodata=False):
        """Trigger a reposync"""
        start_time = datetime.now()
        for (repo_id, url, repo_label, channel_family_id) in self.urls:
            log(0, "Repo URL: %s" % url)
            plugin = None

            # If the repository uses a uln:// URL, switch to the ULN plugin, overriding the command-line
            if url.startswith("uln://"):
                self.repo_plugin = self.load_plugin("uln")

            # pylint: disable=W0703
            try:
                # use modified relative_url as name of repo plugin, because
                # it used as name of cache directory as well

                relative_url = '_'.join(url.split('://')[1].split('/')[1:])
                plugin_name = relative_url.replace("?", "_").replace("&", "_").replace("=", "_")

                plugin = self.repo_plugin(url, plugin_name)

                if update_repodata:
                    plugin.clear_cache()

                if repo_id is not None:
                    keys = rhnSQL.fetchone_dict("""
                        select k1.key as ca_cert, k2.key as client_cert, k3.key as client_key
                        from rhncontentssl
                                join rhncryptokey k1
                                on rhncontentssl.ssl_ca_cert_id = k1.id
                                left outer join rhncryptokey k2
                                on rhncontentssl.ssl_client_cert_id = k2.id
                                left outer join rhncryptokey k3
                                on rhncontentssl.ssl_client_key_id = k3.id
                        where rhncontentssl.content_source_id = :repo_id
                        or rhncontentssl.channel_family_id = :channel_family_id
                        """, repo_id=int(repo_id), channel_family_id=int(channel_family_id))
                    if keys and ('ca_cert' in keys):
                        plugin.set_ssl_options(keys['ca_cert'], keys['client_cert'], keys['client_key'])

                if not self.no_packages:
                    self.import_packages(plugin, repo_id, url)
                    self.import_groups(plugin, url)

                if not self.no_errata:
                    self.import_updates(plugin, url)

                # only for repos obtained from the DB
                if self.sync_kickstart and repo_label:
                    try:
                        self.import_kickstart(plugin, repo_label)
                    except:
                        rhnSQL.rollback()
                        raise
            except Exception:
                e = sys.exc_info()[1]
                log2stderr(0, "ERROR: %s" % e)
            if plugin is not None:
                plugin.clear_ssl_cache()
        if self.regen:
            taskomatic.add_to_repodata_queue_for_channel_package_subscription(
                [self.channel_label], [], "server.app.yumreposync")
            taskomatic.add_to_erratacache_queue(self.channel_label)
        self.update_date()
        rhnSQL.commit()
        elapsed_time = datetime.now() - start_time
        log(0, "Sync of channel completed in %s." % str(elapsed_time).split('.')[0])
        return elapsed_time
Exemple #3
0
    def __init__(self, channel_label, repo_type, url=None, fail=False,
                 filters=None, no_errata=False, sync_kickstart=False, latest=False,
                 metadata_only=False, strict=0, excluded_urls=None, no_packages=False,
                 log_dir="reposync", log_level=None):
        self.regen = False
        self.fail = fail
        self.filters = filters or []
        self.no_packages = no_packages
        self.no_errata = no_errata
        self.sync_kickstart = sync_kickstart
        self.latest = latest
        self.metadata_only = metadata_only
        self.ks_tree_type = 'externally-managed'
        self.ks_install_type = 'generic_rpm'

        initCFG('server.satellite')
        rhnSQL.initDB()

        # setup logging
        log_filename = channel_label + '.log'
        log_path = default_log_location + log_dir + '/' + log_filename
        if log_level is None:
            log_level = 0
        CFG.set('DEBUG', log_level)
        rhnLog.initLOG(log_path, log_level)
        # os.fchown isn't in 2.4 :/
        if isSUSE():
            os.system("chgrp www " + log_path)
        else:
            os.system("chgrp apache " + log_path)

        log2disk(0, "Command: %s" % str(sys.argv))
        log2disk(0, "Sync of channel started.")

        self.channel_label = channel_label
        self.channel = self.load_channel()
        if not self.channel:
            log(0, "Channel %s does not exist." % channel_label)

        if not url:
            # TODO:need to look at user security across orgs
            h = rhnSQL.prepare("""select s.id, s.source_url, s.label, fm.channel_family_id
                                  from rhnContentSource s,
                                       rhnChannelContentSource cs,
                                       rhnChannelFamilyMembers fm
                                 where s.id = cs.source_id
                                   and cs.channel_id = fm.channel_id
                                   and cs.channel_id = :channel_id""")
            h.execute(channel_id=int(self.channel['id']))
            source_data = h.fetchall_dict()
            self.urls = []
            if excluded_urls is None:
                excluded_urls = []
            if source_data:
                for row in source_data:
                    if row['source_url'] not in excluded_urls:
                        self.urls.append((row['id'], row['source_url'], row['label'], row['channel_family_id']))
        else:
            self.urls = [(None, u, None, None) for u in url]

        if not self.urls:
            log2stderr(0, "Channel %s has no URL associated" % channel_label)

        self.repo_plugin = self.load_plugin(repo_type)
        self.strict = strict
        self.all_packages = []
Exemple #4
0
    def import_packages(self, plug, source_id, url):
        if (not self.filters) and source_id:
            h = rhnSQL.prepare("""
                    select flag, filter
                      from rhnContentSourceFilter
                     where source_id = :source_id
                     order by sort_order """)
            h.execute(source_id=source_id)
            filter_data = h.fetchall_dict() or []
            filters = [(row['flag'], re.split(r'[,\s]+', row['filter']))
                       for row in filter_data]
        else:
            filters = self.filters

        packages = plug.list_packages(filters, self.latest)
        self.all_packages.extend(packages)
        to_process = []
        num_passed = len(packages)
        log(0, "Packages in repo:             %5d" % plug.num_packages)
        if plug.num_excluded:
            log(0, "Packages passed filter rules: %5d" % num_passed)
        channel_id = int(self.channel['id'])

        for pack in packages:
            db_pack = rhnPackage.get_info_for_package(
                [pack.name, pack.version, pack.release, pack.epoch, pack.arch],
                channel_id, self.channel['org_id'])

            to_download = True
            to_link = True
            # Package exists in DB
            if db_pack:
                # Path in filesystem is defined
                if db_pack['path']:
                    pack.path = os.path.join(CFG.MOUNT_POINT, db_pack['path'])
                else:
                    pack.path = ""

                if self.metadata_only or self.match_package_checksum(pack.path,
                                                                     pack.checksum_type, pack.checksum):
                    # package is already on disk or not required
                    to_download = False
                    if db_pack['channel_id'] == channel_id:
                        # package is already in the channel
                        to_link = False

                elif db_pack['channel_id'] == channel_id:
                    # different package with SAME NVREA
                    self.disassociate_package(db_pack)

                # just pass data from DB, they will be used if there is no RPM available
                pack.checksum = db_pack['checksum']
                pack.checksum_type = db_pack['checksum_type']
                pack.epoch = db_pack['epoch']

            if to_download or to_link:
                to_process.append((pack, to_download, to_link))

        num_to_process = len(to_process)
        if num_to_process == 0:
            log(0, "No new packages to sync.")
            # If we are just appending, we can exit
            if not self.strict:
                return
        else:
            log(0, "Packages already synced:      %5d" % (num_passed - num_to_process))
            log(0, "Packages to sync:             %5d" % num_to_process)

        self.regen = True
        is_non_local_repo = (url.find("file:/") < 0)

        for (index, what) in enumerate(to_process):
            pack, to_download, to_link = what
            localpath = None
            # pylint: disable=W0703
            try:
                log(0, "%d/%d : %s" % (index + 1, num_to_process, pack.getNVREA()))
                if to_download:
                    pack.path = localpath = plug.get_package(pack, metadata_only=self.metadata_only)
                    pack.load_checksum_from_header()
                    pack.upload_package(self.channel, metadata_only=self.metadata_only)
            except KeyboardInterrupt:
                raise
            except Exception:
                e = sys.exc_info()[1]
                log2stderr(0, e)
                if self.fail:
                    raise
                to_process[index] = (pack, False, False)
                continue
            finally:
                if is_non_local_repo and localpath and os.path.exists(localpath):
                    os.remove(localpath)

        log(0, "Linking packages to channel.")
        if self.strict:
            import_batch = [self.associate_package(pack)
                            for pack in self.all_packages]
        else:
            import_batch = [self.associate_package(pack)
                            for (pack, to_download, to_link) in to_process
                            if to_link]
        backend = SQLBackend()
        caller = "server.app.yumreposync"
        importer = ChannelPackageSubscription(import_batch,
                                              backend, caller=caller, repogen=False,
                                              strict=self.strict)
        importer.run()
        backend.commit()
Exemple #5
0
    def sync(self, update_repodata=False):
        """Trigger a reposync"""
        start_time = datetime.now()
        for (repo_id, url, repo_label) in self.urls:
            log(0, "Repo URL: %s" % url)
            plugin = None

            # If the repository uses a uln:// URL, switch to the ULN plugin, overriding the command-line
            if url.startswith("uln://"):
                self.repo_plugin = self.load_plugin("uln")

            # pylint: disable=W0703
            try:
                # use modified relative_url as name of repo plugin, because
                # it used as name of cache directory as well

                relative_url = '_'.join(url.split('://')[1].split('/')[1:])
                plugin_name = relative_url.replace("?", "_").replace("&", "_").replace("=", "_")

                plugin = self.repo_plugin(url, plugin_name)

                if update_repodata:
                    plugin.clear_cache()

                if repo_id is not None:
                    keys = rhnSQL.fetchone_dict("""
                        select k1.key as ca_cert, k2.key as client_cert, k3.key as client_key
                        from rhncontentsource cs
                                join rhncryptokey k1
                                on cs.ssl_ca_cert_id = k1.id
                                left outer join rhncryptokey k2
                                on cs.ssl_client_cert_id = k2.id
                                left outer join rhncryptokey k3
                                on cs.ssl_client_key_id = k3.id
                        where cs.id = :repo_id
                        """, repo_id=int(repo_id))
                    if keys and ('ca_cert' in keys):
                        plugin.set_ssl_options(keys['ca_cert'], keys['client_cert'], keys['client_key'])

                if not self.no_packages:
                    self.import_packages(plugin, repo_id, url)
                    self.import_groups(plugin, url)

                if not self.no_errata:
                    self.import_updates(plugin, url)

                # only for repos obtained from the DB
                if self.sync_kickstart and repo_label:
                    try:
                        self.import_kickstart(plugin, repo_label)
                    except:
                        rhnSQL.rollback()
                        raise
            except Exception:
                e = sys.exc_info()[1]
                log2stderr(0, "ERROR: %s" % e)
            if plugin is not None:
                plugin.clear_ssl_cache()
        if self.regen:
            taskomatic.add_to_repodata_queue_for_channel_package_subscription(
                [self.channel_label], [], "server.app.yumreposync")
            taskomatic.add_to_erratacache_queue(self.channel_label)
        self.update_date()
        rhnSQL.commit()
        elapsed_time = datetime.now() - start_time
        log(0, "Sync of channel completed in %s." % str(elapsed_time).split('.')[0])
        return elapsed_time
Exemple #6
0
    def __init__(self, channel_label, repo_type, url=None, fail=False,
                 filters=None, no_errata=False, sync_kickstart=False, latest=False,
                 metadata_only=False, strict=0, excluded_urls=None, no_packages=False,
                 log_dir="reposync", log_level=None):
        self.regen = False
        self.fail = fail
        self.filters = filters or []
        self.no_packages = no_packages
        self.no_errata = no_errata
        self.sync_kickstart = sync_kickstart
        self.latest = latest
        self.metadata_only = metadata_only
        self.ks_tree_type = 'externally-managed'
        self.ks_install_type = None

        initCFG('server.satellite')
        rhnSQL.initDB()

        # setup logging
        log_filename = channel_label + '.log'
        log_path = default_log_location + log_dir + '/' + log_filename
        if log_level is None:
            log_level = 0
        CFG.set('DEBUG', log_level)
        rhnLog.initLOG(log_path, log_level)
        # os.fchown isn't in 2.4 :/
        if isSUSE():
            os.system("chgrp www " + log_path)
        else:
            os.system("chgrp apache " + log_path)

        log2disk(0, "Command: %s" % str(sys.argv))
        log2disk(0, "Sync of channel started.")

        self.channel_label = channel_label
        self.channel = self.load_channel()
        if not self.channel:
            log(0, "Channel %s does not exist." % channel_label)

        if self.channel['org_id']:
            self.channel['org_id'] = int(self.channel['org_id'])
        else:
            self.channel['org_id'] = None

        if not url:
            # TODO:need to look at user security across orgs
            h = rhnSQL.prepare("""select s.id, s.source_url, s.label
                                  from rhnContentSource s,
                                       rhnChannelContentSource cs
                                 where s.id = cs.source_id
                                   and cs.channel_id = :channel_id""")
            h.execute(channel_id=int(self.channel['id']))
            source_data = h.fetchall_dict()
            self.urls = []
            if excluded_urls is None:
                excluded_urls = []
            if source_data:
                for row in source_data:
                    if row['source_url'] not in excluded_urls:
                        self.urls.append((row['id'], row['source_url'], row['label']))
        else:
            self.urls = [(None, u, None) for u in url]

        if not self.urls:
            log2stderr(0, "Channel %s has no URL associated" % channel_label)

        self.repo_plugin = self.load_plugin(repo_type)
        self.strict = strict
        self.all_packages = []