def import_packages(self, plug, source_id, url):
if (not self.filters) and source_id:
h = rhnSQL.prepare("""
select flag, filter
from rhnContentSourceFilter
where source_id = :source_id
order by sort_order """)
h.execute(source_id=source_id)
filter_data = h.fetchall_dict() or []
filters = [(row['flag'], re.split(r'[,\s]+', row['filter']))
for row in filter_data]
else:
filters = self.filters
packages = plug.list_packages(filters, self.latest)
self.all_packages.extend(packages)
to_process = []
num_passed = len(packages)
log(0, "Packages in repo: %5d" % plug.num_packages)
if plug.num_excluded:
log(0, "Packages passed filter rules: %5d" % num_passed)
channel_id = int(self.channel['id'])
if self.channel['org_id']:
self.channel['org_id'] = int(self.channel['org_id'])
else:
self.channel['org_id'] = None
for pack in packages:
db_pack = rhnPackage.get_info_for_package(
[pack.name, pack.version, pack.release, pack.epoch, pack.arch],
channel_id, self.channel['org_id'])
to_download = True
to_link = True
# Package exists in DB
if db_pack:
# Path in filesystem is defined
if db_pack['path']:
pack.path = os.path.join(CFG.MOUNT_POINT, db_pack['path'])
else:
pack.path = ""
if self.metadata_only or self.match_package_checksum(pack.path,
pack.checksum_type, pack.checksum):
# package is already on disk or not required
to_download = False
if db_pack['channel_id'] == channel_id:
# package is already in the channel
to_link = False
elif db_pack['channel_id'] == channel_id:
# different package with SAME NVREA
self.disassociate_package(db_pack)
# just pass data from DB, they will be used if there is no RPM available
pack.checksum = db_pack['checksum']
pack.checksum_type = db_pack['checksum_type']
pack.epoch = db_pack['epoch']
if to_download or to_link:
to_process.append((pack, to_download, to_link))
num_to_process = len(to_process)
if num_to_process == 0:
log(0, "No new packages to sync.")
# If we are just appending, we can exit
if not self.strict:
return
else:
log(0, "Packages already synced: %5d" % (num_passed - num_to_process))
log(0, "Packages to sync: %5d" % num_to_process)
self.regen = True
is_non_local_repo = (url.find("file:/") < 0)
for (index, what) in enumerate(to_process):
pack, to_download, to_link = what
localpath = None
# pylint: disable=W0703
try:
log(0, "%d/%d : %s" % (index + 1, num_to_process, pack.getNVREA()))
if to_download:
pack.path = localpath = plug.get_package(pack, metadata_only=self.metadata_only)
pack.load_checksum_from_header()
pack.upload_package(self.channel, metadata_only=self.metadata_only)
except KeyboardInterrupt:
raise
except Exception:
e = sys.exc_info()[1]
log2stderr(0, e)
if self.fail:
raise
to_process[index] = (pack, False, False)
continue
finally:
if is_non_local_repo and localpath and os.path.exists(localpath):
os.remove(localpath)
log(0, "Linking packages to channel.")
if self.strict:
import_batch = [self.associate_package(pack)
for pack in self.all_packages]
else:
import_batch = [self.associate_package(pack)
for (pack, to_download, to_link) in to_process
if to_link]
backend = SQLBackend()
caller = "server.app.yumreposync"
importer = ChannelPackageSubscription(import_batch,
backend, caller=caller, repogen=False,
strict=self.strict)
importer.run()
backend.commit()
def sync(self, update_repodata=False):
"""Trigger a reposync"""
start_time = datetime.now()
for (repo_id, url, repo_label, channel_family_id) in self.urls:
log(0, "Repo URL: %s" % url)
plugin = None
# If the repository uses a uln:// URL, switch to the ULN plugin, overriding the command-line
if url.startswith("uln://"):
self.repo_plugin = self.load_plugin("uln")
# pylint: disable=W0703
try:
# use modified relative_url as name of repo plugin, because
# it used as name of cache directory as well
relative_url = '_'.join(url.split('://')[1].split('/')[1:])
plugin_name = relative_url.replace("?", "_").replace("&", "_").replace("=", "_")
plugin = self.repo_plugin(url, plugin_name)
if update_repodata:
plugin.clear_cache()
if repo_id is not None:
keys = rhnSQL.fetchone_dict("""
select k1.key as ca_cert, k2.key as client_cert, k3.key as client_key
from rhncontentssl
join rhncryptokey k1
on rhncontentssl.ssl_ca_cert_id = k1.id
left outer join rhncryptokey k2
on rhncontentssl.ssl_client_cert_id = k2.id
left outer join rhncryptokey k3
on rhncontentssl.ssl_client_key_id = k3.id
where rhncontentssl.content_source_id = :repo_id
or rhncontentssl.channel_family_id = :channel_family_id
""", repo_id=int(repo_id), channel_family_id=int(channel_family_id))
if keys and ('ca_cert' in keys):
plugin.set_ssl_options(keys['ca_cert'], keys['client_cert'], keys['client_key'])
if not self.no_packages:
self.import_packages(plugin, repo_id, url)
self.import_groups(plugin, url)
if not self.no_errata:
self.import_updates(plugin, url)
# only for repos obtained from the DB
if self.sync_kickstart and repo_label:
try:
self.import_kickstart(plugin, repo_label)
except:
rhnSQL.rollback()
raise
except Exception:
e = sys.exc_info()[1]
log2stderr(0, "ERROR: %s" % e)
if plugin is not None:
plugin.clear_ssl_cache()
if self.regen:
taskomatic.add_to_repodata_queue_for_channel_package_subscription(
[self.channel_label], [], "server.app.yumreposync")
taskomatic.add_to_erratacache_queue(self.channel_label)
self.update_date()
rhnSQL.commit()
elapsed_time = datetime.now() - start_time
log(0, "Sync of channel completed in %s." % str(elapsed_time).split('.')[0])
return elapsed_time
def __init__(self, channel_label, repo_type, url=None, fail=False,
filters=None, no_errata=False, sync_kickstart=False, latest=False,
metadata_only=False, strict=0, excluded_urls=None, no_packages=False,
log_dir="reposync", log_level=None):
self.regen = False
self.fail = fail
self.filters = filters or []
self.no_packages = no_packages
self.no_errata = no_errata
self.sync_kickstart = sync_kickstart
self.latest = latest
self.metadata_only = metadata_only
self.ks_tree_type = 'externally-managed'
self.ks_install_type = 'generic_rpm'
initCFG('server.satellite')
rhnSQL.initDB()
# setup logging
log_filename = channel_label + '.log'
log_path = default_log_location + log_dir + '/' + log_filename
if log_level is None:
log_level = 0
CFG.set('DEBUG', log_level)
rhnLog.initLOG(log_path, log_level)
# os.fchown isn't in 2.4 :/
if isSUSE():
os.system("chgrp www " + log_path)
else:
os.system("chgrp apache " + log_path)
log2disk(0, "Command: %s" % str(sys.argv))
log2disk(0, "Sync of channel started.")
self.channel_label = channel_label
self.channel = self.load_channel()
if not self.channel:
log(0, "Channel %s does not exist." % channel_label)
if not url:
# TODO:need to look at user security across orgs
h = rhnSQL.prepare("""select s.id, s.source_url, s.label, fm.channel_family_id
from rhnContentSource s,
rhnChannelContentSource cs,
rhnChannelFamilyMembers fm
where s.id = cs.source_id
and cs.channel_id = fm.channel_id
and cs.channel_id = :channel_id""")
h.execute(channel_id=int(self.channel['id']))
source_data = h.fetchall_dict()
self.urls = []
if excluded_urls is None:
excluded_urls = []
if source_data:
for row in source_data:
if row['source_url'] not in excluded_urls:
self.urls.append((row['id'], row['source_url'], row['label'], row['channel_family_id']))
else:
self.urls = [(None, u, None, None) for u in url]
if not self.urls:
log2stderr(0, "Channel %s has no URL associated" % channel_label)
self.repo_plugin = self.load_plugin(repo_type)
self.strict = strict
self.all_packages = []
def import_packages(self, plug, source_id, url):
if (not self.filters) and source_id:
h = rhnSQL.prepare("""
select flag, filter
from rhnContentSourceFilter
where source_id = :source_id
order by sort_order """)
h.execute(source_id=source_id)
filter_data = h.fetchall_dict() or []
filters = [(row['flag'], re.split(r'[,\s]+', row['filter']))
for row in filter_data]
else:
filters = self.filters
packages = plug.list_packages(filters, self.latest)
self.all_packages.extend(packages)
to_process = []
num_passed = len(packages)
log(0, "Packages in repo: %5d" % plug.num_packages)
if plug.num_excluded:
log(0, "Packages passed filter rules: %5d" % num_passed)
channel_id = int(self.channel['id'])
for pack in packages:
db_pack = rhnPackage.get_info_for_package(
[pack.name, pack.version, pack.release, pack.epoch, pack.arch],
channel_id, self.channel['org_id'])
to_download = True
to_link = True
# Package exists in DB
if db_pack:
# Path in filesystem is defined
if db_pack['path']:
pack.path = os.path.join(CFG.MOUNT_POINT, db_pack['path'])
else:
pack.path = ""
if self.metadata_only or self.match_package_checksum(pack.path,
pack.checksum_type, pack.checksum):
# package is already on disk or not required
to_download = False
if db_pack['channel_id'] == channel_id:
# package is already in the channel
to_link = False
elif db_pack['channel_id'] == channel_id:
# different package with SAME NVREA
self.disassociate_package(db_pack)
# just pass data from DB, they will be used if there is no RPM available
pack.checksum = db_pack['checksum']
pack.checksum_type = db_pack['checksum_type']
pack.epoch = db_pack['epoch']
if to_download or to_link:
to_process.append((pack, to_download, to_link))
num_to_process = len(to_process)
if num_to_process == 0:
log(0, "No new packages to sync.")
# If we are just appending, we can exit
if not self.strict:
return
else:
log(0, "Packages already synced: %5d" % (num_passed - num_to_process))
log(0, "Packages to sync: %5d" % num_to_process)
self.regen = True
is_non_local_repo = (url.find("file:/") < 0)
for (index, what) in enumerate(to_process):
pack, to_download, to_link = what
localpath = None
# pylint: disable=W0703
try:
log(0, "%d/%d : %s" % (index + 1, num_to_process, pack.getNVREA()))
if to_download:
pack.path = localpath = plug.get_package(pack, metadata_only=self.metadata_only)
pack.load_checksum_from_header()
pack.upload_package(self.channel, metadata_only=self.metadata_only)
except KeyboardInterrupt:
raise
except Exception:
e = sys.exc_info()[1]
log2stderr(0, e)
if self.fail:
raise
to_process[index] = (pack, False, False)
continue
finally:
if is_non_local_repo and localpath and os.path.exists(localpath):
os.remove(localpath)
log(0, "Linking packages to channel.")
if self.strict:
import_batch = [self.associate_package(pack)
for pack in self.all_packages]
else:
import_batch = [self.associate_package(pack)
for (pack, to_download, to_link) in to_process
if to_link]
backend = SQLBackend()
caller = "server.app.yumreposync"
importer = ChannelPackageSubscription(import_batch,
backend, caller=caller, repogen=False,
strict=self.strict)
importer.run()
backend.commit()
def sync(self, update_repodata=False):
"""Trigger a reposync"""
start_time = datetime.now()
for (repo_id, url, repo_label) in self.urls:
log(0, "Repo URL: %s" % url)
plugin = None
# If the repository uses a uln:// URL, switch to the ULN plugin, overriding the command-line
if url.startswith("uln://"):
self.repo_plugin = self.load_plugin("uln")
# pylint: disable=W0703
try:
# use modified relative_url as name of repo plugin, because
# it used as name of cache directory as well
relative_url = '_'.join(url.split('://')[1].split('/')[1:])
plugin_name = relative_url.replace("?", "_").replace("&", "_").replace("=", "_")
plugin = self.repo_plugin(url, plugin_name)
if update_repodata:
plugin.clear_cache()
if repo_id is not None:
keys = rhnSQL.fetchone_dict("""
select k1.key as ca_cert, k2.key as client_cert, k3.key as client_key
from rhncontentsource cs
join rhncryptokey k1
on cs.ssl_ca_cert_id = k1.id
left outer join rhncryptokey k2
on cs.ssl_client_cert_id = k2.id
left outer join rhncryptokey k3
on cs.ssl_client_key_id = k3.id
where cs.id = :repo_id
""", repo_id=int(repo_id))
if keys and ('ca_cert' in keys):
plugin.set_ssl_options(keys['ca_cert'], keys['client_cert'], keys['client_key'])
if not self.no_packages:
self.import_packages(plugin, repo_id, url)
self.import_groups(plugin, url)
if not self.no_errata:
self.import_updates(plugin, url)
# only for repos obtained from the DB
if self.sync_kickstart and repo_label:
try:
self.import_kickstart(plugin, repo_label)
except:
rhnSQL.rollback()
raise
except Exception:
e = sys.exc_info()[1]
log2stderr(0, "ERROR: %s" % e)
if plugin is not None:
plugin.clear_ssl_cache()
if self.regen:
taskomatic.add_to_repodata_queue_for_channel_package_subscription(
[self.channel_label], [], "server.app.yumreposync")
taskomatic.add_to_erratacache_queue(self.channel_label)
self.update_date()
rhnSQL.commit()
elapsed_time = datetime.now() - start_time
log(0, "Sync of channel completed in %s." % str(elapsed_time).split('.')[0])
return elapsed_time
def __init__(self, channel_label, repo_type, url=None, fail=False,
filters=None, no_errata=False, sync_kickstart=False, latest=False,
metadata_only=False, strict=0, excluded_urls=None, no_packages=False,
log_dir="reposync", log_level=None):
self.regen = False
self.fail = fail
self.filters = filters or []
self.no_packages = no_packages
self.no_errata = no_errata
self.sync_kickstart = sync_kickstart
self.latest = latest
self.metadata_only = metadata_only
self.ks_tree_type = 'externally-managed'
self.ks_install_type = None
initCFG('server.satellite')
rhnSQL.initDB()
# setup logging
log_filename = channel_label + '.log'
log_path = default_log_location + log_dir + '/' + log_filename
if log_level is None:
log_level = 0
CFG.set('DEBUG', log_level)
rhnLog.initLOG(log_path, log_level)
# os.fchown isn't in 2.4 :/
if isSUSE():
os.system("chgrp www " + log_path)
else:
os.system("chgrp apache " + log_path)
log2disk(0, "Command: %s" % str(sys.argv))
log2disk(0, "Sync of channel started.")
self.channel_label = channel_label
self.channel = self.load_channel()
if not self.channel:
log(0, "Channel %s does not exist." % channel_label)
if self.channel['org_id']:
self.channel['org_id'] = int(self.channel['org_id'])
else:
self.channel['org_id'] = None
if not url:
# TODO:need to look at user security across orgs
h = rhnSQL.prepare("""select s.id, s.source_url, s.label
from rhnContentSource s,
rhnChannelContentSource cs
where s.id = cs.source_id
and cs.channel_id = :channel_id""")
h.execute(channel_id=int(self.channel['id']))
source_data = h.fetchall_dict()
self.urls = []
if excluded_urls is None:
excluded_urls = []
if source_data:
for row in source_data:
if row['source_url'] not in excluded_urls:
self.urls.append((row['id'], row['source_url'], row['label']))
else:
self.urls = [(None, u, None) for u in url]
if not self.urls:
log2stderr(0, "Channel %s has no URL associated" % channel_label)
self.repo_plugin = self.load_plugin(repo_type)
self.strict = strict
self.all_packages = []