def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = self.synced_channels
# Check channel availability before doing anything
not_available = []
for channel in channels:
if any(channel not in d for d in
[self.channel_metadata, self.channel_to_family]) or (
not self.cdn_repository_manager.check_channel_availability(channel, self.no_kickstarts)):
not_available.append(channel)
if not_available:
raise ChannelNotFoundError(" " + "\n ".join(not_available))
# Need to update channel metadata
self._update_channels_metadata(channels)
# Finally, sync channel content
error_messages = []
total_time = datetime.timedelta()
for channel in channels:
cur_time, ret_code = self._sync_channel(channel)
if ret_code != 0:
error_messages.append("Problems occurred during syncing channel %s. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel))
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
return error_messages
def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = self.synced_channels
# Check channel availability before doing anything
not_available = []
for channel in channels:
if any(channel not in d for d in [
self.channel_metadata, self.channel_to_family,
self.content_source_mapping
]):
not_available.append(channel)
if not_available:
raise ChannelNotFoundError(" " + "\n ".join(not_available))
# Need to update channel metadata
self._update_channels_metadata(channels)
# Finally, sync channel content
total_time = datetime.timedelta()
for channel in channels:
cur_time = self._sync_channel(channel)
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = self.synced_channels
# Check channel availability before doing anything
not_available = []
for channel in channels:
if any(channel not in d for d in
[self.channel_metadata, self.channel_to_family, self.content_source_mapping]):
not_available.append(channel)
if not_available:
raise ChannelNotFoundError(" " + "\n ".join(not_available))
# Need to update channel metadata
self._update_channels_metadata(channels)
# Finally, sync channel content
total_time = datetime.timedelta()
for channel in channels:
cur_time = self._sync_channel(channel)
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = list(self.synced_channels)
# Check channel availability before doing anything
not_available = []
available = []
for channel in channels:
if not self._is_channel_available(channel):
not_available.append(channel)
else:
available.append(channel)
channels = available
error_messages = []
# if we have not_available channels log the error immediately
if not_available:
msg = "ERROR: these channels either do not exist or are not available:\n " + "\n ".join(not_available)
error_messages.append(msg)
# BZ 1434913 - let user know satellite may not be activated if all channels are in not_available
if not available:
msg = "WARNING: Is your Red Hat Satellite activated for CDN?\n"
msg += "(to see details about currently used SSL certificates for accessing CDN:"
msg += " /usr/bin/cdn-sync --cdn-certs)"
error_messages.append(msg)
# Need to update channel metadata
self._update_channels_metadata([ch for ch in channels if ch in self.channel_metadata])
# Make sure custom channels are properly connected with repos
for channel in channels:
if channel in self.synced_channels and self.synced_channels[channel]:
self.cdn_repository_manager.assign_repositories_to_channel(channel)
reposync.clear_ssl_cache()
# Finally, sync channel content
total_time = timedelta()
for channel in channels:
cur_time, failed_packages = self._sync_channel(channel)
if failed_packages < 0:
error_messages.append("Problems occurred during syncing channel %s. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel))
if failed_packages > 0:
error_messages.append("%d packages in channel %s failed to sync. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" % (failed_packages, channel,
channel))
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
return error_messages
def _sync_channel(self, channel):
excluded_urls = []
kickstart_trees = []
if channel in self.kickstart_metadata:
kickstart_trees = self.kickstart_metadata[channel]
excluded_urls.extend(self.cdn_repository_manager.excluded_urls)
if self.no_kickstarts:
kickstart_repos = self.cdn_repository_manager.get_content_sources_kickstart(
channel)
excluded_urls.extend([x['relative_url'] for x in kickstart_repos])
log(0, "======================================")
log(0, "| Channel: %s" % channel)
log(0, "======================================")
# Print note if channel is already EOL
if self._is_channel_eol(channel):
log(
0, "NOTE: This channel reached end-of-life on %s." %
datetime.strptime(self.channel_metadata[channel]['eol'],
"%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%d"))
log(0, "Sync of channel started.")
log2disk(
0,
"Please check 'cdnsync/%s.log' for sync log of this channel." %
channel,
notimeYN=True)
sync = reposync.RepoSync(channel,
repo_type="yum",
url=None,
fail=False,
filters=False,
no_packages=self.no_packages,
no_errata=self.no_errata,
sync_kickstart=(not self.no_kickstarts),
force_all_errata=self.force_all_errata,
force_kickstart=self.force_kickstarts,
latest=False,
metadata_only=self.no_rpms,
excluded_urls=excluded_urls,
strict=self.consider_full,
log_dir="cdnsync",
log_level=self.log_level,
check_ssl_dates=True,
force_null_org_content=True)
sync.set_ks_tree_type('rhn-managed')
if self.import_batch_size:
sync.set_import_batch_size(self.import_batch_size)
if kickstart_trees:
# Assuming all trees have same install type
sync.set_ks_install_type(kickstart_trees[0]['ks_install_type'])
sync.set_urls_prefix(self.mount_point)
return sync.sync()
def _sync_channel(self, channel):
excluded_urls = []
kickstart_trees = []
if channel in self.kickstart_metadata:
kickstart_trees = self.kickstart_metadata[channel]
if self.no_kickstarts:
kickstart_repos = self.cdn_repository_manager.get_content_sources_kickstart(channel)
excluded_urls.extend([x['relative_url'] for x in kickstart_repos])
log(0, "======================================")
log(0, "| Channel: %s" % channel)
log(0, "======================================")
# Print note if channel is already EOL
if self._is_channel_eol(channel):
log(0, "NOTE: This channel reached end-of-life on %s." %
datetime.strptime(self.channel_metadata[channel]['eol'], "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%d"))
log(0, "Sync of channel started.")
log2disk(0, "Please check 'cdnsync/%s.log' for sync log of this channel." % channel, notimeYN=True)
sync = reposync.RepoSync(channel,
repo_type="yum",
url=None,
fail=False,
filters=False,
no_packages=self.no_packages,
no_errata=self.no_errata,
sync_kickstart=(not self.no_kickstarts),
force_all_errata=self.force_all_errata,
force_kickstart=self.force_kickstarts,
latest=False,
metadata_only=self.no_rpms,
excluded_urls=excluded_urls,
strict=self.consider_full,
log_dir="cdnsync",
log_level=self.log_level,
check_ssl_dates=True,
force_null_org_content=True)
sync.set_ks_tree_type('rhn-managed')
if self.import_batch_size:
sync.set_import_batch_size(self.import_batch_size)
if kickstart_trees:
# Assuming all trees have same install type
sync.set_ks_install_type(kickstart_trees[0]['ks_install_type'])
sync.set_urls_prefix(self.mount_point)
return sync.sync()
def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = list(self.synced_channels)
# Check channel availability before doing anything
not_available = []
for channel in channels:
if not self._is_channel_available(channel):
not_available.append(channel)
if not_available:
raise ChannelNotFoundError(" " + "\n ".join(not_available))
# Need to update channel metadata
self._update_channels_metadata(
[ch for ch in channels if ch in self.channel_metadata])
# Make sure custom channels are properly connected with repos
for channel in channels:
if channel in self.synced_channels and self.synced_channels[
channel]:
self.cdn_repository_manager.assign_repositories_to_channel(
channel)
# Finally, sync channel content
error_messages = []
total_time = timedelta()
for channel in channels:
cur_time, failed_packages = self._sync_channel(channel)
if failed_packages < 0:
error_messages.append(
"Problems occurred during syncing channel %s. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" %
(channel, channel))
if failed_packages > 0:
error_messages.append(
"%d packages in channel %s failed to sync. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" %
(failed_packages, channel, channel))
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
return error_messages
def _sync_channel(self, channel):
excluded_urls = []
kickstart_trees = []
if channel in self.kickstart_metadata:
kickstart_trees = self.kickstart_metadata[channel]
if self.no_kickstarts:
kickstart_repos = self.cdn_repository_manager.get_content_sources_kickstart(
channel)
excluded_urls.extend([x['relative_url'] for x in kickstart_repos])
log(0, "======================================")
log(0, "| Channel: %s" % channel)
log(0, "======================================")
log(0, "Sync of channel started.")
log2disk(
0,
"Please check 'cdnsync/%s.log' for sync log of this channel." %
channel,
notimeYN=True)
sync = reposync.RepoSync(channel,
"yum",
url=None,
fail=False,
filters=False,
no_packages=self.no_packages,
no_errata=self.no_errata,
sync_kickstart=(not self.no_kickstarts),
force_all_errata=self.force_all_errata,
force_kickstart=self.force_kickstarts,
latest=False,
metadata_only=self.no_rpms,
excluded_urls=excluded_urls,
strict=self.consider_full,
log_dir="cdnsync",
log_level=self.log_level,
check_ssl_dates=True,
force_null_org_content=True)
sync.set_ks_tree_type('rhn-managed')
if kickstart_trees:
# Assuming all trees have same install type
sync.set_ks_install_type(kickstart_trees[0]['ks_install_type'])
sync.set_urls_prefix(self.mount_point)
return sync.sync()
def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = self.synced_channels
# Check channel availability before doing anything
not_available = []
for channel in channels:
if any(
channel not in d
for d in [self.channel_metadata, self.channel_to_family]
) or (not self.cdn_repository_manager.check_channel_availability(
channel, self.no_kickstarts)):
not_available.append(channel)
if not_available:
raise ChannelNotFoundError(" " + "\n ".join(not_available))
# Need to update channel metadata
self._update_channels_metadata(channels)
# Finally, sync channel content
error_messages = []
total_time = timedelta()
for channel in channels:
cur_time, failed_packages = self._sync_channel(channel)
if failed_packages < 0:
error_messages.append(
"Problems occurred during syncing channel %s. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" %
(channel, channel))
if failed_packages > 0:
error_messages.append(
"%d packages in channel %s failed to sync. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" %
(failed_packages, channel, channel))
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
return error_messages
def _sync_channel(self, channel):
excluded_urls = []
kickstart_trees = []
if channel in self.kickstart_metadata:
kickstart_trees = self.kickstart_metadata[channel]
if self.no_kickstarts:
for tree in kickstart_trees:
tree_label = tree['ks_tree_label']
source = self.kickstart_source_mapping[tree_label][0]
excluded_urls.append(CFG.CDN_ROOT + source['relative_url'])
log(0, "======================================")
log(0, "| Channel: %s" % channel)
log(0, "======================================")
log(0, "Sync of channel started.")
log2disk(
0,
"Please check 'cdnsync/%s.log' for sync log of this channel." %
channel,
notimeYN=True)
sync = reposync.RepoSync(channel,
"yum",
url=None,
fail=True,
filters=False,
no_packages=self.no_packages,
no_errata=self.no_errata,
sync_kickstart=(not self.no_kickstarts),
latest=False,
metadata_only=self.no_rpms,
excluded_urls=excluded_urls,
strict=1,
log_dir="cdnsync",
log_level=self.log_level)
sync.set_ks_tree_type('rhn-managed')
if kickstart_trees:
# Assuming all trees have same install type
sync.set_ks_install_type(kickstart_trees[0]['ks_install_type'])
return sync.sync(update_repodata=True)
def _sync_channel(self, channel):
excluded_urls = []
kickstart_trees = []
if channel in self.kickstart_metadata:
kickstart_trees = self.kickstart_metadata[channel]
if self.no_kickstarts:
kickstart_repos = self.cdn_repository_manager.get_content_sources_kickstart(channel)
excluded_urls.extend([x['relative_url'] for x in kickstart_repos])
log(0, "======================================")
log(0, "| Channel: %s" % channel)
log(0, "======================================")
log(0, "Sync of channel started.")
log2disk(0, "Please check 'cdnsync/%s.log' for sync log of this channel." % channel, notimeYN=True)
sync = reposync.RepoSync(channel,
"yum",
url=None,
fail=False,
filters=False,
no_packages=self.no_packages,
no_errata=self.no_errata,
sync_kickstart=(not self.no_kickstarts),
force_all_errata=self.force_all_errata,
force_kickstart=self.force_kickstarts,
latest=False,
metadata_only=self.no_rpms,
excluded_urls=excluded_urls,
strict=self.consider_full,
log_dir="cdnsync",
log_level=self.log_level,
check_ssl_dates=True,
force_null_org_content=True)
sync.set_ks_tree_type('rhn-managed')
if kickstart_trees:
# Assuming all trees have same install type
sync.set_ks_install_type(kickstart_trees[0]['ks_install_type'])
sync.set_urls_prefix(self.mount_point)
return sync.sync()
def _sync_channel(self, channel):
excluded_urls = []
kickstart_trees = []
if channel in self.kickstart_metadata:
kickstart_trees = self.kickstart_metadata[channel]
if self.no_kickstarts:
for tree in kickstart_trees:
tree_label = tree['ks_tree_label']
source = self.kickstart_source_mapping[tree_label][0]
excluded_urls.append(CFG.CDN_ROOT + source['relative_url'])
log(0, "======================================")
log(0, "| Channel: %s" % channel)
log(0, "======================================")
log(0, "Sync of channel started.")
log2disk(0, "Please check 'cdnsync/%s.log' for sync log of this channel." % channel, notimeYN=True)
sync = reposync.RepoSync(channel,
"yum",
url=None,
fail=True,
filters=False,
no_packages=self.no_packages,
no_errata=self.no_errata,
sync_kickstart=(not self.no_kickstarts),
latest=False,
metadata_only=self.no_rpms,
excluded_urls=excluded_urls,
strict=1,
log_dir="cdnsync",
log_level=self.log_level)
sync.set_ks_tree_type('rhn-managed')
if kickstart_trees:
# Assuming all trees have same install type
sync.set_ks_install_type(kickstart_trees[0]['ks_install_type'])
return sync.sync(update_repodata=True)
def processCommandline():
options = [
Option('--sanity-only',
action='store_true',
help="confirm certificate sanity. Does not activate " +
"the Red Hat Satellite locally or remotely."),
Option('--ignore-expiration',
action='store_true',
help='execute regardless of the expiration ' +
'of the RHN Certificate (not recommended).'),
Option('--ignore-version-mismatch',
action='store_true',
help='execute regardless of version ' +
'mismatch of existing and new certificate.'),
Option('-v',
'--verbose',
action='count',
help='be verbose ' +
'(accumulable: -vvv means "be *really* verbose").'),
Option('--dump-version',
action='store',
help="requested version of XML dump"),
Option('--manifest',
action='store',
help='the RHSM manifest path/filename to activate for CDN'),
Option('--rhn-cert',
action='store',
help='this option is deprecated, use --manifest instead'),
Option('--deactivate',
action='store_true',
help='deactivate CDN-activated Satellite'),
Option('--disconnected',
action='store_true',
help="activate locally, not subscribe to remote repository"),
Option('--manifest-info',
action='store_true',
help="show information about currently activated manifest"),
Option('--manifest-download',
action='store_true',
help="download new manifest from RHSM to temporary location"),
Option('--manifest-refresh',
action='store_true',
help="download new manifest from RHSM and activate it"),
Option('--manifest-reconcile-request',
action='store_true',
help="request regeneration of entitlement certificates")
]
parser = OptionParser(option_list=options)
options, args = parser.parse_args()
initCFG('server.satellite')
if options.verbose is None:
options.verbose = 0
CFG.set('DEBUG', options.verbose)
rhnLog.initLOG(LOG_PATH, options.verbose)
log2disk(0, "Command: %s" % str(sys.argv))
# we take no extra commandline arguments that are not linked to an option
if args:
writeError(
"These arguments make no sense in this context (try --help): %s" %
repr(args))
sys.exit(1)
# No need to check further if deactivating
if options.deactivate:
return options
if options.sanity_only:
options.disconnected = 1
if options.manifest_refresh:
options.manifest_download = 1
if CFG.DISCONNECTED and not options.disconnected:
msg = """Satellite server has been setup to run in disconnected mode.
Either correct server configuration in /etc/rhn/rhn.conf
or use --disconnected to activate it locally."""
writeError(msg)
sys.exit(1)
options.http_proxy = idn_ascii_to_puny(CFG.HTTP_PROXY)
options.http_proxy_username = CFG.HTTP_PROXY_USERNAME
options.http_proxy_password = CFG.HTTP_PROXY_PASSWORD
log(1, 'HTTP_PROXY: %s' % options.http_proxy)
log(1, 'HTTP_PROXY_USERNAME: %s' % options.http_proxy_username)
log(1, 'HTTP_PROXY_PASSWORD: <password>')
return options
def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = set(self.synced_channels)
# Check channel availability before doing anything
not_available = set()
available = set()
all_channel_list = None
for channel in channels:
# Try to expand wildcards in channel labels
if '*' in channel or '?' in channel or '[' in channel:
if all_channel_list is None:
all_channel_list = self._list_available_channels() + [
c for c in self.synced_channels
if self.synced_channels[c]
]
expanded = fnmatch.filter(all_channel_list, channel)
log(
2, "Expanding channel '%s' to: %s" %
(channel, ", ".join(expanded)))
if expanded:
for expanded_channel in expanded:
if not self._is_channel_available(expanded_channel):
not_available.add(expanded_channel)
else:
available.add(expanded_channel)
else:
not_available.add(channel)
elif not self._is_channel_available(channel):
not_available.add(channel)
else:
available.add(channel)
channels = available
error_messages = []
# if we have not_available channels log the error immediately
if not_available:
msg = "ERROR: these channels either do not exist or are not available for synchronization:\n " + \
"\n ".join(not_available)
error_messages.append(msg)
# BZ 1434913 - let user know if system is not activated if no available channels
if not available:
error_messages.extend(self._msg_array_if_not_activated())
# Need to update channel metadata
self._update_channels_metadata(
[ch for ch in channels if ch in self.channel_metadata])
# Make sure custom channels are properly connected with repos
for channel in channels:
if channel in self.synced_channels and self.synced_channels[
channel]:
self.cdn_repository_manager.assign_repositories_to_channel(
channel)
reposync.clear_ssl_cache()
# Finally, sync channel content
total_time = timedelta()
for channel in channels:
cur_time, failed_packages = self._sync_channel(channel)
if failed_packages < 0:
error_messages.append(
"Problems occurred during syncing channel %s. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" %
(channel, channel))
if failed_packages > 0:
error_messages.append(
"%d packages in channel %s failed to sync. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" %
(failed_packages, channel, channel))
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
return error_messages
def __init__(self,
no_packages=False,
no_errata=False,
no_rpms=False,
no_kickstarts=False,
log_level=None,
mount_point=None,
consider_full=False,
force_kickstarts=False,
force_all_errata=False,
email=False,
import_batch_size=None):
if log_level is None:
log_level = 0
self.log_level = log_level
CFG.set('DEBUG', log_level)
self.email = email
if self.email:
initEMAIL_LOG()
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Command: %s" % str(sys.argv))
rhnSQL.initDB()
initCFG('server.satellite')
self.cdn_repository_manager = CdnRepositoryManager(mount_point)
self.no_packages = no_packages
self.no_errata = no_errata
self.no_rpms = no_rpms
if self.no_packages and self.no_rpms:
log(0, "Parameter --no-rpms has no effect.")
self.no_kickstarts = no_kickstarts
self.force_all_errata = force_all_errata
self.force_kickstarts = force_kickstarts
if self.no_kickstarts and self.force_kickstarts:
log(0, "Parameter --force-kickstarts has no effect.")
if mount_point:
self.mount_point = "file://" + mount_point
self.consider_full = consider_full
else:
self.mount_point = CFG.CDN_ROOT
self.consider_full = True
verify_mappings()
f = None
# try block in try block - this is hack for python 2.4 compatibility
# to support finally
try:
try:
# Channel families mapping to channels
f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r')
self.families = json.load(f)
f.close()
# Channel metadata
f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r')
self.channel_metadata = json.load(f)
f.close()
# Dist/Release channel mapping
f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r')
self.channel_dist_mapping = json.load(f)
f.close()
# Kickstart metadata
f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r')
self.kickstart_metadata = json.load(f)
f.close()
except IOError:
e = sys.exc_info()[1]
log(1, "Ignoring channel mappings: %s" % e)
self.families = {}
self.channel_metadata = {}
self.channel_dist_mapping = {}
self.kickstart_metadata = {}
finally:
if f is not None:
f.close()
# Map channels to their channel family
self.channel_to_family = {}
for family in self.families:
for channel in self.families[family]['channels']:
self.channel_to_family[channel] = family
# Set already synced channels, entitled null-org channels and custom channels with associated
# CDN repositories
h = rhnSQL.prepare("""
select distinct c.label, c.org_id
from rhnChannelFamilyPermissions cfp inner join
rhnChannelFamily cf on cfp.channel_family_id = cf.id inner join
rhnChannelFamilyMembers cfm on cf.id = cfm.channel_family_id inner join
rhnChannel c on cfm.channel_id = c.id
where c.org_id is null
or (c.org_id is not null and
exists (
select cs.id
from rhnContentSource cs inner join
rhnChannelContentSource ccs on ccs.source_id = cs.id
where ccs.channel_id = c.id
and cs.org_id is null
)
)
order by c.org_id nulls first, label
""")
h.execute()
channels = h.fetchall_dict() or []
self.synced_channels = {}
for channel in channels:
# Custom channel repositories not available, don't mark as synced
if channel['org_id']:
repos = self.cdn_repository_manager.list_associated_repos(
channel['label'])
if not all([
self.cdn_repository_manager.
check_repository_availability(r) for r in repos
]):
continue
self.synced_channels[channel['label']] = channel['org_id']
# Select available channel families from DB
h = rhnSQL.prepare("""
select distinct label
from rhnChannelFamilyPermissions cfp inner join
rhnChannelFamily cf on cfp.channel_family_id = cf.id
where cf.org_id is null
""")
h.execute()
families = h.fetchall_dict() or []
self.entitled_families = [f['label'] for f in families]
self.import_batch_size = import_batch_size
def __init__(self,
no_packages=False,
no_errata=False,
no_rpms=False,
no_kickstarts=False,
log_level=None):
self.no_packages = no_packages
self.no_errata = no_errata
self.no_rpms = no_rpms
self.no_kickstarts = no_kickstarts
if log_level is None:
log_level = 0
self.log_level = log_level
CFG.set('DEBUG', log_level)
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Command: %s" % str(sys.argv))
rhnSQL.initDB()
initCFG('server.satellite')
f = None
# try block in try block - this is hack for python 2.4 compatibility
# to support finally
try:
try:
# Channel families mapping to channels
f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r')
self.families = json.load(f)
f.close()
# Channel metadata
f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r')
self.channel_metadata = json.load(f)
f.close()
# Dist/Release channel mapping
f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r')
self.channel_dist_mapping = json.load(f)
f.close()
# Channel to repositories mapping
f = open(constants.CONTENT_SOURCE_MAPPING_PATH, 'r')
self.content_source_mapping = json.load(f)
f.close()
# Kickstart metadata
f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r')
self.kickstart_metadata = json.load(f)
f.close()
# Channel to kickstart repositories mapping
f = open(constants.KICKSTART_SOURCE_MAPPING_PATH, 'r')
self.kickstart_source_mapping = json.load(f)
f.close()
except IOError:
e = sys.exc_info()[1]
log2stderr(0, "ERROR: Problem with loading file: %s" % e)
raise CdnMappingsLoadError()
finally:
if f is not None:
f.close()
# Map channels to their channel family
self.channel_to_family = {}
for family in self.families:
for channel in self.families[family]['channels']:
self.channel_to_family[channel] = family
# Set already synced channels
h = rhnSQL.prepare("""
select label from rhnChannel where org_id is null
""")
h.execute()
channels = h.fetchall_dict() or []
self.synced_channels = [ch['label'] for ch in channels]
# Set SSL-keys for channel family
self.family_keys = {}
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False,
log_level=None):
self.no_packages = no_packages
self.no_errata = no_errata
self.no_rpms = no_rpms
self.no_kickstarts = no_kickstarts
if log_level is None:
log_level = 0
self.log_level = log_level
CFG.set('DEBUG', log_level)
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Command: %s" % str(sys.argv))
rhnSQL.initDB()
initCFG('server.satellite')
try:
# Channel families mapping to channels
with open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') as f:
self.families = json.load(f)
# Channel metadata
with open(constants.CHANNEL_DEFINITIONS_PATH, 'r') as f:
self.channel_metadata = json.load(f)
# Dist/Release channel mapping
with open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') as f:
self.channel_dist_mapping = json.load(f)
# Channel to repositories mapping
with open(constants.CONTENT_SOURCE_MAPPING_PATH, 'r') as f:
self.content_source_mapping = json.load(f)
# Kickstart metadata
with open(constants.KICKSTART_DEFINITIONS_PATH, 'r') as f:
self.kickstart_metadata = json.load(f)
# Channel to kickstart repositories mapping
with open(constants.KICKSTART_SOURCE_MAPPING_PATH, 'r') as f:
self.kickstart_source_mapping = json.load(f)
except IOError:
e = sys.exc_info()[1]
log2stderr(0, "ERROR: Problem with loading file: %s" % e)
raise CdnMappingsLoadError()
# Map channels to their channel family
self.channel_to_family = {}
for family in self.families:
for channel in self.families[family]['channels']:
self.channel_to_family[channel] = family
# Set already synced channels
h = rhnSQL.prepare("""
select label from rhnChannel where org_id is null
""")
h.execute()
channels = h.fetchall_dict() or []
self.synced_channels = [ch['label'] for ch in channels]
# Set SSL-keys for channel family
self.family_keys = {}
def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = list(self.synced_channels)
# Check channel availability before doing anything
not_available = []
available = []
for channel in channels:
if not self._is_channel_available(channel):
not_available.append(channel)
else:
available.append(channel)
channels = available
error_messages = []
# if we have not_available channels log the error immediately
if not_available:
msg = "ERROR: these channels either do not exist or are not available:\n " + "\n ".join(
not_available)
error_messages.append(msg)
# BZ 1434913 - let user know satellite may not be activated if all channels are in not_available
if not available:
msg = "WARNING: Is your Red Hat Satellite activated for CDN?\n"
msg += "(to see details about currently used SSL certificates for accessing CDN:"
msg += " /usr/bin/cdn-sync --cdn-certs)"
error_messages.append(msg)
# Need to update channel metadata
self._update_channels_metadata(
[ch for ch in channels if ch in self.channel_metadata])
# Make sure custom channels are properly connected with repos
for channel in channels:
if channel in self.synced_channels and self.synced_channels[
channel]:
self.cdn_repository_manager.assign_repositories_to_channel(
channel)
# Finally, sync channel content
total_time = timedelta()
for channel in channels:
cur_time, failed_packages = self._sync_channel(channel)
if failed_packages < 0:
error_messages.append(
"Problems occurred during syncing channel %s. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" %
(channel, channel))
if failed_packages > 0:
error_messages.append(
"%d packages in channel %s failed to sync. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" %
(failed_packages, channel, channel))
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
return error_messages
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False,
log_level=None, mount_point=None, consider_full=False):
self.cdn_repository_manager = CdnRepositoryManager(mount_point)
self.no_packages = no_packages
self.no_errata = no_errata
self.no_rpms = no_rpms
self.no_kickstarts = no_kickstarts
if log_level is None:
log_level = 0
self.log_level = log_level
if mount_point:
self.mount_point = "file://" + mount_point
self.consider_full = consider_full
else:
self.mount_point = CFG.CDN_ROOT
self.consider_full = True
CFG.set('DEBUG', log_level)
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Command: %s" % str(sys.argv))
rhnSQL.initDB()
initCFG('server.satellite')
verify_mappings()
f = None
# try block in try block - this is hack for python 2.4 compatibility
# to support finally
try:
try:
# Channel families mapping to channels
f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r')
self.families = json.load(f)
f.close()
# Channel metadata
f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r')
self.channel_metadata = json.load(f)
f.close()
# Dist/Release channel mapping
f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r')
self.channel_dist_mapping = json.load(f)
f.close()
# Kickstart metadata
f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r')
self.kickstart_metadata = json.load(f)
f.close()
except IOError:
e = sys.exc_info()[1]
raise CdnMappingsLoadError("Problem with loading file: %s" % e)
finally:
if f is not None:
f.close()
# Map channels to their channel family
self.channel_to_family = {}
for family in self.families:
for channel in self.families[family]['channels']:
self.channel_to_family[channel] = family
# Set already synced channels
h = rhnSQL.prepare("""
select label from rhnChannel where org_id is null
""")
h.execute()
channels = h.fetchall_dict() or []
self.synced_channels = [ch['label'] for ch in channels]
def count_packages(self, channels=None):
start_time = datetime.now()
# Both entitled channels and custom channels with null-org repositories.
channel_list = self._list_available_channels() + [
c for c in self.synced_channels if self.synced_channels[c]
]
# Only some channels specified by parameter
if channels:
channel_list = [c for c in channel_list if c in channels]
log(0, "Number of channels: %d" % len(channel_list))
# Prepare repositories
repo_tree = {}
repository_count = 0
for channel in channel_list:
sources = self.cdn_repository_manager.get_content_sources(channel)
# Custom channel
if not sources:
repos = self.cdn_repository_manager.list_associated_repos(
channel)
sources = []
for index, repo in enumerate(sorted(repos)):
repo_label = "%s-%d" % (channel, index)
sources.append({
'relative_url': repo,
'pulp_repo_label_v2': repo_label
})
repository_count += len(sources)
repo_tree[channel] = sources
log(0, "Number of repositories: %d" % repository_count)
downloader = ThreadedDownloader()
for channel in repo_tree:
for source in repo_tree[channel]:
yum_repo = self._create_yum_repo(source)
params = {}
yum_repo.set_download_parameters(
params, "repodata/repomd.xml",
os.path.join(yum_repo.repo.basecachedir, yum_repo.name,
"repomd.xml.new"))
downloader.add(params)
progress_bar = ProgressBarLogger("Downloading repomd: ",
repository_count)
downloader.set_log_obj(progress_bar)
# Overwrite existing files
downloader.set_force(True)
log2disk(0, "Downloading repomd started.")
downloader.run()
log2disk(0, "Downloading repomd finished.")
progress_bar = ProgressBarLogger("Comparing repomd: ",
len(repo_tree))
to_download_count = 0
repo_tree_to_update = {}
log2disk(0, "Comparing repomd started.")
for channel in repo_tree:
cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT,
channel)
packages_num_path = os.path.join(cdn_repodata_path, "packages_num")
packages_size_path = os.path.join(cdn_repodata_path,
"packages_size")
sources = repo_tree[channel]
yum_repos = [self._create_yum_repo(source) for source in sources]
# packages_num file exists and all cached repomd files are up to date => skip
if os.path.isfile(packages_num_path) and os.path.isfile(
packages_size_path) and all(
[x.repomd_up_to_date() for x in yum_repos]):
progress_bar.log(True, None)
continue
update_channel = False
for yum_repo in yum_repos:
# use new repomd
new_repomd = os.path.join(yum_repo.repo.basecachedir,
yum_repo.name, "repomd.xml.new")
if os.path.isfile(new_repomd):
update_channel = True
os.rename(
new_repomd,
os.path.join(yum_repo.repo.basecachedir, yum_repo.name,
"repomd.xml"))
else:
# it wasn't downloaded
continue
for path, checksum_pair in yum_repo.get_metadata_paths():
params = {}
yum_repo.set_download_parameters(
params,
path,
os.path.join(yum_repo.repo.basecachedir, yum_repo.name,
os.path.basename(path)),
checksum_type=checksum_pair[0],
checksum_value=checksum_pair[1])
downloader.add(params)
to_download_count += 1
# If there is at least one repo with new repomd, pass through this channel
if update_channel:
repo_tree_to_update[channel] = sources
progress_bar.log(True, None)
log2disk(0, "Comparing repomd finished.")
progress_bar = ProgressBarLogger("Downloading metadata:",
to_download_count)
downloader.set_log_obj(progress_bar)
downloader.set_force(False)
log2disk(0, "Downloading metadata started.")
downloader.run()
log2disk(0, "Downloading metadata finished.")
progress_bar = ProgressBarLogger("Counting packages: ",
len(repo_tree_to_update))
log2disk(0, "Counting packages started.")
for channel in repo_tree_to_update:
cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT,
channel)
packages_num_path = os.path.join(cdn_repodata_path, "packages_num")
packages_size_path = os.path.join(cdn_repodata_path,
"packages_size")
sources = repo_tree_to_update[channel]
yum_repos = [self._create_yum_repo(source) for source in sources]
packages = {}
for yum_repo in yum_repos:
for pkg in yum_repo.raw_list_packages():
nvrea = str(pkg)
packages[nvrea] = pkg.packagesize
# create directory for repo data if it doesn't exist
try:
os.makedirs(cdn_repodata_path)
except OSError:
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST and os.path.isdir(
cdn_repodata_path):
pass
else:
raise
f_num_out = open(packages_num_path, 'w')
f_size_out = open(packages_size_path, 'w')
try:
f_num_out.write(str(len(packages)))
f_size_out.write(str(sum(packages.values())))
finally:
if f_num_out is not None:
f_num_out.close()
if f_size_out is not None:
f_size_out.close()
# Delete cache to save space
for yum_repo in yum_repos:
yum_repo.clear_cache(keep_repomd=True)
progress_bar.log(True, None)
log2disk(0, "Counting packages finished.")
end_time = datetime.now()
log(0, "Total time: %s" % str(end_time - start_time).split('.')[0])
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False,
log_level=None, mount_point=None, consider_full=False, force_kickstarts=False,
force_all_errata=False, email=False, import_batch_size=None):
if log_level is None:
log_level = 0
self.log_level = log_level
CFG.set('DEBUG', log_level)
self.email = email
if self.email:
initEMAIL_LOG()
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Command: %s" % str(sys.argv))
rhnSQL.initDB()
initCFG('server.satellite')
self.cdn_repository_manager = CdnRepositoryManager(mount_point)
self.no_packages = no_packages
self.no_errata = no_errata
self.no_rpms = no_rpms
if self.no_packages and self.no_rpms:
log(0, "Parameter --no-rpms has no effect.")
self.no_kickstarts = no_kickstarts
self.force_all_errata = force_all_errata
self.force_kickstarts = force_kickstarts
if self.no_kickstarts and self.force_kickstarts:
log(0, "Parameter --force-kickstarts has no effect.")
if mount_point:
self.mount_point = "file://" + mount_point
self.consider_full = consider_full
else:
self.mount_point = CFG.CDN_ROOT
self.consider_full = True
verify_mappings()
f = None
# try block in try block - this is hack for python 2.4 compatibility
# to support finally
try:
try:
# Channel families mapping to channels
f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r')
self.families = json.load(f)
f.close()
# Channel metadata
f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r')
self.channel_metadata = json.load(f)
f.close()
# Dist/Release channel mapping
f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r')
self.channel_dist_mapping = json.load(f)
f.close()
# Kickstart metadata
f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r')
self.kickstart_metadata = json.load(f)
f.close()
except IOError:
e = sys.exc_info()[1]
log(1, "Ignoring channel mappings: %s" % e)
self.families = {}
self.channel_metadata = {}
self.channel_dist_mapping = {}
self.kickstart_metadata = {}
finally:
if f is not None:
f.close()
# Map channels to their channel family
self.channel_to_family = {}
for family in self.families:
for channel in self.families[family]['channels']:
self.channel_to_family[channel] = family
# Set already synced channels, entitled null-org channels and custom channels with associated
# CDN repositories
h = rhnSQL.prepare("""
select distinct c.label, c.org_id
from rhnChannelFamilyPermissions cfp inner join
rhnChannelFamily cf on cfp.channel_family_id = cf.id inner join
rhnChannelFamilyMembers cfm on cf.id = cfm.channel_family_id inner join
rhnChannel c on cfm.channel_id = c.id
where c.org_id is null
or (c.org_id is not null and
exists (
select cs.id
from rhnContentSource cs inner join
rhnChannelContentSource ccs on ccs.source_id = cs.id
where ccs.channel_id = c.id
and cs.org_id is null
)
)
order by c.org_id nulls first, label
""")
h.execute()
channels = h.fetchall_dict() or []
self.synced_channels = {}
for channel in channels:
# Custom channel repositories not available, don't mark as synced
if channel['org_id']:
repos = self.cdn_repository_manager.list_associated_repos(channel['label'])
if not all([self.cdn_repository_manager.check_repository_availability(r) for r in repos]):
continue
self.synced_channels[channel['label']] = channel['org_id']
# Select available channel families from DB
h = rhnSQL.prepare("""
select distinct label
from rhnChannelFamilyPermissions cfp inner join
rhnChannelFamily cf on cfp.channel_family_id = cf.id
where cf.org_id is null
""")
h.execute()
families = h.fetchall_dict() or []
self.entitled_families = [f['label'] for f in families]
self.import_batch_size = import_batch_size
def sync(self, channels=None):
# If no channels specified, sync already synced channels
if not channels:
channels = set(self.synced_channels)
# Check channel availability before doing anything
not_available = set()
available = set()
all_channel_list = None
for channel in channels:
# Try to expand wildcards in channel labels
if '*' in channel or '?' in channel or '[' in channel:
if all_channel_list is None:
all_channel_list = self._list_available_channels() + [c for c in self.synced_channels
if self.synced_channels[c]]
expanded = fnmatch.filter(all_channel_list, channel)
log(2, "Expanding channel '%s' to: %s" % (channel, ", ".join(expanded)))
if expanded:
for expanded_channel in expanded:
if not self._is_channel_available(expanded_channel):
not_available.add(expanded_channel)
else:
available.add(expanded_channel)
else:
not_available.add(channel)
elif not self._is_channel_available(channel):
not_available.add(channel)
else:
available.add(channel)
channels = available
error_messages = []
# if we have not_available channels log the error immediately
if not_available:
msg = "ERROR: these channels either do not exist or are not available for synchronization:\n " + \
"\n ".join(not_available)
error_messages.append(msg)
# BZ 1434913 - let user know if system is not activated if no available channels
if not available:
error_messages.extend(self._msg_array_if_not_activated())
# Need to update channel metadata
self._update_channels_metadata([ch for ch in channels if ch in self.channel_metadata])
# Make sure custom channels are properly connected with repos
for channel in channels:
if channel in self.synced_channels and self.synced_channels[channel]:
self.cdn_repository_manager.assign_repositories_to_channel(channel)
reposync.clear_ssl_cache()
# Finally, sync channel content
total_time = timedelta()
for channel in channels:
cur_time, failed_packages = self._sync_channel(channel)
if failed_packages < 0:
error_messages.append("Problems occurred during syncing channel %s. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel))
if failed_packages > 0:
error_messages.append("%d packages in channel %s failed to sync. Please check "
"/var/log/rhn/cdnsync/%s.log for the details\n" % (failed_packages, channel,
channel))
total_time += cur_time
# Switch back to cdnsync log
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Sync of channel completed.")
log(0, "Total time: %s" % str(total_time).split('.')[0])
return error_messages
def processCommandline():
options = [
Option('--sanity-only', action='store_true', help="confirm certificate sanity. Does not activate "
+ "the Red Hat Satellite locally or remotely."),
Option('--ignore-expiration', action='store_true', help='execute regardless of the expiration '
+ 'of the RHN Certificate (not recommended).'),
Option('--ignore-version-mismatch', action='store_true', help='execute regardless of version '
+ 'mismatch of existing and new certificate.'),
Option('-v', '--verbose', action='count', help='be verbose '
+ '(accumulable: -vvv means "be *really* verbose").'),
Option('--dump-version', action='store', help="requested version of XML dump"),
Option('--manifest', action='store', help='the RHSM manifest path/filename to activate for CDN'),
Option('--rhn-cert', action='store', help='this option is deprecated, use --manifest instead'),
Option('--deactivate', action='store_true', help='deactivate CDN-activated Satellite'),
Option('--disconnected', action='store_true', help="activate locally, not subscribe to remote repository"),
Option('--manifest-info', action='store_true', help="show information about currently activated manifest"),
Option('--manifest-download', action='store_true',
help="download new manifest from RHSM to temporary location"),
Option('--manifest-refresh', action='store_true', help="download new manifest from RHSM and activate it"),
Option('--manifest-reconcile-request', action='store_true',
help="request regeneration of entitlement certificates")
]
parser = OptionParser(option_list=options)
options, args = parser.parse_args()
initCFG('server.satellite')
if options.verbose is None:
options.verbose = 0
CFG.set('DEBUG', options.verbose)
rhnLog.initLOG(LOG_PATH, options.verbose)
log2disk(0, "Command: %s" % str(sys.argv))
# we take no extra commandline arguments that are not linked to an option
if args:
writeError("These arguments make no sense in this context (try --help): %s" % repr(args))
sys.exit(1)
# No need to check further if deactivating
if options.deactivate:
return options
if options.sanity_only:
options.disconnected = 1
if options.manifest_refresh:
options.manifest_download = 1
if CFG.DISCONNECTED and not options.disconnected:
msg = """Satellite server has been setup to run in disconnected mode.
Either correct server configuration in /etc/rhn/rhn.conf
or use --disconnected to activate it locally."""
writeError(msg)
sys.exit(1)
options.http_proxy = idn_ascii_to_puny(CFG.HTTP_PROXY)
options.http_proxy_username = CFG.HTTP_PROXY_USERNAME
options.http_proxy_password = CFG.HTTP_PROXY_PASSWORD
log(1, 'HTTP_PROXY: %s' % options.http_proxy)
log(1, 'HTTP_PROXY_USERNAME: %s' % options.http_proxy_username)
log(1, 'HTTP_PROXY_PASSWORD: <password>')
return options
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False,
log_level=None, mount_point=None, consider_full=False):
self.cdn_repository_manager = CdnRepositoryManager(mount_point)
self.no_packages = no_packages
self.no_errata = no_errata
self.no_rpms = no_rpms
self.no_kickstarts = no_kickstarts
if log_level is None:
log_level = 0
self.log_level = log_level
if mount_point:
self.mount_point = "file://" + mount_point
self.consider_full = consider_full
else:
self.mount_point = CFG.CDN_ROOT
self.consider_full = True
CFG.set('DEBUG', log_level)
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Command: %s" % str(sys.argv))
rhnSQL.initDB()
initCFG('server.satellite')
verify_mappings()
f = None
# try block in try block - this is hack for python 2.4 compatibility
# to support finally
try:
try:
# Channel families mapping to channels
f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r')
self.families = json.load(f)
f.close()
# Channel metadata
f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r')
self.channel_metadata = json.load(f)
f.close()
# Dist/Release channel mapping
f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r')
self.channel_dist_mapping = json.load(f)
f.close()
# Kickstart metadata
f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r')
self.kickstart_metadata = json.load(f)
f.close()
except IOError:
e = sys.exc_info()[1]
raise CdnMappingsLoadError("Problem with loading file: %s" % e)
finally:
if f is not None:
f.close()
# Map channels to their channel family
self.channel_to_family = {}
for family in self.families:
for channel in self.families[family]['channels']:
self.channel_to_family[channel] = family
# Set already synced channels
h = rhnSQL.prepare("""
select label from rhnChannel where org_id is null
""")
h.execute()
channels = h.fetchall_dict() or []
self.synced_channels = [ch['label'] for ch in channels]