def sync(self, channels=None): # If no channels specified, sync already synced channels if not channels: channels = self.synced_channels # Check channel availability before doing anything not_available = [] for channel in channels: if any(channel not in d for d in [self.channel_metadata, self.channel_to_family]) or ( not self.cdn_repository_manager.check_channel_availability(channel, self.no_kickstarts)): not_available.append(channel) if not_available: raise ChannelNotFoundError(" " + "\n ".join(not_available)) # Need to update channel metadata self._update_channels_metadata(channels) # Finally, sync channel content error_messages = [] total_time = datetime.timedelta() for channel in channels: cur_time, ret_code = self._sync_channel(channel) if ret_code != 0: error_messages.append("Problems occurred during syncing channel %s. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel)) total_time += cur_time # Switch back to cdnsync log rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Sync of channel completed.") log(0, "Total time: %s" % str(total_time).split('.')[0]) return error_messages
def sync(self, channels=None): # If no channels specified, sync already synced channels if not channels: channels = self.synced_channels # Check channel availability before doing anything not_available = [] for channel in channels: if any(channel not in d for d in [ self.channel_metadata, self.channel_to_family, self.content_source_mapping ]): not_available.append(channel) if not_available: raise ChannelNotFoundError(" " + "\n ".join(not_available)) # Need to update channel metadata self._update_channels_metadata(channels) # Finally, sync channel content total_time = datetime.timedelta() for channel in channels: cur_time = self._sync_channel(channel) total_time += cur_time # Switch back to cdnsync log rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Sync of channel completed.") log(0, "Total time: %s" % str(total_time).split('.')[0])
def sync(self, channels=None): # If no channels specified, sync already synced channels if not channels: channels = self.synced_channels # Check channel availability before doing anything not_available = [] for channel in channels: if any(channel not in d for d in [self.channel_metadata, self.channel_to_family, self.content_source_mapping]): not_available.append(channel) if not_available: raise ChannelNotFoundError(" " + "\n ".join(not_available)) # Need to update channel metadata self._update_channels_metadata(channels) # Finally, sync channel content total_time = datetime.timedelta() for channel in channels: cur_time = self._sync_channel(channel) total_time += cur_time # Switch back to cdnsync log rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Sync of channel completed.") log(0, "Total time: %s" % str(total_time).split('.')[0])
def sync(self, channels=None): # If no channels specified, sync already synced channels if not channels: channels = list(self.synced_channels) # Check channel availability before doing anything not_available = [] available = [] for channel in channels: if not self._is_channel_available(channel): not_available.append(channel) else: available.append(channel) channels = available error_messages = [] # if we have not_available channels log the error immediately if not_available: msg = "ERROR: these channels either do not exist or are not available:\n " + "\n ".join(not_available) error_messages.append(msg) # BZ 1434913 - let user know satellite may not be activated if all channels are in not_available if not available: msg = "WARNING: Is your Red Hat Satellite activated for CDN?\n" msg += "(to see details about currently used SSL certificates for accessing CDN:" msg += " /usr/bin/cdn-sync --cdn-certs)" error_messages.append(msg) # Need to update channel metadata self._update_channels_metadata([ch for ch in channels if ch in self.channel_metadata]) # Make sure custom channels are properly connected with repos for channel in channels: if channel in self.synced_channels and self.synced_channels[channel]: self.cdn_repository_manager.assign_repositories_to_channel(channel) reposync.clear_ssl_cache() # Finally, sync channel content total_time = timedelta() for channel in channels: cur_time, failed_packages = self._sync_channel(channel) if failed_packages < 0: error_messages.append("Problems occurred during syncing channel %s. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel)) if failed_packages > 0: error_messages.append("%d packages in channel %s failed to sync. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (failed_packages, channel, channel)) total_time += cur_time # Switch back to cdnsync log rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Sync of channel completed.") log(0, "Total time: %s" % str(total_time).split('.')[0]) return error_messages
def _sync_channel(self, channel): excluded_urls = [] kickstart_trees = [] if channel in self.kickstart_metadata: kickstart_trees = self.kickstart_metadata[channel] excluded_urls.extend(self.cdn_repository_manager.excluded_urls) if self.no_kickstarts: kickstart_repos = self.cdn_repository_manager.get_content_sources_kickstart( channel) excluded_urls.extend([x['relative_url'] for x in kickstart_repos]) log(0, "======================================") log(0, "| Channel: %s" % channel) log(0, "======================================") # Print note if channel is already EOL if self._is_channel_eol(channel): log( 0, "NOTE: This channel reached end-of-life on %s." % datetime.strptime(self.channel_metadata[channel]['eol'], "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%d")) log(0, "Sync of channel started.") log2disk( 0, "Please check 'cdnsync/%s.log' for sync log of this channel." % channel, notimeYN=True) sync = reposync.RepoSync(channel, repo_type="yum", url=None, fail=False, filters=False, no_packages=self.no_packages, no_errata=self.no_errata, sync_kickstart=(not self.no_kickstarts), force_all_errata=self.force_all_errata, force_kickstart=self.force_kickstarts, latest=False, metadata_only=self.no_rpms, excluded_urls=excluded_urls, strict=self.consider_full, log_dir="cdnsync", log_level=self.log_level, check_ssl_dates=True, force_null_org_content=True) sync.set_ks_tree_type('rhn-managed') if self.import_batch_size: sync.set_import_batch_size(self.import_batch_size) if kickstart_trees: # Assuming all trees have same install type sync.set_ks_install_type(kickstart_trees[0]['ks_install_type']) sync.set_urls_prefix(self.mount_point) return sync.sync()
def _sync_channel(self, channel): excluded_urls = [] kickstart_trees = [] if channel in self.kickstart_metadata: kickstart_trees = self.kickstart_metadata[channel] if self.no_kickstarts: kickstart_repos = self.cdn_repository_manager.get_content_sources_kickstart(channel) excluded_urls.extend([x['relative_url'] for x in kickstart_repos]) log(0, "======================================") log(0, "| Channel: %s" % channel) log(0, "======================================") # Print note if channel is already EOL if self._is_channel_eol(channel): log(0, "NOTE: This channel reached end-of-life on %s." % datetime.strptime(self.channel_metadata[channel]['eol'], "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%d")) log(0, "Sync of channel started.") log2disk(0, "Please check 'cdnsync/%s.log' for sync log of this channel." % channel, notimeYN=True) sync = reposync.RepoSync(channel, repo_type="yum", url=None, fail=False, filters=False, no_packages=self.no_packages, no_errata=self.no_errata, sync_kickstart=(not self.no_kickstarts), force_all_errata=self.force_all_errata, force_kickstart=self.force_kickstarts, latest=False, metadata_only=self.no_rpms, excluded_urls=excluded_urls, strict=self.consider_full, log_dir="cdnsync", log_level=self.log_level, check_ssl_dates=True, force_null_org_content=True) sync.set_ks_tree_type('rhn-managed') if self.import_batch_size: sync.set_import_batch_size(self.import_batch_size) if kickstart_trees: # Assuming all trees have same install type sync.set_ks_install_type(kickstart_trees[0]['ks_install_type']) sync.set_urls_prefix(self.mount_point) return sync.sync()
def sync(self, channels=None): # If no channels specified, sync already synced channels if not channels: channels = list(self.synced_channels) # Check channel availability before doing anything not_available = [] for channel in channels: if not self._is_channel_available(channel): not_available.append(channel) if not_available: raise ChannelNotFoundError(" " + "\n ".join(not_available)) # Need to update channel metadata self._update_channels_metadata( [ch for ch in channels if ch in self.channel_metadata]) # Make sure custom channels are properly connected with repos for channel in channels: if channel in self.synced_channels and self.synced_channels[ channel]: self.cdn_repository_manager.assign_repositories_to_channel( channel) # Finally, sync channel content error_messages = [] total_time = timedelta() for channel in channels: cur_time, failed_packages = self._sync_channel(channel) if failed_packages < 0: error_messages.append( "Problems occurred during syncing channel %s. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel)) if failed_packages > 0: error_messages.append( "%d packages in channel %s failed to sync. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (failed_packages, channel, channel)) total_time += cur_time # Switch back to cdnsync log rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Sync of channel completed.") log(0, "Total time: %s" % str(total_time).split('.')[0]) return error_messages
def _sync_channel(self, channel): excluded_urls = [] kickstart_trees = [] if channel in self.kickstart_metadata: kickstart_trees = self.kickstart_metadata[channel] if self.no_kickstarts: kickstart_repos = self.cdn_repository_manager.get_content_sources_kickstart( channel) excluded_urls.extend([x['relative_url'] for x in kickstart_repos]) log(0, "======================================") log(0, "| Channel: %s" % channel) log(0, "======================================") log(0, "Sync of channel started.") log2disk( 0, "Please check 'cdnsync/%s.log' for sync log of this channel." % channel, notimeYN=True) sync = reposync.RepoSync(channel, "yum", url=None, fail=False, filters=False, no_packages=self.no_packages, no_errata=self.no_errata, sync_kickstart=(not self.no_kickstarts), force_all_errata=self.force_all_errata, force_kickstart=self.force_kickstarts, latest=False, metadata_only=self.no_rpms, excluded_urls=excluded_urls, strict=self.consider_full, log_dir="cdnsync", log_level=self.log_level, check_ssl_dates=True, force_null_org_content=True) sync.set_ks_tree_type('rhn-managed') if kickstart_trees: # Assuming all trees have same install type sync.set_ks_install_type(kickstart_trees[0]['ks_install_type']) sync.set_urls_prefix(self.mount_point) return sync.sync()
def sync(self, channels=None): # If no channels specified, sync already synced channels if not channels: channels = self.synced_channels # Check channel availability before doing anything not_available = [] for channel in channels: if any( channel not in d for d in [self.channel_metadata, self.channel_to_family] ) or (not self.cdn_repository_manager.check_channel_availability( channel, self.no_kickstarts)): not_available.append(channel) if not_available: raise ChannelNotFoundError(" " + "\n ".join(not_available)) # Need to update channel metadata self._update_channels_metadata(channels) # Finally, sync channel content error_messages = [] total_time = timedelta() for channel in channels: cur_time, failed_packages = self._sync_channel(channel) if failed_packages < 0: error_messages.append( "Problems occurred during syncing channel %s. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel)) if failed_packages > 0: error_messages.append( "%d packages in channel %s failed to sync. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (failed_packages, channel, channel)) total_time += cur_time # Switch back to cdnsync log rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Sync of channel completed.") log(0, "Total time: %s" % str(total_time).split('.')[0]) return error_messages
def _sync_channel(self, channel): excluded_urls = [] kickstart_trees = [] if channel in self.kickstart_metadata: kickstart_trees = self.kickstart_metadata[channel] if self.no_kickstarts: for tree in kickstart_trees: tree_label = tree['ks_tree_label'] source = self.kickstart_source_mapping[tree_label][0] excluded_urls.append(CFG.CDN_ROOT + source['relative_url']) log(0, "======================================") log(0, "| Channel: %s" % channel) log(0, "======================================") log(0, "Sync of channel started.") log2disk( 0, "Please check 'cdnsync/%s.log' for sync log of this channel." % channel, notimeYN=True) sync = reposync.RepoSync(channel, "yum", url=None, fail=True, filters=False, no_packages=self.no_packages, no_errata=self.no_errata, sync_kickstart=(not self.no_kickstarts), latest=False, metadata_only=self.no_rpms, excluded_urls=excluded_urls, strict=1, log_dir="cdnsync", log_level=self.log_level) sync.set_ks_tree_type('rhn-managed') if kickstart_trees: # Assuming all trees have same install type sync.set_ks_install_type(kickstart_trees[0]['ks_install_type']) return sync.sync(update_repodata=True)
def _sync_channel(self, channel): excluded_urls = [] kickstart_trees = [] if channel in self.kickstart_metadata: kickstart_trees = self.kickstart_metadata[channel] if self.no_kickstarts: kickstart_repos = self.cdn_repository_manager.get_content_sources_kickstart(channel) excluded_urls.extend([x['relative_url'] for x in kickstart_repos]) log(0, "======================================") log(0, "| Channel: %s" % channel) log(0, "======================================") log(0, "Sync of channel started.") log2disk(0, "Please check 'cdnsync/%s.log' for sync log of this channel." % channel, notimeYN=True) sync = reposync.RepoSync(channel, "yum", url=None, fail=False, filters=False, no_packages=self.no_packages, no_errata=self.no_errata, sync_kickstart=(not self.no_kickstarts), force_all_errata=self.force_all_errata, force_kickstart=self.force_kickstarts, latest=False, metadata_only=self.no_rpms, excluded_urls=excluded_urls, strict=self.consider_full, log_dir="cdnsync", log_level=self.log_level, check_ssl_dates=True, force_null_org_content=True) sync.set_ks_tree_type('rhn-managed') if kickstart_trees: # Assuming all trees have same install type sync.set_ks_install_type(kickstart_trees[0]['ks_install_type']) sync.set_urls_prefix(self.mount_point) return sync.sync()
def _sync_channel(self, channel): excluded_urls = [] kickstart_trees = [] if channel in self.kickstart_metadata: kickstart_trees = self.kickstart_metadata[channel] if self.no_kickstarts: for tree in kickstart_trees: tree_label = tree['ks_tree_label'] source = self.kickstart_source_mapping[tree_label][0] excluded_urls.append(CFG.CDN_ROOT + source['relative_url']) log(0, "======================================") log(0, "| Channel: %s" % channel) log(0, "======================================") log(0, "Sync of channel started.") log2disk(0, "Please check 'cdnsync/%s.log' for sync log of this channel." % channel, notimeYN=True) sync = reposync.RepoSync(channel, "yum", url=None, fail=True, filters=False, no_packages=self.no_packages, no_errata=self.no_errata, sync_kickstart=(not self.no_kickstarts), latest=False, metadata_only=self.no_rpms, excluded_urls=excluded_urls, strict=1, log_dir="cdnsync", log_level=self.log_level) sync.set_ks_tree_type('rhn-managed') if kickstart_trees: # Assuming all trees have same install type sync.set_ks_install_type(kickstart_trees[0]['ks_install_type']) return sync.sync(update_repodata=True)
def processCommandline(): options = [ Option('--sanity-only', action='store_true', help="confirm certificate sanity. Does not activate " + "the Red Hat Satellite locally or remotely."), Option('--ignore-expiration', action='store_true', help='execute regardless of the expiration ' + 'of the RHN Certificate (not recommended).'), Option('--ignore-version-mismatch', action='store_true', help='execute regardless of version ' + 'mismatch of existing and new certificate.'), Option('-v', '--verbose', action='count', help='be verbose ' + '(accumulable: -vvv means "be *really* verbose").'), Option('--dump-version', action='store', help="requested version of XML dump"), Option('--manifest', action='store', help='the RHSM manifest path/filename to activate for CDN'), Option('--rhn-cert', action='store', help='this option is deprecated, use --manifest instead'), Option('--deactivate', action='store_true', help='deactivate CDN-activated Satellite'), Option('--disconnected', action='store_true', help="activate locally, not subscribe to remote repository"), Option('--manifest-info', action='store_true', help="show information about currently activated manifest"), Option('--manifest-download', action='store_true', help="download new manifest from RHSM to temporary location"), Option('--manifest-refresh', action='store_true', help="download new manifest from RHSM and activate it"), Option('--manifest-reconcile-request', action='store_true', help="request regeneration of entitlement certificates") ] parser = OptionParser(option_list=options) options, args = parser.parse_args() initCFG('server.satellite') if options.verbose is None: options.verbose = 0 CFG.set('DEBUG', options.verbose) rhnLog.initLOG(LOG_PATH, options.verbose) log2disk(0, "Command: %s" % str(sys.argv)) # we take no extra commandline arguments that are not linked to an option if args: writeError( "These arguments make no sense in this context (try --help): %s" % repr(args)) sys.exit(1) # No need to check further if deactivating if options.deactivate: return options if options.sanity_only: options.disconnected = 1 if options.manifest_refresh: options.manifest_download = 1 if CFG.DISCONNECTED and not options.disconnected: msg = """Satellite server has been setup to run in disconnected mode. Either correct server configuration in /etc/rhn/rhn.conf or use --disconnected to activate it locally.""" writeError(msg) sys.exit(1) options.http_proxy = idn_ascii_to_puny(CFG.HTTP_PROXY) options.http_proxy_username = CFG.HTTP_PROXY_USERNAME options.http_proxy_password = CFG.HTTP_PROXY_PASSWORD log(1, 'HTTP_PROXY: %s' % options.http_proxy) log(1, 'HTTP_PROXY_USERNAME: %s' % options.http_proxy_username) log(1, 'HTTP_PROXY_PASSWORD: <password>') return options
def sync(self, channels=None): # If no channels specified, sync already synced channels if not channels: channels = set(self.synced_channels) # Check channel availability before doing anything not_available = set() available = set() all_channel_list = None for channel in channels: # Try to expand wildcards in channel labels if '*' in channel or '?' in channel or '[' in channel: if all_channel_list is None: all_channel_list = self._list_available_channels() + [ c for c in self.synced_channels if self.synced_channels[c] ] expanded = fnmatch.filter(all_channel_list, channel) log( 2, "Expanding channel '%s' to: %s" % (channel, ", ".join(expanded))) if expanded: for expanded_channel in expanded: if not self._is_channel_available(expanded_channel): not_available.add(expanded_channel) else: available.add(expanded_channel) else: not_available.add(channel) elif not self._is_channel_available(channel): not_available.add(channel) else: available.add(channel) channels = available error_messages = [] # if we have not_available channels log the error immediately if not_available: msg = "ERROR: these channels either do not exist or are not available for synchronization:\n " + \ "\n ".join(not_available) error_messages.append(msg) # BZ 1434913 - let user know if system is not activated if no available channels if not available: error_messages.extend(self._msg_array_if_not_activated()) # Need to update channel metadata self._update_channels_metadata( [ch for ch in channels if ch in self.channel_metadata]) # Make sure custom channels are properly connected with repos for channel in channels: if channel in self.synced_channels and self.synced_channels[ channel]: self.cdn_repository_manager.assign_repositories_to_channel( channel) reposync.clear_ssl_cache() # Finally, sync channel content total_time = timedelta() for channel in channels: cur_time, failed_packages = self._sync_channel(channel) if failed_packages < 0: error_messages.append( "Problems occurred during syncing channel %s. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel)) if failed_packages > 0: error_messages.append( "%d packages in channel %s failed to sync. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (failed_packages, channel, channel)) total_time += cur_time # Switch back to cdnsync log rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Sync of channel completed.") log(0, "Total time: %s" % str(total_time).split('.')[0]) return error_messages
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False, log_level=None, mount_point=None, consider_full=False, force_kickstarts=False, force_all_errata=False, email=False, import_batch_size=None): if log_level is None: log_level = 0 self.log_level = log_level CFG.set('DEBUG', log_level) self.email = email if self.email: initEMAIL_LOG() rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Command: %s" % str(sys.argv)) rhnSQL.initDB() initCFG('server.satellite') self.cdn_repository_manager = CdnRepositoryManager(mount_point) self.no_packages = no_packages self.no_errata = no_errata self.no_rpms = no_rpms if self.no_packages and self.no_rpms: log(0, "Parameter --no-rpms has no effect.") self.no_kickstarts = no_kickstarts self.force_all_errata = force_all_errata self.force_kickstarts = force_kickstarts if self.no_kickstarts and self.force_kickstarts: log(0, "Parameter --force-kickstarts has no effect.") if mount_point: self.mount_point = "file://" + mount_point self.consider_full = consider_full else: self.mount_point = CFG.CDN_ROOT self.consider_full = True verify_mappings() f = None # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: # Channel families mapping to channels f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') self.families = json.load(f) f.close() # Channel metadata f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r') self.channel_metadata = json.load(f) f.close() # Dist/Release channel mapping f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') self.channel_dist_mapping = json.load(f) f.close() # Kickstart metadata f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r') self.kickstart_metadata = json.load(f) f.close() except IOError: e = sys.exc_info()[1] log(1, "Ignoring channel mappings: %s" % e) self.families = {} self.channel_metadata = {} self.channel_dist_mapping = {} self.kickstart_metadata = {} finally: if f is not None: f.close() # Map channels to their channel family self.channel_to_family = {} for family in self.families: for channel in self.families[family]['channels']: self.channel_to_family[channel] = family # Set already synced channels, entitled null-org channels and custom channels with associated # CDN repositories h = rhnSQL.prepare(""" select distinct c.label, c.org_id from rhnChannelFamilyPermissions cfp inner join rhnChannelFamily cf on cfp.channel_family_id = cf.id inner join rhnChannelFamilyMembers cfm on cf.id = cfm.channel_family_id inner join rhnChannel c on cfm.channel_id = c.id where c.org_id is null or (c.org_id is not null and exists ( select cs.id from rhnContentSource cs inner join rhnChannelContentSource ccs on ccs.source_id = cs.id where ccs.channel_id = c.id and cs.org_id is null ) ) order by c.org_id nulls first, label """) h.execute() channels = h.fetchall_dict() or [] self.synced_channels = {} for channel in channels: # Custom channel repositories not available, don't mark as synced if channel['org_id']: repos = self.cdn_repository_manager.list_associated_repos( channel['label']) if not all([ self.cdn_repository_manager. check_repository_availability(r) for r in repos ]): continue self.synced_channels[channel['label']] = channel['org_id'] # Select available channel families from DB h = rhnSQL.prepare(""" select distinct label from rhnChannelFamilyPermissions cfp inner join rhnChannelFamily cf on cfp.channel_family_id = cf.id where cf.org_id is null """) h.execute() families = h.fetchall_dict() or [] self.entitled_families = [f['label'] for f in families] self.import_batch_size = import_batch_size
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False, log_level=None): self.no_packages = no_packages self.no_errata = no_errata self.no_rpms = no_rpms self.no_kickstarts = no_kickstarts if log_level is None: log_level = 0 self.log_level = log_level CFG.set('DEBUG', log_level) rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Command: %s" % str(sys.argv)) rhnSQL.initDB() initCFG('server.satellite') f = None # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: # Channel families mapping to channels f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') self.families = json.load(f) f.close() # Channel metadata f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r') self.channel_metadata = json.load(f) f.close() # Dist/Release channel mapping f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') self.channel_dist_mapping = json.load(f) f.close() # Channel to repositories mapping f = open(constants.CONTENT_SOURCE_MAPPING_PATH, 'r') self.content_source_mapping = json.load(f) f.close() # Kickstart metadata f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r') self.kickstart_metadata = json.load(f) f.close() # Channel to kickstart repositories mapping f = open(constants.KICKSTART_SOURCE_MAPPING_PATH, 'r') self.kickstart_source_mapping = json.load(f) f.close() except IOError: e = sys.exc_info()[1] log2stderr(0, "ERROR: Problem with loading file: %s" % e) raise CdnMappingsLoadError() finally: if f is not None: f.close() # Map channels to their channel family self.channel_to_family = {} for family in self.families: for channel in self.families[family]['channels']: self.channel_to_family[channel] = family # Set already synced channels h = rhnSQL.prepare(""" select label from rhnChannel where org_id is null """) h.execute() channels = h.fetchall_dict() or [] self.synced_channels = [ch['label'] for ch in channels] # Set SSL-keys for channel family self.family_keys = {}
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False, log_level=None): self.no_packages = no_packages self.no_errata = no_errata self.no_rpms = no_rpms self.no_kickstarts = no_kickstarts if log_level is None: log_level = 0 self.log_level = log_level CFG.set('DEBUG', log_level) rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Command: %s" % str(sys.argv)) rhnSQL.initDB() initCFG('server.satellite') try: # Channel families mapping to channels with open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') as f: self.families = json.load(f) # Channel metadata with open(constants.CHANNEL_DEFINITIONS_PATH, 'r') as f: self.channel_metadata = json.load(f) # Dist/Release channel mapping with open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') as f: self.channel_dist_mapping = json.load(f) # Channel to repositories mapping with open(constants.CONTENT_SOURCE_MAPPING_PATH, 'r') as f: self.content_source_mapping = json.load(f) # Kickstart metadata with open(constants.KICKSTART_DEFINITIONS_PATH, 'r') as f: self.kickstart_metadata = json.load(f) # Channel to kickstart repositories mapping with open(constants.KICKSTART_SOURCE_MAPPING_PATH, 'r') as f: self.kickstart_source_mapping = json.load(f) except IOError: e = sys.exc_info()[1] log2stderr(0, "ERROR: Problem with loading file: %s" % e) raise CdnMappingsLoadError() # Map channels to their channel family self.channel_to_family = {} for family in self.families: for channel in self.families[family]['channels']: self.channel_to_family[channel] = family # Set already synced channels h = rhnSQL.prepare(""" select label from rhnChannel where org_id is null """) h.execute() channels = h.fetchall_dict() or [] self.synced_channels = [ch['label'] for ch in channels] # Set SSL-keys for channel family self.family_keys = {}
def sync(self, channels=None): # If no channels specified, sync already synced channels if not channels: channels = list(self.synced_channels) # Check channel availability before doing anything not_available = [] available = [] for channel in channels: if not self._is_channel_available(channel): not_available.append(channel) else: available.append(channel) channels = available error_messages = [] # if we have not_available channels log the error immediately if not_available: msg = "ERROR: these channels either do not exist or are not available:\n " + "\n ".join( not_available) error_messages.append(msg) # BZ 1434913 - let user know satellite may not be activated if all channels are in not_available if not available: msg = "WARNING: Is your Red Hat Satellite activated for CDN?\n" msg += "(to see details about currently used SSL certificates for accessing CDN:" msg += " /usr/bin/cdn-sync --cdn-certs)" error_messages.append(msg) # Need to update channel metadata self._update_channels_metadata( [ch for ch in channels if ch in self.channel_metadata]) # Make sure custom channels are properly connected with repos for channel in channels: if channel in self.synced_channels and self.synced_channels[ channel]: self.cdn_repository_manager.assign_repositories_to_channel( channel) # Finally, sync channel content total_time = timedelta() for channel in channels: cur_time, failed_packages = self._sync_channel(channel) if failed_packages < 0: error_messages.append( "Problems occurred during syncing channel %s. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel)) if failed_packages > 0: error_messages.append( "%d packages in channel %s failed to sync. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (failed_packages, channel, channel)) total_time += cur_time # Switch back to cdnsync log rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Sync of channel completed.") log(0, "Total time: %s" % str(total_time).split('.')[0]) return error_messages
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False, log_level=None, mount_point=None, consider_full=False): self.cdn_repository_manager = CdnRepositoryManager(mount_point) self.no_packages = no_packages self.no_errata = no_errata self.no_rpms = no_rpms self.no_kickstarts = no_kickstarts if log_level is None: log_level = 0 self.log_level = log_level if mount_point: self.mount_point = "file://" + mount_point self.consider_full = consider_full else: self.mount_point = CFG.CDN_ROOT self.consider_full = True CFG.set('DEBUG', log_level) rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Command: %s" % str(sys.argv)) rhnSQL.initDB() initCFG('server.satellite') verify_mappings() f = None # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: # Channel families mapping to channels f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') self.families = json.load(f) f.close() # Channel metadata f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r') self.channel_metadata = json.load(f) f.close() # Dist/Release channel mapping f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') self.channel_dist_mapping = json.load(f) f.close() # Kickstart metadata f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r') self.kickstart_metadata = json.load(f) f.close() except IOError: e = sys.exc_info()[1] raise CdnMappingsLoadError("Problem with loading file: %s" % e) finally: if f is not None: f.close() # Map channels to their channel family self.channel_to_family = {} for family in self.families: for channel in self.families[family]['channels']: self.channel_to_family[channel] = family # Set already synced channels h = rhnSQL.prepare(""" select label from rhnChannel where org_id is null """) h.execute() channels = h.fetchall_dict() or [] self.synced_channels = [ch['label'] for ch in channels]
def count_packages(self, channels=None): start_time = datetime.now() # Both entitled channels and custom channels with null-org repositories. channel_list = self._list_available_channels() + [ c for c in self.synced_channels if self.synced_channels[c] ] # Only some channels specified by parameter if channels: channel_list = [c for c in channel_list if c in channels] log(0, "Number of channels: %d" % len(channel_list)) # Prepare repositories repo_tree = {} repository_count = 0 for channel in channel_list: sources = self.cdn_repository_manager.get_content_sources(channel) # Custom channel if not sources: repos = self.cdn_repository_manager.list_associated_repos( channel) sources = [] for index, repo in enumerate(sorted(repos)): repo_label = "%s-%d" % (channel, index) sources.append({ 'relative_url': repo, 'pulp_repo_label_v2': repo_label }) repository_count += len(sources) repo_tree[channel] = sources log(0, "Number of repositories: %d" % repository_count) downloader = ThreadedDownloader() for channel in repo_tree: for source in repo_tree[channel]: yum_repo = self._create_yum_repo(source) params = {} yum_repo.set_download_parameters( params, "repodata/repomd.xml", os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new")) downloader.add(params) progress_bar = ProgressBarLogger("Downloading repomd: ", repository_count) downloader.set_log_obj(progress_bar) # Overwrite existing files downloader.set_force(True) log2disk(0, "Downloading repomd started.") downloader.run() log2disk(0, "Downloading repomd finished.") progress_bar = ProgressBarLogger("Comparing repomd: ", len(repo_tree)) to_download_count = 0 repo_tree_to_update = {} log2disk(0, "Comparing repomd started.") for channel in repo_tree: cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel) packages_num_path = os.path.join(cdn_repodata_path, "packages_num") packages_size_path = os.path.join(cdn_repodata_path, "packages_size") sources = repo_tree[channel] yum_repos = [self._create_yum_repo(source) for source in sources] # packages_num file exists and all cached repomd files are up to date => skip if os.path.isfile(packages_num_path) and os.path.isfile( packages_size_path) and all( [x.repomd_up_to_date() for x in yum_repos]): progress_bar.log(True, None) continue update_channel = False for yum_repo in yum_repos: # use new repomd new_repomd = os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml.new") if os.path.isfile(new_repomd): update_channel = True os.rename( new_repomd, os.path.join(yum_repo.repo.basecachedir, yum_repo.name, "repomd.xml")) else: # it wasn't downloaded continue for path, checksum_pair in yum_repo.get_metadata_paths(): params = {} yum_repo.set_download_parameters( params, path, os.path.join(yum_repo.repo.basecachedir, yum_repo.name, os.path.basename(path)), checksum_type=checksum_pair[0], checksum_value=checksum_pair[1]) downloader.add(params) to_download_count += 1 # If there is at least one repo with new repomd, pass through this channel if update_channel: repo_tree_to_update[channel] = sources progress_bar.log(True, None) log2disk(0, "Comparing repomd finished.") progress_bar = ProgressBarLogger("Downloading metadata:", to_download_count) downloader.set_log_obj(progress_bar) downloader.set_force(False) log2disk(0, "Downloading metadata started.") downloader.run() log2disk(0, "Downloading metadata finished.") progress_bar = ProgressBarLogger("Counting packages: ", len(repo_tree_to_update)) log2disk(0, "Counting packages started.") for channel in repo_tree_to_update: cdn_repodata_path = os.path.join(constants.CDN_REPODATA_ROOT, channel) packages_num_path = os.path.join(cdn_repodata_path, "packages_num") packages_size_path = os.path.join(cdn_repodata_path, "packages_size") sources = repo_tree_to_update[channel] yum_repos = [self._create_yum_repo(source) for source in sources] packages = {} for yum_repo in yum_repos: for pkg in yum_repo.raw_list_packages(): nvrea = str(pkg) packages[nvrea] = pkg.packagesize # create directory for repo data if it doesn't exist try: os.makedirs(cdn_repodata_path) except OSError: exc = sys.exc_info()[1] if exc.errno == errno.EEXIST and os.path.isdir( cdn_repodata_path): pass else: raise f_num_out = open(packages_num_path, 'w') f_size_out = open(packages_size_path, 'w') try: f_num_out.write(str(len(packages))) f_size_out.write(str(sum(packages.values()))) finally: if f_num_out is not None: f_num_out.close() if f_size_out is not None: f_size_out.close() # Delete cache to save space for yum_repo in yum_repos: yum_repo.clear_cache(keep_repomd=True) progress_bar.log(True, None) log2disk(0, "Counting packages finished.") end_time = datetime.now() log(0, "Total time: %s" % str(end_time - start_time).split('.')[0])
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False, log_level=None, mount_point=None, consider_full=False, force_kickstarts=False, force_all_errata=False, email=False, import_batch_size=None): if log_level is None: log_level = 0 self.log_level = log_level CFG.set('DEBUG', log_level) self.email = email if self.email: initEMAIL_LOG() rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Command: %s" % str(sys.argv)) rhnSQL.initDB() initCFG('server.satellite') self.cdn_repository_manager = CdnRepositoryManager(mount_point) self.no_packages = no_packages self.no_errata = no_errata self.no_rpms = no_rpms if self.no_packages and self.no_rpms: log(0, "Parameter --no-rpms has no effect.") self.no_kickstarts = no_kickstarts self.force_all_errata = force_all_errata self.force_kickstarts = force_kickstarts if self.no_kickstarts and self.force_kickstarts: log(0, "Parameter --force-kickstarts has no effect.") if mount_point: self.mount_point = "file://" + mount_point self.consider_full = consider_full else: self.mount_point = CFG.CDN_ROOT self.consider_full = True verify_mappings() f = None # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: # Channel families mapping to channels f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') self.families = json.load(f) f.close() # Channel metadata f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r') self.channel_metadata = json.load(f) f.close() # Dist/Release channel mapping f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') self.channel_dist_mapping = json.load(f) f.close() # Kickstart metadata f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r') self.kickstart_metadata = json.load(f) f.close() except IOError: e = sys.exc_info()[1] log(1, "Ignoring channel mappings: %s" % e) self.families = {} self.channel_metadata = {} self.channel_dist_mapping = {} self.kickstart_metadata = {} finally: if f is not None: f.close() # Map channels to their channel family self.channel_to_family = {} for family in self.families: for channel in self.families[family]['channels']: self.channel_to_family[channel] = family # Set already synced channels, entitled null-org channels and custom channels with associated # CDN repositories h = rhnSQL.prepare(""" select distinct c.label, c.org_id from rhnChannelFamilyPermissions cfp inner join rhnChannelFamily cf on cfp.channel_family_id = cf.id inner join rhnChannelFamilyMembers cfm on cf.id = cfm.channel_family_id inner join rhnChannel c on cfm.channel_id = c.id where c.org_id is null or (c.org_id is not null and exists ( select cs.id from rhnContentSource cs inner join rhnChannelContentSource ccs on ccs.source_id = cs.id where ccs.channel_id = c.id and cs.org_id is null ) ) order by c.org_id nulls first, label """) h.execute() channels = h.fetchall_dict() or [] self.synced_channels = {} for channel in channels: # Custom channel repositories not available, don't mark as synced if channel['org_id']: repos = self.cdn_repository_manager.list_associated_repos(channel['label']) if not all([self.cdn_repository_manager.check_repository_availability(r) for r in repos]): continue self.synced_channels[channel['label']] = channel['org_id'] # Select available channel families from DB h = rhnSQL.prepare(""" select distinct label from rhnChannelFamilyPermissions cfp inner join rhnChannelFamily cf on cfp.channel_family_id = cf.id where cf.org_id is null """) h.execute() families = h.fetchall_dict() or [] self.entitled_families = [f['label'] for f in families] self.import_batch_size = import_batch_size
def sync(self, channels=None): # If no channels specified, sync already synced channels if not channels: channels = set(self.synced_channels) # Check channel availability before doing anything not_available = set() available = set() all_channel_list = None for channel in channels: # Try to expand wildcards in channel labels if '*' in channel or '?' in channel or '[' in channel: if all_channel_list is None: all_channel_list = self._list_available_channels() + [c for c in self.synced_channels if self.synced_channels[c]] expanded = fnmatch.filter(all_channel_list, channel) log(2, "Expanding channel '%s' to: %s" % (channel, ", ".join(expanded))) if expanded: for expanded_channel in expanded: if not self._is_channel_available(expanded_channel): not_available.add(expanded_channel) else: available.add(expanded_channel) else: not_available.add(channel) elif not self._is_channel_available(channel): not_available.add(channel) else: available.add(channel) channels = available error_messages = [] # if we have not_available channels log the error immediately if not_available: msg = "ERROR: these channels either do not exist or are not available for synchronization:\n " + \ "\n ".join(not_available) error_messages.append(msg) # BZ 1434913 - let user know if system is not activated if no available channels if not available: error_messages.extend(self._msg_array_if_not_activated()) # Need to update channel metadata self._update_channels_metadata([ch for ch in channels if ch in self.channel_metadata]) # Make sure custom channels are properly connected with repos for channel in channels: if channel in self.synced_channels and self.synced_channels[channel]: self.cdn_repository_manager.assign_repositories_to_channel(channel) reposync.clear_ssl_cache() # Finally, sync channel content total_time = timedelta() for channel in channels: cur_time, failed_packages = self._sync_channel(channel) if failed_packages < 0: error_messages.append("Problems occurred during syncing channel %s. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel)) if failed_packages > 0: error_messages.append("%d packages in channel %s failed to sync. Please check " "/var/log/rhn/cdnsync/%s.log for the details\n" % (failed_packages, channel, channel)) total_time += cur_time # Switch back to cdnsync log rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Sync of channel completed.") log(0, "Total time: %s" % str(total_time).split('.')[0]) return error_messages
def processCommandline(): options = [ Option('--sanity-only', action='store_true', help="confirm certificate sanity. Does not activate " + "the Red Hat Satellite locally or remotely."), Option('--ignore-expiration', action='store_true', help='execute regardless of the expiration ' + 'of the RHN Certificate (not recommended).'), Option('--ignore-version-mismatch', action='store_true', help='execute regardless of version ' + 'mismatch of existing and new certificate.'), Option('-v', '--verbose', action='count', help='be verbose ' + '(accumulable: -vvv means "be *really* verbose").'), Option('--dump-version', action='store', help="requested version of XML dump"), Option('--manifest', action='store', help='the RHSM manifest path/filename to activate for CDN'), Option('--rhn-cert', action='store', help='this option is deprecated, use --manifest instead'), Option('--deactivate', action='store_true', help='deactivate CDN-activated Satellite'), Option('--disconnected', action='store_true', help="activate locally, not subscribe to remote repository"), Option('--manifest-info', action='store_true', help="show information about currently activated manifest"), Option('--manifest-download', action='store_true', help="download new manifest from RHSM to temporary location"), Option('--manifest-refresh', action='store_true', help="download new manifest from RHSM and activate it"), Option('--manifest-reconcile-request', action='store_true', help="request regeneration of entitlement certificates") ] parser = OptionParser(option_list=options) options, args = parser.parse_args() initCFG('server.satellite') if options.verbose is None: options.verbose = 0 CFG.set('DEBUG', options.verbose) rhnLog.initLOG(LOG_PATH, options.verbose) log2disk(0, "Command: %s" % str(sys.argv)) # we take no extra commandline arguments that are not linked to an option if args: writeError("These arguments make no sense in this context (try --help): %s" % repr(args)) sys.exit(1) # No need to check further if deactivating if options.deactivate: return options if options.sanity_only: options.disconnected = 1 if options.manifest_refresh: options.manifest_download = 1 if CFG.DISCONNECTED and not options.disconnected: msg = """Satellite server has been setup to run in disconnected mode. Either correct server configuration in /etc/rhn/rhn.conf or use --disconnected to activate it locally.""" writeError(msg) sys.exit(1) options.http_proxy = idn_ascii_to_puny(CFG.HTTP_PROXY) options.http_proxy_username = CFG.HTTP_PROXY_USERNAME options.http_proxy_password = CFG.HTTP_PROXY_PASSWORD log(1, 'HTTP_PROXY: %s' % options.http_proxy) log(1, 'HTTP_PROXY_USERNAME: %s' % options.http_proxy_username) log(1, 'HTTP_PROXY_PASSWORD: <password>') return options