def _is_channel_available(self, label): # Checking channel availability, it means either: # 1. Trying to sync custom channel - in this case, it has to have already associated CDN repositories, # it's ensured by query populating synced_channels variable # 2. Trying to sync channel from mappings - it may not exists so we check requirements from mapping files db_channel = channel_info(label) if db_channel and db_channel['org_id']: # Custom channel doesn't have any null-org repositories assigned if label not in self.synced_channels: log2(0, 0, "ERROR: Custom channel '%s' doesn't contain any CDN repositories." % label, stream=sys.stderr) return False else: if label not in self.channel_metadata: log2(0, 0, "ERROR: Channel '%s' not found in channel metadata mapping." % label, stream=sys.stderr) return False elif label not in self.channel_to_family: log2(0, 0, "ERROR: Channel '%s' not found in channel family mapping." % label, stream=sys.stderr) return False family = self.channel_to_family[label] if family not in self.entitled_families: log2(0, 0, "ERROR: Channel family '%s' containing channel '%s' is not entitled." % (family, label), stream=sys.stderr) return False elif not self.cdn_repository_manager.check_channel_availability(label, self.no_kickstarts): log2(0, 0, "ERROR: Channel '%s' repositories are not available." % label, stream=sys.stderr) return False return True
def import_channel_families(self): """Insert channel family data into DB.""" log(1, "Channel families in manifest: %d" % len(self.sat5_cert.channel_families)) # pylint: disable=E1101 batch = [] for cf in self.sat5_cert.channel_families: # pylint: disable=E1101 label = cf.name try: family = self.families[label] family_object = ChannelFamily() for k in family.keys(): family_object[k] = family[k] family_object['label'] = label batch.append(family_object) self.families_to_import.append(label) except KeyError: # While channel mappings are not consistent with certificate generated on RHN... msg = ("WARNING: Channel family '%s' is provided by manifest but " "was not found in cdn-sync mappings." % label) log2(0, 1, msg, stream=sys.stderr) log(1, "Channel families to import: %d" % len(batch)) # Perform import backend = SQLBackend() importer = ChannelFamilyImport(batch, backend) importer.run()
def _tree_available_channels(self): # collect all channel from available families all_channels = [] channel_tree = {} # Not available parents of child channels not_available_channels = [] for label in self.entitled_families: try: family = self.families[label] except KeyError: log2(2, 2, "WARNING: Can't find channel family in mappings: %s" % label, stream=sys.stderr) continue channels = [c for c in family['channels'] if c is not None] all_channels.extend(channels) # filter available channels all_channels = [x for x in all_channels if self.cdn_repository_manager.check_channel_availability(x, self.no_kickstarts)] for base_channel in [x for x in all_channels if not self.channel_metadata[x]['parent_channel']]: channel_tree[base_channel] = [] for child_channel in [x for x in all_channels if self.channel_metadata[x]['parent_channel']]: parent_channel = self.channel_metadata[child_channel]['parent_channel'] # Parent not available, orphaned child channel if parent_channel not in channel_tree: channel_tree[parent_channel] = [] not_available_channels.append(parent_channel) channel_tree[parent_channel].append(child_channel) return channel_tree, not_available_channels
def _extract_consumer_credentials(self, zip_file): files = zip_file.namelist() consumer_credentials = [] for f in files: if f.startswith( self.UPSTREAM_CONSUMER_PATH) and f.endswith(".json"): consumer_credentials.append(f) if len(consumer_credentials) == 1: upstream_consumer = zip_file.open(consumer_credentials[0]) try: try: data = json.load(upstream_consumer) self.consumer_credentials = Credentials( data['id'], data['cert'], data['key']) except KeyError: log2(0, 0, "ERROR: Cannot access required field in file '%s'" % consumer_credentials[0], stream=sys.stderr) raise finally: upstream_consumer.close() else: raise IncorrectCredentialsError( "ERROR: Single upstream consumer certificate expected, found %d." % len(consumer_credentials))
def check_channel_availability(self, channel_label, no_kickstarts=False): """Checks if all repositories for channel are available.""" if no_kickstarts: sources = self.get_content_sources_regular(channel_label) else: sources = self.get_content_sources(channel_label) # No content, no channel if not sources: return False for source in sources: if not self.check_repository_availability( source['relative_url'], channel_label=channel_label): if source.get('ks_tree_label', None): # don't fail if kickstart is missing, just warn (bz1626797) log2(0, 0, "WARNING: kickstart tree '%s' is unavailable" % source['ks_tree_label'], stream=sys.stderr) self.excluded_urls.append(source['relative_url']) else: return False return True
def _validate(ssl_set): ssl_ca_cert, ssl_cert, ssl_key = ssl_set for certificate_file in (ssl_ca_cert, ssl_cert, ssl_key): if certificate_file and not os.path.isfile(certificate_file): log2(0, 0, "ERROR: Certificate file not found: %s" % certificate_file, stream=sys.stderr) return False return True
def assign_repositories_to_channel(self, channel_label, delete_repos=None, add_repos=None): backend = SQLBackend() self.unlink_all_repos(channel_label, custom_only=True) repos = self.list_associated_repos(channel_label) changed = 0 if delete_repos: for to_delete in delete_repos: if to_delete in repos: repos.remove(to_delete) log(0, "Removing repository '%s' from channel." % to_delete) changed += 1 else: log2(0, 0, "WARNING: Repository '%s' is not attached to channel." % to_delete, stream=sys.stderr) if add_repos: for to_add in add_repos: if to_add not in repos: repos.append(to_add) log(0, "Attaching repository '%s' to channel." % to_add) changed += 1 else: log2(0, 0, "WARNING: Repository '%s' is already attached to channel." % to_add, stream=sys.stderr) # If there are any repositories intended to be attached to channel if repos: content_sources_batch = self.get_content_sources_import_batch( channel_label, backend, repos=sorted(repos)) for content_source in content_sources_batch: content_source['channels'] = [channel_label] importer = ContentSourcesImport(content_sources_batch, backend) importer.run() else: # Make sure everything is unlinked self.unlink_all_repos(channel_label) return changed
def _create_yum_repo(self, repo_source): repo_label = self.cdn_repository_manager.get_content_source_label( repo_source) try: keys = self.cdn_repository_manager.get_repository_crypto_keys( repo_source['relative_url']) except CdnRepositoryNotFoundError: keys = [] log2(1, 1, "WARNING: Repository '%s' was not found." % repo_source['relative_url'], stream=sys.stderr) if keys: (ca_cert_file, client_cert_file, client_key_file) = reposync.write_ssl_set_cache( keys[0]['ca_cert'], keys[0]['client_cert'], keys[0]['client_key']) else: (ca_cert_file, client_cert_file, client_key_file) = (None, None, None) log2( 1, 1, "WARNING: No valid SSL certificates were found for repository '%s'." % repo_source['relative_url'], stream=sys.stderr) return yum_src.ContentSource(self.mount_point + str(repo_source['relative_url']), str(repo_label), org=None, no_mirrors=True, ca_cert_file=ca_cert_file, client_cert_file=client_cert_file, client_key_file=client_key_file)
def _is_channel_available(self, label): # Checking channel availability, it means either: # 1. Trying to sync custom channel - in this case, it has to have already associated CDN repositories, # it's ensured by query populating synced_channels variable # 2. Trying to sync channel from mappings - it may not exists so we check requirements from mapping files db_channel = channel_info(label) if db_channel and db_channel['org_id']: # Custom channel doesn't have any null-org repositories assigned if label not in self.synced_channels: log2(0, 0, "ERROR: Custom channel '%s' doesn't contain any CDN repositories." % label, stream=sys.stderr) return False else: if label not in self.channel_metadata: log2(1, 1, "WARNING: Channel '%s' not found in channel metadata mapping." % label, stream=sys.stderr) return False elif label not in self.channel_to_family: log2(0, 0, "ERROR: Channel '%s' not found in channel family mapping." % label, stream=sys.stderr) return False family = self.channel_to_family[label] if family not in self.entitled_families: log2(0, 0, "ERROR: Channel family '%s' containing channel '%s' is not entitled." % (family, label), stream=sys.stderr) return False elif not self.cdn_repository_manager.check_channel_availability(label, self.no_kickstarts): log2(0, 0, "ERROR: Channel '%s' repositories are not available." % label, stream=sys.stderr) return False return True
def get_file(path, local_base=None): # pylint: disable=W0613 # Called from import_kickstarts, not working for deb repo log2(0, 0, "Unable to download path %s from deb repo." % path, stream=sys.stderr)
def _validate(ssl_set): ssl_ca_cert, ssl_cert, ssl_key = ssl_set for certificate_file in (ssl_ca_cert, ssl_cert, ssl_key): if certificate_file and not os.path.isfile(certificate_file): log2(0, 0, "ERROR: Certificate file not found: %s" % certificate_file, stream=sys.stderr) return False return True
def assign_repositories_to_channel(self, channel_label, delete_repos=None, add_repos=None): backend = SQLBackend() self.unlink_all_repos(channel_label, custom_only=True) repos = self.list_associated_repos(channel_label) changed = 0 if delete_repos: for to_delete in delete_repos: if to_delete in repos: repos.remove(to_delete) log(0, "Removing repository '%s' from channel." % to_delete) changed += 1 else: log2(0, 0, "WARNING: Repository '%s' is not attached to channel." % to_delete, stream=sys.stderr) if add_repos: for to_add in add_repos: if to_add not in repos: repos.append(to_add) log(0, "Attaching repository '%s' to channel." % to_add) changed += 1 else: log2(0, 0, "WARNING: Repository '%s' is already attached to channel." % to_add, stream=sys.stderr) # If there are any repositories intended to be attached to channel if repos: content_sources_batch = self.get_content_sources_import_batch( channel_label, backend, repos=sorted(repos)) for content_source in content_sources_batch: content_source['channels'] = [channel_label] importer = ContentSourcesImport(content_sources_batch, backend) importer.run() else: # Make sure everything is unlinked self.unlink_all_repos(channel_label) return changed
def _update_certificates(self): """Delete and insert certificates needed for syncing from CDN repositories.""" # Remove all previously used certs/keys self._remove_certificates() # Read RHSM cert f = open(constants.CA_CERT_PATH, 'r') try: ca_cert = f.read() finally: if f is not None: f.close() if not satCerts.verify_certificate_dates(str(ca_cert)): log2(0, 0, "WARNING: '%s' certificate is not valid." % constants.CA_CERT_PATH, stream=sys.stderr) # Insert RHSM cert and certs from manifest into DB satCerts.store_rhnCryptoKey( constants.CA_CERT_NAME, ca_cert, None) for entitlement in self.manifest.get_all_entitlements(): creds = entitlement.get_credentials() cert_name = constants.CLIENT_CERT_PREFIX + creds.get_id() key_name = constants.CLIENT_KEY_PREFIX + creds.get_id() if not satCerts.verify_certificate_dates(str(creds.get_cert())): log2(0, 0, "WARNING: '%s' certificate is not valid." % cert_name, stream=sys.stderr) satCerts.store_rhnCryptoKey(cert_name, creds.get_cert(), None) satCerts.store_rhnCryptoKey(key_name, creds.get_key(), None)
def _create_yum_repo(self, repo_source): repo_label = self.cdn_repository_manager.get_content_source_label( repo_source) repo_plugin = yum_src.ContentSource(self.mount_point + str(repo_source['relative_url']), str(repo_label), org=None, no_mirrors=True) try: keys = self.cdn_repository_manager.get_repository_crypto_keys( repo_source['relative_url']) except CdnRepositoryNotFoundError: log2(1, 1, "ERROR: No SSL certificates were found for repository '%s'" % repo_source['relative_url'], stream=sys.stderr) return repo_plugin if len(keys) >= 1: (ca_cert_file, client_cert_file, client_key_file) = reposync.write_ssl_set_cache( keys[0]['ca_cert'], keys[0]['client_cert'], keys[0]['client_key']) repo_plugin.set_ssl_options(ca_cert_file, client_cert_file, client_key_file) else: log2( 1, 1, "ERROR: No valid SSL certificates were found for repository '%s'." % repo_source['relative_url'], stream=sys.stderr) return repo_plugin
def _extract_consumer_info(self, zip_file): files = zip_file.namelist() found = False for f in files: if f == self.CONSUMER_INFO: found = True break if found: consumer_info = zip_file.open(self.CONSUMER_INFO) try: try: data = json.load(consumer_info) self.uuid = data['uuid'] self.name = data['name'] self.ownerid = data['owner']['key'] self.api_url = data['urlApi'] self.web_url = data['urlWeb'] except KeyError: log2(0, 0, "ERROR: Cannot access required field in file '%s'" % self.CONSUMER_INFO, stream=sys.stderr) raise finally: consumer_info.close() else: raise MissingConsumerInfoError()
def _tree_available_channels(self): # collect all channel from available families all_channels = [] channel_tree = {} # Not available parents of child channels not_available_channels = [] for label in self.entitled_families: try: family = self.families[label] except KeyError: log2(2, 2, "WARNING: Can't find channel family in mappings: %s" % label, stream=sys.stderr) continue channels = [c for c in family['channels'] if c is not None] all_channels.extend(channels) # filter available channels all_channels = [x for x in all_channels if self.cdn_repository_manager.check_channel_availability(x, self.no_kickstarts)] for base_channel in [x for x in all_channels if not self.channel_metadata[x]['parent_channel']]: channel_tree[base_channel] = [] for child_channel in [x for x in all_channels if self.channel_metadata[x]['parent_channel']]: parent_channel = self.channel_metadata[child_channel]['parent_channel'] # Parent not available, orphaned child channel if parent_channel not in channel_tree: channel_tree[parent_channel] = [] not_available_channels.append(parent_channel) channel_tree[parent_channel].append(child_channel) return channel_tree, not_available_channels
def refresh_manifest(self, uuid=None): if uuid is None: if self.current_manifest: uuid = self.current_manifest.get_uuid() else: raise ValueError("Uuid is not known.") url = "%s%s/certificates" % (self.base_url, uuid) log(1, "URL: '%s'" % url) response = self._call_api(url, method="put") if response is not None: # pylint: disable=E1101 if response.status_code == requests.codes.ok or response.status_code == requests.codes.no_content: return True else: log2(0, 0, "Status code: %s" % response.status_code, stream=sys.stderr) log2(0, 0, "Message: '%s'" % response.text, stream=sys.stderr) return False
def _extract_consumer_info(self, zip_file): files = zip_file.namelist() found = False for f in files: if f == self.CONSUMER_INFO: found = True break if found: consumer_info = zip_file.open(self.CONSUMER_INFO) try: try: data = json.load(consumer_info) self.uuid = data['uuid'] self.name = data['name'] self.ownerid = data['owner']['key'] self.api_url = data['urlApi'] self.web_url = data['urlWeb'] except KeyError: log2(0, 0, "ERROR: Cannot access required field in file '%s'" % self.CONSUMER_INFO, stream=sys.stderr) raise finally: consumer_info.close() else: raise MissingConsumerInfoError()
def log(self, success, param): self.lock.acquire() self.status += 1 if success: log(0, "%d/%d : %s" % (self.status, self.total, str(param))) else: log2(0, 0, "%d/%d : %s (failed)" % (self.status, self.total, str(param)), stream=sys.stderr) self.lock.release()
def log(self, success, param): self.lock.acquire() self.status += 1 if success: log(0, "%d/%d : %s" % (self.status, self.total, str(param))) else: log2(0, 0, "%d/%d : %s (failed)" % (self.status, self.total, str(param)), stream=sys.stderr) self.lock.release()
def _load_entitlements(self, zip_file): files = zip_file.namelist() entitlements_files = [] for f in files: if f.startswith(self.ENTITLEMENTS_PATH) and f.endswith(".json"): entitlements_files.append(f) if len(entitlements_files) >= 1: self.all_entitlements = [] for entitlement_file in entitlements_files: entitlements = zip_file.open(entitlement_file) # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: data = json.load(entitlements) # Extract credentials certs = data['certificates'] if len(certs) != 1: raise IncorrectEntitlementsFileFormatError( "Single certificate in entitlements file '%s' is expected, found: %d" % (entitlement_file, len(certs))) cert = certs[0] credentials = Credentials(data['id'], cert['cert'], cert['key']) # Extract product IDs products = [] provided_products = data['pool']['providedProducts'] or [] derived_provided_products = data['pool']['derivedProvidedProducts'] or [] product_ids = [provided_product['productId'] for provided_product in provided_products + derived_provided_products] for product_id in set(product_ids): product = Product(product_id) self._fill_product_repositories(zip_file, product) products.append(product) # Skip entitlements not providing any products if products: entitlement = Entitlement(products, credentials) self.all_entitlements.append(entitlement) except KeyError: log2(0, 0, "ERROR: Cannot access required field in file '%s'" % entitlement_file, stream=sys.stderr) raise finally: entitlements.close() else: refer_url = "%s%s" % (self.web_url, self.uuid) if not refer_url.startswith("http"): refer_url = "https://" + refer_url raise IncorrectEntitlementsFileFormatError( "No subscriptions were found in manifest.\n\nPlease refer to %s for setting up subscriptions." % refer_url)
def _fill_product_repositories(self, zip_file, product): product_file = zip_file.open(self.PRODUCTS_PATH + '/' + str(product.get_id()) + '.json') product_data = json.load(product_file) product_file.close() try: for content in product_data['productContent']: content = content['content'] product.add_repository(content['label'], content['contentUrl']) except KeyError: log2(0, 0, "ERROR: Cannot access required field in product '%s'" % product.get_id(), stream=sys.stderr) raise
def _list_available_channels(self): # Select channel families in DB h = rhnSQL.prepare(""" select label from rhnChannelFamilyPermissions cfp inner join rhnChannelFamily cf on cfp.channel_family_id = cf.id where cf.org_id is null """) h.execute() families = h.fetchall_dict() or [] # collect all channel from available families all_channels = [] channel_tree = {} # Not available parents of child channels not_available_channels = [] for family in families: label = family['label'] try: family = self.families[label] except KeyError: log2(0, 1, "ERROR: Unknown channel family: %s" % label, stream=sys.stderr) continue channels = [c for c in family['channels'] if c is not None] all_channels.extend(channels) # filter available channels all_channels = [ x for x in all_channels if self.cdn_repository_manager.check_channel_availability( x, self.no_kickstarts) ] for base_channel in [ x for x in all_channels if not self.channel_metadata[x]['parent_channel'] ]: channel_tree[base_channel] = [] for child_channel in [ x for x in all_channels if self.channel_metadata[x]['parent_channel'] ]: parent_channel = self.channel_metadata[child_channel][ 'parent_channel'] # Parent not available, orphaned child channel if parent_channel not in channel_tree: channel_tree[parent_channel] = [] not_available_channels.append(parent_channel) channel_tree[parent_channel].append(child_channel) return channel_tree, not_available_channels
def _xmlrpc(function, params): try: retval = getattr(BaseWireSource.serverObj, function)(*params) except TypeError: e = sys.exc_info()[1] log(-1, 'ERROR: during "getattr(BaseWireSource.serverObj, %s)(*(%s))"' % (function, params)) raise except rpclib.xmlrpclib.ProtocolError: e = sys.exc_info()[1] log2(-1, 2, 'ERROR: ProtocolError: %s' % e, stream=sys.stderr) raise return retval
def _call_api(self, url, params=None, method="get"): if self.protocol == 'https': verify = CA_CERT_PATH else: verify = False response = None if self.username is not None and self.password is not None: try: if method == "get": response = requests.get(url, params=params, proxies=self._get_proxies(), auth=(self.username, self.password), verify=verify) elif method == "put": response = requests.put(url, params=params, proxies=self._get_proxies(), auth=(self.username, self.password), verify=verify) else: raise ValueError("Unsupported method: '%s'" % method) except requests.RequestException: e = sys.exc_info()[1] log2(0, 0, "ERROR: %s" % str(e), stream=sys.stderr) else: cert = self._write_cert() try: try: if method == "get": response = requests.get(url, params=params, proxies=self._get_proxies(), verify=verify, cert=cert) elif method == "put": response = requests.put(url, params=params, proxies=self._get_proxies(), verify=verify, cert=cert) else: raise ValueError("Unsupported method: '%s'" % method) except requests.RequestException: e = sys.exc_info()[1] log2(0, 0, "ERROR: %s" % str(e), stream=sys.stderr) finally: self._delete_cert(cert) return response
def _load_entitlements(self, zip_file): files = zip_file.namelist() entitlements_files = [] for f in files: if f.startswith(self.ENTITLEMENTS_PATH) and f.endswith(".json"): entitlements_files.append(f) if len(entitlements_files) >= 1: self.all_entitlements = [] for entitlement_file in entitlements_files: entitlements = zip_file.open(entitlement_file) # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: data = json.load(entitlements) # Extract credentials certs = data['certificates'] if len(certs) != 1: raise IncorrectEntitlementsFileFormatError( "ERROR: Single certificate in entitlements file is expected, found: %d" % len(certs)) cert = certs[0] credentials = Credentials(data['id'], cert['cert'], cert['key']) # Extract product IDs products = [] provided_products = data['pool']['providedProducts'] for provided_product in provided_products: product = Product(provided_product['productId']) self._fill_product_repositories(zip_file, product) products.append(product) # Skip entitlements not providing any products if products: entitlement = Entitlement(products, credentials) self.all_entitlements.append(entitlement) except KeyError: log2(0, 0, "ERROR: Cannot access required field in file '%s'" % entitlement_file, stream=sys.stderr) raise finally: entitlements.close() else: raise IncorrectEntitlementsFileFormatError( "ERROR: There has to be at least one entitlements file")
def _login(self): if not self.systemid: raise Exception("systemid not set!") # Set the URL to the one for regular XML-RPC calls self.setServer(CFG.RHN_XMLRPC_HANDLER) try: login_token = self.getServer().authentication.login(self.systemid) except rpclib.xmlrpclib.ProtocolError: e = sys.exc_info()[1] log2(-1, 2, 'ERROR: ProtocolError: %s' % e, stream=sys.stderr) raise return login_token
def print_cdn_certificates_info(self, repos=False): h = rhnSQL.prepare(""" SELECT ck.id, ck.description, ck.key FROM rhnCryptoKeyType ckt, rhnCryptoKey ck WHERE ckt.label = 'SSL' AND ckt.id = ck.crypto_key_type_id AND ck.description LIKE 'CDN_%' AND ck.org_id is NULL ORDER BY ck.description """) h.execute() keys = h.fetchall_dict() or [] if not keys: log2(0, 0, "No SSL certificates were found. Is your %s activated for CDN?" % PRODUCT_NAME, stream=sys.stderr) return for key in keys: log(0, "======================================") log(0, "| Certificate/Key: %s" % key['description']) log(0, "======================================") if constants.CA_CERT_NAME == key['description'] or constants.CLIENT_CERT_PREFIX in key['description']: if not verify_certificate_dates(str(key['key'])): log(0, "WARNING: This certificate is not valid.") cn, serial_number, not_before, not_after = get_certificate_info(str(key['key'])) log(0, "Common name: %s" % str(cn)) log(0, "Serial number: %s" % str(serial_number)) log(0, "Valid from: %s" % str(not_before)) log(0, "Valid to: %s" % str(not_after)) if constants.CLIENT_CERT_PREFIX in key['description']: manager = CdnRepositoryManager(client_cert_id=int(key['id'])) self.cdn_repository_manager = manager log(0, "Provided channels:") channel_tree, not_available_channels = self._tree_available_channels() if not channel_tree: log(0, " NONE") for base_channel in sorted(channel_tree): if base_channel not in not_available_channels: log(0, " * %s" % base_channel) elif channel_tree[base_channel]: log(0, " * %s (only child channels provided)" % base_channel) for child_channel in sorted(channel_tree[base_channel]): log(0, " * %s" % child_channel) if repos: log(0, "Provided repositories:") provided_repos = self.cdn_repository_manager.list_provided_repos(key['id']) for repo in sorted(provided_repos): log(0, " %s" % repo) log(0, "")
def export_manifest(self, uuid=None, ownerid=None, satellite_version=None): """Performs export request to Candlepin API and saves exported manifest to target file. Can take required parameters from current manifest or override them with parameters of this method.""" if uuid is None: if self.current_manifest: uuid = self.current_manifest.get_uuid() else: raise ValueError("Uuid is not known.") if ownerid is None: if self.current_manifest: ownerid = self.current_manifest.get_ownerid() else: raise ValueError("Ownerid is not known.") if satellite_version is None: if self.current_manifest: satellite_version = self.current_manifest.get_satellite_version( ) else: raise ValueError("Satellite version is not known.") url = "%s%s/export" % (self.base_url, uuid) params = { "ext": ["ownerid:%s" % ownerid, "version:%s" % satellite_version] } log(1, "URL: '%s'" % url) log(1, "Parameters: '%s'" % str(params)) response = self._call_api(url, params=params, method="get") if response is not None: # pylint: disable=E1101 if response.status_code == requests.codes.ok: fd, downloaded_manifest = tempfile.mkstemp( prefix="/tmp/manifest-", suffix=".zip") fo = os.fdopen(fd, 'wb') for chunk in response: fo.write(chunk) fo.flush() fo.close() return downloaded_manifest else: log2(0, 0, "Status code: %s" % response.status_code, stream=sys.stderr) log2(0, 0, "Message: '%s'" % response.text, stream=sys.stderr) return None
def print_cdn_certificates_info(self, repos=False): keys = self._get_cdn_certificate_keys_and_certs() if not keys: log2( 0, 0, "No SSL certificates were found. Is your %s activated for CDN?" % PRODUCT_NAME, stream=sys.stderr) sys.exit(1) for key in keys: log(0, "======================================") log(0, "| Certificate/Key: %s" % key['description']) log(0, "======================================") if constants.CA_CERT_NAME == key[ 'description'] or constants.CLIENT_CERT_PREFIX in key[ 'description']: if not verify_certificate_dates(str(key['key'])): log(0, "WARNING: This certificate is not valid.") cn, serial_number, not_before, not_after = get_certificate_info( str(key['key'])) log(0, "Common name: %s" % str(cn)) log(0, "Serial number: %s" % str(serial_number)) log(0, "Valid from: %s" % str(not_before)) log(0, "Valid to: %s" % str(not_after)) if constants.CLIENT_CERT_PREFIX in key['description']: manager = CdnRepositoryManager(client_cert_id=int(key['id'])) self.cdn_repository_manager = manager log(0, "Provided channels:") channel_tree, not_available_channels = self._tree_available_channels( ) if not channel_tree: log(0, " NONE") for base_channel in sorted(channel_tree): if base_channel not in not_available_channels: log(0, " * %s" % base_channel) elif channel_tree[base_channel]: log( 0, " * %s (only child channels provided)" % base_channel) for child_channel in sorted(channel_tree[base_channel]): log(0, " * %s" % child_channel) if repos: log(0, "Provided repositories:") provided_repos = self.cdn_repository_manager.list_provided_repos( key['id']) for repo in sorted(provided_repos): log(0, " %s" % repo) log(0, "")
def get_content_sources_kickstart(self, channel_label): repositories = [] if channel_label in self.kickstart_metadata: for tree in self.kickstart_metadata[channel_label]: tree_label = tree['ks_tree_label'] if tree_label in self.kickstart_source_mapping: # One tree comes from one repo, one repo for each tree is in the mapping, # in future there may be multiple repos for one tree and we will need to select # correct repo repository = self.kickstart_source_mapping[tree_label][0] repository['ks_tree_label'] = tree_label repositories.append(repository) else: log2(1, 1, "WARNING: Can't find repository for kickstart tree in mappings: %s" % tree_label, stream=sys.stderr) return repositories
def _fill_product_repositories(self, zip_file, product): product_file = zip_file.open(self.PRODUCTS_PATH + '/' + str(product.get_id()) + '.json') product_data = json.load(product_file) product_file.close() try: for content in product_data['productContent']: content = content['content'] product.add_repository(content['label'], content['contentUrl']) except KeyError: log2(0, 0, "ERROR: Cannot access required field in product '%s'" % product.get_id(), stream=sys.stderr) raise
def get_content_sources_kickstart(self, channel_label): repositories = [] if channel_label in self.kickstart_metadata: for tree in self.kickstart_metadata[channel_label]: tree_label = tree['ks_tree_label'] if tree_label in self.kickstart_source_mapping: # One tree comes from one repo, one repo for each tree is in the mapping, # in future there may be multiple repos for one tree and we will need to select # correct repo repository = self.kickstart_source_mapping[tree_label][0] repository['ks_tree_label'] = tree_label repositories.append(repository) else: log2(1, 1, "WARNING: Can't find repository for kickstart tree in mappings: %s" % tree_label, stream=sys.stderr) return repositories
def _load_entitlements(self, zip_file): files = zip_file.namelist() entitlements_files = [] for f in files: if f.startswith(self.ENTITLEMENTS_PATH) and f.endswith(".json"): entitlements_files.append(f) if len(entitlements_files) >= 1: self.all_entitlements = [] for entitlement_file in entitlements_files: entitlements = zip_file.open(entitlement_file) # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: data = json.load(entitlements) # Extract credentials certs = data['certificates'] if len(certs) != 1: raise IncorrectEntitlementsFileFormatError( "ERROR: Single certificate in entitlements file is expected, found: %d" % len(certs)) cert = certs[0] credentials = Credentials(data['id'], cert['cert'], cert['key']) # Extract product IDs products = [] provided_products = data['pool']['providedProducts'] for provided_product in provided_products: product = Product(provided_product['productId']) self._fill_product_repositories(zip_file, product) products.append(product) # Skip entitlements not providing any products if products: entitlement = Entitlement(products, credentials) self.all_entitlements.append(entitlement) except KeyError: log2(0, 0, "ERROR: Cannot access required field in file '%s'" % entitlement_file, stream=sys.stderr) raise finally: entitlements.close() else: raise IncorrectEntitlementsFileFormatError( "ERROR: There has to be at least one entitlements file")
def _create_yum_repo(self, repo_source): repo_label = self.cdn_repository_manager.get_content_source_label(repo_source) repo_plugin = yum_src.ContentSource(self.mount_point + str(repo_source['relative_url']), str(repo_label), org=None, no_mirrors=True) try: keys = self.cdn_repository_manager.get_repository_crypto_keys(repo_source['relative_url']) except CdnRepositoryNotFoundError: log2(1, 1, "ERROR: No SSL certificates were found for repository '%s'" % repo_source['relative_url'], stream=sys.stderr) return repo_plugin if len(keys) >= 1: (ca_cert_file, client_cert_file, client_key_file) = reposync.write_ssl_set_cache( keys[0]['ca_cert'], keys[0]['client_cert'], keys[0]['client_key']) repo_plugin.set_ssl_options(ca_cert_file, client_cert_file, client_key_file) else: log2(1, 1, "ERROR: No valid SSL certificates were found for repository '%s'." % repo_source['relative_url'], stream=sys.stderr) return repo_plugin
def _create_yum_repo(self, repo_source): repo_label = self.cdn_repository_manager.get_content_source_label(repo_source) try: keys = self.cdn_repository_manager.get_repository_crypto_keys(repo_source['relative_url']) except CdnRepositoryNotFoundError: keys = [] log2(1, 1, "WARNING: Repository '%s' was not found." % repo_source['relative_url'], stream=sys.stderr) if keys: (ca_cert_file, client_cert_file, client_key_file) = reposync.write_ssl_set_cache( keys[0]['ca_cert'], keys[0]['client_cert'], keys[0]['client_key']) else: (ca_cert_file, client_cert_file, client_key_file) = (None, None, None) log2(1, 1, "WARNING: No valid SSL certificates were found for repository '%s'." % repo_source['relative_url'], stream=sys.stderr) return yum_src.ContentSource(self.mount_point + str(repo_source['relative_url']), str(repo_label), org=None, no_mirrors=True, ca_cert_file=ca_cert_file, client_cert_file=client_cert_file, client_key_file=client_key_file)
def export_manifest(self, uuid=None, ownerid=None, satellite_version=None): """Performs export request to Candlepin API and saves exported manifest to target file. Can take required parameters from current manifest or override them with parameters of this method.""" if uuid is None: if self.current_manifest: uuid = self.current_manifest.get_uuid() else: raise ValueError("Uuid is not known.") if ownerid is None: if self.current_manifest: ownerid = self.current_manifest.get_ownerid() else: raise ValueError("Ownerid is not known.") if satellite_version is None: if self.current_manifest: satellite_version = self.current_manifest.get_satellite_version() else: raise ValueError("Satellite version is not known.") url = "%s%s/export" % (self.base_url, uuid) params = {"ext": ["ownerid:%s" % ownerid, "version:%s" % satellite_version]} log(1, "URL: '%s'" % url) log(1, "Parameters: '%s'" % str(params)) response = self._call_api(url, params=params, method="get") if response is not None: # pylint: disable=E1101 if response.status_code == requests.codes.ok: fd, downloaded_manifest = tempfile.mkstemp(prefix="/tmp/manifest-", suffix=".zip") fo = os.fdopen(fd, 'wb') for chunk in response: fo.write(chunk) fo.flush() fo.close() return downloaded_manifest else: log2(0, 0, "Status code: %s" % response.status_code, stream=sys.stderr) log2(0, 0, "Message: '%s'" % response.text, stream=sys.stderr) return None
def _extract_meta_info(self, zip_file): files = zip_file.namelist() found = False for f in files: if f == self.META_INFO: found = True break if found: meta_info = zip_file.open(self.META_INFO) try: try: data = json.load(meta_info) self.created = data['created'] except KeyError: log2(0, 0, "ERROR: Cannot access required field in file '%s'" % self.META_INFO, stream=sys.stderr) raise finally: meta_info.close() else: raise MissingMetaInfoError()
def check_channel_availability(self, channel_label, no_kickstarts=False): """Checks if all repositories for channel are available.""" if no_kickstarts: sources = self.get_content_sources_regular(channel_label) else: sources = self.get_content_sources(channel_label) # No content, no channel if not sources: return False for source in sources: if not self.check_repository_availability(source['relative_url'], channel_label=channel_label): if source.get('ks_tree_label', None): # don't fail if kickstart is missing, just warn (bz1626797) log2(0, 0, "WARNING: kickstart tree '%s' is unavailable" % source['ks_tree_label'], stream=sys.stderr) self.excluded_urls.append(source['relative_url']) else: return False return True
def _can_add_repos(self, db_channel, repos): # Adding custom repositories to custom channel, need to check: # 1. Repositories availability - if there are SSL certificates for them # 2. Channel is custom # 3. Repositories are not already associated with any channels in mapping files if not db_channel or not db_channel['org_id']: log2(0, 0, "ERROR: Channel doesn't exist or is not custom.", stream=sys.stderr) return False # Repositories can't be part of any channel from mappings channels = [] for repo in repos: channels.extend(self.cdn_repository_manager.list_channels_containing_repository(repo)) if channels: log2(0, 0, "ERROR: Specified repositories can't be synced because they are part of following channels: %s" % ", ".join(channels), stream=sys.stderr) return False # Check availability of repositories not_available = [] for repo in repos: if not self.cdn_repository_manager.check_repository_availability(repo): not_available.append(repo) if not_available: log2(0, 0, "ERROR: Following repositories are not available: %s" % ", ".join(not_available), stream=sys.stderr) return False return True
def __can_retry(self, retry, mirrors, opts, url, e): retrycode = getattr(e, 'errno', None) code = getattr(e, 'code', None) if retry < (self.parent.retries - 1): # No codes at all or some specified codes # 58, 77 - Couple of curl error codes observed in multithreading on RHEL 7 - probably a bug if (retrycode is None and code is None) or (retrycode in opts.retrycodes or code in [58, 77]): log2(0, 2, "ERROR: Download failed: %s - %s. Retrying..." % (url, sys.exc_info()[1]), stream=sys.stderr) return True # 14 - HTTP Error if retry < (mirrors - 1) and retrycode == 14: log2(0, 2, "ERROR: Download failed: %s - %s. Trying next mirror..." % (url, sys.exc_info()[1]), stream=sys.stderr) return True log2(0, 1, "ERROR: Download failed: %s - %s." % (url, sys.exc_info()[1]), stream=sys.stderr) return False
def _can_add_repos(self, db_channel, repos): # Adding custom repositories to custom channel, need to check: # 1. Repositories availability - if there are SSL certificates for them # 2. Channel is custom # 3. Repositories are not already associated with any channels in mapping files if not db_channel or not db_channel['org_id']: log2(0, 0, "ERROR: Channel doesn't exist or is not custom.", stream=sys.stderr) return False # Repositories can't be part of any channel from mappings channels = [] for repo in repos: channels.extend(self.cdn_repository_manager.list_channels_containing_repository(repo)) if channels: log2(0, 0, "ERROR: Specified repositories can't be synced because they are part of following channels: %s" % ", ".join(channels), stream=sys.stderr) return False # Check availability of repositories not_available = [] for repo in repos: if not self.cdn_repository_manager.check_repository_availability(repo): not_available.append(repo) if not_available: log2(0, 0, "ERROR: Following repositories are not available: %s" % ", ".join(not_available), stream=sys.stderr) return False return True
def _extract_consumer_credentials(self, zip_file): files = zip_file.namelist() consumer_credentials = [] for f in files: if f.startswith(self.UPSTREAM_CONSUMER_PATH) and f.endswith(".json"): consumer_credentials.append(f) if len(consumer_credentials) == 1: upstream_consumer = zip_file.open(consumer_credentials[0]) try: try: data = json.load(upstream_consumer) self.consumer_credentials = Credentials(data['id'], data['cert'], data['key']) except KeyError: log2(0, 0, "ERROR: Cannot access required field in file '%s'" % consumer_credentials[0], stream=sys.stderr) raise finally: upstream_consumer.close() else: raise IncorrectCredentialsError( "ERROR: Single upstream consumer certificate expected, found %d." % len(consumer_credentials))
def refresh_manifest(self, uuid=None): if uuid is None: if self.current_manifest: uuid = self.current_manifest.get_uuid() else: raise ValueError("Uuid is not known.") url = "%s%s/certificates" % (self.base_url, uuid) log(1, "URL: '%s'" % url) response = self._call_api(url, method="put") if response is not None: # pylint: disable=E1101 if response.status_code == requests.codes.ok or response.status_code == requests.codes.no_content: return True else: log2(0, 0, "Status code: %s" % response.status_code, stream=sys.stderr) log2(0, 0, "Message: '%s'" % response.text, stream=sys.stderr) return False
def print_cdn_certificates_info(self, repos=False): keys = self._get_cdn_certificate_keys_and_certs() if not keys: log2(0, 0, "No SSL certificates were found. Is your %s activated for CDN?" % PRODUCT_NAME, stream=sys.stderr) sys.exit(1) for key in keys: log(0, "======================================") log(0, "| Certificate/Key: %s" % key['description']) log(0, "======================================") if constants.CA_CERT_NAME == key['description'] or constants.CLIENT_CERT_PREFIX in key['description']: if not verify_certificate_dates(str(key['key'])): log(0, "WARNING: This certificate is not valid.") cn, serial_number, not_before, not_after = get_certificate_info(str(key['key'])) log(0, "Common name: %s" % str(cn)) log(0, "Serial number: %s" % str(serial_number)) log(0, "Valid from: %s" % str(not_before)) log(0, "Valid to: %s" % str(not_after)) if constants.CLIENT_CERT_PREFIX in key['description']: manager = CdnRepositoryManager(client_cert_id=int(key['id'])) self.cdn_repository_manager = manager log(0, "Provided channels:") channel_tree, not_available_channels = self._tree_available_channels() if not channel_tree: log(0, " NONE") for base_channel in sorted(channel_tree): if base_channel not in not_available_channels: log(0, " * %s" % base_channel) elif channel_tree[base_channel]: log(0, " * %s (only child channels provided)" % base_channel) for child_channel in sorted(channel_tree[base_channel]): log(0, " * %s" % child_channel) if repos: log(0, "Provided repositories:") provided_repos = self.cdn_repository_manager.list_provided_repos(key['id']) for repo in sorted(provided_repos): log(0, " %s" % repo) log(0, "")
def _call_api(self, url, params=None, method="get"): if self.protocol == 'https': verify = CA_CERT_PATH else: verify = False response = None if self.username is not None and self.password is not None: try: if method == "get": response = requests.get(url, params=params, proxies=self._get_proxies(), auth=(self.username, self.password), verify=verify) elif method == "put": response = requests.put(url, params=params, proxies=self._get_proxies(), auth=(self.username, self.password), verify=verify) else: raise ValueError("Unsupported method: '%s'" % method) except requests.RequestException: e = sys.exc_info()[1] log2(0, 0, "ERROR: %s" % str(e), stream=sys.stderr) else: cert = self._write_cert() try: try: if method == "get": response = requests.get(url, params=params, proxies=self._get_proxies(), verify=verify, cert=cert) elif method == "put": response = requests.put(url, params=params, proxies=self._get_proxies(), verify=verify, cert=cert) else: raise ValueError("Unsupported method: '%s'" % method) except requests.RequestException: e = sys.exc_info()[1] log2(0, 0, "ERROR: %s" % str(e), stream=sys.stderr) finally: self._delete_cert(cert) return response
def _openSocketStream(self, method, params): """Wraps the gzipstream.GzipStream instantiation in a test block so we can open normally if stream is not gzipped.""" stream = None retryYN = 0 wait = 0.33 lastErrorMsg = '' cfg = config.initUp2dateConfig() for i in range(cfg['networkRetries']): server = self.getServer(retryYN) if server is None: log2(-1, 2, 'ERROR: server unable to initialize, attempt %s' % i, stream=sys.stderr) retryYN = 1 time.sleep(wait) continue func = getattr(server, method) try: stream = func(*params) if CFG.SYNC_TO_TEMP: import tempfile cached = tempfile.NamedTemporaryFile() stream.read_to_file(cached) cached.seek(0) return cached else: return stream except rpclib.xmlrpclib.ProtocolError: e = sys.exc_info()[1] p = tuple(['<the systemid>'] + list(params[1:])) lastErrorMsg = 'ERROR: server.%s%s: %s' % (method, p, e) log2(-1, 2, lastErrorMsg, stream=sys.stderr) retryYN = 1 time.sleep(wait) # do not reraise this exception! except (KeyboardInterrupt, SystemExit): raise except rpclib.xmlrpclib.Fault: e = sys.exc_info()[1] lastErrorMsg = e.faultString break except Exception: # pylint: disable=E0012, W0703 e = sys.exc_info()[1] p = tuple(['<the systemid>'] + list(params[1:])) lastErrorMsg = 'ERROR: server.%s%s: %s' % (method, p, e) log2(-1, 2, lastErrorMsg, stream=sys.stderr) break # do not reraise this exception! if lastErrorMsg: raise_with_tb(RhnSyncException(lastErrorMsg), sys.exc_info()[2]) # Returns a stream # Should never be reached return stream
def _extract_meta_info(self, zip_file): files = zip_file.namelist() found = False for f in files: if f == self.META_INFO: found = True break if found: meta_info = zip_file.open(self.META_INFO) try: try: data = json.load(meta_info) self.created = data['created'] except KeyError: log2(0, 0, "ERROR: Cannot access required field in file '%s'" % self.META_INFO, stream=sys.stderr) raise finally: meta_info.close() else: raise MissingMetaInfoError()
def __can_retry(self, retry, mirrors, opts, url, e): retrycode = getattr(e, 'errno', None) code = getattr(e, 'code', None) if retry < (self.parent.retries - 1): # No codes at all or some specified codes # 58, 77 - Couple of curl error codes observed in multithreading on RHEL 7 - probably a bug if (retrycode is None and code is None) or (retrycode in opts.retrycodes or code in [58, 77]): log2(0, 2, "ERROR: Download failed: %s - %s. Retrying..." % (url, sys.exc_info()[1]), stream=sys.stderr) return True # 14 - HTTP Error if retry < (mirrors - 1) and retrycode == 14: log2(0, 2, "ERROR: Download failed: %s - %s. Trying next mirror..." % (url, sys.exc_info()[1]), stream=sys.stderr) return True log2(0, 1, "ERROR: Download failed: %s - %s." % (url, sys.exc_info()[1]), stream=sys.stderr) return False
def check_repository_availability(self, relative_url, channel_label=None): try: crypto_keys = self.get_repository_crypto_keys(relative_url) except CdnRepositoryNotFoundError: log2(1, 1, "ERROR: No SSL certificates were found for repository '%s'" % relative_url, stream=sys.stderr) return False # Check SSL certificates if not crypto_keys: if channel_label: log2(1, 1, "ERROR: No valid SSL certificates were found for repository '%s'" " required for channel '%s'." % (relative_url, channel_label), stream=sys.stderr) else: log2(1, 1, "ERROR: No valid SSL certificates were found for repository '%s'." % relative_url, stream=sys.stderr) return False # Try to look for repomd file if self.local_mount_point and not os.path.isfile(os.path.join( self.local_mount_point, relative_url[1:], "repodata/repomd.xml")): return False return True
def check_repository_availability(self, relative_url, channel_label=None): try: crypto_keys = self.get_repository_crypto_keys(relative_url) except CdnRepositoryNotFoundError: log2(1, 1, "ERROR: No SSL certificates were found for repository '%s'" % relative_url, stream=sys.stderr) return False # Check SSL certificates if not crypto_keys: if channel_label: log2(1, 1, "ERROR: No valid SSL certificates were found for repository '%s'" " required for channel '%s'." % (relative_url, channel_label), stream=sys.stderr) else: log2(1, 1, "ERROR: No valid SSL certificates were found for repository '%s'." % relative_url, stream=sys.stderr) return False # Try to look for repomd file if self.local_mount_point and not os.path.isfile(os.path.join( self.local_mount_point, relative_url[1:], "repodata/repomd.xml")): return False return True
def get_file(path, local_base=None): # pylint: disable=W0613 # Called from import_kickstarts, not working for deb repo log2(0, 0, "Unable to download path %s from deb repo." % path, stream=sys.stderr) return None
def _load_entitlements(self, zip_file): files = zip_file.namelist() entitlements_files = [] for f in files: if f.startswith(self.ENTITLEMENTS_PATH) and f.endswith(".json"): entitlements_files.append(f) if len(entitlements_files) >= 1: self.all_entitlements = [] for entitlement_file in entitlements_files: entitlements = zip_file.open(entitlement_file) # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: data = json.load(entitlements) # Extract credentials certs = data['certificates'] if len(certs) != 1: raise IncorrectEntitlementsFileFormatError( "Single certificate in entitlements file '%s' is expected, found: %d" % (entitlement_file, len(certs))) cert = certs[0] credentials = Credentials(data['id'], cert['cert'], cert['key']) # Extract product IDs products = [] provided_products = data['pool'][ 'providedProducts'] or [] derived_provided_products = data['pool'][ 'derivedProvidedProducts'] or [] product_ids = [ provided_product['productId'] for provided_product in provided_products + derived_provided_products ] for product_id in set(product_ids): product = Product(product_id) self._fill_product_repositories(zip_file, product) products.append(product) # Skip entitlements not providing any products if products: entitlement = Entitlement(products, credentials) self.all_entitlements.append(entitlement) except KeyError: log2(0, 0, "ERROR: Cannot access required field in file '%s'" % entitlement_file, stream=sys.stderr) raise finally: entitlements.close() else: refer_url = "%s%s" % (self.web_url, self.uuid) if not refer_url.startswith("http"): refer_url = "https://" + refer_url raise IncorrectEntitlementsFileFormatError( "No subscriptions were found in manifest.\n\nPlease refer to %s for setting up subscriptions." % refer_url)
def writeError(e): log2(0, 0, '\nERROR: %s\n' % e, stream=sys.stderr, cleanYN=1)
def writeError(e): log2(0, 0, '\nERROR: %s\n' % e, stream=sys.stderr, cleanYN=1)