def __init__(self, req): """ init with http request object """ # FIXME: should rename some things: # self.bodyFd --> self.body or self.data or ? # self.caChain --> self.caCert self.req = req # turn wsgi.input object into a SmartIO instance so it can be read # more than once if 'wsgi.input' in self.req.headers_in: smartFd = SmartIO(max_mem_size=CFG.MAX_MEM_FILE_SIZE) smartFd.write(self.req.headers_in['wsgi.input'].read()) self.req.headers_in['wsgi.input'] = smartFd self.responseContext = ResponseContext() self.uri = None # '' # Common settings for both the proxy and the redirect # broker and redirect immediately alter these for their own purposes self.caChain = CFG.CA_CHAIN self.httpProxy = CFG.HTTP_PROXY self.httpProxyUsername = CFG.HTTP_PROXY_USERNAME self.httpProxyPassword = CFG.HTTP_PROXY_PASSWORD if not self.httpProxyUsername: self.httpProxyPassword = '' self.rhnParent = CFG.RHN_PARENT or '' self.rhnParent = rhnLib.parseUrl(self.rhnParent)[1].split(':')[0] CFG.set('RHN_PARENT', self.rhnParent)
def accessible(url): """Try if url is accessible :arg url: the url which is tried to access Returns True if url is accessible, otherwise False. """ timeout = 120 if CFG.is_initialized() and CFG.has_key('TIMEOUT'): timeout = CFG.TIMEOUT curl = pycurl.Curl() curl.setopt(pycurl.CONNECTTIMEOUT, timeout) curl.setopt(pycurl.URL, url) curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug) curl.setopt(pycurl.VERBOSE, True) proxy_url, proxy_user, proxy_pass = get_proxy(url) if proxy_url: curl.setopt(pycurl.PROXY, proxy_url) log_debug(2, "Connect to %s" % url) # We implement our own redirection-following, because pycurl # 7.19 doesn't POST after it gets redirected. Ideally we'd be # using pycurl.POSTREDIR here, but that's in 7.21. curl.setopt(pycurl.FOLLOWLOCATION, False) curl.setopt(pycurl.NOBODY, True) try_counter = 5 while try_counter: try_counter -= 1 try: curl.perform() except pycurl.error as e: if e[0] == 56: # Proxy requires authentication log_debug(2, e[1]) if not (proxy_user and proxy_pass): raise TransferException("Proxy requires authentication, " "but reading credentials from " "%s failed." % YAST_PROXY) curl.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (proxy_user, proxy_pass)) else: break status = curl.getinfo(pycurl.HTTP_CODE) # OK or file if status == 200 or (URL(url).scheme == "file" and status == 0): return True elif status in (301, 302): # redirects url = curl.getinfo(pycurl.REDIRECT_URL) log_debug(2, "Got redirect to %s" % url) curl.setopt(pycurl.URL, url) elif status >= 400: break return False
def accessible(url): """Try if url is accessible :arg url: the url which is tried to access Returns True if url is accessible, otherwise False. """ timeout = 120 if CFG.is_initialized() and CFG.has_key('TIMEOUT'): timeout = CFG.TIMEOUT curl = pycurl.Curl() curl.setopt(pycurl.CONNECTTIMEOUT, timeout) curl.setopt(pycurl.URL, url) curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug) curl.setopt(pycurl.VERBOSE, True) proxy_url, proxy_user, proxy_pass = get_proxy(url) if proxy_url: curl.setopt(pycurl.PROXY, proxy_url) log_debug(2, "Connect to %s" % url) # We implement our own redirection-following, because pycurl # 7.19 doesn't POST after it gets redirected. Ideally we'd be # using pycurl.POSTREDIR here, but that's in 7.21. curl.setopt(pycurl.FOLLOWLOCATION, False) curl.setopt(pycurl.NOBODY, True) try_counter = 5 while try_counter: try_counter -= 1 try: curl.perform() except pycurl.error, e: if e[0] == 56: # Proxy requires authentication log_debug(2, e[1]) if not (proxy_user and proxy_pass): raise TransferException("Proxy requires authentication, " "but reading credentials from " "%s failed." % YAST_PROXY) curl.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (proxy_user, proxy_pass)) else: break status = curl.getinfo(pycurl.HTTP_CODE) # OK or file if status == 200 or (URL(url).scheme == "file" and status == 0): return True elif status in (301, 302): # redirects url = curl.getinfo(pycurl.REDIRECT_URL) log_debug(2, "Got redirect to %s" % url) curl.setopt(pycurl.URL, url) elif status >= 400: break
def _read_config(): # we want to change the logging file to 'audit' and set it back # after we finished reading the config file # TODO Changing the component twice on every request is not nice comp = CFG.getComponent() initCFG("audit") enabled = CFG.get("enabled") server_url = CFG.get("server", "") # XXX haven't tested what happens if it's not set back to the original value initCFG(comp) return (enabled, server_url)
def _useProxyFor(url): """Return True if a proxy should be used for given url, otherwise False. This function uses server.satellite.no_proxy variable to check for hosts or domains which should not be connected via a proxy. server.satellite.no_proxy is a comma seperated list. Either an exact match, or the previous character is a '.', so host is within the same domain. A leading '.' in the pattern is ignored. See also 'man curl' """ u = urlparse.urlsplit(url) # pylint can't see inside the SplitResult class # pylint: disable=E1103 if u.scheme == 'file': return False hostname = u.hostname.lower() if hostname in ["localhost", "127.0.0.1", "::1"]: return False comp = CFG.getComponent() if not CFG.has_key("no_proxy"): initCFG("server.satellite") if not CFG.has_key('no_proxy'): initCFG(comp) return True noproxy = CFG.no_proxy initCFG(comp) if not noproxy: return True if not isinstance(noproxy, list): if noproxy == '*': # just an asterisk disables all. return False noproxy = [noproxy] # No proxy: Either an exact match, or the previous character # is a '.', so host is within the same domain. # A leading '.' in the pattern is ignored. Some implementations # need '.foo.ba' to prevent 'foo.ba' from matching 'xfoo.ba'. for domain in noproxy: domain = domain.lower() if domain[0] == '.': domain = domain[1:] if hostname.endswith(domain) and \ (len(hostname) == len(domain) or hostname[len(hostname) - len(domain) - 1] == '.'): return False return True
def _load_proxy_settings(self, url): # read the proxy configuration in /etc/rhn/rhn.conf comp = CFG.getComponent() initCFG('server.satellite') # Get the global HTTP Proxy settings from DB or per-repo # settings on /etc/rhn/spacewalk-repo-sync/zypper.conf if CFG.http_proxy: self.proxy_url, self.proxy_user, self.proxy_pass = get_proxy(url) self.proxy_hostname = self.proxy_url elif os.path.isfile(REPOSYNC_ZYPPER_CONF): zypper_cfg = configparser.ConfigParser() zypper_cfg.read_file(open(REPOSYNC_ZYPPER_CONF)) section_name = None if zypper_cfg.has_section(self.name): section_name = self.name elif zypper_cfg.has_section(channel_label): section_name = channel_label elif zypper_cfg.has_section('main'): section_name = 'main' if section_name: if zypper_cfg.has_option(section_name, option='proxy'): self.proxy_hostname = zypper_cfg.get(section_name, option='proxy') self.proxy_url = "http://%s" % self.proxy_hostname if zypper_cfg.has_option(section_name, 'proxy_username'): self.proxy_user = zypper_cfg.get(section_name, 'proxy_username') if zypper_cfg.has_option(section_name, 'proxy_password'): self.proxy_pass = zypper_cfg.get(section_name, 'proxy_password') # set config component back to original initCFG(comp)
def _create_connection(self): """ Returns a Connection object """ scheme, host, port, _uri = self._parse_url(self.rhnParent) # Build the list of params params = { 'host': host, 'port': port, } if CFG.has_key('timeout'): params['timeout'] = CFG.TIMEOUT if self.httpProxy: params['proxy'] = self.httpProxy params['username'] = self.httpProxyUsername params['password'] = self.httpProxyPassword if scheme == 'https' and self.caChain: params['trusted_certs'] = [self.caChain, ] # Now select the right class if self.httpProxy: if scheme == 'https': conn_class = connections.HTTPSProxyConnection else: conn_class = connections.HTTPProxyConnection else: if scheme == 'https': conn_class = connections.HTTPSConnection else: conn_class = connections.HTTPConnection log_debug(5, "Using connection class", conn_class, 'Params:', params) return conn_class(**params)
def __init__(self, retries=3, log_obj=None, force=False): self.queues = {} comp = CFG.getComponent() initCFG("server.satellite") try: self.threads = int(CFG.REPOSYNC_DOWNLOAD_THREADS) except ValueError: raise ValueError("Number of threads expected, found: '%s'" % CFG.REPOSYNC_DOWNLOAD_THREADS) try: self.timeout = int(CFG.REPOSYNC_TIMEOUT) except ValueError: raise ValueError("Timeout in seconds expected, found: '%s'" % CFG.REPOSYNC_TIMEOUT) try: self.minrate = int(CFG.REPOSYNC_MINRATE) except ValueError: raise ValueError( "Minimal transfer rate in bytes pre second expected, found: '%s'" % CFG.REPOSYNC_MINRATE) if self.threads < 1: raise ValueError("Invalid number of threads: %d" % self.threads) initCFG(comp) self.retries = retries self.log_obj = log_obj self.force = force self.lock = Lock() self.exception = None # WORKAROUND - BZ #1439758 - ensure first item in queue is performed alone to properly setup NSS self.first_in_queue_done = False self.first_in_queue_lock = Lock()
def _create_connection(self): """ Returns a Connection object """ scheme, host, port, _uri = self._parse_url(self.rhnParent) # Build the list of params params = {"host": host, "port": port} if CFG.has_key("timeout"): params["timeout"] = CFG.TIMEOUT if self.httpProxy: params["proxy"] = self.httpProxy params["username"] = self.httpProxyUsername params["password"] = self.httpProxyPassword if scheme == "https" and self.caChain: params["trusted_certs"] = [self.caChain] # Now select the right class if self.httpProxy: if scheme == "https": conn_class = connections.HTTPSProxyConnection else: conn_class = connections.HTTPProxyConnection else: if scheme == "https": conn_class = connections.HTTPSConnection else: conn_class = connections.HTTPConnection log_debug(5, "Using connection class", conn_class, "Params:", params) return conn_class(**params)
def initDB(backend=None, host=None, port=None, username=None, password=None, database=None, sslmode=None, sslrootcert=None): """ Initialize the database. Either we get backend and all parameter which means the caller knows what they are doing, or we populate everything from the config files. """ if backend is None: if CFG is None or not CFG.is_initialized(): initCFG('server') backend = CFG.DB_BACKEND host = CFG.DB_HOST port = CFG.DB_PORT database = CFG.DB_NAME username = CFG.DB_USER password = CFG.DB_PASSWORD sslmode = None sslrootcert = None if CFG.DB_SSL_ENABLED: sslmode = 'verify-full' sslrootcert = CFG.DB_SSLROOTCERT if backend not in SUPPORTED_BACKENDS: raise rhnException("Unsupported database backend", backend) if port: port = int(port) # Hide the password add_to_seclist(password) try: __init__DB(backend, host, port, username, password, database, sslmode, sslrootcert) __init__DB2(backend, host, port, username, password, database, sslmode, sslrootcert) # except (rhnException, SQLError): # raise # pass on, we know those ones # except (KeyboardInterrupt, SystemExit): # raise except SQLConnectError, e: try: global __DB global __DB2 del __DB del __DB2 except NameError: pass raise e
def initDB(backend=None, host=None, port=None, username=None, password=None, database=None, sslmode=None, sslrootcert=None, initsecond=False): """ Initialize the database. Either we get backend and all parameter which means the caller knows what they are doing, or we populate everything from the config files. initsecond: If set to True it initialize a second DB connection. By default only one DB connection is needed. """ if backend is None: if CFG is None or not CFG.is_initialized(): initCFG('server') backend = CFG.DB_BACKEND host = CFG.DB_HOST port = CFG.DB_PORT database = CFG.DB_NAME username = CFG.DB_USER password = CFG.DB_PASSWORD sslmode = None sslrootcert = None if CFG.DB_SSL_ENABLED: sslmode = 'verify-full' sslrootcert = CFG.DB_SSLROOTCERT if backend not in SUPPORTED_BACKENDS: raise rhnException("Unsupported database backend", backend) if port: port = int(port) # Hide the password add_to_seclist(password) try: if initsecond == False: __init__DB(backend, host, port, username, password, database, sslmode, sslrootcert) else: __init__DB2(backend, host, port, username, password, database, sslmode, sslrootcert) # except (rhnException, SQLError): # raise # pass on, we know those ones # except (KeyboardInterrupt, SystemExit): # raise except SQLConnectError: e = sys.exc_info()[1] try: closeDB() except NameError: pass raise e except: raise #e_type, e_value = sys.exc_info()[:2] # raise rhnException("Could not initialize Oracle database connection", # str(e_type), str(e_value)) return 0
def initDB(backend=None, host=None, port=None, username=None, password=None, database=None, sslmode=None, sslrootcert=None, initsecond=False): """ Initialize the database. Either we get backend and all parameter which means the caller knows what they are doing, or we populate everything from the config files. initsecond: If set to True it initialize a second DB connection. By default only one DB connection is needed. """ if backend is None: if CFG is None or not CFG.is_initialized(): initCFG('server') backend = CFG.DB_BACKEND host = CFG.DB_HOST port = CFG.DB_PORT database = CFG.DB_NAME username = CFG.DB_USER password = CFG.DB_PASSWORD sslmode = None sslrootcert = None if CFG.DB_SSL_ENABLED: sslmode = 'verify-full' sslrootcert = CFG.DB_SSLROOTCERT if backend not in SUPPORTED_BACKENDS: raise rhnException("Unsupported database backend", backend) if port: port = int(port) # Hide the password add_to_seclist(password) try: if initsecond == False: __init__DB(backend, host, port, username, password, database, sslmode, sslrootcert) else: __init__DB2(backend, host, port, username, password, database, sslmode, sslrootcert) # except (rhnException, SQLError): # raise # pass on, we know those ones # except (KeyboardInterrupt, SystemExit): # raise except SQLConnectError: e = sys.exc_info()[1] try: closeDB() except NameError: pass raise_with_tb(e, sys.exc_info()[2]) except: raise #e_type, e_value = sys.exc_info()[:2] # raise rhnException("Could not initialize Oracle database connection", # str(e_type), str(e_value)) return 0
def _get_proxy_from_rhn_conf(): """Return a tuple of (url, user, pass) proxy information from rhn config Returns None instead of a tuple if there was no proxy url. user, pass can be None. """ comp = CFG.getComponent() if not CFG.has_key("http_proxy"): initCFG("server.satellite") result = None if CFG.http_proxy: # CFG.http_proxy format is <hostname>[:<port>] in 1.7 url = 'http://%s' % CFG.http_proxy result = (url, CFG.http_proxy_username, CFG.http_proxy_password) initCFG(comp) log_debug(2, "Could not read proxy URL from rhn config.") return result
def getNotificationsTypeDisabled(): """Return list of types which are disabled""" disabledTypes = [] comp = CFG.getComponent() initCFG("java") if CFG.notifications_type_disabled: disabledTypes = CFG.notifications_type_disabled.split(",") initCFG(comp) return disabledTypes
def _createConnection(host, port, scheme): params = {"host": host, "port": port} if CFG.has_key("timeout"): params["timeout"] = CFG.TIMEOUT if scheme == SCHEME_HTTPS: conn_class = connections.HTTPSConnection else: conn_class = connections.HTTPConnection return conn_class(**params)
def _createConnection(host, port, scheme): params = {'host': host, 'port': port} if CFG.has_key('timeout'): params['timeout'] = CFG.TIMEOUT if scheme == SCHEME_HTTPS: conn_class = connections.HTTPSConnection else: conn_class = connections.HTTPConnection return conn_class(**params)
def __init__(self, url, name, insecure=False, interactive=True, yumsrc_conf=None, org="1", channel_label="", no_mirrors=False, ca_cert_file=None, client_cert_file=None, client_key_file=None): # pylint: disable=W0613 self.url = url self.name = name if org: self.org = org else: self.org = "NULL" comp = CFG.getComponent() # read the proxy configuration in /etc/rhn/rhn.conf initCFG('server.satellite') self.proxy_addr, self.proxy_user, self.proxy_pass = get_proxy(self.url) self.authtoken = None # Replace non-valid characters from reponame (only alphanumeric chars allowed) self.reponame = "".join([x if x.isalnum() else "_" for x in self.name]) self.channel_label = channel_label # SUSE vendor repositories belongs to org = NULL # The repository cache root will be "/var/cache/rhn/reposync/REPOSITORY_LABEL/" root = os.path.join(CACHE_DIR, str(org or "NULL"), self.reponame) self.repo = DebRepo(url, root, os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR, self.org, 'stage'), self.proxy_addr, self.proxy_user, self.proxy_pass, gpg_verify=not (insecure)) self.repo.verify() self.num_packages = 0 self.num_excluded = 0 # keep authtokens for mirroring (_scheme, _netloc, _path, query, _fragid) = urlparse.urlsplit(url) if query: self.authtoken = query initCFG(comp)
def __init__(self, req): """ init with http request object """ # FIXME: should rename some things: # self.bodyFd --> self.body or self.data or ? # self.caChain --> self.caCert self.req = req self.responseContext = ResponseContext() self.uri = None # '' # Common settings for both the proxy and the redirect # broker and redirect immediately alter these for their own purposes self.caChain = CFG.CA_CHAIN self.httpProxy = CFG.HTTP_PROXY self.httpProxyUsername = CFG.HTTP_PROXY_USERNAME self.httpProxyPassword = CFG.HTTP_PROXY_PASSWORD if not self.httpProxyUsername: self.httpProxyPassword = '' self.rhnParent = CFG.RHN_PARENT or '' self.rhnParent = rhnLib.parseUrl(self.rhnParent)[1].split(':')[0] CFG.set('RHN_PARENT', self.rhnParent)
def __init__(self, req): """ init with http request object """ # FIXME: should rename some things: # self.bodyFd --> self.body or self.data or ? # self.caChain --> self.caCert self.req = req self.responseContext = ResponseContext() self.uri = None # '' # Common settings for both the proxy and the redirect # broker and redirect immediately alter these for their own purposes self.caChain = CFG.CA_CHAIN self.httpProxy = CFG.HTTP_PROXY self.httpProxyUsername = CFG.HTTP_PROXY_USERNAME self.httpProxyPassword = CFG.HTTP_PROXY_PASSWORD if not self.httpProxyUsername: self.httpProxyPassword = "" self.rhnParent = CFG.RHN_PARENT or "" self.rhnParent = rhnLib.parseUrl(self.rhnParent)[1].split(":")[0] CFG.set("RHN_PARENT", self.rhnParent)
def __init__(self, url, name, insecure=False, interactive=True, yumsrc_conf=None, org="1", channel_label="", no_mirrors=False, ca_cert_file=None, client_cert_file=None, client_key_file=None): # pylint: disable=W0613 self.url = url self.name = name if org: self.org = org else: self.org = "NULL" comp = CFG.getComponent() # read the proxy configuration in /etc/rhn/rhn.conf initCFG('server.satellite') self.proxy_addr, self.proxy_user, self.proxy_pass = get_proxy(self.url) self.authtoken = None self.repo = DebRepo(url, os.path.join(CACHE_DIR, self.org, name), os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR, self.org, 'stage'), self.proxy_addr, self.proxy_user, self.proxy_pass, gpg_verify=not (insecure)) self.repo.verify() self.num_packages = 0 self.num_excluded = 0 # keep authtokens for mirroring (_scheme, _netloc, _path, query, _fragid) = urlparse.urlsplit(url) if query: self.authtoken = query initCFG(comp)
def get_mirror_credentials(): """Return a list of mirror credential tuples (user, pass) N.B. The config values will be read from the global configuration: server.susemanager.mirrcred_user server.susemanager.mirrcred_pass server.susemanager.mirrcred_user_1 server.susemanager.mirrcred_pass_1 etc. The credentials are read sequentially, when the first value is found to be missing, the process is aborted and the list of credentials that have been read so far are returned. For example if server.susemanager.mirrcred_pass_1 can not be read, only the first pair of default mirrcreds will be returned, even though mirrcred_user_2, mirrcred_pass_2 etc. might still exist. """ comp = CFG.getComponent() initCFG("server.susemanager") creds = [] # the default values should at least always be there if not CFG["mirrcred_user"] or not CFG["mirrcred_pass"]: initCFG(comp) raise ConfigParserError("Could not read default mirror credentials: " "server.susemanager.mirrcred_user, " "server.susemanager.mirrcred_pass.") creds.append((CFG["mirrcred_user"], CFG["mirrcred_pass"])) # increment the credentials number, until we can't read one n = 1 while True: try: creds.append((CFG["mirrcred_user_%s" % n], CFG["mirrcred_pass_%s" % n])) except (KeyError, AttributeError): break n += 1 initCFG(comp) return creds
def get_mirror_credentials(): """Return a list of mirror credential tuples (user, pass) N.B. The config values will be read from the global configuration: server.susemanager.mirrcred_user server.susemanager.mirrcred_pass server.susemanager.mirrcred_user_1 server.susemanager.mirrcred_pass_1 etc. The credentials are read sequentially, when the first value is found to be missing, the process is aborted and the list of credentials that have been read so far are returned. For example if server.susemanager.mirrcred_pass_1 can not be read, only the first pair of default mirrcreds will be returned, even though mirrcred_user_2, mirrcred_pass_2 etc. might still exist. """ comp = CFG.getComponent() initCFG("server.susemanager") creds = [] # the default values should at least always be there if not CFG["mirrcred_user"] or not CFG["mirrcred_pass"]: initCFG(comp) raise ConfigParserError("Could not read default mirror credentials: " "server.susemanager.mirrcred_user, " "server.susemanager.mirrcred_pass.") creds.append((CFG["mirrcred_user"], CFG["mirrcred_pass"])) # increment the credentials number, until we can't read one n = 1 while True: try: creds.append( (CFG["mirrcred_user_%s" % n], CFG["mirrcred_pass_%s" % n])) except (KeyError, AttributeError): break n += 1 initCFG(comp) return creds
def __init__(self, retries=3, log_obj=None, force=False): self.queues = {} comp = CFG.getComponent() initCFG('server.satellite') try: self.threads = int(CFG.REPOSYNC_DOWNLOAD_THREADS) except ValueError: initCFG(comp) raise ValueError("Number of threads expected, found: '%s'" % CFG.REPOSYNC_DOWNLOAD_THREADS) else: initCFG(comp) if self.threads < 1: raise ValueError("Invalid number of threads: %d" % self.threads) self.retries = retries self.log_obj = log_obj self.force = force self.lock = Lock() self.exception = None # WORKAROUND - BZ #1439758 - ensure first item in queue is performed alone to properly setup NSS self.first_in_queue_done = False self.first_in_queue_lock = Lock()
def cfg_component(component, root=None, filename=None): """Context manager for rhnConfig. :param comp: The configuration component to use in this context :param root: Root directory location of configuration files, optional :param filename: Configuration file, optional There is a common pattern when using rhnConfig that consists of the following steps: 1. save current component: old = CFG.getComponent() 2. set CFG to another component: initCFG('my_component') 3. Read / Set configuration values 4. set CFG back to the previous component This pattern can now be expressed using the ``with`` statement: with cfg_component('my_component') as CFG: print(CFG.my_value) """ previous = CFG.getComponent() initCFG(component=component, root=root, filename=filename) try: yield CFG finally: initCFG(previous)
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False, log_level=None, mount_point=None, consider_full=False, force_kickstarts=False, force_all_errata=False, email=False, import_batch_size=None): if log_level is None: log_level = 0 self.log_level = log_level CFG.set('DEBUG', log_level) self.email = email if self.email: initEMAIL_LOG() rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Command: %s" % str(sys.argv)) rhnSQL.initDB() initCFG('server.satellite') self.cdn_repository_manager = CdnRepositoryManager(mount_point) self.no_packages = no_packages self.no_errata = no_errata self.no_rpms = no_rpms if self.no_packages and self.no_rpms: log(0, "Parameter --no-rpms has no effect.") self.no_kickstarts = no_kickstarts self.force_all_errata = force_all_errata self.force_kickstarts = force_kickstarts if self.no_kickstarts and self.force_kickstarts: log(0, "Parameter --force-kickstarts has no effect.") if mount_point: self.mount_point = "file://" + mount_point self.consider_full = consider_full else: self.mount_point = CFG.CDN_ROOT self.consider_full = True verify_mappings() f = None # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: # Channel families mapping to channels f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') self.families = json.load(f) f.close() # Channel metadata f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r') self.channel_metadata = json.load(f) f.close() # Dist/Release channel mapping f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') self.channel_dist_mapping = json.load(f) f.close() # Kickstart metadata f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r') self.kickstart_metadata = json.load(f) f.close() except IOError: e = sys.exc_info()[1] log(1, "Ignoring channel mappings: %s" % e) self.families = {} self.channel_metadata = {} self.channel_dist_mapping = {} self.kickstart_metadata = {} finally: if f is not None: f.close() # Map channels to their channel family self.channel_to_family = {} for family in self.families: for channel in self.families[family]['channels']: self.channel_to_family[channel] = family # Set already synced channels, entitled null-org channels and custom channels with associated # CDN repositories h = rhnSQL.prepare(""" select distinct c.label, c.org_id from rhnChannelFamilyPermissions cfp inner join rhnChannelFamily cf on cfp.channel_family_id = cf.id inner join rhnChannelFamilyMembers cfm on cf.id = cfm.channel_family_id inner join rhnChannel c on cfm.channel_id = c.id where c.org_id is null or (c.org_id is not null and exists ( select cs.id from rhnContentSource cs inner join rhnChannelContentSource ccs on ccs.source_id = cs.id where ccs.channel_id = c.id and cs.org_id is null ) ) order by c.org_id nulls first, label """) h.execute() channels = h.fetchall_dict() or [] self.synced_channels = {} for channel in channels: # Custom channel repositories not available, don't mark as synced if channel['org_id']: repos = self.cdn_repository_manager.list_associated_repos( channel['label']) if not all([ self.cdn_repository_manager. check_repository_availability(r) for r in repos ]): continue self.synced_channels[channel['label']] = channel['org_id'] # Select available channel families from DB h = rhnSQL.prepare(""" select distinct label from rhnChannelFamilyPermissions cfp inner join rhnChannelFamily cf on cfp.channel_family_id = cf.id where cf.org_id is null """) h.execute() families = h.fetchall_dict() or [] self.entitled_families = [f['label'] for f in families] self.import_batch_size = import_batch_size
def __init__(self, channel_label, repo_type, url=None, fail=False, filters=None, no_errata=False, sync_kickstart=False, latest=False, metadata_only=False, strict=0, excluded_urls=None, no_packages=False, log_dir="reposync", log_level=None): self.regen = False self.fail = fail self.filters = filters or [] self.no_packages = no_packages self.no_errata = no_errata self.sync_kickstart = sync_kickstart self.latest = latest self.metadata_only = metadata_only self.ks_tree_type = 'externally-managed' self.ks_install_type = 'generic_rpm' initCFG('server.satellite') rhnSQL.initDB() # setup logging log_filename = channel_label + '.log' log_path = default_log_location + log_dir + '/' + log_filename if log_level is None: log_level = 0 CFG.set('DEBUG', log_level) rhnLog.initLOG(log_path, log_level) # os.fchown isn't in 2.4 :/ if isSUSE(): os.system("chgrp www " + log_path) else: os.system("chgrp apache " + log_path) log2disk(0, "Command: %s" % str(sys.argv)) log2disk(0, "Sync of channel started.") self.channel_label = channel_label self.channel = self.load_channel() if not self.channel: log(0, "Channel %s does not exist." % channel_label) if not url: # TODO:need to look at user security across orgs h = rhnSQL.prepare("""select s.id, s.source_url, s.label, fm.channel_family_id from rhnContentSource s, rhnChannelContentSource cs, rhnChannelFamilyMembers fm where s.id = cs.source_id and cs.channel_id = fm.channel_id and cs.channel_id = :channel_id""") h.execute(channel_id=int(self.channel['id'])) source_data = h.fetchall_dict() self.urls = [] if excluded_urls is None: excluded_urls = [] if source_data: for row in source_data: if row['source_url'] not in excluded_urls: self.urls.append((row['id'], row['source_url'], row['label'], row['channel_family_id'])) else: self.urls = [(None, u, None, None) for u in url] if not self.urls: log2stderr(0, "Channel %s has no URL associated" % channel_label) self.repo_plugin = self.load_plugin(repo_type) self.strict = strict self.all_packages = []
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False, log_level=None): self.no_packages = no_packages self.no_errata = no_errata self.no_rpms = no_rpms self.no_kickstarts = no_kickstarts if log_level is None: log_level = 0 self.log_level = log_level CFG.set('DEBUG', log_level) rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Command: %s" % str(sys.argv)) rhnSQL.initDB() initCFG('server.satellite') try: # Channel families mapping to channels with open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') as f: self.families = json.load(f) # Channel metadata with open(constants.CHANNEL_DEFINITIONS_PATH, 'r') as f: self.channel_metadata = json.load(f) # Dist/Release channel mapping with open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') as f: self.channel_dist_mapping = json.load(f) # Channel to repositories mapping with open(constants.CONTENT_SOURCE_MAPPING_PATH, 'r') as f: self.content_source_mapping = json.load(f) # Kickstart metadata with open(constants.KICKSTART_DEFINITIONS_PATH, 'r') as f: self.kickstart_metadata = json.load(f) # Channel to kickstart repositories mapping with open(constants.KICKSTART_SOURCE_MAPPING_PATH, 'r') as f: self.kickstart_source_mapping = json.load(f) except IOError: e = sys.exc_info()[1] log2stderr(0, "ERROR: Problem with loading file: %s" % e) raise CdnMappingsLoadError() # Map channels to their channel family self.channel_to_family = {} for family in self.families: for channel in self.families[family]['channels']: self.channel_to_family[channel] = family # Set already synced channels h = rhnSQL.prepare(""" select label from rhnChannel where org_id is null """) h.execute() channels = h.fetchall_dict() or [] self.synced_channels = [ch['label'] for ch in channels] # Set SSL-keys for channel family self.family_keys = {}
def __redirectToNextLocationNoRetry(self, loopProtection=False): """ This function will perform a redirection to the next location, as specified in the last response's "Location" header. This function will return an actual HTTP response status code. If successful, it will return apache.HTTP_OK, not apache.OK. If unsuccessful, this function will simply return; no retries will be performed. The following error codes can be returned: HTTP_OK,HTTP_PARTIAL_CONTENT - Redirect successful. HTTP_MOVED_TEMPORARILY - Redirect was redirected again by 3rd party. HTTP_MOVED_PERMANENTLY - Redirect was redirected again by 3rd party. HTTP_INTERNAL_SERVER_ERROR - Error extracting redirect information HTTP_SERVICE_UNAVAILABLE - Could not connect to 3rd party server, connection was reset, or a read error occurred during communication. HTTP_* - Any other HTTP status code may also be returned. Upon successful completion of this function, a new responseContext will be created and pushed onto the stack. """ # Obtain the redirect location first before we replace the current # response context. It's contained in the Location header of the # previous response. redirectLocation = self._get_header(rhnConstants.HEADER_LOCATION) # We are about to redirect to a new location so now we'll push a new # response context before we return any errors. self.responseContext.add() # There should always be a redirect URL passed back to us. If not, # there's an error. if not redirectLocation or len(redirectLocation) == 0: log_error(" No redirect location specified!") Traceback(mail=0) return apache.HTTP_INTERNAL_SERVER_ERROR # The _get_header function returns the value as a list. There should # always be exactly one location specified. redirectLocation = redirectLocation[0] log_debug(1, " Redirecting to: ", redirectLocation) # Tear apart the redirect URL. We need the scheme, the host, the # port (if not the default), and the URI. _scheme, host, port, uri = self._parse_url(redirectLocation) # Add any params onto the URI since _parse_url doesn't include them. if redirectLocation.find('?') > -1: uri += redirectLocation[redirectLocation.index('?'):] # Now create a new connection. We'll use SSL if configured to do # so. params = { 'host': host, 'port': port, } if CFG.has_key('timeout'): params['timeout'] = CFG.TIMEOUT if CFG.USE_SSL: log_debug(1, " Redirecting with SSL. Cert= ", self.caChain) params['trusted_certs'] = [self.caChain] connection = connections.HTTPSConnection(**params) else: log_debug(1, " Redirecting withOUT SSL.") connection = connections.HTTPConnection(**params) # Put the connection into the current response context. self.responseContext.setConnection(connection) # Now open the connection to the 3rd party server. log_debug(4, "Attempting to connect to 3rd party server...") try: connection.connect() except socket.error, e: log_error("Error opening redirect connection", redirectLocation, e) Traceback(mail=0) return apache.HTTP_SERVICE_UNAVAILABLE
def __init__(self, channel_label, repo_type, url=None, fail=False, filters=None, no_errata=False, sync_kickstart=False, latest=False, metadata_only=False, strict=0, excluded_urls=None, no_packages=False, log_dir="reposync", log_level=None, force_kickstart=False, force_all_errata=False, check_ssl_dates=False, force_null_org_content=False): self.regen = False self.fail = fail self.filters = filters or [] self.no_packages = no_packages self.no_errata = no_errata self.sync_kickstart = sync_kickstart self.force_all_errata = force_all_errata self.force_kickstart = force_kickstart self.latest = latest self.metadata_only = metadata_only self.ks_tree_type = 'externally-managed' self.ks_install_type = None initCFG('server.satellite') rhnSQL.initDB() # setup logging log_filename = channel_label + '.log' log_path = default_log_location + log_dir + '/' + log_filename if log_level is None: log_level = 0 CFG.set('DEBUG', log_level) rhnLog.initLOG(log_path, log_level) # os.fchown isn't in 2.4 :/ if isSUSE(): os.system("chgrp www " + log_path) else: os.system("chgrp apache " + log_path) log2disk(0, "Command: %s" % str(sys.argv)) log2disk(0, "Sync of channel started.") self.channel_label = channel_label self.channel = self.load_channel() if not self.channel: log(0, "Channel %s does not exist." % channel_label) if not self.channel['org_id'] or force_null_org_content: self.org_id = None else: self.org_id = int(self.channel['org_id']) if not url: # TODO:need to look at user security across orgs h = rhnSQL.prepare("""select s.id, s.source_url, s.label from rhnContentSource s, rhnChannelContentSource cs where s.id = cs.source_id and cs.channel_id = :channel_id""") h.execute(channel_id=int(self.channel['id'])) source_data = h.fetchall_dict() self.urls = [] if excluded_urls is None: excluded_urls = [] if source_data: for row in source_data: if row['source_url'] not in excluded_urls: self.urls.append( (row['id'], row['source_url'], row['label'])) else: self.urls = [(None, u, None) for u in url] if not self.urls: log2(0, 0, "Channel %s has no URL associated" % channel_label, stream=sys.stderr) self.repo_plugin = self.load_plugin(repo_type) self.strict = strict self.all_packages = [] self.check_ssl_dates = check_ssl_dates # Init cache for computed checksums to not compute it on each reposync run again self.checksum_cache = rhnCache.get(checksum_cache_filename) if self.checksum_cache is None: self.checksum_cache = {}
def processCommandline(): options = [ Option('--sanity-only', action='store_true', help="confirm certificate sanity. Does not activate " + "the Red Hat Satellite locally or remotely."), Option('--ignore-expiration', action='store_true', help='execute regardless of the expiration ' + 'of the RHN Certificate (not recommended).'), Option('--ignore-version-mismatch', action='store_true', help='execute regardless of version ' + 'mismatch of existing and new certificate.'), Option('-v', '--verbose', action='count', help='be verbose ' + '(accumulable: -vvv means "be *really* verbose").'), Option('--dump-version', action='store', help="requested version of XML dump"), Option('--manifest', action='store', help='the RHSM manifest path/filename to activate for CDN'), Option('--rhn-cert', action='store', help='this option is deprecated, use --manifest instead'), Option('--deactivate', action='store_true', help='deactivate CDN-activated Satellite'), Option('--disconnected', action='store_true', help="activate locally, not subscribe to remote repository"), Option('--manifest-info', action='store_true', help="show information about currently activated manifest"), Option('--manifest-download', action='store_true', help="download new manifest from RHSM to temporary location"), Option('--manifest-refresh', action='store_true', help="download new manifest from RHSM and activate it"), Option('--manifest-reconcile-request', action='store_true', help="request regeneration of entitlement certificates") ] parser = OptionParser(option_list=options) options, args = parser.parse_args() initCFG('server.satellite') if options.verbose is None: options.verbose = 0 CFG.set('DEBUG', options.verbose) rhnLog.initLOG(LOG_PATH, options.verbose) log2disk(0, "Command: %s" % str(sys.argv)) # we take no extra commandline arguments that are not linked to an option if args: writeError("These arguments make no sense in this context (try --help): %s" % repr(args)) sys.exit(1) # No need to check further if deactivating if options.deactivate: return options if options.sanity_only: options.disconnected = 1 if options.manifest_refresh: options.manifest_download = 1 if CFG.DISCONNECTED and not options.disconnected: msg = """Satellite server has been setup to run in disconnected mode. Either correct server configuration in /etc/rhn/rhn.conf or use --disconnected to activate it locally.""" writeError(msg) sys.exit(1) options.http_proxy = idn_ascii_to_puny(CFG.HTTP_PROXY) options.http_proxy_username = CFG.HTTP_PROXY_USERNAME options.http_proxy_password = CFG.HTTP_PROXY_PASSWORD log(1, 'HTTP_PROXY: %s' % options.http_proxy) log(1, 'HTTP_PROXY_USERNAME: %s' % options.http_proxy_username) log(1, 'HTTP_PROXY_PASSWORD: <password>') return options
def __init__(self, url, name, insecure=False, interactive=True, yumsrc_conf=None, org="1", channel_label="", no_mirrors=False, ca_cert_file=None, client_cert_file=None, client_key_file=None): """ Plugin constructor. """ # pylint: disable=W0613 if urlsplit(url).scheme: self.url = url else: self.url = "file://%s" % url self.name = name self.insecure = insecure self.interactive = interactive self.org = org if org else "NULL" self.proxy_hostname = None self.proxy_url = None self.proxy_user = None self.proxy_pass = None self.authtoken = None self.sslcacert = ca_cert_file self.sslclientcert = client_cert_file self.sslclientkey = client_key_file self.http_headers = {} comp = CFG.getComponent() # read configuration from /etc/rhn/rhn.conf initCFG('server.satellite') # keep authtokens for mirroring (_scheme, _netloc, _path, query, _fragid) = urlsplit(url) if query: self.authtoken = query # load proxy configuration based on the url self._load_proxy_settings(self.url) # Get extra HTTP headers configuration from /etc/rhn/spacewalk-repo-sync/extra_headers.conf if os.path.isfile(REPOSYNC_EXTRA_HTTP_HEADERS_CONF): http_headers_cfg = configparser.ConfigParser() http_headers_cfg.read_file(open(REPOSYNC_EXTRA_HTTP_HEADERS_CONF)) section_name = None if http_headers_cfg.has_section(self.name): section_name = self.name elif http_headers_cfg.has_section(channel_label): section_name = channel_label elif http_headers_cfg.has_section('main'): section_name = 'main' if section_name: for hdr in http_headers_cfg[section_name]: self.http_headers[hdr] = http_headers_cfg.get(section_name, option=hdr) # perform authentication if implemented self._authenticate(url) # Make sure baseurl ends with / and urljoin will work correctly self.urls = [url] if self.urls[0][-1] != '/': self.urls[0] += '/' # Exclude non-valid characters from reponame self.reponame = self.name for chr in ["$", " ", ".", ";"]: self.reponame = self.reponame.replace(chr, "_") self.channel_label = channel_label # SUSE vendor repositories belongs to org = NULL root = os.path.join(CACHE_DIR, str(org or "NULL"), self.channel_label or self.reponame) self.repo = ZypperRepo(root=root, url=self.url, org=self.org) self.num_packages = 0 self.num_excluded = 0 self.gpgkey_autotrust = None self.groupsfile = None # set config component back to original initCFG(comp)
def __init__(self): self._next_poll_interval = None self._notify_threshold = CFG.get('notify_threshold')
def send(url, sendData=None): """Connect to url and return the result as stringIO :arg url: the url where the request will be sent :kwarg sendData: do a post-request when "sendData" is given. Returns the result as stringIO object. """ connect_retries = 10 try_counter = connect_retries timeout = 120 if CFG.is_initialized() and CFG.has_key('TIMEOUT'): timeout = CFG.TIMEOUT curl = pycurl.Curl() curl.setopt(pycurl.CONNECTTIMEOUT, timeout) curl.setopt(pycurl.URL, url) curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug) curl.setopt(pycurl.VERBOSE, True) proxy_url, proxy_user, proxy_pass = get_proxy(url) if proxy_url: curl.setopt(pycurl.PROXY, proxy_url) log_debug(2, "Connect to %s" % url) if sendData is not None: curl.setopt(pycurl.POSTFIELDS, sendData) if (CFG.is_initialized() and CFG.has_key('DISABLE_EXPECT') and CFG.DISABLE_EXPECT): # disable Expect header curl.setopt(pycurl.HTTPHEADER, ['Expect:']) # We implement our own redirection-following, because pycurl # 7.19 doesn't POST after it gets redirected. Ideally we'd be # using pycurl.POSTREDIR here, but that's in 7.21. curl.setopt(pycurl.FOLLOWLOCATION, False) response = StringIO() curl.setopt(pycurl.WRITEFUNCTION, response.write) try_counter = connect_retries while try_counter: try_counter -= 1 try: curl.perform() except pycurl.error, e: if e[0] == 56: # Proxy requires authentication log_debug(2, e[1]) if not (proxy_user and proxy_pass): raise TransferException("Proxy requires authentication, " "but reading credentials from " "%s failed." % YAST_PROXY) curl.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (proxy_user, proxy_pass)) elif e[0] == 60: log_error("Peer certificate could not be authenticated " "with known CA certificates.") raise TransferException("Peer certificate could not be " "authenticated with known CA " "certificates.") else: log_error(e[1]) raise status = curl.getinfo(pycurl.HTTP_CODE) if status == 200 or (URL(url).scheme == "file" and status == 0): # OK or file break elif status in (301, 302): # redirects url = curl.getinfo(pycurl.REDIRECT_URL) log_debug(2, "Got redirect to %s" % url) curl.setopt(pycurl.URL, url)
def main(): # Initialize a command-line processing object with a table of options optionsTable = [ Option('-v', '--verbose', action='count', help='Increase verbosity'), Option('-d', '--dir', action='store', help='Process packages from this directory'), Option('-c', '--channel', action='append', help='Manage this channel'), Option('-n', '--count', action='store', help='Process this number of headers per call', type='int'), Option('-l', '--list', action='store_true', help='Only list the specified channels'), Option('-s', '--sync', action='store_true', help='Check if in sync with the server'), Option('-p', '--printconf', action='store_true', help='Print the configuration and exit'), Option('-X', '--exclude', action="append", help="Exclude packages that match this glob expression"), Option('--newest', action='store_true', help='Only push the files that are newer than the server ones'), Option('--stdin', action='store_true', help='Read the package names from stdin'), Option('--nosig', action='store_true', help="Push unsigned packages"), Option('--username', action='store', help='Use this username to connect to RHN'), Option('--password', action='store', help='Use this password to connect to RHN'), Option('--source', action='store_true', help='Upload source package headers'), Option('--dontcopy', action='store_true', help='Do not copy packages to the local directory'), Option('--copyonly', action='store_true', help="Only copy packages; don't reimport"), Option('--test', action='store_true', help='Only print the packages to be pushed'), Option('-N', '--new-cache', action='store_true', help='Create a new username/password cache'), Option('--no-ssl', action='store_true', help='Turn off SSL (not recommended).'), Option('--no-session-caching', action='store_true', help='Disables session-token authentication.'), Option('-?', '--usage', action='store_true', help="Briefly describe the options"), ] # Process the command line arguments optionParser = OptionParser(option_list=optionsTable, usage="USAGE: %prog [OPTION] [<package>]") options, files = optionParser.parse_args() upload = UploadClass(options, files=files) if options.usage: optionParser.print_usage() sys.exit(0) if options.printconf: CFG.show() return if options.list: upload.list() return if options.sync: upload.checkSync() return if options.copyonly: upload.copyonly() return if options.dir: upload.directory() elif options.stdin: upload.readStdin() if options.exclude: upload.filter_excludes() if options.newest: upload.newest() if not upload.files: upload.die(0, "Nothing to do; exiting. Try --help") if options.test: upload.test() return try: upload.uploadHeaders() except UploadError, e: sys.stderr.write("Upload error: %s\n" % e)
def main(): # Initialize a command-line processing object with a table of options optionsTable = [ Option('-v', '--verbose', action='count', help='Increase verbosity'), Option('-d', '--dir', action='store', help='Process packages from this directory'), Option('-L', '--cache-locally', action='store_true', help='Locally cache packages so that Proxy will not ever need to ' + 'download them. Changes nothing on the upstream server.'), Option('-e', '--from-export', action='store', dest='export_location', help='Process packages from this channel export. Can only be used ' + 'with --cache-locally or --copyonly.'), Option('-c', '--channel', action='append', help='Channel to operate on. When used with --from-export ' + 'specifies channels to cache rpms for, else specifies channels ' + 'that we will be pushing into.'), Option('-n', '--count', action='store', help='Process this number of headers per call', type='int'), Option('-l', '--list', action='store_true', help='Only list the specified channels'), Option('-s', '--sync', action='store_true', help='Check if in sync with the server'), Option('-p', '--printconf', action='store_true', help='Print the configuration and exit'), Option('-X', '--exclude', action="append", help="Exclude packages that match this glob expression"), Option('--newest', action='store_true', help='Only push the files that are newer than the server ones'), Option('--stdin', action='store_true', help='Read the package names from stdin'), Option('--nosig', action='store_true', help="Push unsigned packages"), Option('--username', action='store', help='Use this username to connect to RHN'), Option('--password', action='store', help='Use this password to connect to RHN'), Option('--source', action='store_true', help='Upload source package headers'), Option('--dontcopy', action='store_true', help='Do not copy packages to the local directory'), Option('--copyonly', action='store_true', help="Only copy packages; don't reimport. Same as --cache-locally"), Option('--test', action='store_true', help='Only print the packages to be pushed'), Option('-N', '--new-cache', action='store_true', help='Create a new username/password cache'), Option('--no-ssl', action='store_true', help='Turn off SSL (not recommended).'), Option('--no-session-caching', action='store_true', help='Disables session-token authentication.'), Option('-?', '--usage', action='store_true', help="Briefly describe the options"), ] # Process the command line arguments optionParser = OptionParser(option_list=optionsTable, usage="USAGE: %prog [OPTION] [<package>]") options, files = optionParser.parse_args() upload = UploadClass(options, files=files) if options.usage: optionParser.print_usage() sys.exit(0) if options.printconf: CFG.show() return if options.list: upload.list() return if options.sync: upload.checkSync() return # It's just an alias to copyonly if options.cache_locally: options.copyonly = True # remeber to process dir option before export, export can overwrite dir if options.dir: upload.directory() if options.export_location: if not options.copyonly: upload.die(0, "--from-export can only be used with --cache-locally" + " or --copyonly") if options.source: upload.die(0, "--from-export cannot be used with --source") upload.from_export() if options.stdin: upload.readStdin() # if we're going to allow the user to specify packages by dir *and* export # *and* stdin *and* package list (why not?) then we have to uniquify # the list afterwards. Sort just for user-friendly display. upload.files = sorted(list(set(upload.files))) if options.copyonly: if not upload.files: upload.die(0, "Nothing to do; exiting. Try --help") if options.test: upload.test() return upload.copyonly() return if options.exclude: upload.filter_excludes() if options.newest: upload.newest() if not upload.files: upload.die(0, "Nothing to do; exiting. Try --help") if options.test: upload.test() return try: upload.uploadHeaders() except UploadError, e: sys.stderr.write("Upload error: %s\n" % e)
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False, log_level=None, mount_point=None, consider_full=False): self.cdn_repository_manager = CdnRepositoryManager(mount_point) self.no_packages = no_packages self.no_errata = no_errata self.no_rpms = no_rpms self.no_kickstarts = no_kickstarts if log_level is None: log_level = 0 self.log_level = log_level if mount_point: self.mount_point = "file://" + mount_point self.consider_full = consider_full else: self.mount_point = CFG.CDN_ROOT self.consider_full = True CFG.set('DEBUG', log_level) rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Command: %s" % str(sys.argv)) rhnSQL.initDB() initCFG('server.satellite') verify_mappings() f = None # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: # Channel families mapping to channels f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') self.families = json.load(f) f.close() # Channel metadata f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r') self.channel_metadata = json.load(f) f.close() # Dist/Release channel mapping f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') self.channel_dist_mapping = json.load(f) f.close() # Kickstart metadata f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r') self.kickstart_metadata = json.load(f) f.close() except IOError: e = sys.exc_info()[1] raise CdnMappingsLoadError("Problem with loading file: %s" % e) finally: if f is not None: f.close() # Map channels to their channel family self.channel_to_family = {} for family in self.families: for channel in self.families[family]['channels']: self.channel_to_family[channel] = family # Set already synced channels h = rhnSQL.prepare(""" select label from rhnChannel where org_id is null """) h.execute() channels = h.fetchall_dict() or [] self.synced_channels = [ch['label'] for ch in channels]
def __init__(self, server_address): SocketServer.TCPServer.__init__(self, server_address, None) self._next_poll_interval = None self._notify_threshold = CFG.get('notify_threshold')
def main(): # Initialize a command-line processing object with a table of options optionsTable = [ Option('-v','--verbose', action='count', help='Increase verbosity'), Option('-d','--dir', action='store', help='Process packages from this directory'), Option('-c','--channel', action='append', help='Manage this channel'), Option('-n','--count', action='store', help='Process this number of headers per call', type='int'), Option('-l','--list', action='store_true', help='Only list the specified channels'), Option('-s','--sync', action='store_true', help='Check if in sync with the server'), Option('-p','--printconf', action='store_true', help='Print the configuration and exit'), Option('-X','--exclude', action="append", help="Exclude packages that match this glob expression"), Option( '--newest', action='store_true', help='Only push the files that are newer than the server ones'), Option( '--stdin', action='store_true', help='Read the package names from stdin'), Option( '--nosig', action='store_true', help="Push unsigned packages"), Option( '--username', action='store', help='Use this username to connect to RHN'), Option( '--password', action='store', help='Use this password to connect to RHN'), Option( '--source', action='store_true', help='Upload source package headers'), Option( '--dontcopy', action='store_true', help='Do not copy packages to the local directory'), Option( '--copyonly', action='store_true', help="Only copy packages; don't reimport"), Option( '--test', action='store_true', help='Only print the packages to be pushed'), Option('-N','--new-cache', action='store_true', help='Create a new username/password cache'), Option( '--no-ssl', action='store_true', help='Turn off SSL (not recommended).'), Option( '--no-session-caching', action='store_true', help='Disables session-token authentication.'), Option('-?','--usage', action='store_true', help="Briefly describe the options"), ] # Process the command line arguments optionParser = OptionParser(option_list=optionsTable, usage="USAGE: %prog [OPTION] [<package>]") options, files = optionParser.parse_args() upload = UploadClass(options, files=files) if options.usage: optionParser.print_usage() sys.exit(0) if options.printconf: CFG.show() return if options.list: upload.list() return if options.sync: upload.checkSync() return if options.copyonly: upload.copyonly() return if options.dir: upload.directory() elif options.stdin: upload.readStdin() if options.exclude: upload.filter_excludes() if options.newest: upload.newest() if not upload.files: upload.die(0, "Nothing to do; exiting. Try --help") if options.test: upload.test() return try: upload.uploadHeaders() except UploadError, e: sys.stderr.write("Upload error: %s\n" % e)
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False, log_level=None, mount_point=None, consider_full=False, force_kickstarts=False, force_all_errata=False, email=False, import_batch_size=None): if log_level is None: log_level = 0 self.log_level = log_level CFG.set('DEBUG', log_level) self.email = email if self.email: initEMAIL_LOG() rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Command: %s" % str(sys.argv)) rhnSQL.initDB() initCFG('server.satellite') self.cdn_repository_manager = CdnRepositoryManager(mount_point) self.no_packages = no_packages self.no_errata = no_errata self.no_rpms = no_rpms if self.no_packages and self.no_rpms: log(0, "Parameter --no-rpms has no effect.") self.no_kickstarts = no_kickstarts self.force_all_errata = force_all_errata self.force_kickstarts = force_kickstarts if self.no_kickstarts and self.force_kickstarts: log(0, "Parameter --force-kickstarts has no effect.") if mount_point: self.mount_point = "file://" + mount_point self.consider_full = consider_full else: self.mount_point = CFG.CDN_ROOT self.consider_full = True verify_mappings() f = None # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: # Channel families mapping to channels f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') self.families = json.load(f) f.close() # Channel metadata f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r') self.channel_metadata = json.load(f) f.close() # Dist/Release channel mapping f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') self.channel_dist_mapping = json.load(f) f.close() # Kickstart metadata f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r') self.kickstart_metadata = json.load(f) f.close() except IOError: e = sys.exc_info()[1] log(1, "Ignoring channel mappings: %s" % e) self.families = {} self.channel_metadata = {} self.channel_dist_mapping = {} self.kickstart_metadata = {} finally: if f is not None: f.close() # Map channels to their channel family self.channel_to_family = {} for family in self.families: for channel in self.families[family]['channels']: self.channel_to_family[channel] = family # Set already synced channels, entitled null-org channels and custom channels with associated # CDN repositories h = rhnSQL.prepare(""" select distinct c.label, c.org_id from rhnChannelFamilyPermissions cfp inner join rhnChannelFamily cf on cfp.channel_family_id = cf.id inner join rhnChannelFamilyMembers cfm on cf.id = cfm.channel_family_id inner join rhnChannel c on cfm.channel_id = c.id where c.org_id is null or (c.org_id is not null and exists ( select cs.id from rhnContentSource cs inner join rhnChannelContentSource ccs on ccs.source_id = cs.id where ccs.channel_id = c.id and cs.org_id is null ) ) order by c.org_id nulls first, label """) h.execute() channels = h.fetchall_dict() or [] self.synced_channels = {} for channel in channels: # Custom channel repositories not available, don't mark as synced if channel['org_id']: repos = self.cdn_repository_manager.list_associated_repos(channel['label']) if not all([self.cdn_repository_manager.check_repository_availability(r) for r in repos]): continue self.synced_channels[channel['label']] = channel['org_id'] # Select available channel families from DB h = rhnSQL.prepare(""" select distinct label from rhnChannelFamilyPermissions cfp inner join rhnChannelFamily cf on cfp.channel_family_id = cf.id where cf.org_id is null """) h.execute() families = h.fetchall_dict() or [] self.entitled_families = [f['label'] for f in families] self.import_batch_size = import_batch_size
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False, log_level=None): self.no_packages = no_packages self.no_errata = no_errata self.no_rpms = no_rpms self.no_kickstarts = no_kickstarts if log_level is None: log_level = 0 self.log_level = log_level CFG.set('DEBUG', log_level) rhnLog.initLOG(self.log_path, self.log_level) log2disk(0, "Command: %s" % str(sys.argv)) rhnSQL.initDB() initCFG('server.satellite') f = None # try block in try block - this is hack for python 2.4 compatibility # to support finally try: try: # Channel families mapping to channels f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') self.families = json.load(f) f.close() # Channel metadata f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r') self.channel_metadata = json.load(f) f.close() # Dist/Release channel mapping f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') self.channel_dist_mapping = json.load(f) f.close() # Channel to repositories mapping f = open(constants.CONTENT_SOURCE_MAPPING_PATH, 'r') self.content_source_mapping = json.load(f) f.close() # Kickstart metadata f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r') self.kickstart_metadata = json.load(f) f.close() # Channel to kickstart repositories mapping f = open(constants.KICKSTART_SOURCE_MAPPING_PATH, 'r') self.kickstart_source_mapping = json.load(f) f.close() except IOError: e = sys.exc_info()[1] log2stderr(0, "ERROR: Problem with loading file: %s" % e) raise CdnMappingsLoadError() finally: if f is not None: f.close() # Map channels to their channel family self.channel_to_family = {} for family in self.families: for channel in self.families[family]['channels']: self.channel_to_family[channel] = family # Set already synced channels h = rhnSQL.prepare(""" select label from rhnChannel where org_id is null """) h.execute() channels = h.fetchall_dict() or [] self.synced_channels = [ch['label'] for ch in channels] # Set SSL-keys for channel family self.family_keys = {}
def __redirectToNextLocationNoRetry(self, loopProtection=False): """ This function will perform a redirection to the next location, as specified in the last response's "Location" header. This function will return an actual HTTP response status code. If successful, it will return apache.HTTP_OK, not apache.OK. If unsuccessful, this function will simply return; no retries will be performed. The following error codes can be returned: HTTP_OK,HTTP_PARTIAL_CONTENT - Redirect successful. HTTP_MOVED_TEMPORARILY - Redirect was redirected again by 3rd party. HTTP_MOVED_PERMANENTLY - Redirect was redirected again by 3rd party. HTTP_INTERNAL_SERVER_ERROR - Error extracting redirect information HTTP_SERVICE_UNAVAILABLE - Could not connect to 3rd party server, connection was reset, or a read error occurred during communication. HTTP_* - Any other HTTP status code may also be returned. Upon successful completion of this function, a new responseContext will be created and pushed onto the stack. """ # Obtain the redirect location first before we replace the current # response context. It's contained in the Location header of the # previous response. redirectLocation = self._get_header(rhnConstants.HEADER_LOCATION) # We are about to redirect to a new location so now we'll push a new # response context before we return any errors. self.responseContext.add() # There should always be a redirect URL passed back to us. If not, # there's an error. if not redirectLocation: log_error(" No redirect location specified!") Traceback(mail=0) return apache.HTTP_INTERNAL_SERVER_ERROR # The _get_header function returns the value as a list. There should # always be exactly one location specified. redirectLocation = redirectLocation[0] log_debug(1, " Redirecting to: ", redirectLocation) # Tear apart the redirect URL. We need the scheme, the host, the # port (if not the default), and the URI. _scheme, host, port, uri, query = self._parse_url(redirectLocation) # Add back the query string if query: uri += '?' + query # Now create a new connection. We'll use SSL if configured to do # so. params = { 'host': host, 'port': port, } if CFG.has_key('timeout'): params['timeout'] = CFG.TIMEOUT if CFG.USE_SSL: log_debug(1, " Redirecting with SSL. Cert= ", self.caChain) params['trusted_certs'] = [self.caChain] connection = connections.HTTPSConnection(**params) else: log_debug(1, " Redirecting withOUT SSL.") connection = connections.HTTPConnection(**params) # Put the connection into the current response context. self.responseContext.setConnection(connection) # Now open the connection to the 3rd party server. log_debug(4, "Attempting to connect to 3rd party server...") try: connection.connect() except socket.error as e: log_error("Error opening redirect connection", redirectLocation, e) Traceback(mail=0) return apache.HTTP_SERVICE_UNAVAILABLE log_debug(4, "Connected to 3rd party server:", connection.sock.getpeername()) # Put the request out on the wire. response = None try: # We'll redirect to the URI made in the original request, but with # the new server instead. log_debug(4, "Making request: ", self.req.method, uri) connection.putrequest(self.req.method, uri) # Add some custom headers. if loopProtection: connection.putheader(rhnConstants.HEADER_RHN_REDIRECT, '0') log_debug(4, " Adding original URL header: ", self.rhnParent) connection.putheader(rhnConstants.HEADER_RHN_ORIG_LOC, self.rhnParent) # Add all the other headers in the original request in case we # need to re-authenticate with Hosted. for hdr in list(self.req.headers_in.keys()): if hdr.lower().startswith("x-rhn"): connection.putheader(hdr, self.req.headers_in[hdr]) log_debug(4, "Passing request header: ", hdr, self.req.headers_in[hdr]) connection.endheaders() response = connection.getresponse() except IOError as ioe: # Raised by getresponse() if server closes connection on us. log_error("Redirect connection reset by peer.", redirectLocation, ioe) Traceback(mail=0) # The connection is saved in the current response context, and # will be closed when the caller pops the context. return apache.HTTP_SERVICE_UNAVAILABLE except socket.error as se: # Some socket error occurred. Possibly a read error. log_error("Redirect request failed.", redirectLocation, se) Traceback(mail=0) # The connection is saved in the current response context, and # will be closed when the caller pops the context. return apache.HTTP_SERVICE_UNAVAILABLE # Save the response headers and body FD in the current communication # context. self.responseContext.setBodyFd(response) self.responseContext.setHeaders(response.msg) log_debug(4, "Response headers: ", list(self.responseContext.getHeaders().items())) log_debug(4, "Got redirect response. Status=", response.status) # Return the HTTP status to the caller. return response.status
def processCommandline(): options = [ Option('--sanity-only', action='store_true', help="confirm certificate sanity. Does not activate " + "the Red Hat Satellite locally or remotely."), Option('--ignore-expiration', action='store_true', help='execute regardless of the expiration ' + 'of the RHN Certificate (not recommended).'), Option('--ignore-version-mismatch', action='store_true', help='execute regardless of version ' + 'mismatch of existing and new certificate.'), Option('-v', '--verbose', action='count', help='be verbose ' + '(accumulable: -vvv means "be *really* verbose").'), Option('--dump-version', action='store', help="requested version of XML dump"), Option('--manifest', action='store', help='the RHSM manifest path/filename to activate for CDN'), Option('--rhn-cert', action='store', help='this option is deprecated, use --manifest instead'), Option('--deactivate', action='store_true', help='deactivate CDN-activated Satellite'), Option('--disconnected', action='store_true', help="activate locally, not subscribe to remote repository"), Option('--manifest-info', action='store_true', help="show information about currently activated manifest"), Option('--manifest-download', action='store_true', help="download new manifest from RHSM to temporary location"), Option('--manifest-refresh', action='store_true', help="download new manifest from RHSM and activate it"), Option('--manifest-reconcile-request', action='store_true', help="request regeneration of entitlement certificates") ] parser = OptionParser(option_list=options) options, args = parser.parse_args() initCFG('server.satellite') if options.verbose is None: options.verbose = 0 CFG.set('DEBUG', options.verbose) rhnLog.initLOG(LOG_PATH, options.verbose) log2disk(0, "Command: %s" % str(sys.argv)) # we take no extra commandline arguments that are not linked to an option if args: writeError( "These arguments make no sense in this context (try --help): %s" % repr(args)) sys.exit(1) # No need to check further if deactivating if options.deactivate: return options if options.sanity_only: options.disconnected = 1 if options.manifest_refresh: options.manifest_download = 1 if CFG.DISCONNECTED and not options.disconnected: msg = """Satellite server has been setup to run in disconnected mode. Either correct server configuration in /etc/rhn/rhn.conf or use --disconnected to activate it locally.""" writeError(msg) sys.exit(1) options.http_proxy = idn_ascii_to_puny(CFG.HTTP_PROXY) options.http_proxy_username = CFG.HTTP_PROXY_USERNAME options.http_proxy_password = CFG.HTTP_PROXY_PASSWORD log(1, 'HTTP_PROXY: %s' % options.http_proxy) log(1, 'HTTP_PROXY_USERNAME: %s' % options.http_proxy_username) log(1, 'HTTP_PROXY_PASSWORD: <password>') return options