def __init__(self, req):
""" init with http request object """
# FIXME: should rename some things:
# self.bodyFd --> self.body or self.data or ?
# self.caChain --> self.caCert
self.req = req
# turn wsgi.input object into a SmartIO instance so it can be read
# more than once
if 'wsgi.input' in self.req.headers_in:
smartFd = SmartIO(max_mem_size=CFG.MAX_MEM_FILE_SIZE)
smartFd.write(self.req.headers_in['wsgi.input'].read())
self.req.headers_in['wsgi.input'] = smartFd
self.responseContext = ResponseContext()
self.uri = None # ''
# Common settings for both the proxy and the redirect
# broker and redirect immediately alter these for their own purposes
self.caChain = CFG.CA_CHAIN
self.httpProxy = CFG.HTTP_PROXY
self.httpProxyUsername = CFG.HTTP_PROXY_USERNAME
self.httpProxyPassword = CFG.HTTP_PROXY_PASSWORD
if not self.httpProxyUsername:
self.httpProxyPassword = ''
self.rhnParent = CFG.RHN_PARENT or ''
self.rhnParent = rhnLib.parseUrl(self.rhnParent)[1].split(':')[0]
CFG.set('RHN_PARENT', self.rhnParent)
def __init__(self, req):
""" init with http request object """
# FIXME: should rename some things:
# self.bodyFd --> self.body or self.data or ?
# self.caChain --> self.caCert
self.req = req
# turn wsgi.input object into a SmartIO instance so it can be read
# more than once
if 'wsgi.input' in self.req.headers_in:
smartFd = SmartIO(max_mem_size=CFG.MAX_MEM_FILE_SIZE)
smartFd.write(self.req.headers_in['wsgi.input'].read())
self.req.headers_in['wsgi.input'] = smartFd
self.responseContext = ResponseContext()
self.uri = None # ''
# Common settings for both the proxy and the redirect
# broker and redirect immediately alter these for their own purposes
self.caChain = CFG.CA_CHAIN
self.httpProxy = CFG.HTTP_PROXY
self.httpProxyUsername = CFG.HTTP_PROXY_USERNAME
self.httpProxyPassword = CFG.HTTP_PROXY_PASSWORD
if not self.httpProxyUsername:
self.httpProxyPassword = ''
self.rhnParent = CFG.RHN_PARENT or ''
self.rhnParent = rhnLib.parseUrl(self.rhnParent)[1].split(':')[0]
CFG.set('RHN_PARENT', self.rhnParent)
def accessible(url):
"""Try if url is accessible
:arg url: the url which is tried to access
Returns True if url is accessible, otherwise False.
"""
timeout = 120
if CFG.is_initialized() and CFG.has_key('TIMEOUT'):
timeout = CFG.TIMEOUT
curl = pycurl.Curl()
curl.setopt(pycurl.CONNECTTIMEOUT, timeout)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug)
curl.setopt(pycurl.VERBOSE, True)
proxy_url, proxy_user, proxy_pass = get_proxy(url)
if proxy_url:
curl.setopt(pycurl.PROXY, proxy_url)
log_debug(2, "Connect to %s" % url)
# We implement our own redirection-following, because pycurl
# 7.19 doesn't POST after it gets redirected. Ideally we'd be
# using pycurl.POSTREDIR here, but that's in 7.21.
curl.setopt(pycurl.FOLLOWLOCATION, False)
curl.setopt(pycurl.NOBODY, True)
try_counter = 5
while try_counter:
try_counter -= 1
try:
curl.perform()
except pycurl.error as e:
if e[0] == 56: # Proxy requires authentication
log_debug(2, e[1])
if not (proxy_user and proxy_pass):
raise TransferException("Proxy requires authentication, "
"but reading credentials from "
"%s failed." % YAST_PROXY)
curl.setopt(pycurl.PROXYUSERPWD,
"%s:%s" % (proxy_user, proxy_pass))
else:
break
status = curl.getinfo(pycurl.HTTP_CODE)
# OK or file
if status == 200 or (URL(url).scheme == "file" and status == 0):
return True
elif status in (301, 302): # redirects
url = curl.getinfo(pycurl.REDIRECT_URL)
log_debug(2, "Got redirect to %s" % url)
curl.setopt(pycurl.URL, url)
elif status >= 400:
break
return False
def accessible(url):
"""Try if url is accessible
:arg url: the url which is tried to access
Returns True if url is accessible, otherwise False.
"""
timeout = 120
if CFG.is_initialized() and CFG.has_key('TIMEOUT'):
timeout = CFG.TIMEOUT
curl = pycurl.Curl()
curl.setopt(pycurl.CONNECTTIMEOUT, timeout)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug)
curl.setopt(pycurl.VERBOSE, True)
proxy_url, proxy_user, proxy_pass = get_proxy(url)
if proxy_url:
curl.setopt(pycurl.PROXY, proxy_url)
log_debug(2, "Connect to %s" % url)
# We implement our own redirection-following, because pycurl
# 7.19 doesn't POST after it gets redirected. Ideally we'd be
# using pycurl.POSTREDIR here, but that's in 7.21.
curl.setopt(pycurl.FOLLOWLOCATION, False)
curl.setopt(pycurl.NOBODY, True)
try_counter = 5
while try_counter:
try_counter -= 1
try:
curl.perform()
except pycurl.error, e:
if e[0] == 56: # Proxy requires authentication
log_debug(2, e[1])
if not (proxy_user and proxy_pass):
raise TransferException("Proxy requires authentication, "
"but reading credentials from "
"%s failed." % YAST_PROXY)
curl.setopt(pycurl.PROXYUSERPWD,
"%s:%s" % (proxy_user, proxy_pass))
else:
break
status = curl.getinfo(pycurl.HTTP_CODE)
# OK or file
if status == 200 or (URL(url).scheme == "file" and status == 0):
return True
elif status in (301, 302): # redirects
url = curl.getinfo(pycurl.REDIRECT_URL)
log_debug(2, "Got redirect to %s" % url)
curl.setopt(pycurl.URL, url)
elif status >= 400:
break
def _read_config():
# we want to change the logging file to 'audit' and set it back
# after we finished reading the config file
# TODO Changing the component twice on every request is not nice
comp = CFG.getComponent()
initCFG("audit")
enabled = CFG.get("enabled")
server_url = CFG.get("server", "")
# XXX haven't tested what happens if it's not set back to the original value
initCFG(comp)
return (enabled, server_url)
def _useProxyFor(url):
"""Return True if a proxy should be used for given url, otherwise False.
This function uses server.satellite.no_proxy variable to check for
hosts or domains which should not be connected via a proxy.
server.satellite.no_proxy is a comma seperated list.
Either an exact match, or the previous character
is a '.', so host is within the same domain.
A leading '.' in the pattern is ignored.
See also 'man curl'
"""
u = urlparse.urlsplit(url)
# pylint can't see inside the SplitResult class
# pylint: disable=E1103
if u.scheme == 'file':
return False
hostname = u.hostname.lower()
if hostname in ["localhost", "127.0.0.1", "::1"]:
return False
comp = CFG.getComponent()
if not CFG.has_key("no_proxy"):
initCFG("server.satellite")
if not CFG.has_key('no_proxy'):
initCFG(comp)
return True
noproxy = CFG.no_proxy
initCFG(comp)
if not noproxy:
return True
if not isinstance(noproxy, list):
if noproxy == '*':
# just an asterisk disables all.
return False
noproxy = [noproxy]
# No proxy: Either an exact match, or the previous character
# is a '.', so host is within the same domain.
# A leading '.' in the pattern is ignored. Some implementations
# need '.foo.ba' to prevent 'foo.ba' from matching 'xfoo.ba'.
for domain in noproxy:
domain = domain.lower()
if domain[0] == '.':
domain = domain[1:]
if hostname.endswith(domain) and \
(len(hostname) == len(domain) or
hostname[len(hostname) - len(domain) - 1] == '.'):
return False
return True
def _useProxyFor(url):
"""Return True if a proxy should be used for given url, otherwise False.
This function uses server.satellite.no_proxy variable to check for
hosts or domains which should not be connected via a proxy.
server.satellite.no_proxy is a comma seperated list.
Either an exact match, or the previous character
is a '.', so host is within the same domain.
A leading '.' in the pattern is ignored.
See also 'man curl'
"""
u = urlparse.urlsplit(url)
# pylint can't see inside the SplitResult class
# pylint: disable=E1103
if u.scheme == 'file':
return False
hostname = u.hostname.lower()
if hostname in ["localhost", "127.0.0.1", "::1"]:
return False
comp = CFG.getComponent()
if not CFG.has_key("no_proxy"):
initCFG("server.satellite")
if not CFG.has_key('no_proxy'):
initCFG(comp)
return True
noproxy = CFG.no_proxy
initCFG(comp)
if not noproxy:
return True
if not isinstance(noproxy, list):
if noproxy == '*':
# just an asterisk disables all.
return False
noproxy = [noproxy]
# No proxy: Either an exact match, or the previous character
# is a '.', so host is within the same domain.
# A leading '.' in the pattern is ignored. Some implementations
# need '.foo.ba' to prevent 'foo.ba' from matching 'xfoo.ba'.
for domain in noproxy:
domain = domain.lower()
if domain[0] == '.':
domain = domain[1:]
if hostname.endswith(domain) and \
(len(hostname) == len(domain) or
hostname[len(hostname) - len(domain) - 1] == '.'):
return False
return True
def _load_proxy_settings(self, url):
# read the proxy configuration in /etc/rhn/rhn.conf
comp = CFG.getComponent()
initCFG('server.satellite')
# Get the global HTTP Proxy settings from DB or per-repo
# settings on /etc/rhn/spacewalk-repo-sync/zypper.conf
if CFG.http_proxy:
self.proxy_url, self.proxy_user, self.proxy_pass = get_proxy(url)
self.proxy_hostname = self.proxy_url
elif os.path.isfile(REPOSYNC_ZYPPER_CONF):
zypper_cfg = configparser.ConfigParser()
zypper_cfg.read_file(open(REPOSYNC_ZYPPER_CONF))
section_name = None
if zypper_cfg.has_section(self.name):
section_name = self.name
elif zypper_cfg.has_section(channel_label):
section_name = channel_label
elif zypper_cfg.has_section('main'):
section_name = 'main'
if section_name:
if zypper_cfg.has_option(section_name, option='proxy'):
self.proxy_hostname = zypper_cfg.get(section_name, option='proxy')
self.proxy_url = "http://%s" % self.proxy_hostname
if zypper_cfg.has_option(section_name, 'proxy_username'):
self.proxy_user = zypper_cfg.get(section_name, 'proxy_username')
if zypper_cfg.has_option(section_name, 'proxy_password'):
self.proxy_pass = zypper_cfg.get(section_name, 'proxy_password')
# set config component back to original
initCFG(comp)
def _create_connection(self):
""" Returns a Connection object """
scheme, host, port, _uri = self._parse_url(self.rhnParent)
# Build the list of params
params = {
'host': host,
'port': port,
}
if CFG.has_key('timeout'):
params['timeout'] = CFG.TIMEOUT
if self.httpProxy:
params['proxy'] = self.httpProxy
params['username'] = self.httpProxyUsername
params['password'] = self.httpProxyPassword
if scheme == 'https' and self.caChain:
params['trusted_certs'] = [self.caChain, ]
# Now select the right class
if self.httpProxy:
if scheme == 'https':
conn_class = connections.HTTPSProxyConnection
else:
conn_class = connections.HTTPProxyConnection
else:
if scheme == 'https':
conn_class = connections.HTTPSConnection
else:
conn_class = connections.HTTPConnection
log_debug(5, "Using connection class", conn_class, 'Params:', params)
return conn_class(**params)
def __init__(self, retries=3, log_obj=None, force=False):
self.queues = {}
comp = CFG.getComponent()
initCFG("server.satellite")
try:
self.threads = int(CFG.REPOSYNC_DOWNLOAD_THREADS)
except ValueError:
raise ValueError("Number of threads expected, found: '%s'" %
CFG.REPOSYNC_DOWNLOAD_THREADS)
try:
self.timeout = int(CFG.REPOSYNC_TIMEOUT)
except ValueError:
raise ValueError("Timeout in seconds expected, found: '%s'" %
CFG.REPOSYNC_TIMEOUT)
try:
self.minrate = int(CFG.REPOSYNC_MINRATE)
except ValueError:
raise ValueError(
"Minimal transfer rate in bytes pre second expected, found: '%s'"
% CFG.REPOSYNC_MINRATE)
if self.threads < 1:
raise ValueError("Invalid number of threads: %d" % self.threads)
initCFG(comp)
self.retries = retries
self.log_obj = log_obj
self.force = force
self.lock = Lock()
self.exception = None
# WORKAROUND - BZ #1439758 - ensure first item in queue is performed alone to properly setup NSS
self.first_in_queue_done = False
self.first_in_queue_lock = Lock()
def _create_connection(self):
""" Returns a Connection object """
scheme, host, port, _uri = self._parse_url(self.rhnParent)
# Build the list of params
params = {"host": host, "port": port}
if CFG.has_key("timeout"):
params["timeout"] = CFG.TIMEOUT
if self.httpProxy:
params["proxy"] = self.httpProxy
params["username"] = self.httpProxyUsername
params["password"] = self.httpProxyPassword
if scheme == "https" and self.caChain:
params["trusted_certs"] = [self.caChain]
# Now select the right class
if self.httpProxy:
if scheme == "https":
conn_class = connections.HTTPSProxyConnection
else:
conn_class = connections.HTTPProxyConnection
else:
if scheme == "https":
conn_class = connections.HTTPSConnection
else:
conn_class = connections.HTTPConnection
log_debug(5, "Using connection class", conn_class, "Params:", params)
return conn_class(**params)
def _create_connection(self):
""" Returns a Connection object """
scheme, host, port, _uri = self._parse_url(self.rhnParent)
# Build the list of params
params = {
'host': host,
'port': port,
}
if CFG.has_key('timeout'):
params['timeout'] = CFG.TIMEOUT
if self.httpProxy:
params['proxy'] = self.httpProxy
params['username'] = self.httpProxyUsername
params['password'] = self.httpProxyPassword
if scheme == 'https' and self.caChain:
params['trusted_certs'] = [self.caChain, ]
# Now select the right class
if self.httpProxy:
if scheme == 'https':
conn_class = connections.HTTPSProxyConnection
else:
conn_class = connections.HTTPProxyConnection
else:
if scheme == 'https':
conn_class = connections.HTTPSConnection
else:
conn_class = connections.HTTPConnection
log_debug(5, "Using connection class", conn_class, 'Params:', params)
return conn_class(**params)
def initDB(backend=None,
host=None,
port=None,
username=None,
password=None,
database=None,
sslmode=None,
sslrootcert=None):
"""
Initialize the database.
Either we get backend and all parameter which means the caller
knows what they are doing, or we populate everything from the
config files.
"""
if backend is None:
if CFG is None or not CFG.is_initialized():
initCFG('server')
backend = CFG.DB_BACKEND
host = CFG.DB_HOST
port = CFG.DB_PORT
database = CFG.DB_NAME
username = CFG.DB_USER
password = CFG.DB_PASSWORD
sslmode = None
sslrootcert = None
if CFG.DB_SSL_ENABLED:
sslmode = 'verify-full'
sslrootcert = CFG.DB_SSLROOTCERT
if backend not in SUPPORTED_BACKENDS:
raise rhnException("Unsupported database backend", backend)
if port:
port = int(port)
# Hide the password
add_to_seclist(password)
try:
__init__DB(backend, host, port, username, password, database, sslmode,
sslrootcert)
__init__DB2(backend, host, port, username, password, database, sslmode,
sslrootcert)
# except (rhnException, SQLError):
# raise # pass on, we know those ones
# except (KeyboardInterrupt, SystemExit):
# raise
except SQLConnectError, e:
try:
global __DB
global __DB2
del __DB
del __DB2
except NameError:
pass
raise e
def initDB(backend=None, host=None, port=None, username=None,
password=None, database=None, sslmode=None, sslrootcert=None, initsecond=False):
"""
Initialize the database.
Either we get backend and all parameter which means the caller
knows what they are doing, or we populate everything from the
config files.
initsecond: If set to True it initialize a second DB connection.
By default only one DB connection is needed.
"""
if backend is None:
if CFG is None or not CFG.is_initialized():
initCFG('server')
backend = CFG.DB_BACKEND
host = CFG.DB_HOST
port = CFG.DB_PORT
database = CFG.DB_NAME
username = CFG.DB_USER
password = CFG.DB_PASSWORD
sslmode = None
sslrootcert = None
if CFG.DB_SSL_ENABLED:
sslmode = 'verify-full'
sslrootcert = CFG.DB_SSLROOTCERT
if backend not in SUPPORTED_BACKENDS:
raise rhnException("Unsupported database backend", backend)
if port:
port = int(port)
# Hide the password
add_to_seclist(password)
try:
if initsecond == False:
__init__DB(backend, host, port, username, password, database, sslmode, sslrootcert)
else:
__init__DB2(backend, host, port, username, password, database, sslmode, sslrootcert)
# except (rhnException, SQLError):
# raise # pass on, we know those ones
# except (KeyboardInterrupt, SystemExit):
# raise
except SQLConnectError:
e = sys.exc_info()[1]
try:
closeDB()
except NameError:
pass
raise e
except:
raise
#e_type, e_value = sys.exc_info()[:2]
# raise rhnException("Could not initialize Oracle database connection",
# str(e_type), str(e_value))
return 0
def initDB(backend=None, host=None, port=None, username=None,
password=None, database=None, sslmode=None, sslrootcert=None, initsecond=False):
"""
Initialize the database.
Either we get backend and all parameter which means the caller
knows what they are doing, or we populate everything from the
config files.
initsecond: If set to True it initialize a second DB connection.
By default only one DB connection is needed.
"""
if backend is None:
if CFG is None or not CFG.is_initialized():
initCFG('server')
backend = CFG.DB_BACKEND
host = CFG.DB_HOST
port = CFG.DB_PORT
database = CFG.DB_NAME
username = CFG.DB_USER
password = CFG.DB_PASSWORD
sslmode = None
sslrootcert = None
if CFG.DB_SSL_ENABLED:
sslmode = 'verify-full'
sslrootcert = CFG.DB_SSLROOTCERT
if backend not in SUPPORTED_BACKENDS:
raise rhnException("Unsupported database backend", backend)
if port:
port = int(port)
# Hide the password
add_to_seclist(password)
try:
if initsecond == False:
__init__DB(backend, host, port, username, password, database, sslmode, sslrootcert)
else:
__init__DB2(backend, host, port, username, password, database, sslmode, sslrootcert)
# except (rhnException, SQLError):
# raise # pass on, we know those ones
# except (KeyboardInterrupt, SystemExit):
# raise
except SQLConnectError:
e = sys.exc_info()[1]
try:
closeDB()
except NameError:
pass
raise_with_tb(e, sys.exc_info()[2])
except:
raise
#e_type, e_value = sys.exc_info()[:2]
# raise rhnException("Could not initialize Oracle database connection",
# str(e_type), str(e_value))
return 0
def _get_proxy_from_rhn_conf():
"""Return a tuple of (url, user, pass) proxy information from rhn config
Returns None instead of a tuple if there was no proxy url. user,
pass can be None.
"""
comp = CFG.getComponent()
if not CFG.has_key("http_proxy"):
initCFG("server.satellite")
result = None
if CFG.http_proxy:
# CFG.http_proxy format is <hostname>[:<port>] in 1.7
url = 'http://%s' % CFG.http_proxy
result = (url, CFG.http_proxy_username, CFG.http_proxy_password)
initCFG(comp)
log_debug(2, "Could not read proxy URL from rhn config.")
return result
def _get_proxy_from_rhn_conf():
"""Return a tuple of (url, user, pass) proxy information from rhn config
Returns None instead of a tuple if there was no proxy url. user,
pass can be None.
"""
comp = CFG.getComponent()
if not CFG.has_key("http_proxy"):
initCFG("server.satellite")
result = None
if CFG.http_proxy:
# CFG.http_proxy format is <hostname>[:<port>] in 1.7
url = 'http://%s' % CFG.http_proxy
result = (url, CFG.http_proxy_username, CFG.http_proxy_password)
initCFG(comp)
log_debug(2, "Could not read proxy URL from rhn config.")
return result
def getNotificationsTypeDisabled():
"""Return list of types which are disabled"""
disabledTypes = []
comp = CFG.getComponent()
initCFG("java")
if CFG.notifications_type_disabled:
disabledTypes = CFG.notifications_type_disabled.split(",")
initCFG(comp)
return disabledTypes
def _createConnection(host, port, scheme):
params = {"host": host, "port": port}
if CFG.has_key("timeout"):
params["timeout"] = CFG.TIMEOUT
if scheme == SCHEME_HTTPS:
conn_class = connections.HTTPSConnection
else:
conn_class = connections.HTTPConnection
return conn_class(**params)
def _createConnection(host, port, scheme):
params = {'host': host, 'port': port}
if CFG.has_key('timeout'):
params['timeout'] = CFG.TIMEOUT
if scheme == SCHEME_HTTPS:
conn_class = connections.HTTPSConnection
else:
conn_class = connections.HTTPConnection
return conn_class(**params)
def __init__(self,
url,
name,
insecure=False,
interactive=True,
yumsrc_conf=None,
org="1",
channel_label="",
no_mirrors=False,
ca_cert_file=None,
client_cert_file=None,
client_key_file=None):
# pylint: disable=W0613
self.url = url
self.name = name
if org:
self.org = org
else:
self.org = "NULL"
comp = CFG.getComponent()
# read the proxy configuration in /etc/rhn/rhn.conf
initCFG('server.satellite')
self.proxy_addr, self.proxy_user, self.proxy_pass = get_proxy(self.url)
self.authtoken = None
# Replace non-valid characters from reponame (only alphanumeric chars allowed)
self.reponame = "".join([x if x.isalnum() else "_" for x in self.name])
self.channel_label = channel_label
# SUSE vendor repositories belongs to org = NULL
# The repository cache root will be "/var/cache/rhn/reposync/REPOSITORY_LABEL/"
root = os.path.join(CACHE_DIR, str(org or "NULL"), self.reponame)
self.repo = DebRepo(url,
root,
os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR,
self.org, 'stage'),
self.proxy_addr,
self.proxy_user,
self.proxy_pass,
gpg_verify=not (insecure))
self.repo.verify()
self.num_packages = 0
self.num_excluded = 0
# keep authtokens for mirroring
(_scheme, _netloc, _path, query, _fragid) = urlparse.urlsplit(url)
if query:
self.authtoken = query
initCFG(comp)
def __init__(self, req):
""" init with http request object """
# FIXME: should rename some things:
# self.bodyFd --> self.body or self.data or ?
# self.caChain --> self.caCert
self.req = req
self.responseContext = ResponseContext()
self.uri = None # ''
# Common settings for both the proxy and the redirect
# broker and redirect immediately alter these for their own purposes
self.caChain = CFG.CA_CHAIN
self.httpProxy = CFG.HTTP_PROXY
self.httpProxyUsername = CFG.HTTP_PROXY_USERNAME
self.httpProxyPassword = CFG.HTTP_PROXY_PASSWORD
if not self.httpProxyUsername:
self.httpProxyPassword = ''
self.rhnParent = CFG.RHN_PARENT or ''
self.rhnParent = rhnLib.parseUrl(self.rhnParent)[1].split(':')[0]
CFG.set('RHN_PARENT', self.rhnParent)
def __init__(self, req):
""" init with http request object """
# FIXME: should rename some things:
# self.bodyFd --> self.body or self.data or ?
# self.caChain --> self.caCert
self.req = req
self.responseContext = ResponseContext()
self.uri = None # ''
# Common settings for both the proxy and the redirect
# broker and redirect immediately alter these for their own purposes
self.caChain = CFG.CA_CHAIN
self.httpProxy = CFG.HTTP_PROXY
self.httpProxyUsername = CFG.HTTP_PROXY_USERNAME
self.httpProxyPassword = CFG.HTTP_PROXY_PASSWORD
if not self.httpProxyUsername:
self.httpProxyPassword = ""
self.rhnParent = CFG.RHN_PARENT or ""
self.rhnParent = rhnLib.parseUrl(self.rhnParent)[1].split(":")[0]
CFG.set("RHN_PARENT", self.rhnParent)
def _createConnection(host, port, scheme):
params = {'host': host,
'port': port}
if CFG.has_key('timeout'):
params['timeout'] = CFG.TIMEOUT
if scheme == SCHEME_HTTPS:
conn_class = connections.HTTPSConnection
else:
conn_class = connections.HTTPConnection
return conn_class(**params)
def initDB(backend=None, host=None, port=None, username=None,
password=None, database=None, sslmode=None, sslrootcert=None):
"""
Initialize the database.
Either we get backend and all parameter which means the caller
knows what they are doing, or we populate everything from the
config files.
"""
if backend is None:
if CFG is None or not CFG.is_initialized():
initCFG('server')
backend = CFG.DB_BACKEND
host = CFG.DB_HOST
port = CFG.DB_PORT
database = CFG.DB_NAME
username = CFG.DB_USER
password = CFG.DB_PASSWORD
sslmode = None
sslrootcert = None
if CFG.DB_SSL_ENABLED:
sslmode = 'verify-full'
sslrootcert = CFG.DB_SSLROOTCERT
if backend not in SUPPORTED_BACKENDS:
raise rhnException("Unsupported database backend", backend)
if port:
port = int(port)
# Hide the password
add_to_seclist(password)
try:
__init__DB(backend, host, port, username, password, database, sslmode, sslrootcert)
__init__DB2(backend, host, port, username, password, database, sslmode, sslrootcert)
# except (rhnException, SQLError):
# raise # pass on, we know those ones
# except (KeyboardInterrupt, SystemExit):
# raise
except SQLConnectError, e:
try:
global __DB
global __DB2
del __DB
del __DB2
except NameError:
pass
raise e
def __init__(self,
url,
name,
insecure=False,
interactive=True,
yumsrc_conf=None,
org="1",
channel_label="",
no_mirrors=False,
ca_cert_file=None,
client_cert_file=None,
client_key_file=None):
# pylint: disable=W0613
self.url = url
self.name = name
if org:
self.org = org
else:
self.org = "NULL"
comp = CFG.getComponent()
# read the proxy configuration in /etc/rhn/rhn.conf
initCFG('server.satellite')
self.proxy_addr, self.proxy_user, self.proxy_pass = get_proxy(self.url)
self.authtoken = None
self.repo = DebRepo(url,
os.path.join(CACHE_DIR, self.org, name),
os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR,
self.org, 'stage'),
self.proxy_addr,
self.proxy_user,
self.proxy_pass,
gpg_verify=not (insecure))
self.repo.verify()
self.num_packages = 0
self.num_excluded = 0
# keep authtokens for mirroring
(_scheme, _netloc, _path, query, _fragid) = urlparse.urlsplit(url)
if query:
self.authtoken = query
initCFG(comp)
def get_mirror_credentials():
"""Return a list of mirror credential tuples (user, pass)
N.B. The config values will be read from the global configuration:
server.susemanager.mirrcred_user
server.susemanager.mirrcred_pass
server.susemanager.mirrcred_user_1
server.susemanager.mirrcred_pass_1
etc.
The credentials are read sequentially, when the first value is found
to be missing, the process is aborted and the list of credentials
that have been read so far are returned. For example if
server.susemanager.mirrcred_pass_1 can not be read, only the first
pair of default mirrcreds will be returned, even though
mirrcred_user_2, mirrcred_pass_2 etc. might still exist.
"""
comp = CFG.getComponent()
initCFG("server.susemanager")
creds = []
# the default values should at least always be there
if not CFG["mirrcred_user"] or not CFG["mirrcred_pass"]:
initCFG(comp)
raise ConfigParserError("Could not read default mirror credentials: "
"server.susemanager.mirrcred_user, "
"server.susemanager.mirrcred_pass.")
creds.append((CFG["mirrcred_user"], CFG["mirrcred_pass"]))
# increment the credentials number, until we can't read one
n = 1
while True:
try:
creds.append((CFG["mirrcred_user_%s" % n],
CFG["mirrcred_pass_%s" % n]))
except (KeyError, AttributeError):
break
n += 1
initCFG(comp)
return creds
def get_mirror_credentials():
"""Return a list of mirror credential tuples (user, pass)
N.B. The config values will be read from the global configuration:
server.susemanager.mirrcred_user
server.susemanager.mirrcred_pass
server.susemanager.mirrcred_user_1
server.susemanager.mirrcred_pass_1
etc.
The credentials are read sequentially, when the first value is found
to be missing, the process is aborted and the list of credentials
that have been read so far are returned. For example if
server.susemanager.mirrcred_pass_1 can not be read, only the first
pair of default mirrcreds will be returned, even though
mirrcred_user_2, mirrcred_pass_2 etc. might still exist.
"""
comp = CFG.getComponent()
initCFG("server.susemanager")
creds = []
# the default values should at least always be there
if not CFG["mirrcred_user"] or not CFG["mirrcred_pass"]:
initCFG(comp)
raise ConfigParserError("Could not read default mirror credentials: "
"server.susemanager.mirrcred_user, "
"server.susemanager.mirrcred_pass.")
creds.append((CFG["mirrcred_user"], CFG["mirrcred_pass"]))
# increment the credentials number, until we can't read one
n = 1
while True:
try:
creds.append(
(CFG["mirrcred_user_%s" % n], CFG["mirrcred_pass_%s" % n]))
except (KeyError, AttributeError):
break
n += 1
initCFG(comp)
return creds
def __init__(self, retries=3, log_obj=None, force=False):
self.queues = {}
comp = CFG.getComponent()
initCFG('server.satellite')
try:
self.threads = int(CFG.REPOSYNC_DOWNLOAD_THREADS)
except ValueError:
initCFG(comp)
raise ValueError("Number of threads expected, found: '%s'" %
CFG.REPOSYNC_DOWNLOAD_THREADS)
else:
initCFG(comp)
if self.threads < 1:
raise ValueError("Invalid number of threads: %d" % self.threads)
self.retries = retries
self.log_obj = log_obj
self.force = force
self.lock = Lock()
self.exception = None
# WORKAROUND - BZ #1439758 - ensure first item in queue is performed alone to properly setup NSS
self.first_in_queue_done = False
self.first_in_queue_lock = Lock()
def cfg_component(component, root=None, filename=None):
"""Context manager for rhnConfig.
:param comp: The configuration component to use in this context
:param root: Root directory location of configuration files, optional
:param filename: Configuration file, optional
There is a common pattern when using rhnConfig that consists of the following steps:
1. save current component: old = CFG.getComponent()
2. set CFG to another component: initCFG('my_component')
3. Read / Set configuration values
4. set CFG back to the previous component
This pattern can now be expressed using the ``with`` statement:
with cfg_component('my_component') as CFG:
print(CFG.my_value)
"""
previous = CFG.getComponent()
initCFG(component=component, root=root, filename=filename)
try:
yield CFG
finally:
initCFG(previous)
def __init__(self,
no_packages=False,
no_errata=False,
no_rpms=False,
no_kickstarts=False,
log_level=None,
mount_point=None,
consider_full=False,
force_kickstarts=False,
force_all_errata=False,
email=False,
import_batch_size=None):
if log_level is None:
log_level = 0
self.log_level = log_level
CFG.set('DEBUG', log_level)
self.email = email
if self.email:
initEMAIL_LOG()
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Command: %s" % str(sys.argv))
rhnSQL.initDB()
initCFG('server.satellite')
self.cdn_repository_manager = CdnRepositoryManager(mount_point)
self.no_packages = no_packages
self.no_errata = no_errata
self.no_rpms = no_rpms
if self.no_packages and self.no_rpms:
log(0, "Parameter --no-rpms has no effect.")
self.no_kickstarts = no_kickstarts
self.force_all_errata = force_all_errata
self.force_kickstarts = force_kickstarts
if self.no_kickstarts and self.force_kickstarts:
log(0, "Parameter --force-kickstarts has no effect.")
if mount_point:
self.mount_point = "file://" + mount_point
self.consider_full = consider_full
else:
self.mount_point = CFG.CDN_ROOT
self.consider_full = True
verify_mappings()
f = None
# try block in try block - this is hack for python 2.4 compatibility
# to support finally
try:
try:
# Channel families mapping to channels
f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r')
self.families = json.load(f)
f.close()
# Channel metadata
f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r')
self.channel_metadata = json.load(f)
f.close()
# Dist/Release channel mapping
f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r')
self.channel_dist_mapping = json.load(f)
f.close()
# Kickstart metadata
f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r')
self.kickstart_metadata = json.load(f)
f.close()
except IOError:
e = sys.exc_info()[1]
log(1, "Ignoring channel mappings: %s" % e)
self.families = {}
self.channel_metadata = {}
self.channel_dist_mapping = {}
self.kickstart_metadata = {}
finally:
if f is not None:
f.close()
# Map channels to their channel family
self.channel_to_family = {}
for family in self.families:
for channel in self.families[family]['channels']:
self.channel_to_family[channel] = family
# Set already synced channels, entitled null-org channels and custom channels with associated
# CDN repositories
h = rhnSQL.prepare("""
select distinct c.label, c.org_id
from rhnChannelFamilyPermissions cfp inner join
rhnChannelFamily cf on cfp.channel_family_id = cf.id inner join
rhnChannelFamilyMembers cfm on cf.id = cfm.channel_family_id inner join
rhnChannel c on cfm.channel_id = c.id
where c.org_id is null
or (c.org_id is not null and
exists (
select cs.id
from rhnContentSource cs inner join
rhnChannelContentSource ccs on ccs.source_id = cs.id
where ccs.channel_id = c.id
and cs.org_id is null
)
)
order by c.org_id nulls first, label
""")
h.execute()
channels = h.fetchall_dict() or []
self.synced_channels = {}
for channel in channels:
# Custom channel repositories not available, don't mark as synced
if channel['org_id']:
repos = self.cdn_repository_manager.list_associated_repos(
channel['label'])
if not all([
self.cdn_repository_manager.
check_repository_availability(r) for r in repos
]):
continue
self.synced_channels[channel['label']] = channel['org_id']
# Select available channel families from DB
h = rhnSQL.prepare("""
select distinct label
from rhnChannelFamilyPermissions cfp inner join
rhnChannelFamily cf on cfp.channel_family_id = cf.id
where cf.org_id is null
""")
h.execute()
families = h.fetchall_dict() or []
self.entitled_families = [f['label'] for f in families]
self.import_batch_size = import_batch_size
def __init__(self, channel_label, repo_type, url=None, fail=False,
filters=None, no_errata=False, sync_kickstart=False, latest=False,
metadata_only=False, strict=0, excluded_urls=None, no_packages=False,
log_dir="reposync", log_level=None):
self.regen = False
self.fail = fail
self.filters = filters or []
self.no_packages = no_packages
self.no_errata = no_errata
self.sync_kickstart = sync_kickstart
self.latest = latest
self.metadata_only = metadata_only
self.ks_tree_type = 'externally-managed'
self.ks_install_type = 'generic_rpm'
initCFG('server.satellite')
rhnSQL.initDB()
# setup logging
log_filename = channel_label + '.log'
log_path = default_log_location + log_dir + '/' + log_filename
if log_level is None:
log_level = 0
CFG.set('DEBUG', log_level)
rhnLog.initLOG(log_path, log_level)
# os.fchown isn't in 2.4 :/
if isSUSE():
os.system("chgrp www " + log_path)
else:
os.system("chgrp apache " + log_path)
log2disk(0, "Command: %s" % str(sys.argv))
log2disk(0, "Sync of channel started.")
self.channel_label = channel_label
self.channel = self.load_channel()
if not self.channel:
log(0, "Channel %s does not exist." % channel_label)
if not url:
# TODO:need to look at user security across orgs
h = rhnSQL.prepare("""select s.id, s.source_url, s.label, fm.channel_family_id
from rhnContentSource s,
rhnChannelContentSource cs,
rhnChannelFamilyMembers fm
where s.id = cs.source_id
and cs.channel_id = fm.channel_id
and cs.channel_id = :channel_id""")
h.execute(channel_id=int(self.channel['id']))
source_data = h.fetchall_dict()
self.urls = []
if excluded_urls is None:
excluded_urls = []
if source_data:
for row in source_data:
if row['source_url'] not in excluded_urls:
self.urls.append((row['id'], row['source_url'], row['label'], row['channel_family_id']))
else:
self.urls = [(None, u, None, None) for u in url]
if not self.urls:
log2stderr(0, "Channel %s has no URL associated" % channel_label)
self.repo_plugin = self.load_plugin(repo_type)
self.strict = strict
self.all_packages = []
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False,
log_level=None):
self.no_packages = no_packages
self.no_errata = no_errata
self.no_rpms = no_rpms
self.no_kickstarts = no_kickstarts
if log_level is None:
log_level = 0
self.log_level = log_level
CFG.set('DEBUG', log_level)
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Command: %s" % str(sys.argv))
rhnSQL.initDB()
initCFG('server.satellite')
try:
# Channel families mapping to channels
with open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r') as f:
self.families = json.load(f)
# Channel metadata
with open(constants.CHANNEL_DEFINITIONS_PATH, 'r') as f:
self.channel_metadata = json.load(f)
# Dist/Release channel mapping
with open(constants.CHANNEL_DIST_MAPPING_PATH, 'r') as f:
self.channel_dist_mapping = json.load(f)
# Channel to repositories mapping
with open(constants.CONTENT_SOURCE_MAPPING_PATH, 'r') as f:
self.content_source_mapping = json.load(f)
# Kickstart metadata
with open(constants.KICKSTART_DEFINITIONS_PATH, 'r') as f:
self.kickstart_metadata = json.load(f)
# Channel to kickstart repositories mapping
with open(constants.KICKSTART_SOURCE_MAPPING_PATH, 'r') as f:
self.kickstart_source_mapping = json.load(f)
except IOError:
e = sys.exc_info()[1]
log2stderr(0, "ERROR: Problem with loading file: %s" % e)
raise CdnMappingsLoadError()
# Map channels to their channel family
self.channel_to_family = {}
for family in self.families:
for channel in self.families[family]['channels']:
self.channel_to_family[channel] = family
# Set already synced channels
h = rhnSQL.prepare("""
select label from rhnChannel where org_id is null
""")
h.execute()
channels = h.fetchall_dict() or []
self.synced_channels = [ch['label'] for ch in channels]
# Set SSL-keys for channel family
self.family_keys = {}
def __redirectToNextLocationNoRetry(self, loopProtection=False):
""" This function will perform a redirection to the next location, as
specified in the last response's "Location" header. This function will
return an actual HTTP response status code. If successful, it will
return apache.HTTP_OK, not apache.OK. If unsuccessful, this function
will simply return; no retries will be performed. The following error
codes can be returned:
HTTP_OK,HTTP_PARTIAL_CONTENT - Redirect successful.
HTTP_MOVED_TEMPORARILY - Redirect was redirected again by 3rd party.
HTTP_MOVED_PERMANENTLY - Redirect was redirected again by 3rd party.
HTTP_INTERNAL_SERVER_ERROR - Error extracting redirect information
HTTP_SERVICE_UNAVAILABLE - Could not connect to 3rd party server,
connection was reset, or a read error
occurred during communication.
HTTP_* - Any other HTTP status code may also be
returned.
Upon successful completion of this function, a new responseContext
will be created and pushed onto the stack.
"""
# Obtain the redirect location first before we replace the current
# response context. It's contained in the Location header of the
# previous response.
redirectLocation = self._get_header(rhnConstants.HEADER_LOCATION)
# We are about to redirect to a new location so now we'll push a new
# response context before we return any errors.
self.responseContext.add()
# There should always be a redirect URL passed back to us. If not,
# there's an error.
if not redirectLocation or len(redirectLocation) == 0:
log_error(" No redirect location specified!")
Traceback(mail=0)
return apache.HTTP_INTERNAL_SERVER_ERROR
# The _get_header function returns the value as a list. There should
# always be exactly one location specified.
redirectLocation = redirectLocation[0]
log_debug(1, " Redirecting to: ", redirectLocation)
# Tear apart the redirect URL. We need the scheme, the host, the
# port (if not the default), and the URI.
_scheme, host, port, uri = self._parse_url(redirectLocation)
# Add any params onto the URI since _parse_url doesn't include them.
if redirectLocation.find('?') > -1:
uri += redirectLocation[redirectLocation.index('?'):]
# Now create a new connection. We'll use SSL if configured to do
# so.
params = {
'host': host,
'port': port,
}
if CFG.has_key('timeout'):
params['timeout'] = CFG.TIMEOUT
if CFG.USE_SSL:
log_debug(1, " Redirecting with SSL. Cert= ", self.caChain)
params['trusted_certs'] = [self.caChain]
connection = connections.HTTPSConnection(**params)
else:
log_debug(1, " Redirecting withOUT SSL.")
connection = connections.HTTPConnection(**params)
# Put the connection into the current response context.
self.responseContext.setConnection(connection)
# Now open the connection to the 3rd party server.
log_debug(4, "Attempting to connect to 3rd party server...")
try:
connection.connect()
except socket.error, e:
log_error("Error opening redirect connection", redirectLocation, e)
Traceback(mail=0)
return apache.HTTP_SERVICE_UNAVAILABLE
def __init__(self,
channel_label,
repo_type,
url=None,
fail=False,
filters=None,
no_errata=False,
sync_kickstart=False,
latest=False,
metadata_only=False,
strict=0,
excluded_urls=None,
no_packages=False,
log_dir="reposync",
log_level=None,
force_kickstart=False,
force_all_errata=False,
check_ssl_dates=False,
force_null_org_content=False):
self.regen = False
self.fail = fail
self.filters = filters or []
self.no_packages = no_packages
self.no_errata = no_errata
self.sync_kickstart = sync_kickstart
self.force_all_errata = force_all_errata
self.force_kickstart = force_kickstart
self.latest = latest
self.metadata_only = metadata_only
self.ks_tree_type = 'externally-managed'
self.ks_install_type = None
initCFG('server.satellite')
rhnSQL.initDB()
# setup logging
log_filename = channel_label + '.log'
log_path = default_log_location + log_dir + '/' + log_filename
if log_level is None:
log_level = 0
CFG.set('DEBUG', log_level)
rhnLog.initLOG(log_path, log_level)
# os.fchown isn't in 2.4 :/
if isSUSE():
os.system("chgrp www " + log_path)
else:
os.system("chgrp apache " + log_path)
log2disk(0, "Command: %s" % str(sys.argv))
log2disk(0, "Sync of channel started.")
self.channel_label = channel_label
self.channel = self.load_channel()
if not self.channel:
log(0, "Channel %s does not exist." % channel_label)
if not self.channel['org_id'] or force_null_org_content:
self.org_id = None
else:
self.org_id = int(self.channel['org_id'])
if not url:
# TODO:need to look at user security across orgs
h = rhnSQL.prepare("""select s.id, s.source_url, s.label
from rhnContentSource s,
rhnChannelContentSource cs
where s.id = cs.source_id
and cs.channel_id = :channel_id""")
h.execute(channel_id=int(self.channel['id']))
source_data = h.fetchall_dict()
self.urls = []
if excluded_urls is None:
excluded_urls = []
if source_data:
for row in source_data:
if row['source_url'] not in excluded_urls:
self.urls.append(
(row['id'], row['source_url'], row['label']))
else:
self.urls = [(None, u, None) for u in url]
if not self.urls:
log2(0,
0,
"Channel %s has no URL associated" % channel_label,
stream=sys.stderr)
self.repo_plugin = self.load_plugin(repo_type)
self.strict = strict
self.all_packages = []
self.check_ssl_dates = check_ssl_dates
# Init cache for computed checksums to not compute it on each reposync run again
self.checksum_cache = rhnCache.get(checksum_cache_filename)
if self.checksum_cache is None:
self.checksum_cache = {}
def processCommandline():
options = [
Option('--sanity-only', action='store_true', help="confirm certificate sanity. Does not activate "
+ "the Red Hat Satellite locally or remotely."),
Option('--ignore-expiration', action='store_true', help='execute regardless of the expiration '
+ 'of the RHN Certificate (not recommended).'),
Option('--ignore-version-mismatch', action='store_true', help='execute regardless of version '
+ 'mismatch of existing and new certificate.'),
Option('-v', '--verbose', action='count', help='be verbose '
+ '(accumulable: -vvv means "be *really* verbose").'),
Option('--dump-version', action='store', help="requested version of XML dump"),
Option('--manifest', action='store', help='the RHSM manifest path/filename to activate for CDN'),
Option('--rhn-cert', action='store', help='this option is deprecated, use --manifest instead'),
Option('--deactivate', action='store_true', help='deactivate CDN-activated Satellite'),
Option('--disconnected', action='store_true', help="activate locally, not subscribe to remote repository"),
Option('--manifest-info', action='store_true', help="show information about currently activated manifest"),
Option('--manifest-download', action='store_true',
help="download new manifest from RHSM to temporary location"),
Option('--manifest-refresh', action='store_true', help="download new manifest from RHSM and activate it"),
Option('--manifest-reconcile-request', action='store_true',
help="request regeneration of entitlement certificates")
]
parser = OptionParser(option_list=options)
options, args = parser.parse_args()
initCFG('server.satellite')
if options.verbose is None:
options.verbose = 0
CFG.set('DEBUG', options.verbose)
rhnLog.initLOG(LOG_PATH, options.verbose)
log2disk(0, "Command: %s" % str(sys.argv))
# we take no extra commandline arguments that are not linked to an option
if args:
writeError("These arguments make no sense in this context (try --help): %s" % repr(args))
sys.exit(1)
# No need to check further if deactivating
if options.deactivate:
return options
if options.sanity_only:
options.disconnected = 1
if options.manifest_refresh:
options.manifest_download = 1
if CFG.DISCONNECTED and not options.disconnected:
msg = """Satellite server has been setup to run in disconnected mode.
Either correct server configuration in /etc/rhn/rhn.conf
or use --disconnected to activate it locally."""
writeError(msg)
sys.exit(1)
options.http_proxy = idn_ascii_to_puny(CFG.HTTP_PROXY)
options.http_proxy_username = CFG.HTTP_PROXY_USERNAME
options.http_proxy_password = CFG.HTTP_PROXY_PASSWORD
log(1, 'HTTP_PROXY: %s' % options.http_proxy)
log(1, 'HTTP_PROXY_USERNAME: %s' % options.http_proxy_username)
log(1, 'HTTP_PROXY_PASSWORD: <password>')
return options
def __init__(self, url, name, insecure=False, interactive=True,
yumsrc_conf=None, org="1", channel_label="",
no_mirrors=False, ca_cert_file=None, client_cert_file=None,
client_key_file=None):
"""
Plugin constructor.
"""
# pylint: disable=W0613
if urlsplit(url).scheme:
self.url = url
else:
self.url = "file://%s" % url
self.name = name
self.insecure = insecure
self.interactive = interactive
self.org = org if org else "NULL"
self.proxy_hostname = None
self.proxy_url = None
self.proxy_user = None
self.proxy_pass = None
self.authtoken = None
self.sslcacert = ca_cert_file
self.sslclientcert = client_cert_file
self.sslclientkey = client_key_file
self.http_headers = {}
comp = CFG.getComponent()
# read configuration from /etc/rhn/rhn.conf
initCFG('server.satellite')
# keep authtokens for mirroring
(_scheme, _netloc, _path, query, _fragid) = urlsplit(url)
if query:
self.authtoken = query
# load proxy configuration based on the url
self._load_proxy_settings(self.url)
# Get extra HTTP headers configuration from /etc/rhn/spacewalk-repo-sync/extra_headers.conf
if os.path.isfile(REPOSYNC_EXTRA_HTTP_HEADERS_CONF):
http_headers_cfg = configparser.ConfigParser()
http_headers_cfg.read_file(open(REPOSYNC_EXTRA_HTTP_HEADERS_CONF))
section_name = None
if http_headers_cfg.has_section(self.name):
section_name = self.name
elif http_headers_cfg.has_section(channel_label):
section_name = channel_label
elif http_headers_cfg.has_section('main'):
section_name = 'main'
if section_name:
for hdr in http_headers_cfg[section_name]:
self.http_headers[hdr] = http_headers_cfg.get(section_name, option=hdr)
# perform authentication if implemented
self._authenticate(url)
# Make sure baseurl ends with / and urljoin will work correctly
self.urls = [url]
if self.urls[0][-1] != '/':
self.urls[0] += '/'
# Exclude non-valid characters from reponame
self.reponame = self.name
for chr in ["$", " ", ".", ";"]:
self.reponame = self.reponame.replace(chr, "_")
self.channel_label = channel_label
# SUSE vendor repositories belongs to org = NULL
root = os.path.join(CACHE_DIR, str(org or "NULL"), self.channel_label or self.reponame)
self.repo = ZypperRepo(root=root, url=self.url, org=self.org)
self.num_packages = 0
self.num_excluded = 0
self.gpgkey_autotrust = None
self.groupsfile = None
# set config component back to original
initCFG(comp)
def __init__(self):
self._next_poll_interval = None
self._notify_threshold = CFG.get('notify_threshold')
def __init__(self):
self._next_poll_interval = None
self._notify_threshold = CFG.get('notify_threshold')
def send(url, sendData=None):
"""Connect to url and return the result as stringIO
:arg url: the url where the request will be sent
:kwarg sendData: do a post-request when "sendData" is given.
Returns the result as stringIO object.
"""
connect_retries = 10
try_counter = connect_retries
timeout = 120
if CFG.is_initialized() and CFG.has_key('TIMEOUT'):
timeout = CFG.TIMEOUT
curl = pycurl.Curl()
curl.setopt(pycurl.CONNECTTIMEOUT, timeout)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug)
curl.setopt(pycurl.VERBOSE, True)
proxy_url, proxy_user, proxy_pass = get_proxy(url)
if proxy_url:
curl.setopt(pycurl.PROXY, proxy_url)
log_debug(2, "Connect to %s" % url)
if sendData is not None:
curl.setopt(pycurl.POSTFIELDS, sendData)
if (CFG.is_initialized() and
CFG.has_key('DISABLE_EXPECT') and
CFG.DISABLE_EXPECT):
# disable Expect header
curl.setopt(pycurl.HTTPHEADER, ['Expect:'])
# We implement our own redirection-following, because pycurl
# 7.19 doesn't POST after it gets redirected. Ideally we'd be
# using pycurl.POSTREDIR here, but that's in 7.21.
curl.setopt(pycurl.FOLLOWLOCATION, False)
response = StringIO()
curl.setopt(pycurl.WRITEFUNCTION, response.write)
try_counter = connect_retries
while try_counter:
try_counter -= 1
try:
curl.perform()
except pycurl.error, e:
if e[0] == 56: # Proxy requires authentication
log_debug(2, e[1])
if not (proxy_user and proxy_pass):
raise TransferException("Proxy requires authentication, "
"but reading credentials from "
"%s failed." % YAST_PROXY)
curl.setopt(pycurl.PROXYUSERPWD,
"%s:%s" % (proxy_user, proxy_pass))
elif e[0] == 60:
log_error("Peer certificate could not be authenticated "
"with known CA certificates.")
raise TransferException("Peer certificate could not be "
"authenticated with known CA "
"certificates.")
else:
log_error(e[1])
raise
status = curl.getinfo(pycurl.HTTP_CODE)
if status == 200 or (URL(url).scheme == "file" and status == 0):
# OK or file
break
elif status in (301, 302): # redirects
url = curl.getinfo(pycurl.REDIRECT_URL)
log_debug(2, "Got redirect to %s" % url)
curl.setopt(pycurl.URL, url)
def main():
# Initialize a command-line processing object with a table of options
optionsTable = [
Option('-v', '--verbose', action='count', help='Increase verbosity'),
Option('-d',
'--dir',
action='store',
help='Process packages from this directory'),
Option('-c', '--channel', action='append', help='Manage this channel'),
Option('-n',
'--count',
action='store',
help='Process this number of headers per call',
type='int'),
Option('-l',
'--list',
action='store_true',
help='Only list the specified channels'),
Option('-s',
'--sync',
action='store_true',
help='Check if in sync with the server'),
Option('-p',
'--printconf',
action='store_true',
help='Print the configuration and exit'),
Option('-X',
'--exclude',
action="append",
help="Exclude packages that match this glob expression"),
Option('--newest',
action='store_true',
help='Only push the files that are newer than the server ones'),
Option('--stdin',
action='store_true',
help='Read the package names from stdin'),
Option('--nosig', action='store_true', help="Push unsigned packages"),
Option('--username',
action='store',
help='Use this username to connect to RHN'),
Option('--password',
action='store',
help='Use this password to connect to RHN'),
Option('--source',
action='store_true',
help='Upload source package headers'),
Option('--dontcopy',
action='store_true',
help='Do not copy packages to the local directory'),
Option('--copyonly',
action='store_true',
help="Only copy packages; don't reimport"),
Option('--test',
action='store_true',
help='Only print the packages to be pushed'),
Option('-N',
'--new-cache',
action='store_true',
help='Create a new username/password cache'),
Option('--no-ssl',
action='store_true',
help='Turn off SSL (not recommended).'),
Option('--no-session-caching',
action='store_true',
help='Disables session-token authentication.'),
Option('-?',
'--usage',
action='store_true',
help="Briefly describe the options"),
]
# Process the command line arguments
optionParser = OptionParser(option_list=optionsTable,
usage="USAGE: %prog [OPTION] [<package>]")
options, files = optionParser.parse_args()
upload = UploadClass(options, files=files)
if options.usage:
optionParser.print_usage()
sys.exit(0)
if options.printconf:
CFG.show()
return
if options.list:
upload.list()
return
if options.sync:
upload.checkSync()
return
if options.copyonly:
upload.copyonly()
return
if options.dir:
upload.directory()
elif options.stdin:
upload.readStdin()
if options.exclude:
upload.filter_excludes()
if options.newest:
upload.newest()
if not upload.files:
upload.die(0, "Nothing to do; exiting. Try --help")
if options.test:
upload.test()
return
try:
upload.uploadHeaders()
except UploadError, e:
sys.stderr.write("Upload error: %s\n" % e)
def main():
# Initialize a command-line processing object with a table of options
optionsTable = [
Option('-v', '--verbose', action='count', help='Increase verbosity'),
Option('-d', '--dir', action='store', help='Process packages from this directory'),
Option('-L', '--cache-locally', action='store_true',
help='Locally cache packages so that Proxy will not ever need to '
+ 'download them. Changes nothing on the upstream server.'),
Option('-e', '--from-export', action='store', dest='export_location',
help='Process packages from this channel export. Can only be used '
+ 'with --cache-locally or --copyonly.'),
Option('-c', '--channel', action='append',
help='Channel to operate on. When used with --from-export '
+ 'specifies channels to cache rpms for, else specifies channels '
+ 'that we will be pushing into.'),
Option('-n', '--count', action='store', help='Process this number of headers per call', type='int'),
Option('-l', '--list', action='store_true', help='Only list the specified channels'),
Option('-s', '--sync', action='store_true', help='Check if in sync with the server'),
Option('-p', '--printconf', action='store_true', help='Print the configuration and exit'),
Option('-X', '--exclude', action="append", help="Exclude packages that match this glob expression"),
Option('--newest', action='store_true', help='Only push the files that are newer than the server ones'),
Option('--stdin', action='store_true', help='Read the package names from stdin'),
Option('--nosig', action='store_true', help="Push unsigned packages"),
Option('--username', action='store', help='Use this username to connect to RHN'),
Option('--password', action='store', help='Use this password to connect to RHN'),
Option('--source', action='store_true', help='Upload source package headers'),
Option('--dontcopy', action='store_true', help='Do not copy packages to the local directory'),
Option('--copyonly', action='store_true',
help="Only copy packages; don't reimport. Same as --cache-locally"),
Option('--test', action='store_true', help='Only print the packages to be pushed'),
Option('-N', '--new-cache', action='store_true', help='Create a new username/password cache'),
Option('--no-ssl', action='store_true', help='Turn off SSL (not recommended).'),
Option('--no-session-caching', action='store_true',
help='Disables session-token authentication.'),
Option('-?', '--usage', action='store_true', help="Briefly describe the options"),
]
# Process the command line arguments
optionParser = OptionParser(option_list=optionsTable, usage="USAGE: %prog [OPTION] [<package>]")
options, files = optionParser.parse_args()
upload = UploadClass(options, files=files)
if options.usage:
optionParser.print_usage()
sys.exit(0)
if options.printconf:
CFG.show()
return
if options.list:
upload.list()
return
if options.sync:
upload.checkSync()
return
# It's just an alias to copyonly
if options.cache_locally:
options.copyonly = True
# remeber to process dir option before export, export can overwrite dir
if options.dir:
upload.directory()
if options.export_location:
if not options.copyonly:
upload.die(0, "--from-export can only be used with --cache-locally"
+ " or --copyonly")
if options.source:
upload.die(0, "--from-export cannot be used with --source")
upload.from_export()
if options.stdin:
upload.readStdin()
# if we're going to allow the user to specify packages by dir *and* export
# *and* stdin *and* package list (why not?) then we have to uniquify
# the list afterwards. Sort just for user-friendly display.
upload.files = sorted(list(set(upload.files)))
if options.copyonly:
if not upload.files:
upload.die(0, "Nothing to do; exiting. Try --help")
if options.test:
upload.test()
return
upload.copyonly()
return
if options.exclude:
upload.filter_excludes()
if options.newest:
upload.newest()
if not upload.files:
upload.die(0, "Nothing to do; exiting. Try --help")
if options.test:
upload.test()
return
try:
upload.uploadHeaders()
except UploadError, e:
sys.stderr.write("Upload error: %s\n" % e)
def main():
# Initialize a command-line processing object with a table of options
optionsTable = [
Option('-v', '--verbose', action='count', help='Increase verbosity'),
Option('-d', '--dir', action='store', help='Process packages from this directory'),
Option('-L', '--cache-locally', action='store_true',
help='Locally cache packages so that Proxy will not ever need to '
+ 'download them. Changes nothing on the upstream server.'),
Option('-e', '--from-export', action='store', dest='export_location',
help='Process packages from this channel export. Can only be used '
+ 'with --cache-locally or --copyonly.'),
Option('-c', '--channel', action='append',
help='Channel to operate on. When used with --from-export '
+ 'specifies channels to cache rpms for, else specifies channels '
+ 'that we will be pushing into.'),
Option('-n', '--count', action='store', help='Process this number of headers per call', type='int'),
Option('-l', '--list', action='store_true', help='Only list the specified channels'),
Option('-s', '--sync', action='store_true', help='Check if in sync with the server'),
Option('-p', '--printconf', action='store_true', help='Print the configuration and exit'),
Option('-X', '--exclude', action="append", help="Exclude packages that match this glob expression"),
Option('--newest', action='store_true', help='Only push the files that are newer than the server ones'),
Option('--stdin', action='store_true', help='Read the package names from stdin'),
Option('--nosig', action='store_true', help="Push unsigned packages"),
Option('--username', action='store', help='Use this username to connect to RHN'),
Option('--password', action='store', help='Use this password to connect to RHN'),
Option('--source', action='store_true', help='Upload source package headers'),
Option('--dontcopy', action='store_true', help='Do not copy packages to the local directory'),
Option('--copyonly', action='store_true',
help="Only copy packages; don't reimport. Same as --cache-locally"),
Option('--test', action='store_true', help='Only print the packages to be pushed'),
Option('-N', '--new-cache', action='store_true', help='Create a new username/password cache'),
Option('--no-ssl', action='store_true', help='Turn off SSL (not recommended).'),
Option('--no-session-caching', action='store_true',
help='Disables session-token authentication.'),
Option('-?', '--usage', action='store_true', help="Briefly describe the options"),
]
# Process the command line arguments
optionParser = OptionParser(option_list=optionsTable, usage="USAGE: %prog [OPTION] [<package>]")
options, files = optionParser.parse_args()
upload = UploadClass(options, files=files)
if options.usage:
optionParser.print_usage()
sys.exit(0)
if options.printconf:
CFG.show()
return
if options.list:
upload.list()
return
if options.sync:
upload.checkSync()
return
# It's just an alias to copyonly
if options.cache_locally:
options.copyonly = True
# remeber to process dir option before export, export can overwrite dir
if options.dir:
upload.directory()
if options.export_location:
if not options.copyonly:
upload.die(0, "--from-export can only be used with --cache-locally"
+ " or --copyonly")
if options.source:
upload.die(0, "--from-export cannot be used with --source")
upload.from_export()
if options.stdin:
upload.readStdin()
# if we're going to allow the user to specify packages by dir *and* export
# *and* stdin *and* package list (why not?) then we have to uniquify
# the list afterwards. Sort just for user-friendly display.
upload.files = sorted(list(set(upload.files)))
if options.copyonly:
if not upload.files:
upload.die(0, "Nothing to do; exiting. Try --help")
if options.test:
upload.test()
return
upload.copyonly()
return
if options.exclude:
upload.filter_excludes()
if options.newest:
upload.newest()
if not upload.files:
upload.die(0, "Nothing to do; exiting. Try --help")
if options.test:
upload.test()
return
try:
upload.uploadHeaders()
except UploadError, e:
sys.stderr.write("Upload error: %s\n" % e)
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False,
log_level=None, mount_point=None, consider_full=False):
self.cdn_repository_manager = CdnRepositoryManager(mount_point)
self.no_packages = no_packages
self.no_errata = no_errata
self.no_rpms = no_rpms
self.no_kickstarts = no_kickstarts
if log_level is None:
log_level = 0
self.log_level = log_level
if mount_point:
self.mount_point = "file://" + mount_point
self.consider_full = consider_full
else:
self.mount_point = CFG.CDN_ROOT
self.consider_full = True
CFG.set('DEBUG', log_level)
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Command: %s" % str(sys.argv))
rhnSQL.initDB()
initCFG('server.satellite')
verify_mappings()
f = None
# try block in try block - this is hack for python 2.4 compatibility
# to support finally
try:
try:
# Channel families mapping to channels
f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r')
self.families = json.load(f)
f.close()
# Channel metadata
f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r')
self.channel_metadata = json.load(f)
f.close()
# Dist/Release channel mapping
f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r')
self.channel_dist_mapping = json.load(f)
f.close()
# Kickstart metadata
f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r')
self.kickstart_metadata = json.load(f)
f.close()
except IOError:
e = sys.exc_info()[1]
raise CdnMappingsLoadError("Problem with loading file: %s" % e)
finally:
if f is not None:
f.close()
# Map channels to their channel family
self.channel_to_family = {}
for family in self.families:
for channel in self.families[family]['channels']:
self.channel_to_family[channel] = family
# Set already synced channels
h = rhnSQL.prepare("""
select label from rhnChannel where org_id is null
""")
h.execute()
channels = h.fetchall_dict() or []
self.synced_channels = [ch['label'] for ch in channels]
def __init__(self, server_address):
SocketServer.TCPServer.__init__(self, server_address, None)
self._next_poll_interval = None
self._notify_threshold = CFG.get('notify_threshold')
def __redirectToNextLocationNoRetry(self, loopProtection=False):
""" This function will perform a redirection to the next location, as
specified in the last response's "Location" header. This function will
return an actual HTTP response status code. If successful, it will
return apache.HTTP_OK, not apache.OK. If unsuccessful, this function
will simply return; no retries will be performed. The following error
codes can be returned:
HTTP_OK,HTTP_PARTIAL_CONTENT - Redirect successful.
HTTP_MOVED_TEMPORARILY - Redirect was redirected again by 3rd party.
HTTP_MOVED_PERMANENTLY - Redirect was redirected again by 3rd party.
HTTP_INTERNAL_SERVER_ERROR - Error extracting redirect information
HTTP_SERVICE_UNAVAILABLE - Could not connect to 3rd party server,
connection was reset, or a read error
occurred during communication.
HTTP_* - Any other HTTP status code may also be
returned.
Upon successful completion of this function, a new responseContext
will be created and pushed onto the stack.
"""
# Obtain the redirect location first before we replace the current
# response context. It's contained in the Location header of the
# previous response.
redirectLocation = self._get_header(rhnConstants.HEADER_LOCATION)
# We are about to redirect to a new location so now we'll push a new
# response context before we return any errors.
self.responseContext.add()
# There should always be a redirect URL passed back to us. If not,
# there's an error.
if not redirectLocation or len(redirectLocation) == 0:
log_error(" No redirect location specified!")
Traceback(mail=0)
return apache.HTTP_INTERNAL_SERVER_ERROR
# The _get_header function returns the value as a list. There should
# always be exactly one location specified.
redirectLocation = redirectLocation[0]
log_debug(1, " Redirecting to: ", redirectLocation)
# Tear apart the redirect URL. We need the scheme, the host, the
# port (if not the default), and the URI.
_scheme, host, port, uri = self._parse_url(redirectLocation)
# Add any params onto the URI since _parse_url doesn't include them.
if redirectLocation.find('?') > -1:
uri += redirectLocation[redirectLocation.index('?'):]
# Now create a new connection. We'll use SSL if configured to do
# so.
params = {
'host': host,
'port': port,
}
if CFG.has_key('timeout'):
params['timeout'] = CFG.TIMEOUT
if CFG.USE_SSL:
log_debug(1, " Redirecting with SSL. Cert= ", self.caChain)
params['trusted_certs'] = [self.caChain]
connection = connections.HTTPSConnection(**params)
else:
log_debug(1, " Redirecting withOUT SSL.")
connection = connections.HTTPConnection(**params)
# Put the connection into the current response context.
self.responseContext.setConnection(connection)
# Now open the connection to the 3rd party server.
log_debug(4, "Attempting to connect to 3rd party server...")
try:
connection.connect()
except socket.error, e:
log_error("Error opening redirect connection", redirectLocation, e)
Traceback(mail=0)
return apache.HTTP_SERVICE_UNAVAILABLE
def main():
# Initialize a command-line processing object with a table of options
optionsTable = [
Option('-v','--verbose', action='count', help='Increase verbosity'),
Option('-d','--dir', action='store', help='Process packages from this directory'),
Option('-c','--channel', action='append', help='Manage this channel'),
Option('-n','--count', action='store', help='Process this number of headers per call', type='int'),
Option('-l','--list', action='store_true', help='Only list the specified channels'),
Option('-s','--sync', action='store_true', help='Check if in sync with the server'),
Option('-p','--printconf', action='store_true', help='Print the configuration and exit'),
Option('-X','--exclude', action="append", help="Exclude packages that match this glob expression"),
Option( '--newest', action='store_true', help='Only push the files that are newer than the server ones'),
Option( '--stdin', action='store_true', help='Read the package names from stdin'),
Option( '--nosig', action='store_true', help="Push unsigned packages"),
Option( '--username', action='store', help='Use this username to connect to RHN'),
Option( '--password', action='store', help='Use this password to connect to RHN'),
Option( '--source', action='store_true', help='Upload source package headers'),
Option( '--dontcopy', action='store_true', help='Do not copy packages to the local directory'),
Option( '--copyonly', action='store_true', help="Only copy packages; don't reimport"),
Option( '--test', action='store_true', help='Only print the packages to be pushed'),
Option('-N','--new-cache', action='store_true', help='Create a new username/password cache'),
Option( '--no-ssl', action='store_true', help='Turn off SSL (not recommended).'),
Option( '--no-session-caching', action='store_true',
help='Disables session-token authentication.'),
Option('-?','--usage', action='store_true', help="Briefly describe the options"),
]
# Process the command line arguments
optionParser = OptionParser(option_list=optionsTable, usage="USAGE: %prog [OPTION] [<package>]")
options, files = optionParser.parse_args()
upload = UploadClass(options, files=files)
if options.usage:
optionParser.print_usage()
sys.exit(0)
if options.printconf:
CFG.show()
return
if options.list:
upload.list()
return
if options.sync:
upload.checkSync()
return
if options.copyonly:
upload.copyonly()
return
if options.dir:
upload.directory()
elif options.stdin:
upload.readStdin()
if options.exclude:
upload.filter_excludes()
if options.newest:
upload.newest()
if not upload.files:
upload.die(0, "Nothing to do; exiting. Try --help")
if options.test:
upload.test()
return
try:
upload.uploadHeaders()
except UploadError, e:
sys.stderr.write("Upload error: %s\n" % e)
def __init__(self, no_packages=False, no_errata=False, no_rpms=False, no_kickstarts=False,
log_level=None, mount_point=None, consider_full=False, force_kickstarts=False,
force_all_errata=False, email=False, import_batch_size=None):
if log_level is None:
log_level = 0
self.log_level = log_level
CFG.set('DEBUG', log_level)
self.email = email
if self.email:
initEMAIL_LOG()
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Command: %s" % str(sys.argv))
rhnSQL.initDB()
initCFG('server.satellite')
self.cdn_repository_manager = CdnRepositoryManager(mount_point)
self.no_packages = no_packages
self.no_errata = no_errata
self.no_rpms = no_rpms
if self.no_packages and self.no_rpms:
log(0, "Parameter --no-rpms has no effect.")
self.no_kickstarts = no_kickstarts
self.force_all_errata = force_all_errata
self.force_kickstarts = force_kickstarts
if self.no_kickstarts and self.force_kickstarts:
log(0, "Parameter --force-kickstarts has no effect.")
if mount_point:
self.mount_point = "file://" + mount_point
self.consider_full = consider_full
else:
self.mount_point = CFG.CDN_ROOT
self.consider_full = True
verify_mappings()
f = None
# try block in try block - this is hack for python 2.4 compatibility
# to support finally
try:
try:
# Channel families mapping to channels
f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r')
self.families = json.load(f)
f.close()
# Channel metadata
f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r')
self.channel_metadata = json.load(f)
f.close()
# Dist/Release channel mapping
f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r')
self.channel_dist_mapping = json.load(f)
f.close()
# Kickstart metadata
f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r')
self.kickstart_metadata = json.load(f)
f.close()
except IOError:
e = sys.exc_info()[1]
log(1, "Ignoring channel mappings: %s" % e)
self.families = {}
self.channel_metadata = {}
self.channel_dist_mapping = {}
self.kickstart_metadata = {}
finally:
if f is not None:
f.close()
# Map channels to their channel family
self.channel_to_family = {}
for family in self.families:
for channel in self.families[family]['channels']:
self.channel_to_family[channel] = family
# Set already synced channels, entitled null-org channels and custom channels with associated
# CDN repositories
h = rhnSQL.prepare("""
select distinct c.label, c.org_id
from rhnChannelFamilyPermissions cfp inner join
rhnChannelFamily cf on cfp.channel_family_id = cf.id inner join
rhnChannelFamilyMembers cfm on cf.id = cfm.channel_family_id inner join
rhnChannel c on cfm.channel_id = c.id
where c.org_id is null
or (c.org_id is not null and
exists (
select cs.id
from rhnContentSource cs inner join
rhnChannelContentSource ccs on ccs.source_id = cs.id
where ccs.channel_id = c.id
and cs.org_id is null
)
)
order by c.org_id nulls first, label
""")
h.execute()
channels = h.fetchall_dict() or []
self.synced_channels = {}
for channel in channels:
# Custom channel repositories not available, don't mark as synced
if channel['org_id']:
repos = self.cdn_repository_manager.list_associated_repos(channel['label'])
if not all([self.cdn_repository_manager.check_repository_availability(r) for r in repos]):
continue
self.synced_channels[channel['label']] = channel['org_id']
# Select available channel families from DB
h = rhnSQL.prepare("""
select distinct label
from rhnChannelFamilyPermissions cfp inner join
rhnChannelFamily cf on cfp.channel_family_id = cf.id
where cf.org_id is null
""")
h.execute()
families = h.fetchall_dict() or []
self.entitled_families = [f['label'] for f in families]
self.import_batch_size = import_batch_size
def __init__(self,
no_packages=False,
no_errata=False,
no_rpms=False,
no_kickstarts=False,
log_level=None):
self.no_packages = no_packages
self.no_errata = no_errata
self.no_rpms = no_rpms
self.no_kickstarts = no_kickstarts
if log_level is None:
log_level = 0
self.log_level = log_level
CFG.set('DEBUG', log_level)
rhnLog.initLOG(self.log_path, self.log_level)
log2disk(0, "Command: %s" % str(sys.argv))
rhnSQL.initDB()
initCFG('server.satellite')
f = None
# try block in try block - this is hack for python 2.4 compatibility
# to support finally
try:
try:
# Channel families mapping to channels
f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r')
self.families = json.load(f)
f.close()
# Channel metadata
f = open(constants.CHANNEL_DEFINITIONS_PATH, 'r')
self.channel_metadata = json.load(f)
f.close()
# Dist/Release channel mapping
f = open(constants.CHANNEL_DIST_MAPPING_PATH, 'r')
self.channel_dist_mapping = json.load(f)
f.close()
# Channel to repositories mapping
f = open(constants.CONTENT_SOURCE_MAPPING_PATH, 'r')
self.content_source_mapping = json.load(f)
f.close()
# Kickstart metadata
f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r')
self.kickstart_metadata = json.load(f)
f.close()
# Channel to kickstart repositories mapping
f = open(constants.KICKSTART_SOURCE_MAPPING_PATH, 'r')
self.kickstart_source_mapping = json.load(f)
f.close()
except IOError:
e = sys.exc_info()[1]
log2stderr(0, "ERROR: Problem with loading file: %s" % e)
raise CdnMappingsLoadError()
finally:
if f is not None:
f.close()
# Map channels to their channel family
self.channel_to_family = {}
for family in self.families:
for channel in self.families[family]['channels']:
self.channel_to_family[channel] = family
# Set already synced channels
h = rhnSQL.prepare("""
select label from rhnChannel where org_id is null
""")
h.execute()
channels = h.fetchall_dict() or []
self.synced_channels = [ch['label'] for ch in channels]
# Set SSL-keys for channel family
self.family_keys = {}
def __redirectToNextLocationNoRetry(self, loopProtection=False):
""" This function will perform a redirection to the next location, as
specified in the last response's "Location" header. This function will
return an actual HTTP response status code. If successful, it will
return apache.HTTP_OK, not apache.OK. If unsuccessful, this function
will simply return; no retries will be performed. The following error
codes can be returned:
HTTP_OK,HTTP_PARTIAL_CONTENT - Redirect successful.
HTTP_MOVED_TEMPORARILY - Redirect was redirected again by 3rd party.
HTTP_MOVED_PERMANENTLY - Redirect was redirected again by 3rd party.
HTTP_INTERNAL_SERVER_ERROR - Error extracting redirect information
HTTP_SERVICE_UNAVAILABLE - Could not connect to 3rd party server,
connection was reset, or a read error
occurred during communication.
HTTP_* - Any other HTTP status code may also be
returned.
Upon successful completion of this function, a new responseContext
will be created and pushed onto the stack.
"""
# Obtain the redirect location first before we replace the current
# response context. It's contained in the Location header of the
# previous response.
redirectLocation = self._get_header(rhnConstants.HEADER_LOCATION)
# We are about to redirect to a new location so now we'll push a new
# response context before we return any errors.
self.responseContext.add()
# There should always be a redirect URL passed back to us. If not,
# there's an error.
if not redirectLocation:
log_error(" No redirect location specified!")
Traceback(mail=0)
return apache.HTTP_INTERNAL_SERVER_ERROR
# The _get_header function returns the value as a list. There should
# always be exactly one location specified.
redirectLocation = redirectLocation[0]
log_debug(1, " Redirecting to: ", redirectLocation)
# Tear apart the redirect URL. We need the scheme, the host, the
# port (if not the default), and the URI.
_scheme, host, port, uri, query = self._parse_url(redirectLocation)
# Add back the query string
if query:
uri += '?' + query
# Now create a new connection. We'll use SSL if configured to do
# so.
params = {
'host': host,
'port': port,
}
if CFG.has_key('timeout'):
params['timeout'] = CFG.TIMEOUT
if CFG.USE_SSL:
log_debug(1, " Redirecting with SSL. Cert= ", self.caChain)
params['trusted_certs'] = [self.caChain]
connection = connections.HTTPSConnection(**params)
else:
log_debug(1, " Redirecting withOUT SSL.")
connection = connections.HTTPConnection(**params)
# Put the connection into the current response context.
self.responseContext.setConnection(connection)
# Now open the connection to the 3rd party server.
log_debug(4, "Attempting to connect to 3rd party server...")
try:
connection.connect()
except socket.error as e:
log_error("Error opening redirect connection", redirectLocation, e)
Traceback(mail=0)
return apache.HTTP_SERVICE_UNAVAILABLE
log_debug(4, "Connected to 3rd party server:",
connection.sock.getpeername())
# Put the request out on the wire.
response = None
try:
# We'll redirect to the URI made in the original request, but with
# the new server instead.
log_debug(4, "Making request: ", self.req.method, uri)
connection.putrequest(self.req.method, uri)
# Add some custom headers.
if loopProtection:
connection.putheader(rhnConstants.HEADER_RHN_REDIRECT, '0')
log_debug(4, " Adding original URL header: ", self.rhnParent)
connection.putheader(rhnConstants.HEADER_RHN_ORIG_LOC,
self.rhnParent)
# Add all the other headers in the original request in case we
# need to re-authenticate with Hosted.
for hdr in list(self.req.headers_in.keys()):
if hdr.lower().startswith("x-rhn"):
connection.putheader(hdr, self.req.headers_in[hdr])
log_debug(4, "Passing request header: ", hdr,
self.req.headers_in[hdr])
connection.endheaders()
response = connection.getresponse()
except IOError as ioe:
# Raised by getresponse() if server closes connection on us.
log_error("Redirect connection reset by peer.", redirectLocation,
ioe)
Traceback(mail=0)
# The connection is saved in the current response context, and
# will be closed when the caller pops the context.
return apache.HTTP_SERVICE_UNAVAILABLE
except socket.error as se:
# Some socket error occurred. Possibly a read error.
log_error("Redirect request failed.", redirectLocation, se)
Traceback(mail=0)
# The connection is saved in the current response context, and
# will be closed when the caller pops the context.
return apache.HTTP_SERVICE_UNAVAILABLE
# Save the response headers and body FD in the current communication
# context.
self.responseContext.setBodyFd(response)
self.responseContext.setHeaders(response.msg)
log_debug(4, "Response headers: ",
list(self.responseContext.getHeaders().items()))
log_debug(4, "Got redirect response. Status=", response.status)
# Return the HTTP status to the caller.
return response.status
def __init__(self, server_address):
SocketServer.TCPServer.__init__(self, server_address, None)
self._next_poll_interval = None
self._notify_threshold = CFG.get('notify_threshold')
def processCommandline():
options = [
Option('--sanity-only',
action='store_true',
help="confirm certificate sanity. Does not activate " +
"the Red Hat Satellite locally or remotely."),
Option('--ignore-expiration',
action='store_true',
help='execute regardless of the expiration ' +
'of the RHN Certificate (not recommended).'),
Option('--ignore-version-mismatch',
action='store_true',
help='execute regardless of version ' +
'mismatch of existing and new certificate.'),
Option('-v',
'--verbose',
action='count',
help='be verbose ' +
'(accumulable: -vvv means "be *really* verbose").'),
Option('--dump-version',
action='store',
help="requested version of XML dump"),
Option('--manifest',
action='store',
help='the RHSM manifest path/filename to activate for CDN'),
Option('--rhn-cert',
action='store',
help='this option is deprecated, use --manifest instead'),
Option('--deactivate',
action='store_true',
help='deactivate CDN-activated Satellite'),
Option('--disconnected',
action='store_true',
help="activate locally, not subscribe to remote repository"),
Option('--manifest-info',
action='store_true',
help="show information about currently activated manifest"),
Option('--manifest-download',
action='store_true',
help="download new manifest from RHSM to temporary location"),
Option('--manifest-refresh',
action='store_true',
help="download new manifest from RHSM and activate it"),
Option('--manifest-reconcile-request',
action='store_true',
help="request regeneration of entitlement certificates")
]
parser = OptionParser(option_list=options)
options, args = parser.parse_args()
initCFG('server.satellite')
if options.verbose is None:
options.verbose = 0
CFG.set('DEBUG', options.verbose)
rhnLog.initLOG(LOG_PATH, options.verbose)
log2disk(0, "Command: %s" % str(sys.argv))
# we take no extra commandline arguments that are not linked to an option
if args:
writeError(
"These arguments make no sense in this context (try --help): %s" %
repr(args))
sys.exit(1)
# No need to check further if deactivating
if options.deactivate:
return options
if options.sanity_only:
options.disconnected = 1
if options.manifest_refresh:
options.manifest_download = 1
if CFG.DISCONNECTED and not options.disconnected:
msg = """Satellite server has been setup to run in disconnected mode.
Either correct server configuration in /etc/rhn/rhn.conf
or use --disconnected to activate it locally."""
writeError(msg)
sys.exit(1)
options.http_proxy = idn_ascii_to_puny(CFG.HTTP_PROXY)
options.http_proxy_username = CFG.HTTP_PROXY_USERNAME
options.http_proxy_password = CFG.HTTP_PROXY_PASSWORD
log(1, 'HTTP_PROXY: %s' % options.http_proxy)
log(1, 'HTTP_PROXY_USERNAME: %s' % options.http_proxy_username)
log(1, 'HTTP_PROXY_PASSWORD: <password>')
return options
def send(url, sendData=None):
"""Connect to url and return the result as stringIO
:arg url: the url where the request will be sent
:kwarg sendData: do a post-request when "sendData" is given.
Returns the result as stringIO object.
"""
connect_retries = 10
try_counter = connect_retries
timeout = 120
if CFG.is_initialized() and CFG.has_key('TIMEOUT'):
timeout = CFG.TIMEOUT
curl = pycurl.Curl()
curl.setopt(pycurl.CONNECTTIMEOUT, timeout)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug)
curl.setopt(pycurl.VERBOSE, True)
proxy_url, proxy_user, proxy_pass = get_proxy(url)
if proxy_url:
curl.setopt(pycurl.PROXY, proxy_url)
log_debug(2, "Connect to %s" % url)
if sendData is not None:
curl.setopt(pycurl.POSTFIELDS, sendData)
if (CFG.is_initialized() and CFG.has_key('DISABLE_EXPECT')
and CFG.DISABLE_EXPECT):
# disable Expect header
curl.setopt(pycurl.HTTPHEADER, ['Expect:'])
# We implement our own redirection-following, because pycurl
# 7.19 doesn't POST after it gets redirected. Ideally we'd be
# using pycurl.POSTREDIR here, but that's in 7.21.
curl.setopt(pycurl.FOLLOWLOCATION, False)
response = StringIO()
curl.setopt(pycurl.WRITEFUNCTION, response.write)
try_counter = connect_retries
while try_counter:
try_counter -= 1
try:
curl.perform()
except pycurl.error, e:
if e[0] == 56: # Proxy requires authentication
log_debug(2, e[1])
if not (proxy_user and proxy_pass):
raise TransferException("Proxy requires authentication, "
"but reading credentials from "
"%s failed." % YAST_PROXY)
curl.setopt(pycurl.PROXYUSERPWD,
"%s:%s" % (proxy_user, proxy_pass))
elif e[0] == 60:
log_error("Peer certificate could not be authenticated "
"with known CA certificates.")
raise TransferException("Peer certificate could not be "
"authenticated with known CA "
"certificates.")
else:
log_error(e[1])
raise
status = curl.getinfo(pycurl.HTTP_CODE)
if status == 200 or (URL(url).scheme == "file" and status == 0):
# OK or file
break
elif status in (301, 302): # redirects
url = curl.getinfo(pycurl.REDIRECT_URL)
log_debug(2, "Got redirect to %s" % url)
curl.setopt(pycurl.URL, url)
