def __init__(self, retries=3, log_obj=None, force=False):
self.queues = {}
comp = CFG.getComponent()
initCFG("server.satellite")
try:
self.threads = int(CFG.REPOSYNC_DOWNLOAD_THREADS)
except ValueError:
raise ValueError("Number of threads expected, found: '%s'" %
CFG.REPOSYNC_DOWNLOAD_THREADS)
try:
self.timeout = int(CFG.REPOSYNC_TIMEOUT)
except ValueError:
raise ValueError("Timeout in seconds expected, found: '%s'" %
CFG.REPOSYNC_TIMEOUT)
try:
self.minrate = int(CFG.REPOSYNC_MINRATE)
except ValueError:
raise ValueError(
"Minimal transfer rate in bytes pre second expected, found: '%s'"
% CFG.REPOSYNC_MINRATE)
if self.threads < 1:
raise ValueError("Invalid number of threads: %d" % self.threads)
initCFG(comp)
self.retries = retries
self.log_obj = log_obj
self.force = force
self.lock = Lock()
self.exception = None
# WORKAROUND - BZ #1439758 - ensure first item in queue is performed alone to properly setup NSS
self.first_in_queue_done = False
self.first_in_queue_lock = Lock()
def _load_proxy_settings(self, url):
# read the proxy configuration in /etc/rhn/rhn.conf
comp = CFG.getComponent()
initCFG('server.satellite')
# Get the global HTTP Proxy settings from DB or per-repo
# settings on /etc/rhn/spacewalk-repo-sync/zypper.conf
if CFG.http_proxy:
self.proxy_url, self.proxy_user, self.proxy_pass = get_proxy(url)
self.proxy_hostname = self.proxy_url
elif os.path.isfile(REPOSYNC_ZYPPER_CONF):
zypper_cfg = configparser.ConfigParser()
zypper_cfg.read_file(open(REPOSYNC_ZYPPER_CONF))
section_name = None
if zypper_cfg.has_section(self.name):
section_name = self.name
elif zypper_cfg.has_section(channel_label):
section_name = channel_label
elif zypper_cfg.has_section('main'):
section_name = 'main'
if section_name:
if zypper_cfg.has_option(section_name, option='proxy'):
self.proxy_hostname = zypper_cfg.get(section_name, option='proxy')
self.proxy_url = "http://%s" % self.proxy_hostname
if zypper_cfg.has_option(section_name, 'proxy_username'):
self.proxy_user = zypper_cfg.get(section_name, 'proxy_username')
if zypper_cfg.has_option(section_name, 'proxy_password'):
self.proxy_pass = zypper_cfg.get(section_name, 'proxy_password')
# set config component back to original
initCFG(comp)
def getNotificationsTypeDisabled():
"""Return list of types which are disabled"""
disabledTypes = []
comp = CFG.getComponent()
initCFG("java")
if CFG.notifications_type_disabled:
disabledTypes = CFG.notifications_type_disabled.split(",")
initCFG(comp)
return disabledTypes
def __init__(self,
url,
name,
insecure=False,
interactive=True,
yumsrc_conf=None,
org="1",
channel_label="",
no_mirrors=False,
ca_cert_file=None,
client_cert_file=None,
client_key_file=None):
# pylint: disable=W0613
self.url = url
self.name = name
if org:
self.org = org
else:
self.org = "NULL"
comp = CFG.getComponent()
# read the proxy configuration in /etc/rhn/rhn.conf
initCFG('server.satellite')
self.proxy_addr, self.proxy_user, self.proxy_pass = get_proxy(self.url)
self.authtoken = None
# Replace non-valid characters from reponame (only alphanumeric chars allowed)
self.reponame = "".join([x if x.isalnum() else "_" for x in self.name])
self.channel_label = channel_label
# SUSE vendor repositories belongs to org = NULL
# The repository cache root will be "/var/cache/rhn/reposync/REPOSITORY_LABEL/"
root = os.path.join(CACHE_DIR, str(org or "NULL"), self.reponame)
self.repo = DebRepo(url,
root,
os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR,
self.org, 'stage'),
self.proxy_addr,
self.proxy_user,
self.proxy_pass,
gpg_verify=not (insecure))
self.repo.verify()
self.num_packages = 0
self.num_excluded = 0
# keep authtokens for mirroring
(_scheme, _netloc, _path, query, _fragid) = urlparse.urlsplit(url)
if query:
self.authtoken = query
initCFG(comp)
def _read_config():
# we want to change the logging file to 'audit' and set it back
# after we finished reading the config file
# TODO Changing the component twice on every request is not nice
comp = CFG.getComponent()
initCFG("audit")
enabled = CFG.get("enabled")
server_url = CFG.get("server", "")
# XXX haven't tested what happens if it's not set back to the original value
initCFG(comp)
return (enabled, server_url)
def _useProxyFor(url):
"""Return True if a proxy should be used for given url, otherwise False.
This function uses server.satellite.no_proxy variable to check for
hosts or domains which should not be connected via a proxy.
server.satellite.no_proxy is a comma seperated list.
Either an exact match, or the previous character
is a '.', so host is within the same domain.
A leading '.' in the pattern is ignored.
See also 'man curl'
"""
u = urlparse.urlsplit(url)
# pylint can't see inside the SplitResult class
# pylint: disable=E1103
if u.scheme == 'file':
return False
hostname = u.hostname.lower()
if hostname in ["localhost", "127.0.0.1", "::1"]:
return False
comp = CFG.getComponent()
if not CFG.has_key("no_proxy"):
initCFG("server.satellite")
if not CFG.has_key('no_proxy'):
initCFG(comp)
return True
noproxy = CFG.no_proxy
initCFG(comp)
if not noproxy:
return True
if not isinstance(noproxy, list):
if noproxy == '*':
# just an asterisk disables all.
return False
noproxy = [noproxy]
# No proxy: Either an exact match, or the previous character
# is a '.', so host is within the same domain.
# A leading '.' in the pattern is ignored. Some implementations
# need '.foo.ba' to prevent 'foo.ba' from matching 'xfoo.ba'.
for domain in noproxy:
domain = domain.lower()
if domain[0] == '.':
domain = domain[1:]
if hostname.endswith(domain) and \
(len(hostname) == len(domain) or
hostname[len(hostname) - len(domain) - 1] == '.'):
return False
return True
def _useProxyFor(url):
"""Return True if a proxy should be used for given url, otherwise False.
This function uses server.satellite.no_proxy variable to check for
hosts or domains which should not be connected via a proxy.
server.satellite.no_proxy is a comma seperated list.
Either an exact match, or the previous character
is a '.', so host is within the same domain.
A leading '.' in the pattern is ignored.
See also 'man curl'
"""
u = urlparse.urlsplit(url)
# pylint can't see inside the SplitResult class
# pylint: disable=E1103
if u.scheme == 'file':
return False
hostname = u.hostname.lower()
if hostname in ["localhost", "127.0.0.1", "::1"]:
return False
comp = CFG.getComponent()
if not CFG.has_key("no_proxy"):
initCFG("server.satellite")
if not CFG.has_key('no_proxy'):
initCFG(comp)
return True
noproxy = CFG.no_proxy
initCFG(comp)
if not noproxy:
return True
if not isinstance(noproxy, list):
if noproxy == '*':
# just an asterisk disables all.
return False
noproxy = [noproxy]
# No proxy: Either an exact match, or the previous character
# is a '.', so host is within the same domain.
# A leading '.' in the pattern is ignored. Some implementations
# need '.foo.ba' to prevent 'foo.ba' from matching 'xfoo.ba'.
for domain in noproxy:
domain = domain.lower()
if domain[0] == '.':
domain = domain[1:]
if hostname.endswith(domain) and \
(len(hostname) == len(domain) or
hostname[len(hostname) - len(domain) - 1] == '.'):
return False
return True
def __init__(self,
url,
name,
insecure=False,
interactive=True,
yumsrc_conf=None,
org="1",
channel_label="",
no_mirrors=False,
ca_cert_file=None,
client_cert_file=None,
client_key_file=None):
# pylint: disable=W0613
self.url = url
self.name = name
if org:
self.org = org
else:
self.org = "NULL"
comp = CFG.getComponent()
# read the proxy configuration in /etc/rhn/rhn.conf
initCFG('server.satellite')
self.proxy_addr, self.proxy_user, self.proxy_pass = get_proxy(self.url)
self.authtoken = None
self.repo = DebRepo(url,
os.path.join(CACHE_DIR, self.org, name),
os.path.join(CFG.MOUNT_POINT, CFG.PREPENDED_DIR,
self.org, 'stage'),
self.proxy_addr,
self.proxy_user,
self.proxy_pass,
gpg_verify=not (insecure))
self.repo.verify()
self.num_packages = 0
self.num_excluded = 0
# keep authtokens for mirroring
(_scheme, _netloc, _path, query, _fragid) = urlparse.urlsplit(url)
if query:
self.authtoken = query
initCFG(comp)
def _get_proxy_from_rhn_conf():
"""Return a tuple of (url, user, pass) proxy information from rhn config
Returns None instead of a tuple if there was no proxy url. user,
pass can be None.
"""
comp = CFG.getComponent()
if not CFG.has_key("http_proxy"):
initCFG("server.satellite")
result = None
if CFG.http_proxy:
# CFG.http_proxy format is <hostname>[:<port>] in 1.7
url = 'http://%s' % CFG.http_proxy
result = (url, CFG.http_proxy_username, CFG.http_proxy_password)
initCFG(comp)
log_debug(2, "Could not read proxy URL from rhn config.")
return result
def _get_proxy_from_rhn_conf():
"""Return a tuple of (url, user, pass) proxy information from rhn config
Returns None instead of a tuple if there was no proxy url. user,
pass can be None.
"""
comp = CFG.getComponent()
if not CFG.has_key("http_proxy"):
initCFG("server.satellite")
result = None
if CFG.http_proxy:
# CFG.http_proxy format is <hostname>[:<port>] in 1.7
url = 'http://%s' % CFG.http_proxy
result = (url, CFG.http_proxy_username, CFG.http_proxy_password)
initCFG(comp)
log_debug(2, "Could not read proxy URL from rhn config.")
return result
def get_mirror_credentials():
"""Return a list of mirror credential tuples (user, pass)
N.B. The config values will be read from the global configuration:
server.susemanager.mirrcred_user
server.susemanager.mirrcred_pass
server.susemanager.mirrcred_user_1
server.susemanager.mirrcred_pass_1
etc.
The credentials are read sequentially, when the first value is found
to be missing, the process is aborted and the list of credentials
that have been read so far are returned. For example if
server.susemanager.mirrcred_pass_1 can not be read, only the first
pair of default mirrcreds will be returned, even though
mirrcred_user_2, mirrcred_pass_2 etc. might still exist.
"""
comp = CFG.getComponent()
initCFG("server.susemanager")
creds = []
# the default values should at least always be there
if not CFG["mirrcred_user"] or not CFG["mirrcred_pass"]:
initCFG(comp)
raise ConfigParserError("Could not read default mirror credentials: "
"server.susemanager.mirrcred_user, "
"server.susemanager.mirrcred_pass.")
creds.append((CFG["mirrcred_user"], CFG["mirrcred_pass"]))
# increment the credentials number, until we can't read one
n = 1
while True:
try:
creds.append((CFG["mirrcred_user_%s" % n],
CFG["mirrcred_pass_%s" % n]))
except (KeyError, AttributeError):
break
n += 1
initCFG(comp)
return creds
def get_mirror_credentials():
"""Return a list of mirror credential tuples (user, pass)
N.B. The config values will be read from the global configuration:
server.susemanager.mirrcred_user
server.susemanager.mirrcred_pass
server.susemanager.mirrcred_user_1
server.susemanager.mirrcred_pass_1
etc.
The credentials are read sequentially, when the first value is found
to be missing, the process is aborted and the list of credentials
that have been read so far are returned. For example if
server.susemanager.mirrcred_pass_1 can not be read, only the first
pair of default mirrcreds will be returned, even though
mirrcred_user_2, mirrcred_pass_2 etc. might still exist.
"""
comp = CFG.getComponent()
initCFG("server.susemanager")
creds = []
# the default values should at least always be there
if not CFG["mirrcred_user"] or not CFG["mirrcred_pass"]:
initCFG(comp)
raise ConfigParserError("Could not read default mirror credentials: "
"server.susemanager.mirrcred_user, "
"server.susemanager.mirrcred_pass.")
creds.append((CFG["mirrcred_user"], CFG["mirrcred_pass"]))
# increment the credentials number, until we can't read one
n = 1
while True:
try:
creds.append(
(CFG["mirrcred_user_%s" % n], CFG["mirrcred_pass_%s" % n]))
except (KeyError, AttributeError):
break
n += 1
initCFG(comp)
return creds
def __init__(self, retries=3, log_obj=None, force=False):
self.queues = {}
comp = CFG.getComponent()
initCFG('server.satellite')
try:
self.threads = int(CFG.REPOSYNC_DOWNLOAD_THREADS)
except ValueError:
initCFG(comp)
raise ValueError("Number of threads expected, found: '%s'" %
CFG.REPOSYNC_DOWNLOAD_THREADS)
else:
initCFG(comp)
if self.threads < 1:
raise ValueError("Invalid number of threads: %d" % self.threads)
self.retries = retries
self.log_obj = log_obj
self.force = force
self.lock = Lock()
self.exception = None
# WORKAROUND - BZ #1439758 - ensure first item in queue is performed alone to properly setup NSS
self.first_in_queue_done = False
self.first_in_queue_lock = Lock()
def cfg_component(component, root=None, filename=None):
"""Context manager for rhnConfig.
:param comp: The configuration component to use in this context
:param root: Root directory location of configuration files, optional
:param filename: Configuration file, optional
There is a common pattern when using rhnConfig that consists of the following steps:
1. save current component: old = CFG.getComponent()
2. set CFG to another component: initCFG('my_component')
3. Read / Set configuration values
4. set CFG back to the previous component
This pattern can now be expressed using the ``with`` statement:
with cfg_component('my_component') as CFG:
print(CFG.my_value)
"""
previous = CFG.getComponent()
initCFG(component=component, root=root, filename=filename)
try:
yield CFG
finally:
initCFG(previous)
def __init__(self, url, name, insecure=False, interactive=True,
yumsrc_conf=None, org="1", channel_label="",
no_mirrors=False, ca_cert_file=None, client_cert_file=None,
client_key_file=None):
"""
Plugin constructor.
"""
# pylint: disable=W0613
if urlsplit(url).scheme:
self.url = url
else:
self.url = "file://%s" % url
self.name = name
self.insecure = insecure
self.interactive = interactive
self.org = org if org else "NULL"
self.proxy_hostname = None
self.proxy_url = None
self.proxy_user = None
self.proxy_pass = None
self.authtoken = None
self.sslcacert = ca_cert_file
self.sslclientcert = client_cert_file
self.sslclientkey = client_key_file
self.http_headers = {}
comp = CFG.getComponent()
# read configuration from /etc/rhn/rhn.conf
initCFG('server.satellite')
# ensure the config namespace will be switched back in any case
try:
# keep authtokens for mirroring
(_scheme, _netloc, _path, query, _fragid) = urlsplit(url)
if query:
self.authtoken = query
# load proxy configuration based on the url
self._load_proxy_settings(self.url)
# Get extra HTTP headers configuration from /etc/rhn/spacewalk-repo-sync/extra_headers.conf
if os.path.isfile(REPOSYNC_EXTRA_HTTP_HEADERS_CONF):
http_headers_cfg = configparser.ConfigParser()
http_headers_cfg.read_file(open(REPOSYNC_EXTRA_HTTP_HEADERS_CONF))
section_name = None
if http_headers_cfg.has_section(self.name):
section_name = self.name
elif http_headers_cfg.has_section(channel_label):
section_name = channel_label
elif http_headers_cfg.has_section('main'):
section_name = 'main'
if section_name:
for hdr in http_headers_cfg[section_name]:
self.http_headers[hdr] = http_headers_cfg.get(section_name, option=hdr)
# perform authentication if implemented
self._authenticate(url)
# Make sure baseurl ends with / and urljoin will work correctly
self.urls = [url]
if self.urls[0][-1] != '/':
self.urls[0] += '/'
# Replace non-valid characters from reponame (only alphanumeric chars allowed)
self.reponame = "".join([x if x.isalnum() else "_" for x in self.name])
self.channel_label = channel_label
# SUSE vendor repositories belongs to org = NULL
# The repository cache root will be "/var/cache/rhn/reposync/REPOSITORY_LABEL/"
root = os.path.join(CACHE_DIR, str(org or "NULL"), self.reponame)
self.repo = ZypperRepo(root=root, url=self.url, org=self.org)
self.num_packages = 0
self.num_excluded = 0
self.gpgkey_autotrust = None
self.groupsfile = None
# configure network connection
try:
# bytes per second
self.minrate = int(CFG.REPOSYNC_MINRATE)
except ValueError:
self.minrate = 1000
try:
# seconds
self.timeout = int(CFG.REPOSYNC_TIMEOUT)
except ValueError:
self.timeout = 300
finally:
# set config component back to original
initCFG(comp)
def __init__(self,
url,
name,
insecure=False,
interactive=False,
yumsrc_conf=YUMSRC_CONF,
org="1",
channel_label="",
no_mirrors=True,
ca_cert_file=None,
client_cert_file=None,
client_key_file=None):
# insecure and interactive are not implemented for this module.
"""
Plugin constructor.
"""
name = re.sub('[^a-zA-Z0-9_.:-]+', '_', name)
if urlsplit(url).scheme:
self.url = url
else:
self.url = "file://%s" % url
self.name = name
self.insecure = insecure
self.interactive = interactive
self.org = org if org else "NULL"
self.proxy_hostname = None
self.proxy_url = None
self.proxy_user = None
self.proxy_pass = None
self.authtoken = None
self.sslcacert = ca_cert_file
self.sslclientcert = client_cert_file
self.sslclientkey = client_key_file
self.http_headers = {}
self.dnfbase = dnf.Base()
self.dnfbase.conf.read(yumsrc_conf)
if not os.path.exists(yumsrc_conf):
self.dnfbase.conf.read('/dev/null')
self.configparser = ConfigParser(
) # Reading config file directly as dnf only ready MAIN section.
self.configparser.setSubstitutions(dnf.Base().conf.substitutions)
self.configparser.read(yumsrc_conf)
self.dnfbase.conf.cachedir = os.path.join(CACHE_DIR, self.org)
# store the configuration and restore it at the end.
comp = CFG.getComponent()
# read the proxy configuration
# /etc/rhn/rhn.conf has more priority than yum.conf
initCFG('server.satellite')
# ensure the config namespace will be switched back in any case
try:
# keep authtokens for mirroring
(_scheme, _netloc, _path, query, _fragid) = urlsplit(url)
if query:
self.authtoken = query
# load proxy configuration based on the url
self._load_proxy_settings(self.url)
# perform authentication if implemented
self._authenticate(url)
# Check for settings in yum configuration files (for custom repos/channels only)
if org:
repos = self.dnfbase.repos
else:
repos = None
if repos and name in repos:
repo = repos[name]
elif repos and channel_label in repos:
repo = repos[channel_label]
# In case we are using Repo object based on channel config, override it's id to name of the repo
# To not create channel directories in cache directory
repo.id = name
else:
# Not using values from config files
repo = dnf.repo.Repo(name, self.dnfbase.conf)
repo.repofile = yumsrc_conf
# pylint: disable=W0212
repo._populate(self.configparser, name, yumsrc_conf)
self.repo = repo
self.yumbase = self.dnfbase # for compatibility
self.setup_repo(repo, no_mirrors, ca_cert_file, client_cert_file,
client_key_file)
self.num_packages = 0
self.num_excluded = 0
self.groupsfile = None
self.repo = self.dnfbase.repos[self.repoid]
self.get_metadata_paths()
finally:
# set config component back to original
initCFG(comp)
def __init__(self, url, name, insecure=False, interactive=True,
yumsrc_conf=None, org="1", channel_label="",
no_mirrors=False, ca_cert_file=None, client_cert_file=None,
client_key_file=None):
"""
Plugin constructor.
"""
# pylint: disable=W0613
if urlsplit(url).scheme:
self.url = url
else:
self.url = "file://%s" % url
self.name = name
self.insecure = insecure
self.interactive = interactive
self.org = org if org else "NULL"
self.proxy_hostname = None
self.proxy_url = None
self.proxy_user = None
self.proxy_pass = None
self.authtoken = None
self.sslcacert = ca_cert_file
self.sslclientcert = client_cert_file
self.sslclientkey = client_key_file
self.http_headers = {}
comp = CFG.getComponent()
# read configuration from /etc/rhn/rhn.conf
initCFG('server.satellite')
# keep authtokens for mirroring
(_scheme, _netloc, _path, query, _fragid) = urlsplit(url)
if query:
self.authtoken = query
# load proxy configuration based on the url
self._load_proxy_settings(self.url)
# Get extra HTTP headers configuration from /etc/rhn/spacewalk-repo-sync/extra_headers.conf
if os.path.isfile(REPOSYNC_EXTRA_HTTP_HEADERS_CONF):
http_headers_cfg = configparser.ConfigParser()
http_headers_cfg.read_file(open(REPOSYNC_EXTRA_HTTP_HEADERS_CONF))
section_name = None
if http_headers_cfg.has_section(self.name):
section_name = self.name
elif http_headers_cfg.has_section(channel_label):
section_name = channel_label
elif http_headers_cfg.has_section('main'):
section_name = 'main'
if section_name:
for hdr in http_headers_cfg[section_name]:
self.http_headers[hdr] = http_headers_cfg.get(section_name, option=hdr)
# perform authentication if implemented
self._authenticate(url)
# Make sure baseurl ends with / and urljoin will work correctly
self.urls = [url]
if self.urls[0][-1] != '/':
self.urls[0] += '/'
# Exclude non-valid characters from reponame
self.reponame = self.name
for chr in ["$", " ", ".", ";"]:
self.reponame = self.reponame.replace(chr, "_")
self.channel_label = channel_label
# SUSE vendor repositories belongs to org = NULL
root = os.path.join(CACHE_DIR, str(org or "NULL"), self.channel_label or self.reponame)
self.repo = ZypperRepo(root=root, url=self.url, org=self.org)
self.num_packages = 0
self.num_excluded = 0
self.gpgkey_autotrust = None
self.groupsfile = None
# set config component back to original
initCFG(comp)
