def runTest(self):
"""
Tests that verify the kavecommon library and simple functions in that library
"""
import kavecommon as kc
sources = []
for mirror in kc.mirrors():
sources.append(kc.repo_url("", repo=mirror))
sources.append(kc.repo_url(""))
kc.failover_source(sources)
self.assertRaises(IOError, kc.failover_source, ["non-existing-file-88989381923813.file"]) # ,"bug ABK-112"
# make some directory structure to check the permissions changing...
import tempfile
import os
tempdir = tempfile.mkdtemp()
os.system("mkdir -p " + tempdir + "/this/is/a/test1")
os.system("mkdir -p " + tempdir + "/this/is/a/test2")
os.system("mkdir -p " + tempdir + "/this/is/a/test3")
os.system("chmod -R 555 " + tempdir + "/this ")
kc.chmod_up(tempdir + "/this/is/a/test1", "511", seen=[tempdir, tempdir + "/this"])
self.assertFalse(os.access(tempdir + "/this/is/a", os.W_OK), tempdir + " permissions settings failed")
kc.chmod_up(tempdir + "/this/is/a/test1", "744", seen=[tempdir])
self.assertTrue(os.access(tempdir + "/this/is/a", os.W_OK), tempdir + " permissions settings failed")
self.assertFalse(os.access(tempdir + "/this/is/a/test2", os.W_OK), tempdir + " permissions settings failed")
# create a file in this directory to test the copy/caching, at least for things without wget
os.system("touch " + tempdir + "/this/is/test.test")
topd = os.path.realpath(os.curdir)
os.chdir(tempdir)
kc.copy_or_cache(sources=[tempdir + "/this/is/test.test"], filename="test.test", cache_dir=tempdir + "/this/")
os.chdir(topd)
self.assertTrue(os.path.exists(tempdir + "/test.test") and os.path.exists(
tempdir + "/this/is/test.test") and os.path.exists(tempdir + "/this/test.test"),
tempdir + " copy/caching failed")
# Test the trueorfalse method
cnv = {'true': True, 'y': True, 'ye': True, 'yes': True,
'false': False, 'n': False, 'no': False, 'none': False,
' false': False, 'y ': True}
for k, v in cnv.iteritems():
self.assertTrue(kc.trueorfalse(k) == v)
self.assertTrue(kc.trueorfalse(k.upper()) == v)
self.assertRaises(TypeError, kc.trueorfalse, {})
self.assertRaises(TypeError, kc.trueorfalse, 'GAAAH')
# remove this temporary file when done
if os.path.exists(tempdir) and len(tempdir) > 4:
os.system("rm -rf " + tempdir)
def verify_bpjson(self, jbp, bp):
"""
Check that a given x.blueprint.json file is OK
jbp = the json dictionary
check that the jbp has the needed config params
"""
self.assertTrue("host_groups" in jbp, bp + " blueprint misses host_groups!")
self.assertTrue("Blueprints" in jbp, bp + " blueprint misses Blueprints directive!")
all_services = []
for hostgroup in jbp["host_groups"]:
all_services = all_services + [component["name"] for component in hostgroup["components"]]
supplied_configs = {}
if "configurations" in jbp:
for aconf in jbp["configurations"]:
for k, v in aconf.iteritems():
supplied_configs[k] = v
# find the parameters which must be forced for these services
required_configs = {}
known_services = find_services()
for service in all_services:
for sname, dir in known_services:
if service.split('_')[0] not in sname:
continue
cfg_name = glob.glob(dir + '/configuration/*.xml')[0]
# Load XML and parse
tree = ET.ElementTree(file=cfg_name)
for property in tree.getroot():
if property.tag != 'property':
continue
name = property.find('name').text
isRequired = (
'require-input' in property.attrib and
kc.trueorfalse(property.attrib['require-input'])
)
if isRequired:
try:
required_configs[cfg_name.split('/')[-1].split('.')[0]].append(name)
except KeyError:
required_configs[cfg_name.split('/')[-1].split('.')[0]] = [name]
missing = []
for k, v in required_configs.iteritems():
for req in v:
try:
supplied_configs[k][req]
except KeyError:
missing.append((k, req))
if len(missing):
print "Blueprint:", bp, jbp["Blueprints"]["blueprint_name"]
print "Services:", all_services
print "Required:", required_configs
print "Supplied:", supplied_configs
print "Missing:", missing
self.assertFalse(len(missing),
bp + " missing required configurations in default group! \n\t"
+ '\n\t'.join([str(x) for x in set(missing)]))
return True
def verify_bpjson(self, jbp, bp):
"""
Check that a given x.blueprint.json file is OK
jbp = the json dictionary
check that the jbp has the needed config params
"""
self.assertTrue("host_groups" in jbp, bp + " blueprint misses host_groups!")
self.assertTrue("Blueprints" in jbp, bp + " blueprint misses Blueprints directive!")
all_services = []
for hostgroup in jbp["host_groups"]:
all_services = all_services + [component["name"] for component in hostgroup["components"]]
supplied_configs = {}
if "configurations" in jbp:
for aconf in jbp["configurations"]:
for k, v in aconf.iteritems():
supplied_configs[k] = v
# find the parameters which must be forced for these services
required_configs = {}
known_services = find_services()
for service in all_services:
for sname, dir in known_services:
if service.split('_')[0] not in sname:
continue
cfg_name = glob.glob(dir + '/configuration/*.xml')[0]
# Load XML and parse
tree = ET.ElementTree(file=cfg_name)
for property in tree.getroot():
if property.tag != 'property':
continue
name = property.find('name').text
is_required = (
'require-input' in property.attrib and
kc.trueorfalse(property.attrib['require-input'])
)
if is_required:
try:
required_configs[cfg_name.split('/')[-1].split('.')[0]].append(name)
except KeyError:
required_configs[cfg_name.split('/')[-1].split('.')[0]] = [name]
missing = []
for k, v in required_configs.iteritems():
for req in v:
try:
supplied_configs[k][req]
except KeyError:
missing.append((k, req))
if len(missing):
print "Blueprint:", bp, jbp["Blueprints"]["blueprint_name"]
print "Services:", all_services
print "Required:", required_configs
print "Supplied:", supplied_configs
print "Missing:", missing
self.assertFalse(len(missing),
bp + " missing required configurations in default group! \n\t"
+ '\n\t'.join([str(x) for x in set(missing)]))
return True
hostname = config["hostname"]
gitlab_conf_file = "/etc/gitlab/gitlab.rb"
gitlab_ssl_port = kc.default("configurations/gitlab/gitlab_ssl_port", "443", kc.is_valid_port)
gitlab_url = kc.default("configurations/gitlab/gitlab_url", hostname, kc.is_valid_hostname)
unicorn_port = kc.default("configurations/gitlab/unicorn_port", "8080", kc.is_valid_port)
unicorn_interface = default("configurations/gitlab/unicorn_interface", '127.0.0.1')
if gitlab_url == 'hostname':
gitlab_url = hostname
if not gitlab_url:
raise Exception('gitlab_url set to an unusable value \'%s\'' % gitlab_url)
gitlab_signin_enabled = default('configurations/gitlab/gitlab_signin_enabled', 'true')
gitlab_signin_enabled = kc.trueorfalse(gitlab_signin_enabled)
gitlab_admin_password = config['configurations']['gitlab']['gitlab_admin_password']
Logger.sensitive_strings[gitlab_admin_password] = "[PROTECTED]"
restrict_public_projects = default('configurations/gitlab/restrict_public_projects', 'true')
restrict_public_projects = kc.trueorfalse(restrict_public_projects)
use_external_postgres = default('configurations/gitlab/use_external_postgres', 'false')
use_external_postgres = kc.trueorfalse(use_external_postgres)
postgres_database_name = default('configurations/gitlab/postgres_database_name', 'gitlabhq_production')
postgres_database_port = default('configurations/gitlab/postgres_database_port', '5432')
postgres_database_host = default('configurations/gitlab/postgres_database_host', '127.0.0.1')
postgres_database_user = default('configurations/gitlab/postgres_database_user', 'gitlab')
# doesn't work at the moment, check in gitlabs explicitly and throw an exception instead
ldap_group = ''
ldap_admin_group = ''
def runTest(self):
"""Check that properties are either forced to be entered or have a default set,
in case a default is set, check the same default exists in the params file"""
import os
import string
failingxmlfiles = {}
failingpyfiles = {}
defaults = {}
# First, part 1: check that all params are either required or have a default
# Also fill defaults with the list of defaults to check against the params file
for root, dirs, files in os.walk(os.path.realpath(__file__ + '/../../../')):
for f in [os.path.join(root, f) for f in files if f.endswith('.xml') and f not in self.skip]:
if "configuration" not in root:
continue
tree = ET.ElementTree(file=f)
for property in tree.getroot():
if property.tag != 'property':
continue
name = property.find('name').text
is_required = (
'require-input' in property.attrib and
kc.trueorfalse(property.attrib['require-input'])
)
has_default = False
for child in property:
if child.tag != 'value':
continue
if child.text is not None and len(child.text.strip()):
has_default = True
if not is_required:
try:
defaults[f][name] = child.text.strip()
except KeyError:
defaults[f] = {name: child.text.strip()}
break
if not has_default and not is_required:
failingxmlfiles[f] = name
self.assertEqual(len(failingxmlfiles), 0,
"Found " + str(len(failingxmlfiles))
+ " xml file missing defaults/required "
+ str(failingxmlfiles.keys())
+ " \n" + str(failingxmlfiles)
)
# Part 2: now check in the params file to see that the same default is set there
for root, dirs, files in os.walk(os.path.realpath(__file__ + '/../../../')):
for f in [os.path.join(root, f) for f in files if f == 'params.py' and f not in self.skip]:
confname = None
existingdefaultslist = []
for confname in defaults:
if f.startswith(confname[:confname.find('configuration')]):
existingdefaultslist.append(confname)
if not existingdefaultslist:
continue
for existingdefaults in existingdefaultslist:
configname = existingdefaults.split('/')[-1].lower().split('.')[0]
all_params = ""
with open(f) as fp:
all_params = fp.read()
all_params = self.rm_unchecked_chars(all_params)
for defaultp, defaultv in defaults[existingdefaults].iteritems():
if configname + '/' + defaultp in self.skip_prop:
continue
defaultvs = self.rm_unchecked_chars(defaultv)
search_string = 'default(configurations/' + configname + '/' + defaultp + ',' + defaultvs
if search_string not in all_params:
failingpyfiles[f + '/' + configname + '/' + defaultp] = search_string
# If the default is longer than 80 characters it' very difficult to debug, and best
# if I return just some substring instead
if len(search_string) > 80:
begin = 'default(configurations/' + configname + '/' + defaultp + ','
if begin not in all_params:
failingpyfiles[f + '/' + configname + '/' + defaultp] = search_string[:80] + '... )'
else:
# find the first non-matching character and return 80 chars including 10 before
search_string = 'default(configurations/' + configname + '/' + defaultp + ','
this_default = all_params[all_params.find(search_string) + len(search_string):]
print this_default[:10]
start = 0
for start in range(len(defaultvs)):
if len(this_default) < start:
start = 0
break
if defaultvs[start] != this_default[start]:
start = max(start - 10, 0)
break
failingpyfiles[f + '/' + configname + '/' + defaultp
] = ('( ... '
+ this_default[start:start + 80]
+ '... )'
)
self.assertEqual(len(failingpyfiles), 0,
"Found " + str(len(failingpyfiles))
+ " python params file missing defaults {file/service/property: missing_str_approx }"
+ " \n" + str(failingpyfiles).replace("',", "',\n")
)
ldap_bind_password = default('configurations/freeipa/ldap_bind_password', False)
if not ldap_bind_password or len(ldap_bind_password) < 8:
raise Exception('FreeIPA ldap_bind_password: \'%s\' isn\'t acceptable (min 8 char long)' % ldap_bind_password)
else:
Logger.sensitive_strings[ldap_bind_password] = "[PROTECTED]"
hostname_components = config["hostname"].split('.')
if len(hostname_components) < 3:
raise Exception('FreeIPA hostname is not a FQDN. installation not possible')
domain = '.'.join(hostname_components[1:])
realm = '.'.join(hostname_components[1:]).upper()
realm_ldap = 'dc=' + ',dc='.join(hostname_components[1:])
install_with_dns = default('configurations/freeipa/install_with_dns', True)
install_with_dns = kc.trueorfalse(install_with_dns)
default_shell = default('configurations/freeipa/default_shell', '/bin/bash')
# Only except IPv4 for now
forwarders = default('configurations/freeipa/forwarders', '8.8.8.8').split(',')
forwarders = [forwarder.strip() for forwarder in forwarders]
forwarders = [forwarder for forwarder in forwarders if re.match('\\d+\\.\\d+\\.\\d+\\.\\d+', forwarder)]
client_init_wait = default('configurations/freeipa/client_init_wait', 600)
all_hosts = default("/clusterHostInfo/all_hosts", [hostname])
ldap_bind_user = default('configurations/freeipa/ldap_bind_user', 'kave_bind_user')
ldap_bind_services = ['twiki', 'gitlab', 'jenkins']
initial_users_and_groups = default('configurations/freeipa/initial_users_and_groups', '{"Users": [], "Groups" : {}}')
import os
import kavecommon as kc
config = Script.get_config()
hostname = config["hostname"]
top_dir = kc.default("configurations/kavetoolbox/top_dir", "/opt/",
kc.is_valid_directory)
releaseversion = default('configurations/kavetoolbox/releaseversion',
"3.0-Beta")
alternative_download = default(
'configurations/kavetoolbox/alternative_download', "none")
ignore_missing_groups = default(
'configurations/kavetoolbox/ignore_missing_groups', "False")
ignore_missing_groups = kc.trueorfalse(ignore_missing_groups)
command_line_args = default('configurations/kavetoolbox/command_line_args',
"False")
try:
command_line_args = kc.trueorfalse(command_line_args)
except TypeError, ValueError:
if type(command_line_args) is str:
pass
else:
print "could not interpret value of command_line_args correctly"
raise
custom_install_template_default = """
# -------------------------------
import kavedefaults as cnf
cnf.li.InstallTopDir="{{top_dir}}"
hostname = config["hostname"]
gitlab_conf_file = "/etc/gitlab/gitlab.rb"
gitlab_port = kc.default("configurations/gitlab/gitlab_port", "80", kc.is_valid_port)
gitlab_url = kc.default("configurations/gitlab/gitlab_url", hostname, kc.is_valid_hostname)
unicorn_port = kc.default("configurations/gitlab/unicorn_port", "8080", kc.is_valid_port)
unicorn_interface = default("configurations/gitlab/unicorn_interface", '127.0.0.1')
if gitlab_url == 'hostname':
gitlab_url = hostname
if not gitlab_url:
raise Exception('gitlab_url set to an unusable value \'%s\'' % gitlab_url)
gitlab_signin_enabled = default('configurations/gitlab/gitlab_signin_enabled', 'true')
gitlab_signin_enabled = kc.trueorfalse(gitlab_signin_enabled)
gitlab_admin_password = config['configurations']['gitlab']['gitlab_admin_password']
Logger.sensitive_strings[gitlab_admin_password] = "[PROTECTED]"
restrict_public_projects = default('configurations/gitlab/restrict_public_projects', 'true')
restrict_public_projects = kc.trueorfalse(restrict_public_projects)
# postgre configuration in case it is already installed!
postgre_disabled = False
# doesn't work at the moment, check in gitlabs explicitly and throw an exception instead
ldap_group = ''
ldap_admin_group = ''
# ldap configuration
ldap_enabled = False
freeipa_host = default('/clusterHostInfo/freeipa_server_hosts', [False])[0]
if freeipa_host:
#
##############################################################################
from resource_management import *
from resource_management.core.system import System
import os
import kavecommon as kc
config = Script.get_config()
hostname = config["hostname"]
top_dir = kc.default("configurations/kavetoolbox/top_dir", "/opt/", kc.is_valid_directory)
releaseversion = default('configurations/kavetoolbox/releaseversion', "3.7-Beta")
alternative_download = default('configurations/kavetoolbox/alternative_download', "none")
ignore_missing_groups = default('configurations/kavetoolbox/ignore_missing_groups', "False")
ignore_missing_groups = kc.trueorfalse(ignore_missing_groups)
command_line_args = default('configurations/kavetoolbox/command_line_args', "False")
try:
command_line_args = kc.trueorfalse(command_line_args)
except TypeError, ValueError:
if type(command_line_args) is str:
pass
else:
print "could not interpret value of command_line_args correctly"
raise
kave_custom_environment = default('configurations/kavetoolbox/kave_custom_environment', """
# -------------------------------
PY4JSRC="/usr/hdp/current/spark2-client/python/lib/py4j-0.10.6-src.zip"
PYSPARK="/usr/hdp/current/spark2-client/python/lib/pyspark.zip"
if [ -f $PY4JSRC ]; then
raise Exception(
'FreeIPA ldap_bind_password: \'%s\' isn\'t acceptable (min 8 char long)'
% ldap_bind_password)
else:
Logger.sensitive_strings[ldap_bind_password] = "[PROTECTED]"
hostname_components = config["hostname"].split('.')
if len(hostname_components) < 3:
raise Exception(
'FreeIPA hostname is not a FQDN. installation not possible')
domain = '.'.join(hostname_components[1:])
realm = '.'.join(hostname_components[1:]).upper()
realm_ldap = 'dc=' + ',dc='.join(hostname_components[1:])
install_with_dns = default('configurations/freeipa/install_with_dns', True)
install_with_dns = kc.trueorfalse(install_with_dns)
default_shell = default('configurations/freeipa/default_shell', '/bin/bash')
pki_insecure_port = kc.default('configurations/freeipa/pki_insecure_port',
'8081', kc.is_valid_port)
pki_secure_port = kc.default('configurations/freeipa/pki_secure_port', '8444',
kc.is_valid_port)
# Only except IPv4 for now
forwarders = default('configurations/freeipa/forwarders', '8.8.8.8').split(',')
forwarders = [forwarder.strip() for forwarder in forwarders]
forwarders = [
forwarder for forwarder in forwarders
if re.match('\\d+\\.\\d+\\.\\d+\\.\\d+', forwarder)
]
# limitations under the License.
#
##############################################################################
from resource_management import *
from resource_management.core.system import System
import os
import random
import string
import kavecommon as kc
config = Script.get_config()
hostname = config["hostname"]
enable_pam_auth = default('configurations/hue/enable_pam_auth', 'True')
enable_pam_auth = kc.trueorfalse(enable_pam_auth)
server_user = default('configurations/hue/server_user', 'hue')
# Copied from knox configuration!!
namenode_hosts = default("/clusterHostInfo/namenode_host", None)
if type(namenode_hosts) is list:
namenode_host = namenode_hosts[0]
else:
namenode_host = namenode_hosts
has_namenode = (namenode_host is not None)
namenode_http_port = "50070"
namenode_rpc_port = "8020"
if has_namenode:
hostname = config["hostname"]
gitlab_conf_file = "/etc/gitlab/gitlab.rb"
gitlab_port = default("configurations/gitlab/gitlab_port", "80")
gitlab_url = default("configurations/gitlab/gitlab_url", hostname)
unicorn_port = default("configurations/gitlab/unicorn_port", "8080")
unicorn_interface = default("configurations/gitlab/unicorn_interface", '127.0.0.1')
if gitlab_url == 'hostname':
gitlab_url = hostname
if not gitlab_url:
raise Exception('gitlab_url set to an unusable value \'%s\'' % gitlab_url)
gitlab_signin_enabled = default('configurations/gitlab/gitlab_signin_enabled', 'true')
gitlab_signin_enabled = kc.trueorfalse(gitlab_signin_enabled)
gitlab_admin_password = config['configurations']['gitlab']['gitlab_admin_password']
Logger.sensitive_strings[gitlab_admin_password] = "[PROTECTED]"
restrict_public_projects = default('configurations/gitlab/restrict_public_projects', 'true')
restrict_public_projects = kc.trueorfalse(restrict_public_projects)
# postgre configuration in case it is already installed!
postgre_disabled = False
# doesn't work at the moment, check in gitlabs explicitly and throw an exception instead
ldap_group = ''
ldap_admin_group = ''
# ldap configuration
ldap_enabled = False
freeipa_host = default('/clusterHostInfo/freeipa_server_hosts', [False])[0]
if freeipa_host:
def runTest(self):
"""Check that properties are either forced to be entered or have a default set,
in case a default is set, check the same default exists in the params file"""
import os
import string
failingxmlfiles = {}
failingpyfiles = {}
defaults = {}
# First, part 1: check that all params are either required or have a default
# Also fill defaults with the list of defaults to check against the params file
for root, dirs, files in os.walk(os.path.realpath(__file__ + '/../../../')):
for f in [os.path.join(root, f) for f in files if f.endswith('.xml') and f not in self.skip]:
if "configuration" not in root:
continue
tree = ET.ElementTree(file=f)
for property in tree.getroot():
if property.tag != 'property':
continue
name = property.find('name').text
isRequired = (
'require-input' in property.attrib and
kc.trueorfalse(property.attrib['require-input'])
)
hasDefault = False
for child in property:
if child.tag != 'value':
continue
if child.text is not None and len(child.text.strip()):
hasDefault = True
if not isRequired:
try:
defaults[f][name] = child.text.strip()
except KeyError:
defaults[f] = {name: child.text.strip()}
break
if not hasDefault and not isRequired:
failingxmlfiles[f] = name
self.assertEqual(len(failingxmlfiles), 0,
"Found " + str(len(failingxmlfiles))
+ " xml file missing defaults/required "
+ str(failingxmlfiles.keys())
+ " \n" + str(failingxmlfiles)
)
# Part 2: now check in the params file to see that the same default is set there
for root, dirs, files in os.walk(os.path.realpath(__file__ + '/../../../')):
for f in [os.path.join(root, f) for f in files if f == 'params.py' and f not in self.skip]:
existingdefaults = None
for existingdefaults in defaults:
if f.startswith(existingdefaults[:existingdefaults.find('configuration')]):
break
if not existingdefaults:
continue
servicename = existingdefaults.split('/')[-1].lower().split('.')[0]
all_params = ""
with open(f) as fp:
all_params = fp.read()
all_params = self.rm_unchecked_chars(all_params)
for defaultp, defaultv in defaults[existingdefaults].iteritems():
if servicename + '/' + defaultp in self.skip_prop:
continue
defaultvs = self.rm_unchecked_chars(defaultv)
search_string = 'default(configurations/' + servicename + '/' + defaultp + ',' + defaultvs + ')'
if search_string not in all_params:
failingpyfiles[f + '/' + servicename + '/' + defaultp] = search_string
self.assertEqual(len(failingpyfiles), 0,
"Found " + str(len(failingpyfiles))
+ " python params file missing defaults {file/service/property: missing_str_approx }"
+ " \n" + str(failingpyfiles).replace("',", "',\n")
)
kc.is_valid_port)
gitlab_url = kc.default("configurations/gitlab/gitlab_url", hostname,
kc.is_valid_hostname)
unicorn_port = kc.default("configurations/gitlab/unicorn_port", "8080",
kc.is_valid_port)
unicorn_interface = default("configurations/gitlab/unicorn_interface",
'127.0.0.1')
if gitlab_url == 'hostname':
gitlab_url = hostname
if not gitlab_url:
raise Exception('gitlab_url set to an unusable value \'%s\'' % gitlab_url)
gitlab_signin_enabled = default('configurations/gitlab/gitlab_signin_enabled',
'true')
gitlab_signin_enabled = kc.trueorfalse(gitlab_signin_enabled)
gitlab_admin_password = config['configurations']['gitlab'][
'gitlab_admin_password']
Logger.sensitive_strings[gitlab_admin_password] = "[PROTECTED]"
restrict_public_projects = default(
'configurations/gitlab/restrict_public_projects', 'true')
restrict_public_projects = kc.trueorfalse(restrict_public_projects)
use_external_postgres = default('configurations/gitlab/use_external_postgres',
'false')
use_external_postgres = kc.trueorfalse(use_external_postgres)
postgres_database_name = default(
'configurations/gitlab/postgres_database_name', 'gitlabhq_production')
postgres_database_port = default(
'configurations/gitlab/postgres_database_port', '5432')
postgres_database_host = default(
'configurations/gitlab/postgres_database_host', '127.0.0.1')
def runTest(self):
"""Check that properties are either forced to be entered or have a default set,
in case a default is set, check the same default exists in the params file"""
import os
import string
failingxmlfiles = {}
failingpyfiles = {}
defaults = {}
# First, part 1: check that all params are either required or have a default
# Also fill defaults with the list of defaults to check against the params file
for root, dirs, files in os.walk(os.path.realpath(__file__ + '/../../../')):
for f in [os.path.join(root, f) for f in files if f.endswith('.xml') and f not in self.skip]:
if "configuration" not in root:
continue
tree = ET.ElementTree(file=f)
for property in tree.getroot():
if property.tag != 'property':
continue
name = property.find('name').text
is_required = (
'require-input' in property.attrib and
kc.trueorfalse(property.attrib['require-input'])
)
has_default = False
for child in property:
if child.tag != 'value':
continue
if child.text is not None and len(child.text.strip()):
has_default = True
if not is_required:
try:
defaults[f][name] = child.text.strip()
except KeyError:
defaults[f] = {name: child.text.strip()}
break
if not has_default and not is_required:
failingxmlfiles[f] = name
self.assertEqual(len(failingxmlfiles), 0,
"Found " + str(len(failingxmlfiles))
+ " xml file missing defaults/required "
+ str(failingxmlfiles.keys())
+ " \n" + str(failingxmlfiles)
)
# Part 2: now check in the params file to see that the same default is set there
for root, dirs, files in os.walk(os.path.realpath(__file__ + '/../../../')):
for f in [os.path.join(root, f) for f in files if f == 'params.py' and f not in self.skip]:
existingdefaults = None
for existingdefaults in defaults:
if f.startswith(existingdefaults[:existingdefaults.find('configuration')]):
break
if not existingdefaults:
continue
servicename = existingdefaults.split('/')[-1].lower().split('.')[0]
all_params = ""
with open(f) as fp:
all_params = fp.read()
all_params = self.rm_unchecked_chars(all_params)
for defaultp, defaultv in defaults[existingdefaults].iteritems():
if servicename + '/' + defaultp in self.skip_prop:
continue
defaultvs = self.rm_unchecked_chars(defaultv)
search_string = 'default(configurations/' + servicename + '/' + defaultp + ',' + defaultvs
if search_string not in all_params:
failingpyfiles[f + '/' + servicename + '/' + defaultp] = search_string
# If the default is longer than 80 characters it' very difficult to debug, and best if I return
# just some substring instead
if len(search_string) > 80:
begin = 'default(configurations/' + servicename + '/' + defaultp + ','
if begin not in all_params:
failingpyfiles[f + '/' + servicename + '/' + defaultp] = search_string[:80] + '... )'
else:
# find the first non-matching character and return 80 chars including 10 before
search_string = 'default(configurations/' + servicename + '/' + defaultp + ','
this_default = all_params[all_params.find(search_string) + len(search_string):]
print this_default[:10]
start = 0
for start in range(len(defaultvs)):
if len(this_default) < start:
start = 0
break
if defaultvs[start] != this_default[start]:
start = max(start - 10, 0)
break
failingpyfiles[f + '/' + servicename + '/' + defaultp
] = ('( ... '
+ this_default[start:start + 80]
+ '... )'
)
self.assertEqual(len(failingpyfiles), 0,
"Found " + str(len(failingpyfiles))
+ " python params file missing defaults {file/service/property: missing_str_approx }"
+ " \n" + str(failingpyfiles).replace("',", "',\n")
)
raise Exception(
'FreeIPA ldap_bind_password: \'%s\' isn\'t acceptable (min 8 char long)'
% ldap_bind_password)
else:
Logger.sensitive_strings[ldap_bind_password] = "[PROTECTED]"
domain_components = ipa_domain.split('.')
if len(domain_components) < 2:
raise Exception('FreeIPA domain is not a FQDN. installation not possible')
domain = ipa_domain.lower()
realm = ipa_domain.upper()
realm_ldap = 'dc=' + ',dc='.join(domain_components)
install_with_dns = default('configurations/freeipa/install_with_dns', False)
install_with_dns = kc.trueorfalse(install_with_dns)
default_shell = default('configurations/freeipa/default_shell', '/bin/bash')
# Only except IPv4 for now
forwarders = default('configurations/freeipa/forwarders',
'10.0.0.10').split(',')
forwarders = [forwarder.strip() for forwarder in forwarders]
forwarders = [
forwarder for forwarder in forwarders
if re.match('\\d+\\.\\d+\\.\\d+\\.\\d+', forwarder)
]
other_nameservers = '\n'.join(['nameserver ' + f for f in forwarders])
client_init_wait = default('configurations/freeipa/client_init_wait', 600)
ldap_bind_password = default('configurations/freeipa/ldap_bind_password', False)
if not ldap_bind_password or len(ldap_bind_password) < 8:
raise Exception('FreeIPA ldap_bind_password: \'%s\' isn\'t acceptable (min 8 char long)' % ldap_bind_password)
else:
Logger.sensitive_strings[ldap_bind_password] = "[PROTECTED]"
domain_components = ipa_domain.split('.')
if len(domain_components) < 2:
raise Exception('FreeIPA domain is not a FQDN. installation not possible')
domain = ipa_domain.lower()
realm = ipa_domain.upper()
realm_ldap = 'dc=' + ',dc='.join(domain_components)
install_with_dns = default('configurations/freeipa/install_with_dns', False)
install_with_dns = kc.trueorfalse(install_with_dns)
default_shell = default('configurations/freeipa/default_shell', '/bin/bash')
# Only except IPv4 for now
forwarders = default('configurations/freeipa/forwarders', '10.0.0.10').split(',')
forwarders = [forwarder.strip() for forwarder in forwarders]
forwarders = [forwarder for forwarder in forwarders if re.match('\\d+\\.\\d+\\.\\d+\\.\\d+', forwarder)]
other_nameservers = '\n'.join(['nameserver ' + f for f in forwarders])
client_init_wait = default('configurations/freeipa/client_init_wait', 600)
all_hosts = default("/clusterHostInfo/all_hosts", None)
ldap_bind_user = kc.default('configurations/freeipa/ldap_bind_user', 'kave_bind_user', kc.is_valid_username)
ldap_bind_services = ['twiki', 'gitlab', 'jenkins']