def groom(self, model):
extRolePath = appendPath(self.path, "roles.yml")
if os.path.exists(extRolePath):
pathList = yaml.load(open(extRolePath), Loader=yaml.SafeLoader)
if not isinstance(pathList, list):
ERROR(
"File {} must contain a list of path".format(extRolePath))
for p in pathList:
model['data']["rolePaths"].add(appendPath(self.path, p))
else:
rolesPath = appendPath(self.path, "roles")
if os.path.exists(rolesPath):
model['data']["rolePaths"].add(rolesPath)
codeFile = appendPath(self.path, "groomer.py")
if os.path.exists(codeFile):
logger.debug("Will load '{0}' as python code".format(codeFile))
self.groomer = imp.load_source(self.name, codeFile)
if hasattr(self.groomer, "groom"):
method = getattr(self.groomer, "groom")
logger.debug("FOUND '{0}' method".format(str(method)))
ret = method(self, model)
if ret == None or not isinstance(ret, bool):
ERROR(
"Invalid plugin '{}'. groom(model) must return a boolean (enabled yes/no)."
.format(self.name))
else:
self.enabled = ret
def groom(plugin, model):
setDefaultInMap(model["cluster"]["confluent"], "disabled", False)
if model["cluster"]["confluent"]["disabled"]:
return False
lookupRepository(model, "confluent")
if "confluent" not in model["config"] or "ansible_repo_folder" not in model["config"]["confluent"]:
ERROR("Missing 'confluent.ansible_repo_folder' in configuration file")
for node in model['cluster']['nodes']:
if "kafka_log_dirs" in node:
if len(node["kafka_log_dirs"]) == 0:
del(node["kafka_log_dirs"])
else:
if "kafka_log_dirs" in model["data"]["roleByName"][node["role"]]:
node["kafka_log_dirs"] = model["data"]["roleByName"][node["role"]]["kafka_log_dirs"]
ansible_repo_folder = appendPath(os.path.dirname(model["data"]["configFile"]), model["config"]["confluent"]["ansible_repo_folder"])
model["config"]["confluent"]["ansible_repo_folder"] = ansible_repo_folder
model["data"]["rolePaths"].add(appendPath(ansible_repo_folder, "roles"))
# We need to define an ansible group "preflight" hosting all nodes
preflight = []
for node in model["cluster"]["nodes"]:
preflight.append(node["name"])
model["data"]["groupByName"]["preflight"] = preflight
return True
def initVault(model):
global vault
if VAULT_ID in model[CLUSTER]:
vaultId = model[CLUSTER][VAULT_ID]
if VAULTS not in model["config"]:
ERROR("{} is missing from configuration while encryption id required ('vault_id' is defined)".format(VAULTS))
l = filter(lambda x: x["vault_id"] == vaultId, model["config"][VAULTS])
if len(l) > 1:
ERROR("{}: vault_id '{}' is defined twice in configuration file!".format(VAULTS, vaultId))
if len(l) != 1:
ERROR("{}: vault_id '{}' is not defined in configuration file!".format(VAULTS, vaultId))
f = appendPath(os.path.dirname(model[DATA][CONFIG_FILE]), l[0][PASSWORD_FILE])
if not (os.path.isfile(f) and os.access(f, os.R_OK)):
ERROR("Non existing or not accessible vault password file '{}'.".format(f))
pwd = file2String(f)
pwd = pwd.strip()
model[DATA][VAULT_PASSWORD_FILE] = f
vault = Vault(pwd)
if SAFE_CONFIFG_FILE in l[0]:
scFileName = appendPath(os.path.dirname(model[DATA][CONFIG_FILE]), l[0][SAFE_CONFIFG_FILE])
model[DATA][_SAFE_CONFIG_FILE_] = scFileName
if not (os.path.isfile(scFileName) and os.access(scFileName, os.R_OK)):
ERROR("Non existing or not accessible safe config file '{}'.".format(scFileName))
logger.info("Loading safe config from '{}'".format(scFileName))
data, was_encrypted = vault.encryptedFile2String(scFileName)
safeConfig = yaml.load(data, Loader=yaml.SafeLoader)
model[SAFE_CONFIG] = safeConfig
if not was_encrypted:
print("\n'{}' was not encrypted. Will encrypt it".format(scFileName))
vault.stringToEncryptedFile(data, scFileName)
else:
vault = None
def groom(_plugin, model):
model[DATA][AWS] = {}
#model[DATA][AWS][REFERENCE_SUBNET]= model[CLUSTER][NODES][0][AWS][SUBNET]
setDefaultInMap(model[CLUSTER][AWS], KEY_PAIR, "default")
kp = lookupKeyPair(model, model[CLUSTER][AWS][KEY_PAIR])
model[DATA][AWS][DATA_KEY_PAIR] = kp[KEY_PAIR_NAME]
if PRIVATE_KEY_PATH in kp:
# If path is relative, adjust to config file location
model[DATA][AWS][DATA_PRIVATE_KEY_PATH] = appendPath(os.path.dirname(model["data"]["configFile"]), kp[PRIVATE_KEY_PATH])
model[DATA][AWS][DATA_ROUTE53] = lookupRoute53(model, model[CLUSTER][AWS][ROUTE53])
groomSecurityGroups(model)
groomRoles(model)
groomNodes(model)
model["data"]["buildScript"] = appendPath(model["data"]["targetFolder"], "build.sh")
return True # Always enabled
def groom(plugin, model):
repoInConfig = "repositories" in model["config"] and "vagrant" in model[
"config"]["repositories"] and "yum_repo_base_url" in model["config"][
"repositories"]["vagrant"]
setDefaultInMap(model["cluster"]["vagrant"], "local_yum_repo",
repoInConfig)
if model["cluster"]["vagrant"]["local_yum_repo"] and not repoInConfig:
ERROR(
"'repositories.vagrant.repo_yum_base_url' is not defined in config file while 'vagrant.local_yum_repo' is set to True in '{}'"
.format(model["data"]["sourceFileDir"]))
if repoInConfig:
# All plugins are lookinhg up their repositories in model["data"]. So does the vagrant one.
setDefaultInMap(model["data"], "repositories", {})
setDefaultInMap(model["data"]["repositories"], "vagrant", {})
model["data"]["repositories"]["vagrant"]["yum_repo_base_url"] = model[
"config"]["repositories"]["vagrant"]["yum_repo_base_url"]
for node in model['cluster']['nodes']:
if not SYNCED_FOLDERS in node:
node[SYNCED_FOLDERS] = []
role = model["data"]["roleByName"][node["role"]]
if SYNCED_FOLDERS in role:
node[SYNCED_FOLDERS] += role[SYNCED_FOLDERS]
if SYNCED_FOLDERS in model["cluster"]["vagrant"]:
node[SYNCED_FOLDERS] += model["cluster"]["vagrant"][SYNCED_FOLDERS]
model["data"]["buildScript"] = appendPath(model["data"]["targetFolder"],
"build.sh")
return True # Always enabled
def groom(_plugin, model):
setDefaultInMap(model[CLUSTER][DRPROXY], DISABLED, False)
if model[CLUSTER][DRPROXY][DISABLED]:
return False
else:
for f in [CERT_FILE, KEY_FILE, ROOT_CA_FILE]:
model[CLUSTER][DRPROXY][f] = appendPath(model[DATA][SOURCE_FILE_DIR], model[CLUSTER][DRPROXY][f])
if not os.path.isfile(model[CLUSTER][DRPROXY][f]):
ERROR("Unable to find '{}'!".format(model[CLUSTER][DRPROXY][f]))
return True
def groom(_plugin, model):
setDefaultInMap(model[CLUSTER], K8S, {})
setDefaultInMap(model[CLUSTER][K8S], KUBESPRAY, {})
setDefaultInMap(model[CLUSTER][K8S][KUBESPRAY], DISABLED, False)
setDefaultInMap(model[CLUSTER][K8S][KUBESPRAY], METRICS_SERVER, True)
setDefaultInMap(model[CLUSTER][K8S][KUBESPRAY], AUDIT, False)
setDefaultInMap(model[CLUSTER][K8S][KUBESPRAY], POD_SECURITY_POLICIES, True)
if model[CLUSTER][K8S][KUBESPRAY][DISABLED]:
return False
else:
lookupRepository(model, None, "docker_yum", model[CLUSTER][K8S][KUBESPRAY]['docker_yum_repo_id'])
if K9S_REPO_ID in model[CLUSTER][K8S][KUBESPRAY]:
lookupRepository(model, "k9s", repoId = model[CLUSTER][K8S][KUBESPRAY][K9S_REPO_ID])
if HELM_REPO_ID in model[CLUSTER][K8S][KUBESPRAY]:
lookupRepository(model, "helm", repoId = model[CLUSTER][K8S][KUBESPRAY][HELM_REPO_ID])
lookupHelper(model, KUBESPRAY, helperId=model[CLUSTER][K8S][KUBESPRAY]["helper_id"])
lookupHttpProxy(model, model[CLUSTER][K8S][KUBESPRAY]["docker_proxy_id"] if "docker_proxy_id" in model[CLUSTER][K8S][KUBESPRAY] else None, "docker")
lookupHttpProxy(model, model[CLUSTER][K8S][KUBESPRAY]["master_root_proxy_id"] if "master_root_proxy_id" in model[CLUSTER][K8S][KUBESPRAY] else None, "master_root")
lookupHttpProxy(model, model[CLUSTER][K8S][KUBESPRAY]["yumproxy_id"] if "yum_proxy_id" in model[CLUSTER][K8S][KUBESPRAY] else None, "yum")
if FILES_REPO_ID in model[CLUSTER][K8S][KUBESPRAY]:
lookupRepository(model, "kubespray_files", repoId=model[CLUSTER][K8S][KUBESPRAY][FILES_REPO_ID])
model[DATA][ROLE_PATHS].add(appendPath(model[DATA][HELPERS][KUBESPRAY][FOLDER], "roles"))
model[DATA]["dnsNbrDots"] = model[CLUSTER][K8S][KUBESPRAY][CLUSTER_NAME].count(".") + 1
certByName = {}
if DOCKER_CERTIFICATES in model["config"]:
for cert in model["config"][DOCKER_CERTIFICATES]:
cert["path"] = appendPath(os.path.dirname(model[DATA][CONFIG_FILE]), cert["path"])
if not os.path.isfile(cert["path"]) or not os.access(cert["path"], os.R_OK):
ERROR("Configuration error: docker_certificates.{}: Invalid path '{}'".format(cert["name"], cert["path"]))
certByName[cert["name"]] = cert
model[DATA][DOCKER_CERTIFICATES] = []
if DOCKER_CERTIFICATES in model[CLUSTER][K8S][KUBESPRAY]:
for certName in model[CLUSTER][K8S][KUBESPRAY][DOCKER_CERTIFICATES]:
if certName in certByName:
cert = certByName[certName]
if "port" in cert:
cert["endpoint"] = "{}:{}".format(cert["host"], cert['port'])
else:
cert["endoint"] = cert["host"]
model[DATA][DOCKER_CERTIFICATES].append(cert)
else:
ERROR("docker_certificates '{}' is not defined in configuration file!".format(certName))
return True
def walk(self, targetFileByName):
""" Enrich the targetFileByName structure with file from this plugin """
#logger.debug(self.path + "<----")
snippetsPath = appendPath(self.path, "snippets")
pref = len(snippetsPath) + 1
for dirpath, dirnames, filenames in os.walk(
snippetsPath): # @UnusedVariable
#logger.debug("dirpath:{} dirnames:{} filename:{}".format(dirpath, dirnames, filenames))
for filename in filenames:
#logger.debug(filename)
if not filename == ".gitignore":
sourceFile = os.path.join(dirpath, filename)
targetFileName = sourceFile[pref:]
if targetFileName.count(".") < 2:
# We pass throught non super-suffixed files
order = 0
ftype = "txt"
else:
# Handle the type and eventual suffix (Used as short comment)
pos = targetFileName.rfind(".")
suffix = targetFileName[pos + 1:]
targetFileName = targetFileName[:pos]
pos = suffix.find("-")
if pos != -1:
ftype = suffix[:pos]
suffix = suffix[pos + 1:]
else:
ftype = suffix
suffix = None
# Now order number
pos = targetFileName.rfind(".")
idx = targetFileName[pos + 1:]
targetFileName = targetFileName[:pos]
try:
order = int(idx)
except ValueError:
ERROR("'{0}' is not a valid file part".format(
sourceFile))
logger.debug(sourceFile + "-->" + targetFileName + "(" +
str(idx) + ")")
if targetFileName not in targetFileByName:
targetFileByName[targetFileName] = {}
#targetFileByName[targetFileName].name = targetFileName
targetFileByName[targetFileName]['fileParts'] = []
fp = {}
fp['name'] = sourceFile
fp['order'] = order
fp['plugin'] = self.name
fp['type'] = ftype
if suffix != None:
fp["suffix"] = suffix
targetFileByName[targetFileName]['fileParts'].append(fp)
def buildConfig(sourceFileDir, baseConfigFile):
configFile = findUpward(baseConfigFile, sourceFileDir)
logger.info("Using '{}' as configuration file".format(configFile))
config = yaml.load(open(configFile), Loader=yaml.SafeLoader)
if PLUGINS_PATH not in config:
ERROR("Missing '{}' in configuration file".format(PLUGINS_PATH))
# Adjust plugin path relative to the config file
baseDir = os.path.dirname(configFile)
for index, path in enumerate(config[PLUGINS_PATH]):
config[PLUGINS_PATH][index] = misc.appendPath(baseDir, path)
return (config, configFile)
def groom(plugin, model):
for node in model[CLUSTER][NODES]:
if ANSIBLE_USER not in node:
if INVENTORY not in model[CLUSTER] or ANSIBLE_USER not in model[
CLUSTER][INVENTORY][DEFAULTS]:
ERROR(
"'{}' is not defined either in node '{}' and in inventory.defaults!"
.format(ANSIBLE_USER, node[NAME]))
else:
node[ANSIBLE_USER] = model[CLUSTER][INVENTORY][DEFAULTS][
ANSIBLE_USER]
if ANSIBLE_BECOME not in node:
if INVENTORY not in model[CLUSTER] or ANSIBLE_BECOME not in model[
CLUSTER][INVENTORY][DEFAULTS]:
ERROR(
"'{}' is not defined either in node '{}' and in inventory.defaults!"
.format(ANSIBLE_BECOME, node[NAME]))
else:
node[ANSIBLE_BECOME] = model[CLUSTER][INVENTORY][DEFAULTS][
ANSIBLE_BECOME]
if ANSIBLE_PRIVATE_KEY not in node:
if INVENTORY not in model[
CLUSTER] or ANSIBLE_PRIVATE_KEY not in model[CLUSTER][
INVENTORY][DEFAULTS]:
ERROR(
"'{}' is not defined either in node '{}' and in inventory.defaults!"
.format(ANSIBLE_PRIVATE_KEY, node[NAME]))
else:
node[ANSIBLE_PRIVATE_KEY] = model[CLUSTER][INVENTORY][
DEFAULTS][ANSIBLE_PRIVATE_KEY]
node[ANSIBLE_PRIVATE_KEY] = appendPath(model[DATA][SOURCE_FILE_DIR],
node[ANSIBLE_PRIVATE_KEY])
if not os.path.isfile(node[ANSIBLE_PRIVATE_KEY]) or not os.access(
node[ANSIBLE_PRIVATE_KEY], os.R_OK):
ERROR("Node '{}': Invalid private key path:'{}'".format(
node[NAME], node[ANSIBLE_PRIVATE_KEY]))
model["data"]["buildScript"] = appendPath(model["data"]["targetFolder"],
"build.sh")
return True # Always enabled
def groom(plugin, model):
if ANSIBLE in model[CLUSTER]:
setDefaultInMap(model[CLUSTER][ANSIBLE], DISABLED, False)
if model[CLUSTER][ANSIBLE][DISABLED]:
return False
if PLAYBOOKS in model[CLUSTER][ANSIBLE]:
for idx in range(0, len(model[CLUSTER][ANSIBLE][PLAYBOOKS])):
model[CLUSTER][ANSIBLE][PLAYBOOKS][idx][FILE] = appendPath(
model[DATA][SOURCE_FILE_DIR],
model[CLUSTER][ANSIBLE][PLAYBOOKS][idx][FILE])
if ROLES_PATHS in model[CLUSTER][ANSIBLE]:
for rp in model[CLUSTER][ANSIBLE][ROLES_PATHS]:
model[DATA]["rolePaths"].add(
appendPath(model[DATA][SOURCE_FILE_DIR], rp))
if ROLES_PATHS_FROM_PLUGINS in model[CLUSTER][ANSIBLE]:
for pluginName in model[CLUSTER][ANSIBLE][
ROLES_PATHS_FROM_PLUGINS]:
plugin = lookupPlugin(pluginName,
model[CONFIG]["plugins_paths"])
if plugin != None:
rolesPath = appendPath(plugin.path, "roles")
if os.path.exists(rolesPath):
model['data']["rolePaths"].add(rolesPath)
else:
ERROR(
"ansible.{}: There is no 'roles' folder in plugin '{}'"
.format(ROLES_PATHS_FROM_PLUGINS, pluginName))
else:
ERROR("ansible.{}: plugin '{}' not found".format(
ROLES_PATHS_FROM_PLUGINS, pluginName))
if ROLES in model[CLUSTER][ANSIBLE]:
for role in model[CLUSTER][ANSIBLE][ROLES]:
setDefaultInMap(role, SCOPE, "all")
if ANSIBLE in model[CONFIG] and ROLES_PATHS in model[CONFIG][ANSIBLE]:
for rp in model[CONFIG][ANSIBLE][ROLES_PATHS]:
model[DATA]["rolePaths"].add(
appendPath(os.path.dirname(model[DATA]["configFile"]), rp))
return True
def groom(plugin, model):
setDefaultInMap(model["cluster"], "kubespray", {})
setDefaultInMap(model["cluster"]["kubespray"], "disabled", False)
if model["cluster"]["kubespray"]["disabled"]:
return False
else:
if "kubespray" not in model[
"config"] or "ansible_repo_folder" not in model["config"][
"kubespray"]:
ERROR(
"Missing 'kubespray.ansible_repo_folder' in configuration file"
)
ansible_repo_folder = appendPath(
os.path.dirname(model["data"]["configFile"]),
model["config"]["kubespray"]["ansible_repo_folder"])
model["config"]["kubespray"][
"ansible_repo_folder"] = ansible_repo_folder
model["data"]["rolePaths"].add(appendPath(ansible_repo_folder,
"roles"))
model["data"]["dnsNbrDots"] = model["cluster"]["domain"].count(".") + 1
return True
def groom(_plugin, model):
setDefaultInMap(model[CLUSTER], HARBOR, {})
setDefaultInMap(model[CLUSTER][HARBOR], DISABLED, False)
if model[CLUSTER][HARBOR][DISABLED]:
return False
else:
lookupRepository(model,
"harbor",
repoId=model[CLUSTER][HARBOR][REPO_ID])
model[CLUSTER][HARBOR][SSL_CERT_SRC] = appendPath(
model[DATA][SOURCE_FILE_DIR], model[CLUSTER][HARBOR][SSL_CERT_SRC])
if not os.path.isfile(model[CLUSTER][HARBOR][SSL_CERT_SRC]):
ERROR("Unable to find '{}'!".format(
model[CLUSTER][HARBOR][SSL_CERT_SRC]))
model[CLUSTER][HARBOR][SSL_KEY_SRC] = appendPath(
model[DATA][SOURCE_FILE_DIR], model[CLUSTER][HARBOR][SSL_KEY_SRC])
if not os.path.isfile(model[CLUSTER][HARBOR][SSL_KEY_SRC]):
ERROR("Unable to find '{}'!".format(
model[CLUSTER][HARBOR][SSL_KEY_SRC]))
setDefaultInMap(model[CLUSTER][HARBOR], VALIDATE_API_CERT, False)
setDefaultInMap(model[CLUSTER][HARBOR], HOSTNAME, "{{ ansible_fqdn }}")
return True
def groom(_plugin, model):
setDefaultInMap(model[CLUSTER][REGISTER_CA], DISABLED, False)
if model[CLUSTER][REGISTER_CA][DISABLED]:
return False
else:
if FROM_PATHS in model[CLUSTER][REGISTER_CA]:
for idx, p in enumerate(model[CLUSTER][REGISTER_CA][FROM_PATHS]):
model[CLUSTER][REGISTER_CA][FROM_PATHS][idx][SRC] = appendPath(
model[DATA][SOURCE_FILE_DIR], p[SRC])
if not os.path.isfile(
model[CLUSTER][REGISTER_CA][FROM_PATHS][idx][SRC]):
ERROR("Unable to find '{}'!".format(
model[CLUSTER][REGISTER_CA][FROM_PATHS][idx][SRC]))
return True
def groom(_plugin, model):
setDefaultInMap(model[CLUSTER], FREEIPA, {})
setDefaultInMap(model[CLUSTER][FREEIPA], DISABLED, False)
if model[CLUSTER][FREEIPA][DISABLED]:
return False
else:
model[DATA][FREEIPA] = {}
lookupHelper(model,
FREEIPA,
helperId=model[CLUSTER][FREEIPA]["helper_id"])
model[DATA][ROLE_PATHS].add(
appendPath(model[DATA][HELPERS][FREEIPA][FOLDER], "roles"))
if CERT_FILES in model[CLUSTER][FREEIPA] and len(
model[CLUSTER][FREEIPA][CERT_FILES]) > 0:
# NB: ipaserver_external_cert_files_from_controller does not works! (Missing basename in the copy). Will handle ourself before
# In fact, was unable to transfer root authority from one install to another. A new CA is generated on each freeipa build.
model[DATA][FREEIPA][FILES_TO_COPY] = []
model[DATA][FREEIPA][EXTERNAL_CERT_FILES] = []
for fn in model[CLUSTER][FREEIPA][CERT_FILES]:
fc = {}
fc[SRC] = appendPath(model[DATA][SOURCE_FILE_DIR], fn)
if not os.path.isfile(fc[SRC]):
ERROR("Unable to find '{}'!".format(fc[SRC]))
fc[DEST] = os.path.join("/root/", os.path.basename(fc[SRC]))
model[DATA][FREEIPA][FILES_TO_COPY].append(fc)
model[DATA][FREEIPA][EXTERNAL_CERT_FILES].append(fc[DEST])
if USERS in model[CLUSTER][FREEIPA]:
for user in model[CLUSTER][FREEIPA][USERS]:
setDefaultInMap(user, UPDATE_PASSWORD, "on_create")
# We better provide some default here than letting freeipa doing it. This for better control, and for updating (freeipi does not modify them once set).
n = "{} {}".format(user[FIRSTNAME], user[LASTNAME])
setDefaultInMap(user, CN, n)
setDefaultInMap(user, DISPLAYNAME, n)
setDefaultInMap(user, INITALS, (user[FIRSTNAME][0] +
user[LASTNAME][0]).upper())
return True
def groom(_plugin, model):
repoInConfig = "repositories" in model["config"] and "vagrant" in model["config"]["repositories"] and "yum_repo_base_url" in model["config"]["repositories"]["vagrant"]
if model["cluster"]["vagrant"]["yum_repo"] == "local" and not repoInConfig:
ERROR("'repositories.vagrant.repo_yum_base_url' is not defined in config file while 'vagrant.yum_repo' is set to 'local' in '{}'".format(model["data"]["sourceFileDir"]))
if repoInConfig:
# All plugins are lookinhg up their repositories in model["data"]. So does the vagrant one.
setDefaultInMap(model["data"], "repositories", {})
setDefaultInMap(model["data"]["repositories"], "vagrant", {})
model["data"]["repositories"]["vagrant"]["yum_repo_base_url"] = model["config"]["repositories"]["vagrant"]["yum_repo_base_url"]
groomRoles(model)
groomNodes(model)
model["data"]["buildScript"] = appendPath(model["data"]["targetFolder"], "build.sh")
return True # Always enabled
def groom(_plugin, model):
setDefaultInMap(model[CLUSTER], K8S, {})
setDefaultInMap(model[CLUSTER][K8S], LOCAL_STATIC_PVS, {})
setDefaultInMap(model[CLUSTER][K8S][LOCAL_STATIC_PVS], DISABLED, False)
if model[CLUSTER][K8S][LOCAL_STATIC_PVS][DISABLED]:
return False
# Lookup interesting storage classes
localSaticStorageClassByName = {}
model[DATA][LOCAL_STATIC_STORAGE_CLASSES] = []
for sc in model[CLUSTER][K8S][LOCAL_STATIC_PVS][STORAGE_CLASSES]:
if nameCheckRegex.match(sc["name"]) is None:
ERROR("Invalid storage_class name '{}': DNS-1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character".format(sc["name"]))
if sc[TYPE] == SC_TYPE_LOCAL_STATIC:
localSaticStorageClassByName[sc[NAME]] = sc
model[DATA][LOCAL_STATIC_STORAGE_CLASSES].append(sc)
model[DATA][PV_MOUNT_FOLDERS] = []
for _, role in model[DATA][ROLE_BY_NAME].items():
# Lookup interesting raw dataDsisk
dataDiskByRef = {}
if DATA_DISKS in role:
for ddisk in role[DATA_DISKS]:
if REF in ddisk:
dataDiskByRef[ddisk[REF]] = ddisk
ddisk[SPLITS] = []
storageClasses = Set()
hostDirs = Set()
role[LVM_SPLITTERS] = []
role[BIND_MOUNTS] = []
splitCount = 0
if K8S in role and LOCAL_STATIC_PVS in role[K8S]:
for lpv in role[K8S][LOCAL_STATIC_PVS]:
if lpv[STORAGE_CLASS] not in localSaticStorageClassByName:
ERROR("role['{}'].k8s.local_static_pv.host_dir['{}']': Undefined storage_class '{}'".format(role[NAME], lpv[HOST_DIR], lpv[STORAGE_CLASS]))
if lpv[STORAGE_CLASS] in storageClasses:
ERROR("role['{}'].k8s.local_static_pv: storage_class '{}' is used twice".format(role[NAME], lpv[STORAGE_CLASS]))
storageClasses.add(lpv[STORAGE_CLASS])
if lpv[HOST_DIR] in hostDirs:
ERROR("role['{}'].k8s.local_static_pv: host_dir '{}' is used twice".format(role[NAME], lpv[HOST_DIR]))
hostDirs.add(lpv[HOST_DIR])
for index, source in enumerate(lpv[SOURCES]):
if FOLDER in source:
if DATA_DISK_REF in source or SPLITS in source or COUNT not in source:
invalidSourceError(role, lpv, index)
for _ in range(0, source[COUNT]):
mount = {}
role[BIND_MOUNTS].append(mount)
name = "vol{:03}".format(len(role[BIND_MOUNTS]))
mount["path"] = appendPath(lpv[HOST_DIR], name)
mount["src"] = appendPath(source[FOLDER], name)
elif DATA_DISK_REF in source:
if FOLDER in source or COUNT in source or SPLITS not in source:
invalidSourceError(role, lpv, index)
if source[DATA_DISK_REF] in dataDiskByRef:
ddisk = dataDiskByRef[source[DATA_DISK_REF]]
for size in source[SPLITS]:
split = {}
ddisk[SPLITS].append(split)
splitCount += 1
split[NAME] = "split{:03}".format(splitCount)
split["_size"] = size
split["mount"] = appendPath(lpv[HOST_DIR], split[NAME])
else:
ERROR("role['{}'].k8s.local_static_pv.host_dir['{}'][{}]': Undefined data_disk_ref '{}'".format(role[NAME], lpv[HOST_DIR], index, source[DATA_DISK_REF]))
else:
invalidSourceError(role, lpv, index)
pvMountFolder = {}
pvMountFolder["className"] = lpv[STORAGE_CLASS]
pvMountFolder["hostDir"] = lpv[HOST_DIR]
model[DATA][PV_MOUNT_FOLDERS].append(pvMountFolder)
# Now, loop on data_disk to build splitter
for ref, ddisk in dataDiskByRef.items():
if len(ddisk[SPLITS]) > 0:
requiredSize = 0
splitter = {}
role[LVM_SPLITTERS].append(splitter)
splitter["physical_volumes"] = [ (ddisk[DEVICE] if ddisk[DEVICE].startswith("/dev") else "/dev/{}".format(ddisk[DEVICE])) ]
splitter["vg_name"] = "vg{}".format( ddisk[DEVICE][5:] if ddisk[DEVICE].startswith("/dev/") else ddisk[DEVICE] )
splitter["logical_volumes"] = []
for split in ddisk[SPLITS]:
splitter["logical_volumes"].append(split)
split["size"] = "{}G".format(split["_size"])
split["fstype"] = "xfs"
split["mount_options"] = "defaults,noatime"
split["fsopts"] = ""
requiredSize += split["_size"]
if requiredSize > ddisk[SIZE]:
ERROR("role['{}'].data_disk.ref[{}]: Required size ({}G) exceed disk size ({}G)'".format(role[NAME], ref, requiredSize, ddisk[SIZE]))
return True
def groom(plugin, model):
setDefaultInMap(model[CLUSTER][HORTONWORKS], DISABLED, False)
if model[CLUSTER][HORTONWORKS][DISABLED]:
return False
else:
lookupRepository(model, HORTONWORKS)
if HORTONWORKS not in model[CONFIG] or ANSIBLE_REPO_FOLDER not in model[
CONFIG][HORTONWORKS]:
ERROR(
"Missing 'hortonworks.ansible_repo_folder' in configuration file"
)
ansible_repo_folder = appendPath(
os.path.dirname(model[DATA][CONFIG_FILE]),
model[CONFIG][HORTONWORKS][ANSIBLE_REPO_FOLDER])
model[CONFIG][HORTONWORKS][ANSIBLE_REPO_FOLDER] = ansible_repo_folder
model[DATA][ROLE_PATHS].add(appendPath(ansible_repo_folder, "roles"))
# ------------- We need to define some groups for the intention of external tools.
zookeepers = []
kafka_brokers = []
model[DATA][EXTRA_GROUP_BY_NAME] = {}
for role in model[CLUSTER][ROLES]:
if HW_SERVICES in role:
if "ZOOKEEPER_SERVER" in role[HW_SERVICES]:
zookeepers.extend(map(lambda x: x[NAME], role[NODES]))
if "KAFKA_BROKER" in role[HW_SERVICES]:
kafka_brokers.extend(map(lambda x: x[NAME], role[NODES]))
if ZOOKEEPERS not in model[DATA][GROUP_BY_NAME]:
model[DATA][EXTRA_GROUP_BY_NAME][ZOOKEEPERS] = zookeepers
if KAFKA_BROKERS not in model[DATA][GROUP_BY_NAME]:
model[DATA][EXTRA_GROUP_BY_NAME][KAFKA_BROKERS] = kafka_brokers
# ---------------------------- Handle java
#setDefaultInMap(model[CLUSTER][HORTONWORKS], JAVA, EMBEDDED)
if model[CLUSTER][HORTONWORKS][JAVA] == ORACLEJDK:
if ORACLEJDK_TARBALL_LOCATION not in model[DATA][REPOSITORIES][
HORTONWORKS]:
ERROR(
"'hortonworks.java' is set to 'oraclejdk' while there is no 'repositories.hortonworks.oraclejdk_tarball_location' defined in configuration file!"
)
if ORACLEJDK_JCE_LOCATION not in model[DATA][REPOSITORIES][
HORTONWORKS]:
ERROR(
"'hortonworks.java' is set to 'oraclejdk' while there is no 'repositories.hortonworks.oraclejdk_jce_location' defined in configuration file!"
)
# ---------------------------- Handle database
if model[CLUSTER][HORTONWORKS][DATABASE][TYPE] != "embedded":
if MODE not in model[CLUSTER][HORTONWORKS][DATABASE]:
ERROR(
"hostonworks.database.mode must be defined if type != 'embedded'!"
)
if model[CLUSTER][HORTONWORKS][DATABASE][
MODE] != "included" and model[CLUSTER][HORTONWORKS][
DATABASE][TYPE] != 'postgres':
ERROR(
"hostonworks.database: Only 'postgres' type is supported in 'internal' or 'external' mode "
)
if model[CLUSTER][HORTONWORKS][DATABASE][
MODE] == 'external' and SERVER not in model[CLUSTER][
HORTONWORKS][DATABASE]:
ERROR(
"hostonworks.database.server must be defined in 'external' mode "
)
if model[CLUSTER][HORTONWORKS][DATABASE][
MODE] == 'internal' and SERVER in model[CLUSTER][
HORTONWORKS][DATABASE]:
ERROR(
"hostonworks.database.server must NOT be defined in 'internal' mode "
)
setDefaultInMap(model[CLUSTER][HORTONWORKS][DATABASE], ADD_REPO,
True)
if model[CLUSTER][HORTONWORKS][DATABASE][MODE] == 'internal':
if not POSTGRESQL_SERVER in model[DATA][GROUP_BY_NAME]:
ERROR(
"hostonworks.database.mode == 'internal', but no group '{}' was defined"
.format(POSTGRESQL_SERVER))
model[CLUSTER][HORTONWORKS][DATABASE][SERVER] = model[DATA][
GROUP_BY_NAME][POSTGRESQL_SERVER][0]
# -------------------------------------------------Handle database
# We need to create two layout.
# - One to create databases and users on db server.
# - One to provide info to group_vars/all
setDefaultInMap(model[CLUSTER][HORTONWORKS], WEAK_PASSWORDS, False)
setDefaultInMap(model[DATA], HORTONWORKS, {})
setDefaultInMap(model[DATA][HORTONWORKS], DATABASES, {})
setDefaultInMap(model, PASSWORDS, {})
setDefaultInMap(model[PASSWORDS], DATABASES, {})
tags = Set()
tags.add("ambari")
for role in model[CLUSTER][ROLES]:
if HW_SERVICES in role:
if "HIVE_METASTORE" in role[HW_SERVICES]:
tags.add("hive")
if "OOZIE_SERVER" in role[HW_SERVICES]:
tags.add("oozie")
if "DRUID_BROKER" in role[
HW_SERVICES] or "DRUID_OVERLORD" in role[HW_SERVICES]:
tags.add("druid")
if "SUPERSET" in role[HW_SERVICES]:
tags.add("superset")
if "RANGER_ADMIN" in role[HW_SERVICES]:
tags.add("rangeradmin")
if "RANGER_KMS_SERVER" in role[HW_SERVICES]:
tags.add("rangerkms")
if "REGISTRY_SERVER" in role[HW_SERVICES]:
tags.add("registry")
if "STREAMLINE_SERVER" in role[HW_SERVICES]:
tags.add("streamline")
model[DATA][HORTONWORKS][DATABASES_TO_CREATE] = tags
for tag in [
"ambari", "hive", "oozie", "druid", "superset", "rangeradmin",
"rangerkms", "registry", "streamline"
]:
user = tag
if model[CLUSTER][HORTONWORKS][WEAK_PASSWORDS]:
password = user
else:
password = genaratePassword()
md5Password = "******" + hashlib.md5(password + user).hexdigest()
model[DATA][HORTONWORKS][DATABASES][tag] = {
'user': user,
'database': tag,
'md5Password': md5Password
}
model[PASSWORDS][DATABASES][tag] = password
return True
def groom(_plugin, model):
setDefaultInMap(model[CLUSTER], K8S, {})
setDefaultInMap(model[CLUSTER][K8S], KOOMGR, {})
setDefaultInMap(model[CLUSTER][K8S][KOOMGR], DISABLED, False)
if model[CLUSTER][K8S][KOOMGR][DISABLED]:
return False
else:
setDefaultInMap(model[CLUSTER][K8S][KOOMGR], LOG_LEVEL, 0)
setDefaultInMap(model[CLUSTER][K8S][KOOMGR], ADMIN_GROUP, "kooadmin")
setDefaultInMap(model[DATA], K8S, {})
setDefaultInMap(model[DATA][K8S], KOOMGR, {})
providerByName = {}
if KOOMGR in model[CONFIG]:
if STATIC_PROVIDERS in model[CONFIG][KOOMGR]:
for prvd in model[CONFIG][KOOMGR][STATIC_PROVIDERS]:
prvd[TYPE] = "static"
if prvd[NAME] in providerByName:
ERROR("There is two providers of name '{}'".format(
prvd[NAME]))
providerByName[prvd[NAME]] = prvd
if LDAP_PROVIDERS in model[CONFIG][KOOMGR]:
for prvd in model[CONFIG][KOOMGR][LDAP_PROVIDERS]:
prvd[TYPE] = "ldap"
if prvd[NAME] in providerByName:
ERROR("There is two providers of name '{}'".format(
prvd[NAME]))
providerByName[prvd[NAME]] = prvd
if ROOTCA in prvd:
prvd[ROOTCA] = appendPath(
os.path.dirname(model[DATA][CONFIG_FILE]),
prvd[ROOTCA])
if not os.path.isfile(prvd[ROOTCA]):
ERROR("Unable to find '{}'!".format(prvd[ROOTCA]))
if CRD_PROVIDERS in model[CONFIG][KOOMGR]:
for prvd in model[CONFIG][KOOMGR][CRD_PROVIDERS]:
prvd[TYPE] = "crd"
if prvd[NAME] in providerByName:
ERROR("There is two providers of name '{}'".format(
prvd[NAME]))
providerByName[prvd[NAME]] = prvd
if LOCAL_MANIFESTS in model[CLUSTER][K8S][KOOMGR]:
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][CRD] = appendPath(
model[DATA][SOURCE_FILE_DIR],
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][CRD])
if not os.path.isfile(
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][CRD]):
ERROR("Unable to find '{}'!".format(
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][CRD]))
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][DEPLOY] = appendPath(
model[DATA][SOURCE_FILE_DIR],
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][DEPLOY])
if not os.path.isfile(
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][DEPLOY]):
ERROR("Unable to find '{}'!".format(
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][DEPLOY]))
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][RBAC] = appendPath(
model[DATA][SOURCE_FILE_DIR],
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][RBAC])
if not os.path.isfile(
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][RBAC]):
ERROR("Unable to find '{}'!".format(
model[CLUSTER][K8S][KOOMGR][LOCAL_MANIFESTS][RBAC]))
model[DATA][K8S][KOOMGR][PROVIDERS] = []
for pname in model[CLUSTER][K8S][KOOMGR][PROVIDERS]:
if pname not in providerByName:
ERROR(
"Provider '{}' does not exists in global configuration file!"
.format(pname))
prvd = providerByName[pname]
model[DATA][K8S][KOOMGR][PROVIDERS].append(prvd)
return True
def groom(plugin, model):
setDefaultInMap(model[CLUSTER][ELASTICSEARCH], DISABLED, False)
if model[CLUSTER][ELASTICSEARCH][DISABLED]:
return False
lookupRepository(model, ELASTICSEARCH)
if ELASTICSEARCH not in model[CONFIG] or ANSIBLE_REPO_FOLDER not in model[CONFIG][ELASTICSEARCH]:
ERROR("Missing 'elasticsearch.ansible_repo_folder' in configuration file")
ansible_repo_folder = appendPath(os.path.dirname(model[DATA]["configFile"]), model[CONFIG][ELASTICSEARCH][ANSIBLE_REPO_FOLDER])
model[CONFIG][ELASTICSEARCH][ANSIBLE_REPO_FOLDER] = ansible_repo_folder
model[DATA]["rolePaths"].add(ansible_repo_folder)
f = os.path.join(plugin.path, "default.yml")
if os.path.exists(f):
base = yaml.load(open(f))
else:
base = {}
model[DATA][ESNODES] = []
"""
For each es_node, will merge elasticsearch vars from:
- Plugin default configuration file
- global from cluster definition file
- parent role
- es_node """
for role in model[CLUSTER][ROLES]:
if ELASTICSEARCH in role and NODES in role[ELASTICSEARCH]:
index = -1
for esnode in role[ELASTICSEARCH][NODES]:
index += 1
map = copy.deepcopy(base)
# Add repository info. There is two reasons to use a package url:
# - It will be faster if the repo is local
# - Seems yum install is bugged on current role:
# TASK [ansible-elasticsearch : RedHat - Install Elasticsearch] **************************************************************************************************
# fatal: [w2]: FAILED! => {"msg": "The conditional check 'redhat_elasticsearch_install_from_repo.rc == 0' failed. The error was: error while evaluating conditional (redhat_elasticsearch_install_from_repo.rc == 0): 'dict object' has no attribute 'rc'"}
map["es_custom_package_url"] = model[DATA][REPOSITORIES][ELASTICSEARCH]["elasticsearch_package_url"]
map["es_use_repository"] = False
# Add global value
if ELASTICSEARCH in model[CLUSTER] and PLAYBOOK_VARS in model[CLUSTER][ELASTICSEARCH]:
if not isinstance(model[CLUSTER][ELASTICSEARCH][PLAYBOOK_VARS], dict):
ERROR("Invalid global '{}.{}' definition: not a dictionary".format(ELASTICSEARCH, PLAYBOOK_VARS))
else:
map = schemaMerge(map, model[CLUSTER][ELASTICSEARCH][PLAYBOOK_VARS])
# Add the role specific value
if PLAYBOOK_VARS in role[ELASTICSEARCH]:
if not isinstance(role[ELASTICSEARCH][PLAYBOOK_VARS], dict):
ERROR("Invalid role definition ('{}'): '{}.{}' is not a dictionary".format(role[NAME], ELASTICSEARCH,PLAYBOOK_VARS))
else:
map = schemaMerge(map, role[ELASTICSEARCH][PLAYBOOK_VARS])
# And get the es_node specific value
if not isinstance(esnode, dict):
ERROR("Invalid node definition in role '{}': item#{} is not a dictionary".format(role[NAME], index))
else:
map = schemaMerge(map, esnode)
if not ES_CONFIG in map or not NODE_MASTER in map[ES_CONFIG]:
ERROR("Invalid es_node definition in role '{}, item#{}: es_config.'node.master' must be defined".format(role[NAME], index))
if not ES_CONFIG in map or not NODE_DATA in map[ES_CONFIG]:
ERROR("Invalid es_node definition in role '{}, item#{}: es_config.'node.data' must be defined".format(role[NAME], index))
if not ES_INSTANCE_NAME in map:
ERROR("Invalid es_node definition in role '{}, item#{}: es_instance_name must be defined".format(role[NAME], index))
map[ES_VERSION] = model[DATA][REPOSITORIES][ELASTICSEARCH][VERSION]
map[ES_MAJOR_VERSION] = map[ES_VERSION][:2] + "X"
esn = {}
esn[ROLE] = role[NAME]
esn[VARS] = map
model[DATA][ESNODES].append(esn)
# We must arrange for master nodes to be deployed first.
model[DATA][ESNODES].sort(key=keyFromEsNode, reverse=False)
# We need to define an ansible group "_elasticsearch_" hosting all nodes with elasticsearch installed
elasticGroup = []
for role in model[CLUSTER][ROLES]:
if ELASTICSEARCH in role and NODES in role[ELASTICSEARCH]:
for node in role[NODES]:
elasticGroup.append(node[NAME])
model[DATA][GROUP_BY_NAME][_ELASTICSEARCH_] = elasticGroup
return True
def main():
global vaultFactory
mydir = os.path.dirname(os.path.realpath(__file__))
parser = argparse.ArgumentParser()
parser.add_argument('--src', required=True)
parser.add_argument('--mark', choices=["none", "both", "start", "end"])
parser.add_argument('--dump', action='store_true')
parser.add_argument('--dumpPasswords', action='store_true')
parser.add_argument('--out') # Generate a file to set some variable
param = parser.parse_args()
loggingConfFile = os.path.join(mydir, "./logging.yml")
logging.config.dictConfig(
yaml.load(open(loggingConfFile), Loader=yaml.SafeLoader))
sourceFile = os.path.normpath(os.path.abspath(param.src))
if not os.path.isfile(sourceFile):
ERROR("File '{}' does not exists".format(sourceFile))
logger.info("Will handle '{}'".format(sourceFile))
sourceFileDir = os.path.dirname(sourceFile)
cluster = yaml.load(open(sourceFile), Loader=yaml.SafeLoader)
targetFolder = misc.appendPath(
sourceFileDir,
cluster["build_folder"] if "build_folder" in cluster else "build")
misc.ensureFolder(targetFolder)
logger.info("Build folder: '{}'".format(targetFolder))
if "config_file" in cluster:
baseConfigFile = cluster["config_file"]
else:
baseConfigFile = "ezconfig.yml"
config, configFile = buildConfig(sourceFileDir, baseConfigFile)
plugins = []
plugins.append(Plugin("core", misc.appendPath(mydir, "../plugins/core")))
logger.debug("Plugins path:'{}'".format(config[PLUGINS_PATH]))
appendPlugins(plugins, cluster, config[PLUGINS_PATH])
schema = buildSchema(mydir, plugins)
configSchema, safeConfigSchema = buildConfigSchema(mydir,
config[PLUGINS_PATH])
if param.dump:
dumper = Dumper(targetFolder, param.dumpPasswords)
dumper.dump("schema.json", schema)
dumper.dump("config-schema.json", configSchema)
dumper.dump("safe-config-schema.json", safeConfigSchema)
else:
dumper = None
k = kwalify(source_data=cluster, schema_data=schema)
k.validate(raise_exception=False)
if len(k.errors) != 0:
ERROR("Problem {0}: {1}".format(sourceFile, k.errors))
k = kwalify(source_data=config, schema_data=configSchema)
k.validate(raise_exception=False)
if len(k.errors) != 0:
ERROR("Configuration problem {0}: {1}".format(configFile, k.errors))
data = {}
data['sourceFileDir'] = sourceFileDir
data["targetFolder"] = targetFolder
data['ezclusterHome'] = misc.appendPath(mydir, "..")
data["rolePaths"] = set()
data["configFile"] = configFile
model = {}
model['cluster'] = cluster
model["config"] = config
model['data'] = data
initVault(model)
if SAFE_CONFIG in model and safeConfigSchema != None:
k = kwalify(source_data=model[SAFE_CONFIG],
schema_data=safeConfigSchema)
k.validate(raise_exception=False)
if len(k.errors) != 0:
ERROR("Configuration problem {0}: {1}".format(
model["data"][_SAFE_CONFIG_FILE_], k.errors))
for plugin in plugins:
plugin.groom(model)
for plugin in plugins:
plugin.groom2(model)
targetFileByName = buildTargetFileByName(plugins)
if param.dump:
dumper.dump("cluster.json", model['cluster'])
dumper.dump("data.json", model['data'])
dumper.dump("targetFileByName.json", targetFileByName)
dumper.dump("config.json", config)
if SAFE_CONFIG in model and dumper.unsafe:
dumper.dump("safeConfig.json", model[SAFE_CONFIG])
for plugin in plugins:
plugin.dump(model, dumper)
generate(targetFileByName, targetFolder, model, param.mark, dumper)
if "out" in param:
f = open(param.out, "w+")
f.write("# Generated by ezcluster:\n")
if "buildScript" in model["data"]:
f.write('BUILD_SCRIPT="{}"\n'.format(model["data"]["buildScript"]))
f.close()