def groomMavenRepositories(context):
model = context.model
misc.ensureObjectInMaps(model[DATA], [MAVEN_REPO_BY_NAME], {})
if MAVEN_REPOSITORIES in model[SRC]:
for repo in model[SRC][MAVEN_REPOSITORIES]:
model[DATA][MAVEN_REPO_BY_NAME][repo["name"]] = repo
misc.setDefaultInMap(repo, VALIDATE_CERTS, True)
misc.setDefaultInMap(repo, TIMEOUT, 10)
def onGrooming(self):
misc.ensureObjectInMaps(self.context.model[DATA], [SUPERVISORS, SCOPE_BY_NAME], {})
misc.applyWhenOnList(self.context.model[SRC], SUPERVISORS)
misc.applyWhenOnList(self.context.model[SRC], SUPERVISOR_PROGRAMS)
misc.applyWhenOnList(self.context.model[SRC], SUPERVISOR_GROUPS)
self.groomSupervisors()
self.groomPrograms()
self.groomGroups()
def onGrooming(self):
if self.context.toExclude(SCOPE_YARN):
return
misc.applyWhenOnSingle(self.context.model[SRC], YARN_RELAY)
misc.applyWhenOnList(self.context.model[SRC], YARN_SERVICES)
misc.ensureObjectInMaps(self.context.model[DATA], [YARN], {})
groomYarnRelay(self.context.model)
groomYarnServices(self.context.model)
def onGrooming(self):
misc.applyWhenOnList(self.context.model[SRC], USERS)
misc.applyWhenOnList(self.context.model[SRC], GROUPS)
if self.context.toExclude(SCOPE_USERS):
return
misc.ensureObjectInMaps(self.context.model[DATA], [USERS, SCOPE_BY_NAME], {})
groomUsers(self.context)
groomGroups(self.context)
def onGrooming(self):
misc.applyWhenOnSingle(self.context.model[SRC], KAFKA_RELAY)
misc.applyWhenOnList(self.context.model[SRC], KAFKA_TOPICS)
if self.context.toExclude(SCOPE_KAFKA):
return
self.buildHelper()
misc.ensureObjectInMaps(self.context.model[DATA], [KAFKA], {})
groomKafkaRelay(self.context.model)
groomKafkaTopics(self.context.model)
def grabHBaseRangerPoliciesFromTables(model):
if HBASE_TABLES in model[SRC]:
for table in model[SRC][HBASE_TABLES]:
if RANGER_POLICY in table:
policy = table[RANGER_POLICY]
policy[TABLES] = [ table[NAMESPACE] +':' + table[NAME] ]
misc.setDefaultInMap(policy, NAME, DEFAULT_HBASE_TABLE_POLICY_NAME.format(table[NAMESPACE], table[NAME]))
policy[NO_REMOVE] = table[NO_REMOVE]
misc.ensureObjectInMaps( model[SRC], [HBASE_RANGER_POLICIES], [])
model[SRC][HBASE_RANGER_POLICIES].append(policy)
def grabHBaseRangerPoliciesFromNamespaces(model):
if HBASE_NAMESPACES in model[SRC]:
for namespace in model[SRC][HBASE_NAMESPACES]:
if RANGER_POLICY in namespace:
policy = namespace[RANGER_POLICY]
policy[TABLES] = [ namespace[NAME] + ":*" ]
misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(namespace[NAME]))
policy[NO_REMOVE] = namespace[NO_REMOVE]
misc.ensureObjectInMaps( model[SRC], [HBASE_RANGER_POLICIES], [])
model[SRC][HBASE_RANGER_POLICIES].append(policy)
def grabStormRangerPoliciesFromTopologies(model):
if STORM_TOPOLOGIES in model[SRC]:
for topology in model[SRC][STORM_TOPOLOGIES]:
if RANGER_POLICY in topology:
policy = topology[RANGER_POLICY]
policy[TOPOLOGIES] = [ topology[NAME] ]
misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(topology[NAME]))
policy[NO_REMOVE] = topology[NO_REMOVE]
misc.ensureObjectInMaps( model[SRC], [STORM_RANGER_POLICIES], [])
model[SRC][STORM_RANGER_POLICIES].append(policy)
def grabKafkaRangerPoliciesFromTopics(model):
if KAFKA_TOPICS in model[SRC]:
for topic in model[SRC][KAFKA_TOPICS]:
if RANGER_POLICY in topic:
policy = topic[RANGER_POLICY]
policy[TOPICS] = [ topic[NAME] ]
misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(topic[NAME]))
policy[NO_REMOVE] = topic[NO_REMOVE]
misc.ensureObjectInMaps( model[SRC], [KAFKA_RANGER_POLICIES], [])
model[SRC][KAFKA_RANGER_POLICIES].append(policy)
def onGrooming(self):
misc.applyWhenOnList(self.context.model[SRC], ELASTICSEARCH_SERVERS)
misc.applyWhenOnList(self.context.model[SRC], ELASTICSEARCH_INDICES)
misc.applyWhenOnList(self.context.model[SRC], ELASTICSEARCH_TEMPLATES)
if self.context.toExclude(SCOPE_ELASTIC):
return
misc.ensureObjectInMaps(self.context.model[DATA], [ELASTIC], {})
groomElasticServers(self.context.model)
groomElasticIndices(self.context.model)
groomElasticTemplates(self.context.model)
def buildHelper(self):
helper = {}
helper[DIR] = os.path.normpath(os.path.join(self.path, "helpers"))
jdchivejars = glob.glob(
os.path.join(helper[DIR], "jdchive/jdchive_uber*.jar"))
if len(jdchivejars) < 1:
misc.ERROR(
"Unable to find helper for Hive.Please, refer to the documentation about Installation"
)
helper[JDCHIVE_JAR] = os.path.basename(jdchivejars[0])
misc.ensureObjectInMaps(self.context.model, [HELPER, HIVE], helper)
def onGrooming(self):
misc.applyWhenOnSingle(self.context.model[SRC], HIVE_RELAY)
misc.applyWhenOnList(self.context.model[SRC], HIVE_DATABASES)
misc.applyWhenOnList(self.context.model[SRC], HIVE_TABLES)
if self.context.toExclude(SCOPE_HIVE):
return
self.buildHelper()
misc.ensureObjectInMaps(self.context.model[DATA], [HIVE], {})
groomHiveRelay(self.context.model)
groomHiveDatabases(self.context.model)
groomHiveTables(self.context.model)
def grabHdfsRangerPoliciesFromTrees(model):
if TREES in model[SRC]:
for tree in model[SRC][TREES]:
if RANGER_POLICY in tree:
if tree[SCOPE] != HDFS:
misc.ERROR("Can't setup Apache Ranger policy on tree '{0}' as scope is not hdfs".format(tree[DEST_FOLDER]))
policy = tree[RANGER_POLICY]
policy[PATHS] = [ tree[DEST_FOLDER] ]
policy[NO_REMOVE] = tree[NO_REMOVE]
misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(tree[DEST_FOLDER]))
misc.ensureObjectInMaps( model[SRC], [HDFS_RANGER_POLICIES], [])
model[SRC][HDFS_RANGER_POLICIES].append(policy)
def grabHdfsRangerPoliciesFromFolders(model):
if FOLDERS in model[SRC]:
for folder in model[SRC][FOLDERS]:
if RANGER_POLICY in folder:
if folder[SCOPE] != HDFS:
misc.ERROR("Can't setup Apache Ranger policy on folder '{0}' as scope is not hdfs".format(folder[PATH]))
policy = folder[RANGER_POLICY]
policy[PATHS] = [ folder[PATH] ]
policy[NO_REMOVE] = folder[NO_REMOVE]
misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(folder[PATH]))
misc.ensureObjectInMaps( model[SRC], [HDFS_RANGER_POLICIES], [])
model[SRC][HDFS_RANGER_POLICIES].append(policy)
def grabHiveRangerPoliciesFromTables(model):
if HIVE_TABLES in model[SRC]:
for table in model[SRC][HIVE_TABLES]:
if RANGER_POLICY in table:
policy = table[RANGER_POLICY]
policy[DATABASES] = [ table[DATABASE] ]
policy[TABLES] = [ table[NAME] ]
policy[COLUMNS] = [ "*" ]
misc.setDefaultInMap(policy, NAME, DEFAULT_HIVE_TABLE_POLICY_NAME.format(table[DATABASE], table[NAME]))
policy[NO_REMOVE] = table[NO_REMOVE]
misc.ensureObjectInMaps( model[SRC], [HIVE_RANGER_POLICIES], [])
model[SRC][HIVE_RANGER_POLICIES].append(policy)
def grabHiveRangerPoliciesFromDatabase(model):
if HIVE_DATABASES in model[SRC]:
for database in model[SRC][HIVE_DATABASES]:
if RANGER_POLICY in database:
policy = database[RANGER_POLICY]
policy[DATABASES] = [ database[NAME] ]
policy[TABLES] = [ "*" ]
policy[COLUMNS] = [ "*" ]
misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(database[NAME]))
policy[NO_REMOVE] = database[NO_REMOVE]
misc.ensureObjectInMaps( model[SRC], [HIVE_RANGER_POLICIES], [])
model[SRC][HIVE_RANGER_POLICIES].append(policy)
def grabHdfsRangerPoliciesFromFiles(model):
if FILES in model[SRC]:
for xfile in model[SRC][FILES]:
if RANGER_POLICY in xfile:
if xfile[SCOPE] != HDFS:
misc.ERROR("Can't setup Apache Ranger policy on file '{0}' as scope is not hdfs".format(os.path.join(xfile[DEST_FOLDER], xfile[DEST_NAME])))
policy = xfile[RANGER_POLICY]
misc.setDefaultInMap(policy, RECURSIVE, False)
policy[PATHS] = [ os.path.join(xfile[DEST_FOLDER], xfile[DEST_NAME]) ] # groomFiles should have been called before
misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format( policy[PATHS][0]))
policy[NO_REMOVE] = xfile[NO_REMOVE]
misc.ensureObjectInMaps( model[SRC], [HDFS_RANGER_POLICIES], [])
model[SRC][HDFS_RANGER_POLICIES].append(policy)
def onGrooming(self):
misc.applyWhenOnSingle(self.context.model[SRC], HBASE_RELAY)
misc.applyWhenOnList(self.context.model[SRC], HBASE_NAMESPACES)
misc.applyWhenOnList(self.context.model[SRC], HBASE_TABLES)
misc.applyWhenOnList(self.context.model[SRC], HBASE_DATASETS)
if self.context.toExclude(SCOPE_HBASE):
return
self.buildHelper()
misc.ensureObjectInMaps(self.context.model[DATA], [HBASE], {})
groomHbaseRelay(self.context.model)
groomHBaseNamespaces(self.context.model)
groomHBaseTables(self.context.model)
groomHBaseDatasets(self.context.model)
def groomPrograms(self):
if self.context.toExclude(SCOPE_SUPERVISOR):
return
model = self.context.model
if SUPERVISOR_PROGRAMS in model[SRC]:
for prg in model[SRC][SUPERVISOR_PROGRAMS]:
if not SUPERVISOR_BY_NAME in model[DATA][SUPERVISORS] or not prg[SUPERVISOR] in model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME]:
misc.ERROR("supervisor_program '{}' refer to an undefined supervisor '{}'".format(prg[NAME], prg[SUPERVISOR]))
else:
supervisord = model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME][prg[SUPERVISOR]]
if prg[NAME] in supervisord[PROGRAM_BY_NAME]:
misc.ERROR("supervisor_program '{}' is defined twice in supervisor '{}'".format(prg[NAME], supervisord[NAME]))
# Register in model.data
supervisord[PROGRAM_BY_NAME][prg[NAME]] = prg
#model[DATA][SUPERVISORS][SCOPE_BY_NAME][supervisord[SCOPE]][PROGRAMS].append(prg)
# Adjust attributes
misc.setDefaultInMap(prg, NO_REMOVE, False)
if prg[NO_REMOVE] and not supervisord[NO_REMOVE]:
misc.ERROR("Supervisor_program '{}' has no remove flag set while its supervisor ({}) has not!".format(prg[NAME], supervisord[NAME]))
if CONF_FILE_SRC in prg:
path, _, errMsg = lookupSrc(model, prg[CONF_FILE_SRC])
if path != None:
prg[CONF_FILE_SRC_JJ2] = path
else:
misc.ERROR("Supervisor_program '{0}': {1}".format(prg[NAME], errMsg))
else:
prg[CONF_FILE_SRC_JJ2] = os.path.join(self.path, "templates/program.conf.jj2")
if COMMAND not in prg:
misc.ERROR("Supervisor_program '{}': A 'command' parameter must be provided if using the default configuration file (No 'conf_file_src' parameter".format(prg[NAME]))
prg[CONF_FILE_SRC_J2] = "supervisor_{}_program_{}.conf".format(supervisord[NAME], prg[NAME])
prg[CONF_FILE_DST] = os.path.join(supervisord[INCLUDE_DIR], "{}_prg.ini".format(prg[NAME]))
prg[SUPERVISOR_OWNER] = supervisord[USER]
prg[SUPERVISOR_GROUP] = supervisord[GROUP]
prg[SUPERVISOR_CONF] = supervisord[CONF_FILE_DST]
misc.setDefaultInMap(prg, STATE, ST_STARTED)
if NUMPROCS in prg and prg[NUMPROCS] > 1:
prg[_NAME_] = prg[NAME] + ":" # This is in fact a group of process
else:
prg[_NAME_] = prg[NAME]
if prg[STATE] not in validState:
misc.ERROR("Supervisor_program {0}: state value '{1}' is not valid. Must be one of {2}".format(prg[NAME], prg[STATE], validState))
misc.setDefaultInMap(prg, AUTOSTART, prg[STATE] == ST_STARTED)
if SCOPE not in prg:
prg[SCOPE] = supervisord[SCOPE]
# Note we don't set prg[USER], as we want to be unset in config file if not set
# ---------------------- Insert in scope
misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [prg[SCOPE], PROGRAMS_TO_MANAGE], [])
model[DATA][SUPERVISORS][SCOPE_BY_NAME][prg[SCOPE]][PROGRAMS_TO_MANAGE].append(prg)
if not prg[NO_REMOVE]:
misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [prg[SCOPE], PROGRAMS_TO_REMOVE], [])
model[DATA][SUPERVISORS][SCOPE_BY_NAME][prg[SCOPE]][PROGRAMS_TO_REMOVE].append(prg)
def onGrooming(self):
model = self.context.model
#logger.debug("Called self.onGrooming() for plugin '{0}'".format(self.name))
misc.ensureObjectInMaps(model[SRC], [PLUGINS_PATHS], [])
# Add our internal plugins at the end of the list
model[SRC][PLUGINS_PATHS].append(
os.path.normpath(
os.path.join(os.path.dirname(__file__), "../../plugins")))
if not PLUGINS in model[SRC]:
model[SRC][PLUGINS] = DEFAULT_PLUGINS
# ---------------------------- Check encrypted vars
if ENCRYPTED_VARS in model[SRC]:
for k, v in model[SRC][ENCRYPTED_VARS].iteritems():
if not (isinstance(v, basestring)
and v.startswith("$ANSIBLE_VAULT")):
misc.ERROR(
"Encrypted variable '{0}' does not seems to provide a valid encrypted value"
.format(k))
def buildHelper(self):
helper = {}
helper[DIR] = os.path.normpath(os.path.join(self.path, "helpers"))
jdchtablejars = glob.glob(
os.path.join(helper[DIR], "jdchtable/jdchtable_uber*.jar"))
if len(jdchtablejars) < 1:
misc.ERROR(
"Unable to find helper for HBase.Please, refer to the documentation about Installation"
)
helper[JDCHTABLE_JAR] = os.path.basename(jdchtablejars[0])
hbloadjars = glob.glob(
os.path.join(helper[DIR], "hbload/hbload_uber*.jar"))
if len(hbloadjars) < 1:
misc.ERROR(
"Unable to find helper for HBase datasets loader. Please, refer to the documentation about Installation"
)
helper[HBLOAD_JAR] = os.path.basename(hbloadjars[0])
misc.ensureObjectInMaps(self.context.model, [HELPER, HBASE], helper)
def buildHelper(self):
if KAFKA_RELAY in self.context.model[SRC]:
helper = {}
helper[DIR] = os.path.normpath(os.path.join(self.path, "helpers"))
jarPattern = "jdctopic/jdctopic.{}-*-uber.jar".format(
self.context.model[SRC][KAFKA_RELAY][KAFKA_VERSION])
jdctopicjars = glob.glob(os.path.join(helper[DIR], jarPattern))
if len(jdctopicjars) < 1:
misc.ERROR(
"Unable to find helper for Kafka.Please, refer to the documentation about Installation"
)
if len(jdctopicjars) > 1:
misc.ERROR(
"Several version of kafka helper jar in {}. Please, cleanup."
.format(helper[DIR]))
helper[JDCTOPIC_JAR] = os.path.basename(jdctopicjars[0])
misc.ensureObjectInMaps(self.context.model, [HELPER, KAFKA],
helper)
def groomGroups(self):
if self.context.toExclude(SCOPE_SUPERVISOR):
return
model = self.context.model
if SUPERVISOR_GROUPS in model[SRC]:
for grp in model[SRC][SUPERVISOR_GROUPS]:
if not SUPERVISOR_BY_NAME in model[DATA][SUPERVISORS] or not grp[SUPERVISOR] in model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME]:
misc.ERROR("supervisor_group '{}' refer to an undefined supervisor '{}'".format(grp[NAME], grp[SUPERVISOR]))
else:
supervisord = model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME][grp[SUPERVISOR]]
if grp[NAME] in supervisord[GROUP_BY_NAME]:
misc.ERROR("supervisor_group '{}' is defined twice in supervisor '{}'".format(grp[NAME], supervisord[NAME]))
supervisord[GROUP_BY_NAME][grp[NAME]] = grp
for prgName in grp[PROGRAMS]:
if prgName not in supervisord[PROGRAM_BY_NAME]:
misc.ERROR("supervisor_group '{}' refer to an undefined program '{}'".format(grp[NAME], prgName))
else:
prg = supervisord[PROGRAM_BY_NAME][prgName]
# The program name must be patched:
prg[_NAME_] = grp[NAME] + ":" + supervisord[PROGRAM_BY_NAME][prgName][_NAME_]
if _SCOPE_ in grp:
if grp[_SCOPE_] != prg[SCOPE]:
misc.ERROR("supervisor_group '{}' host programs with different scope ({} != {}). Must be same".format(grp[NAME], grp[_SCOPE_], prg[SCOPE]))
else:
grp[_SCOPE_] = prg[SCOPE]
misc.setDefaultInMap(grp, NO_REMOVE, False)
if grp[NO_REMOVE] and not supervisord[NO_REMOVE]:
misc.ERROR("Supervisor_group '{}' has no remove flag set while its supervisor ({}) has not!".format(grp[NAME], supervisord[NAME]))
grp[CONF_FILE_SRC_JJ2] = os.path.join(self.path, "templates/group.conf.jj2")
grp[CONF_FILE_SRC_J2] = "supervisor_{}_group_{}.conf".format(supervisord[NAME], grp[NAME])
grp[CONF_FILE_DST] = os.path.join(supervisord[INCLUDE_DIR], "{}_grp.ini".format(grp[NAME]))
grp[SUPERVISOR_OWNER] = supervisord[USER]
grp[SUPERVISOR_GROUP] = supervisord[GROUP]
grp[SUPERVISOR_CONF] = supervisord[CONF_FILE_DST]
grp[_NAME_] = grp[NAME] + ":"
# ---------------------- Insert in scope
misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [grp[_SCOPE_], GROUPS_TO_MANAGE], [])
model[DATA][SUPERVISORS][SCOPE_BY_NAME][grp[_SCOPE_]][GROUPS_TO_MANAGE].append(grp)
if not grp[NO_REMOVE]:
misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [grp[_SCOPE_], GROUPS_TO_REMOVE], [])
model[DATA][SUPERVISORS][SCOPE_BY_NAME][grp[_SCOPE_]][GROUPS_TO_REMOVE].append(grp)
def groomSystemd(self):
if self.context.toExclude(SCOPE_SYSTEMD):
return
model = self.context.model
unitNames = Set()
if SYSTEMD_UNITS in model[SRC]:
for unit in model[SRC][SYSTEMD_UNITS]:
if unit[NAME] in unitNames:
misc.ERROR("systemd_unit '{0}' is defined twice!".format(
unit[NAME]))
unitNames.add(unit[NAME])
misc.setDefaultInMap(unit, NO_REMOVE, False)
misc.setDefaultInMap(unit, ENABLED, True)
misc.setDefaultInMap(unit, STATE, ST_CURRENT)
if unit[STATE] not in validState:
misc.ERROR(
"Systemd_unit {0}: state value '{1}' is not valid. Must be one of {2}"
.format(unit[NAME], unit[STATE], validState))
misc.setDefaultInMap(unit, ACTION_ON_NOTIFY, AON_RESTART)
if unit[ACTION_ON_NOTIFY] not in validAon:
misc.ERROR(
"Systemd_unit {0}: action_on_notify value '{1}' is not valid. Must be one of {2}"
.format(unit[NAME], unit[ACTION_ON_NOTIFY], validAon))
# ---------------------- Lookup unit file
path, displaySrc, errMsg = lookupSrc(model, unit[UNIT_FILE])
if path != None:
unit[_UNIT_FILE_] = path
unit[_DISPLAY_SRC_] = displaySrc
else:
misc.ERROR("Systemd_unit '{0}': {1}".format(
unit[NAME], errMsg))
# ---------------------- Insert in scope
if not self.context.checkScope(unit[SCOPE]):
misc.ERROR(
"Systemd_unit {0}: scope attribute '{1}' does not match any host or host_group!"
.format(unit[NAME], unit[SCOPE]))
else:
misc.ensureObjectInMaps(
self.context.model[DATA][SYSTEMD][SCOPE_BY_NAME],
[unit[SCOPE], SYSTEMD], [])
model[DATA][SYSTEMD][SCOPE_BY_NAME][
unit[SCOPE]][SYSTEMD].append(unit)
def groomSupervisors(self):
if self.context.toExclude(SCOPE_SUPERVISOR):
return
model = self.context.model
if SUPERVISORS in model[SRC]:
misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS], [SUPERVISOR_BY_NAME], {})
for supervisord in model[SRC][SUPERVISORS]:
if supervisord[NAME] in self.context.model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME]:
misc.ERROR("supervisor '{0}' is defined twice!".format(supervisord[NAME]))
self.context.model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME][supervisord[NAME]] = supervisord
misc.setDefaultInMap(supervisord, MANAGED, True)
self.groomOneSupervisord(model, supervisord)
# ---------------------- Insert in scope
if not self.context.checkScope(supervisord[SCOPE]):
misc.ERROR("Supervisor {0}: scope attribute '{1}' does not match any host or host_group!".format(supervisord[NAME], supervisord[SCOPE]))
else:
#misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [supervisord[SCOPE], SUPERVISORS], [])
#misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [supervisord[SCOPE], PROGRAMS], [])
#model[DATA][SUPERVISORS][SCOPE_BY_NAME][supervisord[SCOPE]][SUPERVISORS].append(supervisord)
if supervisord[MANAGED]:
misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [supervisord[SCOPE], SUPERVISORS_TO_MANAGE], [])
model[DATA][SUPERVISORS][SCOPE_BY_NAME][supervisord[SCOPE]][SUPERVISORS_TO_MANAGE].append(supervisord)
if not supervisord[NO_REMOVE]:
misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [supervisord[SCOPE], SUPERVISORS_TO_REMOVE], [])
model[DATA][SUPERVISORS][SCOPE_BY_NAME][supervisord[SCOPE]][SUPERVISORS_TO_REMOVE].append(supervisord)
def groomSourceHostCredentials(model):
misc.ensureObjectInMaps(model[DATA], [HDFS, CREDENTIAL_BY_HOST], {})
if SOURCE_HOST_CREDENTIALS in model[SRC]:
for hcredential in model[SRC][SOURCE_HOST_CREDENTIALS]:
key = hcredential[HOST]
model[DATA][HDFS][CREDENTIAL_BY_HOST][key] = hcredential
del model[DATA][HDFS][CREDENTIAL_BY_HOST][key][
HOST] # No need to keep hostname, as it is the key
misc.setDefaultInMap(hcredential, KDEBUG, False)
if LOCAL_KEYTAB_PATH not in hcredential and NODE_KEYTAB_PATH not in hcredential:
misc.ERROR(
"source_host_credential for host {0}: Please provide a 'local_keytab_path' and/or a 'node_keytab_path'"
.format(key))
if LOCAL_KEYTAB_PATH in hcredential:
if not os.path.exists(hcredential[LOCAL_KEYTAB_PATH]):
misc.ERROR(
"source_host_credential for host {0}: local_keytab_file '{1}' does not exists!"
.format(key, hcredential[LOCAL_KEYTAB_PATH]))
if NODE_KEYTAB_PATH not in hcredential:
hcredential[
_NODE_KEYTAB_FOLDER_] = DEFAULT_HDFS_KEYTABS_FOLDER
hcredential[NODE_KEYTAB_PATH] = os.path.join(
hcredential[_NODE_KEYTAB_FOLDER_],
os.path.basename(hcredential[LOCAL_KEYTAB_PATH]))
def groomElasticServers(model):
misc.ensureObjectInMaps(model[DATA][ELASTIC], [SERVER_BY_NAME], {})
if ELASTICSEARCH_SERVERS in model[SRC]:
relayFiles = Set() # To check uniqueness
for server in model[SRC][ELASTICSEARCH_SERVERS]:
if server[NAME] in model[DATA][ELASTIC][SERVER_BY_NAME]:
misc.ERROR(
"Elasticsearch server '{}' is defined twice!".format(
server[NAME]))
misc.ensureObjectInMaps(server, [INDICES], [])
misc.ensureObjectInMaps(server, [TEMPLATES], [])
model[DATA][ELASTIC][SERVER_BY_NAME][server[NAME]] = server
misc.setDefaultInMap(server, VALIDATE_CERTS, True)
misc.setDefaultInMap(server, NO_LOG, True)
if (USERNAME in server) != (PASSWORD in server):
misc.ERROR(
"Elasticsearch server '{}': 'username' and 'password' must be defined together or not at all!"
.format(server[NAME]))
if CA_BUNDLE_LOCAL_FILE in server:
if not os.path.exists(server[CA_BUNDLE_LOCAL_FILE]):
misc.ERROR(
"Elasticsearch server '{}': ca_bundle_local_file: {0} does not exists"
.format(server[NAME], server[CA_BUNDLE_LOCAL_FILE]))
if CA_BUNDLE_RELAY_FILE not in server:
misc.ERROR(
"Elasticsearch server '{}': If a ca_bundle_local_file is defined, then a ca_bundle_relay_file must also be defined"
.format(server[NAME]))
if not os.path.isabs(server[CA_BUNDLE_RELAY_FILE]):
misc.ERROR(
"Elasticsearch server '{}': ca_bundle_relay_file: {0} must be absolute!"
.format(server[NAME], server[CA_BUNDLE_RELAY_FILE]))
if server[CA_BUNDLE_RELAY_FILE] in relayFiles:
misc.ERROR(
"ca_bundle_relay_file: {} is defined by several elasticsearch_server"
.format(server[CA_BUNDLE_RELAY_FILE]))
relayFiles.add(server[CA_BUNDLE_RELAY_FILE])
server[CA_BUNDLE_RELAY_FOLDER] = os.path.dirname(
server[CA_BUNDLE_RELAY_FILE])
def ensureScope(context, scope):
root = context.model[DATA][USERS][SCOPE_BY_NAME]
if not scope in root:
misc.ensureObjectInMaps(root, [scope, USERS], [])
misc.ensureObjectInMaps(root, [scope, GROUPS], [])
def onGrooming(self):
misc.ensureObjectInMaps(self.context.model[DATA],
[SYSTEMD, SCOPE_BY_NAME], {})
misc.applyWhenOnList(self.context.model[SRC], SYSTEMD_UNITS)
self.groomSystemd()
def onGrooming(self):
misc.applyWhenOnList(self.context.model[SRC], ANSIBLE_PLAYBOOKS)
misc.applyWhenOnList(self.context.model[SRC], ANSIBLE_ROLES)
""" Main job is to build a referential playbookByActionByPriority"""
playbooksByActionByPriority = {}
if not self.context.toExclude(SCOPE_ANSIBLE):
src = self.context.model[SRC]
if ANSIBLE_PLAYBOOKS in src:
for pl in src[ANSIBLE_PLAYBOOKS]:
action = pl[FOR_ACTION]
priority = pl[PRIORITY]
misc.ensureObjectInMaps(playbooksByActionByPriority, [action], {})
misc.ensureObjectInMaps(playbooksByActionByPriority[action], [priority], {})
if PLAYBOOK_FILE in pl:
playbookFile = self.lookupPathInFolderList(pl[PLAYBOOK_FILE], PLAYBOOKS_FOLDERS, "playbook")
misc.ensureObjectInMaps(playbooksByActionByPriority[action][priority], [PLAYBOOK_FILES], [])
playbooksByActionByPriority[action][priority][PLAYBOOK_FILES].append(playbookFile)
elif PLAYBOOK_TEXT in pl:
misc.ensureObjectInMaps(playbooksByActionByPriority[action][priority], [PLAYBOOK_TEXTS], [])
playbooksByActionByPriority[action][priority][PLAYBOOK_TEXTS].append(pl[PLAYBOOK_TEXT])
else:
misc.ERROR("There is at least one ansible_playbook whith neither playbook_text not playbook_file. (action:'{0}', priority:'{1}'".format(action, priority))
if ANSIBLE_ROLES in src:
for rl in src[ANSIBLE_ROLES]:
if not self.context.checkScope(rl[SCOPE]):
misc.ERROR("Invalid scope '{0}' in ansible role '{1}'".format(rl[SCOPE], rl[ROLE]))
action = rl[FOR_ACTION]
priority = rl[PRIORITY]
misc.ensureObjectInMaps(playbooksByActionByPriority, [action], {})
misc.ensureObjectInMaps(playbooksByActionByPriority[action], [priority], {})
misc.ensureObjectInMaps(playbooksByActionByPriority[action][priority], [ROLES], [])
playbooksByActionByPriority[action][priority][ROLES].append( { ROLE: rl[ROLE], SCOPE: rl[SCOPE] } )
self.referential = playbooksByActionByPriority
# We don't need to expose this to template, but this can be usefull for debugging
self.context.model[DATA][PLAYBOOKS_BY_ACTION_BY_PRIORITY] = playbooksByActionByPriority
