def onGrooming(self):
if self.context.toExclude(SCOPE_STORM):
return
misc.applyWhenOnSingle(self.context.model[SRC], STORM_RELAY)
misc.applyWhenOnList(self.context.model[SRC], STORM_TOPOLOGIES)
groomStormRelay(self.context.model)
groomStormTopologies(self.context.model)
def onGrooming(self):
misc.applyWhenOnList(self.context.model[SRC], ANSIBLE_PLAYBOOKS)
misc.applyWhenOnList(self.context.model[SRC], ANSIBLE_ROLES)
""" Main job is to build a referential playbookByActionByPriority"""
playbooksByActionByPriority = {}
if not self.context.toExclude(SCOPE_ANSIBLE):
src = self.context.model[SRC]
if ANSIBLE_PLAYBOOKS in src:
for pl in src[ANSIBLE_PLAYBOOKS]:
action = pl[FOR_ACTION]
priority = pl[PRIORITY]
misc.ensureObjectInMaps(playbooksByActionByPriority, [action], {})
misc.ensureObjectInMaps(playbooksByActionByPriority[action], [priority], {})
if PLAYBOOK_FILE in pl:
playbookFile = self.lookupPathInFolderList(pl[PLAYBOOK_FILE], PLAYBOOKS_FOLDERS, "playbook")
misc.ensureObjectInMaps(playbooksByActionByPriority[action][priority], [PLAYBOOK_FILES], [])
playbooksByActionByPriority[action][priority][PLAYBOOK_FILES].append(playbookFile)
elif PLAYBOOK_TEXT in pl:
misc.ensureObjectInMaps(playbooksByActionByPriority[action][priority], [PLAYBOOK_TEXTS], [])
playbooksByActionByPriority[action][priority][PLAYBOOK_TEXTS].append(pl[PLAYBOOK_TEXT])
else:
misc.ERROR("There is at least one ansible_playbook whith neither playbook_text not playbook_file. (action:'{0}', priority:'{1}'".format(action, priority))
if ANSIBLE_ROLES in src:
for rl in src[ANSIBLE_ROLES]:
if not self.context.checkScope(rl[SCOPE]):
misc.ERROR("Invalid scope '{0}' in ansible role '{1}'".format(rl[SCOPE], rl[ROLE]))
action = rl[FOR_ACTION]
priority = rl[PRIORITY]
misc.ensureObjectInMaps(playbooksByActionByPriority, [action], {})
misc.ensureObjectInMaps(playbooksByActionByPriority[action], [priority], {})
misc.ensureObjectInMaps(playbooksByActionByPriority[action][priority], [ROLES], [])
playbooksByActionByPriority[action][priority][ROLES].append( { ROLE: rl[ROLE], SCOPE: rl[SCOPE] } )
self.referential = playbooksByActionByPriority
# We don't need to expose this to template, but this can be usefull for debugging
self.context.model[DATA][PLAYBOOKS_BY_ACTION_BY_PRIORITY] = playbooksByActionByPriority
def onGrooming(self):
if self.context.toExclude(SCOPE_YARN):
return
misc.applyWhenOnSingle(self.context.model[SRC], YARN_RELAY)
misc.applyWhenOnList(self.context.model[SRC], YARN_SERVICES)
misc.ensureObjectInMaps(self.context.model[DATA], [YARN], {})
groomYarnRelay(self.context.model)
groomYarnServices(self.context.model)
def onGrooming(self):
misc.applyWhenOnList(self.context.model[SRC], USERS)
misc.applyWhenOnList(self.context.model[SRC], GROUPS)
if self.context.toExclude(SCOPE_USERS):
return
misc.ensureObjectInMaps(self.context.model[DATA], [USERS, SCOPE_BY_NAME], {})
groomUsers(self.context)
groomGroups(self.context)
def onGrooming(self):
misc.applyWhenOnSingle(self.context.model[SRC], KAFKA_RELAY)
misc.applyWhenOnList(self.context.model[SRC], KAFKA_TOPICS)
if self.context.toExclude(SCOPE_KAFKA):
return
self.buildHelper()
misc.ensureObjectInMaps(self.context.model[DATA], [KAFKA], {})
groomKafkaRelay(self.context.model)
groomKafkaTopics(self.context.model)
def onGrooming(self):
misc.applyWhenOnSingle(self.context.model[SRC], HIVE_RELAY)
misc.applyWhenOnList(self.context.model[SRC], HIVE_DATABASES)
misc.applyWhenOnList(self.context.model[SRC], HIVE_TABLES)
if self.context.toExclude(SCOPE_HIVE):
return
self.buildHelper()
misc.ensureObjectInMaps(self.context.model[DATA], [HIVE], {})
groomHiveRelay(self.context.model)
groomHiveDatabases(self.context.model)
groomHiveTables(self.context.model)
def onGrooming(self):
if ANSIBLE_INVENTORIES in self.context.model[SRC]:
misc.applyWhenOnList(self.context.model[SRC], ANSIBLE_INVENTORIES)
for inventory in self.context.model[SRC][ANSIBLE_INVENTORIES]:
if not os.path.exists(inventory[FILE]):
misc.ERROR(
"Ansible inventory file '{0}' does not exists!".format(
inventory[FILE]))
else:
populateModelFromInventory(self.context.model[SRC],
inventory)
def onGrooming(self):
model = self.context.model
misc.ensureObjectInMaps(model[DATA], [FILES, SCOPE_BY_NAME], {})
if HDFS in self.context.pluginByName:
# We need to anticipate on works performed by hdfs plugin, as we need it right now
misc.ensureObjectInMaps(model[DATA], [HDFS, NODE_TO_HDFS_BY_NAME],
{})
misc.ensureObjectInMaps(model[DATA], [HDFS, FILES], [])
misc.ensureObjectInMaps(model[DATA], [HDFS, FOLDERS], [])
misc.ensureObjectInMaps(model[DATA], [HDFS, TREES], [])
misc.ensureObjectInMaps(model[DATA], [HDFS, CACHEFOLDERS], Set())
misc.applyWhenOnSingle(self.context.model[SRC], HDFS_RELAY)
misc.applyWhenOnList(self.context.model[SRC],
SOURCE_HOST_CREDENTIALS)
if HDFS_RELAY in model[SRC]:
misc.setDefaultInMap(model[SRC][HDFS_RELAY], CACHE_FOLDER,
DEFAULT_HDFS_RELAY_CACHE_FOLDER)
misc.applyWhenOnList(self.context.model[SRC], MAVEN_REPOSITORIES)
misc.applyWhenOnList(self.context.model[SRC], FOLDERS)
misc.applyWhenOnList(self.context.model[SRC], FILES)
misc.applyWhenOnList(self.context.model[SRC], TREES)
groomMavenRepositories(self.context)
groomFolders(self.context)
groomFiles(self.context)
groomTrees(self.context)
# Handle scope exclusion
if self.context.toExclude(SCOPE_FILES):
scopeToRemove = []
for scope in model[DATA][FILES][SCOPE_BY_NAME]:
if (self.context.toExclude(scope)):
scopeToRemove.append(scope)
for scope in scopeToRemove:
del (model[DATA][FILES][SCOPE_BY_NAME][scope])
if (self.context.toExclude(SCOPE_HDFS) and HDFS in model[DATA]):
model[DATA][HDFS][FILES] = []
model[DATA][HDFS][FOLDERS] = []
model[DATA][HDFS][TREES] = []
model[DATA][HDFS][NODE_TO_HDFS_BY_NAME] = {}
if HDFS in model[DATA] and len(
model[DATA][HDFS][NODE_TO_HDFS_BY_NAME]) == 0 and len(
model[DATA][HDFS][FILES]) == 0 and len(
model[DATA][HDFS][FOLDERS]) == 0 and len(
model[DATA][HDFS][TREES]) == 0:
# Optimization for execution time
if HDFS_RELAY in model[SRC]:
del (model[SRC][HDFS_RELAY])
setWillUseMavenRepo(model)
setServiceNotifications(self.context)
def onGrooming(self):
self.context.model[DATA][INVENTORY] = {}
misc.applyWhenOnList(self.context.model[SRC], HOSTS)
misc.applyWhenOnList(self.context.model[SRC], HOST_GROUPS)
misc.applyWhenOnList(self.context.model[SRC], HOST_OVERRIDES)
misc.applyWhenOnList(self.context.model[SRC], HOST_GROUP_OVERRIDES)
buildHostDicts(self.context.model)
flattenGroups(self.context.model)
handleHostOverrides(self.context.model)
handleHostGroupOverrides(self.context.model)
check(self.context.model)
prepareAnsibleModel(self.context.model)
misc.setDefaultInMap(self.context.model[SRC], EXIT_ON_FAIL, True)
def onGrooming(self):
misc.ensureObjectInMaps(self.context.model[DATA], [SUPERVISORS, SCOPE_BY_NAME], {})
misc.applyWhenOnList(self.context.model[SRC], SUPERVISORS)
misc.applyWhenOnList(self.context.model[SRC], SUPERVISOR_PROGRAMS)
misc.applyWhenOnList(self.context.model[SRC], SUPERVISOR_GROUPS)
self.groomSupervisors()
self.groomPrograms()
self.groomGroups()
def onGrooming(self):
misc.applyWhenOnList(self.context.model[SRC], ELASTICSEARCH_SERVERS)
misc.applyWhenOnList(self.context.model[SRC], ELASTICSEARCH_INDICES)
misc.applyWhenOnList(self.context.model[SRC], ELASTICSEARCH_TEMPLATES)
if self.context.toExclude(SCOPE_ELASTIC):
return
misc.ensureObjectInMaps(self.context.model[DATA], [ELASTIC], {})
groomElasticServers(self.context.model)
groomElasticIndices(self.context.model)
groomElasticTemplates(self.context.model)
def onGrooming(self):
misc.applyWhenOnSingle(self.context.model[SRC], HBASE_RELAY)
misc.applyWhenOnList(self.context.model[SRC], HBASE_NAMESPACES)
misc.applyWhenOnList(self.context.model[SRC], HBASE_TABLES)
misc.applyWhenOnList(self.context.model[SRC], HBASE_DATASETS)
if self.context.toExclude(SCOPE_HBASE):
return
self.buildHelper()
misc.ensureObjectInMaps(self.context.model[DATA], [HBASE], {})
groomHbaseRelay(self.context.model)
groomHBaseNamespaces(self.context.model)
groomHBaseTables(self.context.model)
groomHBaseDatasets(self.context.model)
def onGrooming(self):
misc.applyWhenOnSingle(self.context.model[SRC], RANGER_RELAY)
if 'hdfs' in self.context.pluginByName:
misc.applyWhenOnList(self.context.model[SRC], HDFS_RANGER_POLICIES)
if 'hbase' in self.context.pluginByName:
misc.applyWhenOnList(self.context.model[SRC], HBASE_RANGER_POLICIES)
if 'kafka' in self.context.pluginByName:
misc.applyWhenOnList(self.context.model[SRC], KAFKA_RANGER_POLICIES)
if 'hive' in self.context.pluginByName:
misc.applyWhenOnList(self.context.model[SRC], HIVE_RANGER_POLICIES)
if 'storm' in self.context.pluginByName:
misc.applyWhenOnList(self.context.model[SRC], STORM_RANGER_POLICIES)
misc.applyWhenOnList(self.context.model[SRC], YARN_RANGER_POLICIES)
if self.context.toExclude(SCOPE_RANGER):
return
groomRangerRelay(self.context.model)
if 'hdfs' in self.context.pluginByName:
groomRangerHdfsPolicies(self.context.model)
if 'hbase' in self.context.pluginByName:
groomRangerHBasePolicies(self.context.model)
if 'kafka' in self.context.pluginByName:
groomRangerKafkaPolicies(self.context.model)
if 'hive' in self.context.pluginByName:
groomRangerHivePolicies(self.context.model)
if 'storm' in self.context.pluginByName:
groomRangerStormPolicies(self.context.model)
groomRangerYarnPolicies(self.context.model)
def onGrooming(self):
misc.ensureObjectInMaps(self.context.model[DATA],
[SYSTEMD, SCOPE_BY_NAME], {})
misc.applyWhenOnList(self.context.model[SRC], SYSTEMD_UNITS)
self.groomSystemd()