def onGrooming(self):
misc.applyWhenOnSingle(self.context.model[SRC], RANGER_RELAY)
if 'hdfs' in self.context.pluginByName:
misc.applyWhenOnList(self.context.model[SRC], HDFS_RANGER_POLICIES)
if 'hbase' in self.context.pluginByName:
misc.applyWhenOnList(self.context.model[SRC], HBASE_RANGER_POLICIES)
if 'kafka' in self.context.pluginByName:
misc.applyWhenOnList(self.context.model[SRC], KAFKA_RANGER_POLICIES)
if 'hive' in self.context.pluginByName:
misc.applyWhenOnList(self.context.model[SRC], HIVE_RANGER_POLICIES)
if 'storm' in self.context.pluginByName:
misc.applyWhenOnList(self.context.model[SRC], STORM_RANGER_POLICIES)
misc.applyWhenOnList(self.context.model[SRC], YARN_RANGER_POLICIES)
if self.context.toExclude(SCOPE_RANGER):
return
groomRangerRelay(self.context.model)
if 'hdfs' in self.context.pluginByName:
groomRangerHdfsPolicies(self.context.model)
if 'hbase' in self.context.pluginByName:
groomRangerHBasePolicies(self.context.model)
if 'kafka' in self.context.pluginByName:
groomRangerKafkaPolicies(self.context.model)
if 'hive' in self.context.pluginByName:
groomRangerHivePolicies(self.context.model)
if 'storm' in self.context.pluginByName:
groomRangerStormPolicies(self.context.model)
groomRangerYarnPolicies(self.context.model)
def onGrooming(self):
if self.context.toExclude(SCOPE_STORM):
return
misc.applyWhenOnSingle(self.context.model[SRC], STORM_RELAY)
misc.applyWhenOnList(self.context.model[SRC], STORM_TOPOLOGIES)
groomStormRelay(self.context.model)
groomStormTopologies(self.context.model)
def onGrooming(self):
if self.context.toExclude(SCOPE_YARN):
return
misc.applyWhenOnSingle(self.context.model[SRC], YARN_RELAY)
misc.applyWhenOnList(self.context.model[SRC], YARN_SERVICES)
misc.ensureObjectInMaps(self.context.model[DATA], [YARN], {})
groomYarnRelay(self.context.model)
groomYarnServices(self.context.model)
def onGrooming(self):
misc.applyWhenOnSingle(self.context.model[SRC], KAFKA_RELAY)
misc.applyWhenOnList(self.context.model[SRC], KAFKA_TOPICS)
if self.context.toExclude(SCOPE_KAFKA):
return
self.buildHelper()
misc.ensureObjectInMaps(self.context.model[DATA], [KAFKA], {})
groomKafkaRelay(self.context.model)
groomKafkaTopics(self.context.model)
def onGrooming(self):
misc.applyWhenOnSingle(self.context.model[SRC], HIVE_RELAY)
misc.applyWhenOnList(self.context.model[SRC], HIVE_DATABASES)
misc.applyWhenOnList(self.context.model[SRC], HIVE_TABLES)
if self.context.toExclude(SCOPE_HIVE):
return
self.buildHelper()
misc.ensureObjectInMaps(self.context.model[DATA], [HIVE], {})
groomHiveRelay(self.context.model)
groomHiveDatabases(self.context.model)
groomHiveTables(self.context.model)
def onGrooming(self):
misc.applyWhenOnSingle(self.context.model[SRC], HBASE_RELAY)
misc.applyWhenOnList(self.context.model[SRC], HBASE_NAMESPACES)
misc.applyWhenOnList(self.context.model[SRC], HBASE_TABLES)
misc.applyWhenOnList(self.context.model[SRC], HBASE_DATASETS)
if self.context.toExclude(SCOPE_HBASE):
return
self.buildHelper()
misc.ensureObjectInMaps(self.context.model[DATA], [HBASE], {})
groomHbaseRelay(self.context.model)
groomHBaseNamespaces(self.context.model)
groomHBaseTables(self.context.model)
groomHBaseDatasets(self.context.model)
def onGrooming(self):
model = self.context.model
misc.ensureObjectInMaps(model[DATA], [FILES, SCOPE_BY_NAME], {})
if HDFS in self.context.pluginByName:
# We need to anticipate on works performed by hdfs plugin, as we need it right now
misc.ensureObjectInMaps(model[DATA], [HDFS, NODE_TO_HDFS_BY_NAME],
{})
misc.ensureObjectInMaps(model[DATA], [HDFS, FILES], [])
misc.ensureObjectInMaps(model[DATA], [HDFS, FOLDERS], [])
misc.ensureObjectInMaps(model[DATA], [HDFS, TREES], [])
misc.ensureObjectInMaps(model[DATA], [HDFS, CACHEFOLDERS], Set())
misc.applyWhenOnSingle(self.context.model[SRC], HDFS_RELAY)
misc.applyWhenOnList(self.context.model[SRC],
SOURCE_HOST_CREDENTIALS)
if HDFS_RELAY in model[SRC]:
misc.setDefaultInMap(model[SRC][HDFS_RELAY], CACHE_FOLDER,
DEFAULT_HDFS_RELAY_CACHE_FOLDER)
misc.applyWhenOnList(self.context.model[SRC], MAVEN_REPOSITORIES)
misc.applyWhenOnList(self.context.model[SRC], FOLDERS)
misc.applyWhenOnList(self.context.model[SRC], FILES)
misc.applyWhenOnList(self.context.model[SRC], TREES)
groomMavenRepositories(self.context)
groomFolders(self.context)
groomFiles(self.context)
groomTrees(self.context)
# Handle scope exclusion
if self.context.toExclude(SCOPE_FILES):
scopeToRemove = []
for scope in model[DATA][FILES][SCOPE_BY_NAME]:
if (self.context.toExclude(scope)):
scopeToRemove.append(scope)
for scope in scopeToRemove:
del (model[DATA][FILES][SCOPE_BY_NAME][scope])
if (self.context.toExclude(SCOPE_HDFS) and HDFS in model[DATA]):
model[DATA][HDFS][FILES] = []
model[DATA][HDFS][FOLDERS] = []
model[DATA][HDFS][TREES] = []
model[DATA][HDFS][NODE_TO_HDFS_BY_NAME] = {}
if HDFS in model[DATA] and len(
model[DATA][HDFS][NODE_TO_HDFS_BY_NAME]) == 0 and len(
model[DATA][HDFS][FILES]) == 0 and len(
model[DATA][HDFS][FOLDERS]) == 0 and len(
model[DATA][HDFS][TREES]) == 0:
# Optimization for execution time
if HDFS_RELAY in model[SRC]:
del (model[SRC][HDFS_RELAY])
setWillUseMavenRepo(model)
setServiceNotifications(self.context)