Ejemplo n.º 1
0
 def onGrooming(self):
     misc.applyWhenOnSingle(self.context.model[SRC], RANGER_RELAY)
     if 'hdfs' in self.context.pluginByName:
         misc.applyWhenOnList(self.context.model[SRC], HDFS_RANGER_POLICIES)
     if 'hbase' in self.context.pluginByName:
         misc.applyWhenOnList(self.context.model[SRC], HBASE_RANGER_POLICIES)
     if 'kafka' in self.context.pluginByName:
         misc.applyWhenOnList(self.context.model[SRC], KAFKA_RANGER_POLICIES)
     if 'hive' in self.context.pluginByName:
         misc.applyWhenOnList(self.context.model[SRC], HIVE_RANGER_POLICIES)
     if 'storm' in self.context.pluginByName:
         misc.applyWhenOnList(self.context.model[SRC], STORM_RANGER_POLICIES)
     misc.applyWhenOnList(self.context.model[SRC], YARN_RANGER_POLICIES)
     if self.context.toExclude(SCOPE_RANGER):
         return
     groomRangerRelay(self.context.model)
     if 'hdfs' in self.context.pluginByName:
         groomRangerHdfsPolicies(self.context.model)
     if 'hbase' in self.context.pluginByName:
         groomRangerHBasePolicies(self.context.model)
     if 'kafka' in self.context.pluginByName:
         groomRangerKafkaPolicies(self.context.model)
     if 'hive' in self.context.pluginByName:
         groomRangerHivePolicies(self.context.model)
     if 'storm' in self.context.pluginByName:
         groomRangerStormPolicies(self.context.model)
     groomRangerYarnPolicies(self.context.model)
Ejemplo n.º 2
0
 def onGrooming(self):
     if self.context.toExclude(SCOPE_STORM):
         return
     misc.applyWhenOnSingle(self.context.model[SRC], STORM_RELAY)
     misc.applyWhenOnList(self.context.model[SRC], STORM_TOPOLOGIES)
     groomStormRelay(self.context.model)
     groomStormTopologies(self.context.model)
Ejemplo n.º 3
0
 def onGrooming(self):
     if self.context.toExclude(SCOPE_YARN):
         return
     misc.applyWhenOnSingle(self.context.model[SRC], YARN_RELAY)
     misc.applyWhenOnList(self.context.model[SRC], YARN_SERVICES)
     misc.ensureObjectInMaps(self.context.model[DATA], [YARN], {})
     groomYarnRelay(self.context.model)
     groomYarnServices(self.context.model)
Ejemplo n.º 4
0
 def onGrooming(self):
     misc.applyWhenOnSingle(self.context.model[SRC], KAFKA_RELAY)
     misc.applyWhenOnList(self.context.model[SRC], KAFKA_TOPICS)
     if self.context.toExclude(SCOPE_KAFKA):
         return
     self.buildHelper()
     misc.ensureObjectInMaps(self.context.model[DATA], [KAFKA], {})
     groomKafkaRelay(self.context.model)
     groomKafkaTopics(self.context.model)
Ejemplo n.º 5
0
 def onGrooming(self):
     misc.applyWhenOnSingle(self.context.model[SRC], HIVE_RELAY)
     misc.applyWhenOnList(self.context.model[SRC], HIVE_DATABASES)
     misc.applyWhenOnList(self.context.model[SRC], HIVE_TABLES)
     if self.context.toExclude(SCOPE_HIVE):
         return
     self.buildHelper()
     misc.ensureObjectInMaps(self.context.model[DATA], [HIVE], {})
     groomHiveRelay(self.context.model)
     groomHiveDatabases(self.context.model)
     groomHiveTables(self.context.model)
Ejemplo n.º 6
0
 def onGrooming(self):
     misc.applyWhenOnSingle(self.context.model[SRC], HBASE_RELAY)
     misc.applyWhenOnList(self.context.model[SRC], HBASE_NAMESPACES)
     misc.applyWhenOnList(self.context.model[SRC], HBASE_TABLES)
     misc.applyWhenOnList(self.context.model[SRC], HBASE_DATASETS)
     if self.context.toExclude(SCOPE_HBASE):
         return
     self.buildHelper()
     misc.ensureObjectInMaps(self.context.model[DATA], [HBASE], {})
     groomHbaseRelay(self.context.model)
     groomHBaseNamespaces(self.context.model)
     groomHBaseTables(self.context.model)
     groomHBaseDatasets(self.context.model)
Ejemplo n.º 7
0
    def onGrooming(self):
        model = self.context.model
        misc.ensureObjectInMaps(model[DATA], [FILES, SCOPE_BY_NAME], {})
        if HDFS in self.context.pluginByName:
            # We need to anticipate on works performed by hdfs plugin, as we need it right now
            misc.ensureObjectInMaps(model[DATA], [HDFS, NODE_TO_HDFS_BY_NAME],
                                    {})
            misc.ensureObjectInMaps(model[DATA], [HDFS, FILES], [])
            misc.ensureObjectInMaps(model[DATA], [HDFS, FOLDERS], [])
            misc.ensureObjectInMaps(model[DATA], [HDFS, TREES], [])
            misc.ensureObjectInMaps(model[DATA], [HDFS, CACHEFOLDERS], Set())
            misc.applyWhenOnSingle(self.context.model[SRC], HDFS_RELAY)
            misc.applyWhenOnList(self.context.model[SRC],
                                 SOURCE_HOST_CREDENTIALS)
        if HDFS_RELAY in model[SRC]:
            misc.setDefaultInMap(model[SRC][HDFS_RELAY], CACHE_FOLDER,
                                 DEFAULT_HDFS_RELAY_CACHE_FOLDER)
        misc.applyWhenOnList(self.context.model[SRC], MAVEN_REPOSITORIES)
        misc.applyWhenOnList(self.context.model[SRC], FOLDERS)
        misc.applyWhenOnList(self.context.model[SRC], FILES)
        misc.applyWhenOnList(self.context.model[SRC], TREES)
        groomMavenRepositories(self.context)
        groomFolders(self.context)
        groomFiles(self.context)
        groomTrees(self.context)

        # Handle scope exclusion
        if self.context.toExclude(SCOPE_FILES):
            scopeToRemove = []
            for scope in model[DATA][FILES][SCOPE_BY_NAME]:
                if (self.context.toExclude(scope)):
                    scopeToRemove.append(scope)
            for scope in scopeToRemove:
                del (model[DATA][FILES][SCOPE_BY_NAME][scope])
        if (self.context.toExclude(SCOPE_HDFS) and HDFS in model[DATA]):
            model[DATA][HDFS][FILES] = []
            model[DATA][HDFS][FOLDERS] = []
            model[DATA][HDFS][TREES] = []
            model[DATA][HDFS][NODE_TO_HDFS_BY_NAME] = {}

        if HDFS in model[DATA] and len(
                model[DATA][HDFS][NODE_TO_HDFS_BY_NAME]) == 0 and len(
                    model[DATA][HDFS][FILES]) == 0 and len(
                        model[DATA][HDFS][FOLDERS]) == 0 and len(
                            model[DATA][HDFS][TREES]) == 0:
            # Optimization for execution time
            if HDFS_RELAY in model[SRC]:
                del (model[SRC][HDFS_RELAY])
        setWillUseMavenRepo(model)
        setServiceNotifications(self.context)