def groomMavenRepositories(context): model = context.model misc.ensureObjectInMaps(model[DATA], [MAVEN_REPO_BY_NAME], {}) if MAVEN_REPOSITORIES in model[SRC]: for repo in model[SRC][MAVEN_REPOSITORIES]: model[DATA][MAVEN_REPO_BY_NAME][repo["name"]] = repo misc.setDefaultInMap(repo, VALIDATE_CERTS, True) misc.setDefaultInMap(repo, TIMEOUT, 10)
def onGrooming(self): misc.ensureObjectInMaps(self.context.model[DATA], [SUPERVISORS, SCOPE_BY_NAME], {}) misc.applyWhenOnList(self.context.model[SRC], SUPERVISORS) misc.applyWhenOnList(self.context.model[SRC], SUPERVISOR_PROGRAMS) misc.applyWhenOnList(self.context.model[SRC], SUPERVISOR_GROUPS) self.groomSupervisors() self.groomPrograms() self.groomGroups()
def onGrooming(self): if self.context.toExclude(SCOPE_YARN): return misc.applyWhenOnSingle(self.context.model[SRC], YARN_RELAY) misc.applyWhenOnList(self.context.model[SRC], YARN_SERVICES) misc.ensureObjectInMaps(self.context.model[DATA], [YARN], {}) groomYarnRelay(self.context.model) groomYarnServices(self.context.model)
def onGrooming(self): misc.applyWhenOnList(self.context.model[SRC], USERS) misc.applyWhenOnList(self.context.model[SRC], GROUPS) if self.context.toExclude(SCOPE_USERS): return misc.ensureObjectInMaps(self.context.model[DATA], [USERS, SCOPE_BY_NAME], {}) groomUsers(self.context) groomGroups(self.context)
def onGrooming(self): misc.applyWhenOnSingle(self.context.model[SRC], KAFKA_RELAY) misc.applyWhenOnList(self.context.model[SRC], KAFKA_TOPICS) if self.context.toExclude(SCOPE_KAFKA): return self.buildHelper() misc.ensureObjectInMaps(self.context.model[DATA], [KAFKA], {}) groomKafkaRelay(self.context.model) groomKafkaTopics(self.context.model)
def grabHBaseRangerPoliciesFromTables(model): if HBASE_TABLES in model[SRC]: for table in model[SRC][HBASE_TABLES]: if RANGER_POLICY in table: policy = table[RANGER_POLICY] policy[TABLES] = [ table[NAMESPACE] +':' + table[NAME] ] misc.setDefaultInMap(policy, NAME, DEFAULT_HBASE_TABLE_POLICY_NAME.format(table[NAMESPACE], table[NAME])) policy[NO_REMOVE] = table[NO_REMOVE] misc.ensureObjectInMaps( model[SRC], [HBASE_RANGER_POLICIES], []) model[SRC][HBASE_RANGER_POLICIES].append(policy)
def grabHBaseRangerPoliciesFromNamespaces(model): if HBASE_NAMESPACES in model[SRC]: for namespace in model[SRC][HBASE_NAMESPACES]: if RANGER_POLICY in namespace: policy = namespace[RANGER_POLICY] policy[TABLES] = [ namespace[NAME] + ":*" ] misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(namespace[NAME])) policy[NO_REMOVE] = namespace[NO_REMOVE] misc.ensureObjectInMaps( model[SRC], [HBASE_RANGER_POLICIES], []) model[SRC][HBASE_RANGER_POLICIES].append(policy)
def grabStormRangerPoliciesFromTopologies(model): if STORM_TOPOLOGIES in model[SRC]: for topology in model[SRC][STORM_TOPOLOGIES]: if RANGER_POLICY in topology: policy = topology[RANGER_POLICY] policy[TOPOLOGIES] = [ topology[NAME] ] misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(topology[NAME])) policy[NO_REMOVE] = topology[NO_REMOVE] misc.ensureObjectInMaps( model[SRC], [STORM_RANGER_POLICIES], []) model[SRC][STORM_RANGER_POLICIES].append(policy)
def grabKafkaRangerPoliciesFromTopics(model): if KAFKA_TOPICS in model[SRC]: for topic in model[SRC][KAFKA_TOPICS]: if RANGER_POLICY in topic: policy = topic[RANGER_POLICY] policy[TOPICS] = [ topic[NAME] ] misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(topic[NAME])) policy[NO_REMOVE] = topic[NO_REMOVE] misc.ensureObjectInMaps( model[SRC], [KAFKA_RANGER_POLICIES], []) model[SRC][KAFKA_RANGER_POLICIES].append(policy)
def onGrooming(self): misc.applyWhenOnList(self.context.model[SRC], ELASTICSEARCH_SERVERS) misc.applyWhenOnList(self.context.model[SRC], ELASTICSEARCH_INDICES) misc.applyWhenOnList(self.context.model[SRC], ELASTICSEARCH_TEMPLATES) if self.context.toExclude(SCOPE_ELASTIC): return misc.ensureObjectInMaps(self.context.model[DATA], [ELASTIC], {}) groomElasticServers(self.context.model) groomElasticIndices(self.context.model) groomElasticTemplates(self.context.model)
def buildHelper(self): helper = {} helper[DIR] = os.path.normpath(os.path.join(self.path, "helpers")) jdchivejars = glob.glob( os.path.join(helper[DIR], "jdchive/jdchive_uber*.jar")) if len(jdchivejars) < 1: misc.ERROR( "Unable to find helper for Hive.Please, refer to the documentation about Installation" ) helper[JDCHIVE_JAR] = os.path.basename(jdchivejars[0]) misc.ensureObjectInMaps(self.context.model, [HELPER, HIVE], helper)
def onGrooming(self): misc.applyWhenOnSingle(self.context.model[SRC], HIVE_RELAY) misc.applyWhenOnList(self.context.model[SRC], HIVE_DATABASES) misc.applyWhenOnList(self.context.model[SRC], HIVE_TABLES) if self.context.toExclude(SCOPE_HIVE): return self.buildHelper() misc.ensureObjectInMaps(self.context.model[DATA], [HIVE], {}) groomHiveRelay(self.context.model) groomHiveDatabases(self.context.model) groomHiveTables(self.context.model)
def grabHdfsRangerPoliciesFromTrees(model): if TREES in model[SRC]: for tree in model[SRC][TREES]: if RANGER_POLICY in tree: if tree[SCOPE] != HDFS: misc.ERROR("Can't setup Apache Ranger policy on tree '{0}' as scope is not hdfs".format(tree[DEST_FOLDER])) policy = tree[RANGER_POLICY] policy[PATHS] = [ tree[DEST_FOLDER] ] policy[NO_REMOVE] = tree[NO_REMOVE] misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(tree[DEST_FOLDER])) misc.ensureObjectInMaps( model[SRC], [HDFS_RANGER_POLICIES], []) model[SRC][HDFS_RANGER_POLICIES].append(policy)
def grabHdfsRangerPoliciesFromFolders(model): if FOLDERS in model[SRC]: for folder in model[SRC][FOLDERS]: if RANGER_POLICY in folder: if folder[SCOPE] != HDFS: misc.ERROR("Can't setup Apache Ranger policy on folder '{0}' as scope is not hdfs".format(folder[PATH])) policy = folder[RANGER_POLICY] policy[PATHS] = [ folder[PATH] ] policy[NO_REMOVE] = folder[NO_REMOVE] misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(folder[PATH])) misc.ensureObjectInMaps( model[SRC], [HDFS_RANGER_POLICIES], []) model[SRC][HDFS_RANGER_POLICIES].append(policy)
def grabHiveRangerPoliciesFromTables(model): if HIVE_TABLES in model[SRC]: for table in model[SRC][HIVE_TABLES]: if RANGER_POLICY in table: policy = table[RANGER_POLICY] policy[DATABASES] = [ table[DATABASE] ] policy[TABLES] = [ table[NAME] ] policy[COLUMNS] = [ "*" ] misc.setDefaultInMap(policy, NAME, DEFAULT_HIVE_TABLE_POLICY_NAME.format(table[DATABASE], table[NAME])) policy[NO_REMOVE] = table[NO_REMOVE] misc.ensureObjectInMaps( model[SRC], [HIVE_RANGER_POLICIES], []) model[SRC][HIVE_RANGER_POLICIES].append(policy)
def grabHiveRangerPoliciesFromDatabase(model): if HIVE_DATABASES in model[SRC]: for database in model[SRC][HIVE_DATABASES]: if RANGER_POLICY in database: policy = database[RANGER_POLICY] policy[DATABASES] = [ database[NAME] ] policy[TABLES] = [ "*" ] policy[COLUMNS] = [ "*" ] misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format(database[NAME])) policy[NO_REMOVE] = database[NO_REMOVE] misc.ensureObjectInMaps( model[SRC], [HIVE_RANGER_POLICIES], []) model[SRC][HIVE_RANGER_POLICIES].append(policy)
def grabHdfsRangerPoliciesFromFiles(model): if FILES in model[SRC]: for xfile in model[SRC][FILES]: if RANGER_POLICY in xfile: if xfile[SCOPE] != HDFS: misc.ERROR("Can't setup Apache Ranger policy on file '{0}' as scope is not hdfs".format(os.path.join(xfile[DEST_FOLDER], xfile[DEST_NAME]))) policy = xfile[RANGER_POLICY] misc.setDefaultInMap(policy, RECURSIVE, False) policy[PATHS] = [ os.path.join(xfile[DEST_FOLDER], xfile[DEST_NAME]) ] # groomFiles should have been called before misc.setDefaultInMap(policy, NAME, DEFAULT_POLICY_NAME.format( policy[PATHS][0])) policy[NO_REMOVE] = xfile[NO_REMOVE] misc.ensureObjectInMaps( model[SRC], [HDFS_RANGER_POLICIES], []) model[SRC][HDFS_RANGER_POLICIES].append(policy)
def onGrooming(self): misc.applyWhenOnSingle(self.context.model[SRC], HBASE_RELAY) misc.applyWhenOnList(self.context.model[SRC], HBASE_NAMESPACES) misc.applyWhenOnList(self.context.model[SRC], HBASE_TABLES) misc.applyWhenOnList(self.context.model[SRC], HBASE_DATASETS) if self.context.toExclude(SCOPE_HBASE): return self.buildHelper() misc.ensureObjectInMaps(self.context.model[DATA], [HBASE], {}) groomHbaseRelay(self.context.model) groomHBaseNamespaces(self.context.model) groomHBaseTables(self.context.model) groomHBaseDatasets(self.context.model)
def groomPrograms(self): if self.context.toExclude(SCOPE_SUPERVISOR): return model = self.context.model if SUPERVISOR_PROGRAMS in model[SRC]: for prg in model[SRC][SUPERVISOR_PROGRAMS]: if not SUPERVISOR_BY_NAME in model[DATA][SUPERVISORS] or not prg[SUPERVISOR] in model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME]: misc.ERROR("supervisor_program '{}' refer to an undefined supervisor '{}'".format(prg[NAME], prg[SUPERVISOR])) else: supervisord = model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME][prg[SUPERVISOR]] if prg[NAME] in supervisord[PROGRAM_BY_NAME]: misc.ERROR("supervisor_program '{}' is defined twice in supervisor '{}'".format(prg[NAME], supervisord[NAME])) # Register in model.data supervisord[PROGRAM_BY_NAME][prg[NAME]] = prg #model[DATA][SUPERVISORS][SCOPE_BY_NAME][supervisord[SCOPE]][PROGRAMS].append(prg) # Adjust attributes misc.setDefaultInMap(prg, NO_REMOVE, False) if prg[NO_REMOVE] and not supervisord[NO_REMOVE]: misc.ERROR("Supervisor_program '{}' has no remove flag set while its supervisor ({}) has not!".format(prg[NAME], supervisord[NAME])) if CONF_FILE_SRC in prg: path, _, errMsg = lookupSrc(model, prg[CONF_FILE_SRC]) if path != None: prg[CONF_FILE_SRC_JJ2] = path else: misc.ERROR("Supervisor_program '{0}': {1}".format(prg[NAME], errMsg)) else: prg[CONF_FILE_SRC_JJ2] = os.path.join(self.path, "templates/program.conf.jj2") if COMMAND not in prg: misc.ERROR("Supervisor_program '{}': A 'command' parameter must be provided if using the default configuration file (No 'conf_file_src' parameter".format(prg[NAME])) prg[CONF_FILE_SRC_J2] = "supervisor_{}_program_{}.conf".format(supervisord[NAME], prg[NAME]) prg[CONF_FILE_DST] = os.path.join(supervisord[INCLUDE_DIR], "{}_prg.ini".format(prg[NAME])) prg[SUPERVISOR_OWNER] = supervisord[USER] prg[SUPERVISOR_GROUP] = supervisord[GROUP] prg[SUPERVISOR_CONF] = supervisord[CONF_FILE_DST] misc.setDefaultInMap(prg, STATE, ST_STARTED) if NUMPROCS in prg and prg[NUMPROCS] > 1: prg[_NAME_] = prg[NAME] + ":" # This is in fact a group of process else: prg[_NAME_] = prg[NAME] if prg[STATE] not in validState: misc.ERROR("Supervisor_program {0}: state value '{1}' is not valid. Must be one of {2}".format(prg[NAME], prg[STATE], validState)) misc.setDefaultInMap(prg, AUTOSTART, prg[STATE] == ST_STARTED) if SCOPE not in prg: prg[SCOPE] = supervisord[SCOPE] # Note we don't set prg[USER], as we want to be unset in config file if not set # ---------------------- Insert in scope misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [prg[SCOPE], PROGRAMS_TO_MANAGE], []) model[DATA][SUPERVISORS][SCOPE_BY_NAME][prg[SCOPE]][PROGRAMS_TO_MANAGE].append(prg) if not prg[NO_REMOVE]: misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [prg[SCOPE], PROGRAMS_TO_REMOVE], []) model[DATA][SUPERVISORS][SCOPE_BY_NAME][prg[SCOPE]][PROGRAMS_TO_REMOVE].append(prg)
def onGrooming(self): model = self.context.model #logger.debug("Called self.onGrooming() for plugin '{0}'".format(self.name)) misc.ensureObjectInMaps(model[SRC], [PLUGINS_PATHS], []) # Add our internal plugins at the end of the list model[SRC][PLUGINS_PATHS].append( os.path.normpath( os.path.join(os.path.dirname(__file__), "../../plugins"))) if not PLUGINS in model[SRC]: model[SRC][PLUGINS] = DEFAULT_PLUGINS # ---------------------------- Check encrypted vars if ENCRYPTED_VARS in model[SRC]: for k, v in model[SRC][ENCRYPTED_VARS].iteritems(): if not (isinstance(v, basestring) and v.startswith("$ANSIBLE_VAULT")): misc.ERROR( "Encrypted variable '{0}' does not seems to provide a valid encrypted value" .format(k))
def buildHelper(self): helper = {} helper[DIR] = os.path.normpath(os.path.join(self.path, "helpers")) jdchtablejars = glob.glob( os.path.join(helper[DIR], "jdchtable/jdchtable_uber*.jar")) if len(jdchtablejars) < 1: misc.ERROR( "Unable to find helper for HBase.Please, refer to the documentation about Installation" ) helper[JDCHTABLE_JAR] = os.path.basename(jdchtablejars[0]) hbloadjars = glob.glob( os.path.join(helper[DIR], "hbload/hbload_uber*.jar")) if len(hbloadjars) < 1: misc.ERROR( "Unable to find helper for HBase datasets loader. Please, refer to the documentation about Installation" ) helper[HBLOAD_JAR] = os.path.basename(hbloadjars[0]) misc.ensureObjectInMaps(self.context.model, [HELPER, HBASE], helper)
def buildHelper(self): if KAFKA_RELAY in self.context.model[SRC]: helper = {} helper[DIR] = os.path.normpath(os.path.join(self.path, "helpers")) jarPattern = "jdctopic/jdctopic.{}-*-uber.jar".format( self.context.model[SRC][KAFKA_RELAY][KAFKA_VERSION]) jdctopicjars = glob.glob(os.path.join(helper[DIR], jarPattern)) if len(jdctopicjars) < 1: misc.ERROR( "Unable to find helper for Kafka.Please, refer to the documentation about Installation" ) if len(jdctopicjars) > 1: misc.ERROR( "Several version of kafka helper jar in {}. Please, cleanup." .format(helper[DIR])) helper[JDCTOPIC_JAR] = os.path.basename(jdctopicjars[0]) misc.ensureObjectInMaps(self.context.model, [HELPER, KAFKA], helper)
def groomGroups(self): if self.context.toExclude(SCOPE_SUPERVISOR): return model = self.context.model if SUPERVISOR_GROUPS in model[SRC]: for grp in model[SRC][SUPERVISOR_GROUPS]: if not SUPERVISOR_BY_NAME in model[DATA][SUPERVISORS] or not grp[SUPERVISOR] in model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME]: misc.ERROR("supervisor_group '{}' refer to an undefined supervisor '{}'".format(grp[NAME], grp[SUPERVISOR])) else: supervisord = model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME][grp[SUPERVISOR]] if grp[NAME] in supervisord[GROUP_BY_NAME]: misc.ERROR("supervisor_group '{}' is defined twice in supervisor '{}'".format(grp[NAME], supervisord[NAME])) supervisord[GROUP_BY_NAME][grp[NAME]] = grp for prgName in grp[PROGRAMS]: if prgName not in supervisord[PROGRAM_BY_NAME]: misc.ERROR("supervisor_group '{}' refer to an undefined program '{}'".format(grp[NAME], prgName)) else: prg = supervisord[PROGRAM_BY_NAME][prgName] # The program name must be patched: prg[_NAME_] = grp[NAME] + ":" + supervisord[PROGRAM_BY_NAME][prgName][_NAME_] if _SCOPE_ in grp: if grp[_SCOPE_] != prg[SCOPE]: misc.ERROR("supervisor_group '{}' host programs with different scope ({} != {}). Must be same".format(grp[NAME], grp[_SCOPE_], prg[SCOPE])) else: grp[_SCOPE_] = prg[SCOPE] misc.setDefaultInMap(grp, NO_REMOVE, False) if grp[NO_REMOVE] and not supervisord[NO_REMOVE]: misc.ERROR("Supervisor_group '{}' has no remove flag set while its supervisor ({}) has not!".format(grp[NAME], supervisord[NAME])) grp[CONF_FILE_SRC_JJ2] = os.path.join(self.path, "templates/group.conf.jj2") grp[CONF_FILE_SRC_J2] = "supervisor_{}_group_{}.conf".format(supervisord[NAME], grp[NAME]) grp[CONF_FILE_DST] = os.path.join(supervisord[INCLUDE_DIR], "{}_grp.ini".format(grp[NAME])) grp[SUPERVISOR_OWNER] = supervisord[USER] grp[SUPERVISOR_GROUP] = supervisord[GROUP] grp[SUPERVISOR_CONF] = supervisord[CONF_FILE_DST] grp[_NAME_] = grp[NAME] + ":" # ---------------------- Insert in scope misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [grp[_SCOPE_], GROUPS_TO_MANAGE], []) model[DATA][SUPERVISORS][SCOPE_BY_NAME][grp[_SCOPE_]][GROUPS_TO_MANAGE].append(grp) if not grp[NO_REMOVE]: misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [grp[_SCOPE_], GROUPS_TO_REMOVE], []) model[DATA][SUPERVISORS][SCOPE_BY_NAME][grp[_SCOPE_]][GROUPS_TO_REMOVE].append(grp)
def groomSystemd(self): if self.context.toExclude(SCOPE_SYSTEMD): return model = self.context.model unitNames = Set() if SYSTEMD_UNITS in model[SRC]: for unit in model[SRC][SYSTEMD_UNITS]: if unit[NAME] in unitNames: misc.ERROR("systemd_unit '{0}' is defined twice!".format( unit[NAME])) unitNames.add(unit[NAME]) misc.setDefaultInMap(unit, NO_REMOVE, False) misc.setDefaultInMap(unit, ENABLED, True) misc.setDefaultInMap(unit, STATE, ST_CURRENT) if unit[STATE] not in validState: misc.ERROR( "Systemd_unit {0}: state value '{1}' is not valid. Must be one of {2}" .format(unit[NAME], unit[STATE], validState)) misc.setDefaultInMap(unit, ACTION_ON_NOTIFY, AON_RESTART) if unit[ACTION_ON_NOTIFY] not in validAon: misc.ERROR( "Systemd_unit {0}: action_on_notify value '{1}' is not valid. Must be one of {2}" .format(unit[NAME], unit[ACTION_ON_NOTIFY], validAon)) # ---------------------- Lookup unit file path, displaySrc, errMsg = lookupSrc(model, unit[UNIT_FILE]) if path != None: unit[_UNIT_FILE_] = path unit[_DISPLAY_SRC_] = displaySrc else: misc.ERROR("Systemd_unit '{0}': {1}".format( unit[NAME], errMsg)) # ---------------------- Insert in scope if not self.context.checkScope(unit[SCOPE]): misc.ERROR( "Systemd_unit {0}: scope attribute '{1}' does not match any host or host_group!" .format(unit[NAME], unit[SCOPE])) else: misc.ensureObjectInMaps( self.context.model[DATA][SYSTEMD][SCOPE_BY_NAME], [unit[SCOPE], SYSTEMD], []) model[DATA][SYSTEMD][SCOPE_BY_NAME][ unit[SCOPE]][SYSTEMD].append(unit)
def groomSupervisors(self): if self.context.toExclude(SCOPE_SUPERVISOR): return model = self.context.model if SUPERVISORS in model[SRC]: misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS], [SUPERVISOR_BY_NAME], {}) for supervisord in model[SRC][SUPERVISORS]: if supervisord[NAME] in self.context.model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME]: misc.ERROR("supervisor '{0}' is defined twice!".format(supervisord[NAME])) self.context.model[DATA][SUPERVISORS][SUPERVISOR_BY_NAME][supervisord[NAME]] = supervisord misc.setDefaultInMap(supervisord, MANAGED, True) self.groomOneSupervisord(model, supervisord) # ---------------------- Insert in scope if not self.context.checkScope(supervisord[SCOPE]): misc.ERROR("Supervisor {0}: scope attribute '{1}' does not match any host or host_group!".format(supervisord[NAME], supervisord[SCOPE])) else: #misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [supervisord[SCOPE], SUPERVISORS], []) #misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [supervisord[SCOPE], PROGRAMS], []) #model[DATA][SUPERVISORS][SCOPE_BY_NAME][supervisord[SCOPE]][SUPERVISORS].append(supervisord) if supervisord[MANAGED]: misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [supervisord[SCOPE], SUPERVISORS_TO_MANAGE], []) model[DATA][SUPERVISORS][SCOPE_BY_NAME][supervisord[SCOPE]][SUPERVISORS_TO_MANAGE].append(supervisord) if not supervisord[NO_REMOVE]: misc.ensureObjectInMaps(self.context.model[DATA][SUPERVISORS][SCOPE_BY_NAME], [supervisord[SCOPE], SUPERVISORS_TO_REMOVE], []) model[DATA][SUPERVISORS][SCOPE_BY_NAME][supervisord[SCOPE]][SUPERVISORS_TO_REMOVE].append(supervisord)
def groomSourceHostCredentials(model): misc.ensureObjectInMaps(model[DATA], [HDFS, CREDENTIAL_BY_HOST], {}) if SOURCE_HOST_CREDENTIALS in model[SRC]: for hcredential in model[SRC][SOURCE_HOST_CREDENTIALS]: key = hcredential[HOST] model[DATA][HDFS][CREDENTIAL_BY_HOST][key] = hcredential del model[DATA][HDFS][CREDENTIAL_BY_HOST][key][ HOST] # No need to keep hostname, as it is the key misc.setDefaultInMap(hcredential, KDEBUG, False) if LOCAL_KEYTAB_PATH not in hcredential and NODE_KEYTAB_PATH not in hcredential: misc.ERROR( "source_host_credential for host {0}: Please provide a 'local_keytab_path' and/or a 'node_keytab_path'" .format(key)) if LOCAL_KEYTAB_PATH in hcredential: if not os.path.exists(hcredential[LOCAL_KEYTAB_PATH]): misc.ERROR( "source_host_credential for host {0}: local_keytab_file '{1}' does not exists!" .format(key, hcredential[LOCAL_KEYTAB_PATH])) if NODE_KEYTAB_PATH not in hcredential: hcredential[ _NODE_KEYTAB_FOLDER_] = DEFAULT_HDFS_KEYTABS_FOLDER hcredential[NODE_KEYTAB_PATH] = os.path.join( hcredential[_NODE_KEYTAB_FOLDER_], os.path.basename(hcredential[LOCAL_KEYTAB_PATH]))
def groomElasticServers(model): misc.ensureObjectInMaps(model[DATA][ELASTIC], [SERVER_BY_NAME], {}) if ELASTICSEARCH_SERVERS in model[SRC]: relayFiles = Set() # To check uniqueness for server in model[SRC][ELASTICSEARCH_SERVERS]: if server[NAME] in model[DATA][ELASTIC][SERVER_BY_NAME]: misc.ERROR( "Elasticsearch server '{}' is defined twice!".format( server[NAME])) misc.ensureObjectInMaps(server, [INDICES], []) misc.ensureObjectInMaps(server, [TEMPLATES], []) model[DATA][ELASTIC][SERVER_BY_NAME][server[NAME]] = server misc.setDefaultInMap(server, VALIDATE_CERTS, True) misc.setDefaultInMap(server, NO_LOG, True) if (USERNAME in server) != (PASSWORD in server): misc.ERROR( "Elasticsearch server '{}': 'username' and 'password' must be defined together or not at all!" .format(server[NAME])) if CA_BUNDLE_LOCAL_FILE in server: if not os.path.exists(server[CA_BUNDLE_LOCAL_FILE]): misc.ERROR( "Elasticsearch server '{}': ca_bundle_local_file: {0} does not exists" .format(server[NAME], server[CA_BUNDLE_LOCAL_FILE])) if CA_BUNDLE_RELAY_FILE not in server: misc.ERROR( "Elasticsearch server '{}': If a ca_bundle_local_file is defined, then a ca_bundle_relay_file must also be defined" .format(server[NAME])) if not os.path.isabs(server[CA_BUNDLE_RELAY_FILE]): misc.ERROR( "Elasticsearch server '{}': ca_bundle_relay_file: {0} must be absolute!" .format(server[NAME], server[CA_BUNDLE_RELAY_FILE])) if server[CA_BUNDLE_RELAY_FILE] in relayFiles: misc.ERROR( "ca_bundle_relay_file: {} is defined by several elasticsearch_server" .format(server[CA_BUNDLE_RELAY_FILE])) relayFiles.add(server[CA_BUNDLE_RELAY_FILE]) server[CA_BUNDLE_RELAY_FOLDER] = os.path.dirname( server[CA_BUNDLE_RELAY_FILE])
def ensureScope(context, scope): root = context.model[DATA][USERS][SCOPE_BY_NAME] if not scope in root: misc.ensureObjectInMaps(root, [scope, USERS], []) misc.ensureObjectInMaps(root, [scope, GROUPS], [])
def onGrooming(self): misc.ensureObjectInMaps(self.context.model[DATA], [SYSTEMD, SCOPE_BY_NAME], {}) misc.applyWhenOnList(self.context.model[SRC], SYSTEMD_UNITS) self.groomSystemd()
def onGrooming(self): misc.applyWhenOnList(self.context.model[SRC], ANSIBLE_PLAYBOOKS) misc.applyWhenOnList(self.context.model[SRC], ANSIBLE_ROLES) """ Main job is to build a referential playbookByActionByPriority""" playbooksByActionByPriority = {} if not self.context.toExclude(SCOPE_ANSIBLE): src = self.context.model[SRC] if ANSIBLE_PLAYBOOKS in src: for pl in src[ANSIBLE_PLAYBOOKS]: action = pl[FOR_ACTION] priority = pl[PRIORITY] misc.ensureObjectInMaps(playbooksByActionByPriority, [action], {}) misc.ensureObjectInMaps(playbooksByActionByPriority[action], [priority], {}) if PLAYBOOK_FILE in pl: playbookFile = self.lookupPathInFolderList(pl[PLAYBOOK_FILE], PLAYBOOKS_FOLDERS, "playbook") misc.ensureObjectInMaps(playbooksByActionByPriority[action][priority], [PLAYBOOK_FILES], []) playbooksByActionByPriority[action][priority][PLAYBOOK_FILES].append(playbookFile) elif PLAYBOOK_TEXT in pl: misc.ensureObjectInMaps(playbooksByActionByPriority[action][priority], [PLAYBOOK_TEXTS], []) playbooksByActionByPriority[action][priority][PLAYBOOK_TEXTS].append(pl[PLAYBOOK_TEXT]) else: misc.ERROR("There is at least one ansible_playbook whith neither playbook_text not playbook_file. (action:'{0}', priority:'{1}'".format(action, priority)) if ANSIBLE_ROLES in src: for rl in src[ANSIBLE_ROLES]: if not self.context.checkScope(rl[SCOPE]): misc.ERROR("Invalid scope '{0}' in ansible role '{1}'".format(rl[SCOPE], rl[ROLE])) action = rl[FOR_ACTION] priority = rl[PRIORITY] misc.ensureObjectInMaps(playbooksByActionByPriority, [action], {}) misc.ensureObjectInMaps(playbooksByActionByPriority[action], [priority], {}) misc.ensureObjectInMaps(playbooksByActionByPriority[action][priority], [ROLES], []) playbooksByActionByPriority[action][priority][ROLES].append( { ROLE: rl[ROLE], SCOPE: rl[SCOPE] } ) self.referential = playbooksByActionByPriority # We don't need to expose this to template, but this can be usefull for debugging self.context.model[DATA][PLAYBOOKS_BY_ACTION_BY_PRIORITY] = playbooksByActionByPriority