def install_ranger(self): import params hbase_user = params.config['configurations']['hbase-env']['hbase_user'] user_group = params.config['configurations']['cluster-env'][ "user_group"] splicemachine_conf_dir = '/etc/splicemachine/conf' hdfs_audit_spool = params.config['configurations'][ 'ranger-splicemachine-audit'][ 'xasecure.audit.destination.hdfs.batch.filespool.dir'] solr_audit_spool = params.config['configurations'][ 'ranger-splicemachine-audit'][ 'xasecure.audit.destination.solr.batch.filespool.dir'] policy_cache_dir = params.config['configurations'][ 'ranger-splicemachine-security'][ 'ranger.plugin.splicemachine.policy.cache.dir'] hdfs_audit_dir = params.config['configurations'][ 'ranger-splicemachine-audit'][ 'xasecure.audit.destination.hdfs.dir'] Directory(splicemachine_conf_dir, owner=hbase_user, group=user_group, create_parents=True) Directory(hdfs_audit_spool, owner=hbase_user, group=user_group, create_parents=True) Directory(solr_audit_spool, owner=hbase_user, group=user_group, create_parents=True) Directory(policy_cache_dir, owner=hbase_user, group=user_group, create_parents=True) XmlConfig( "ranger-splicemachine-security.xml", conf_dir=splicemachine_conf_dir, configurations=params.config['configurations'] ['ranger-splicemachine-security'], configuration_attributes=params.config['configuration_attributes'] ['ranger-splicemachine-security'], owner=hbase_user, group=user_group, ) XmlConfig( "ranger-splicemachine-audit.xml", conf_dir=splicemachine_conf_dir, configurations=params.config['configurations'] ['ranger-splicemachine-audit'], configuration_attributes=params.config['configuration_attributes'] ['ranger-splicemachine-audit'], owner=hbase_user, group=user_group, )
def mahout(): import params # ensure that matching LZO libraries are installed for Mahout lzo_utils.install_lzo_if_needed() Directory(params.mahout_conf_dir, create_parents=True, owner=params.mahout_user, group=params.user_group) XmlConfig( "yarn-site.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['yarn-site'], configuration_attributes=params.config['configuration_attributes'] ['yarn-site'], owner=params.yarn_user, group=params.user_group, mode=0644) if not is_empty(params.log4j_props): File(format("{params.mahout_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.mahout_user, content=params.log4j_props) elif (os.path.exists(format("{params.mahout_conf_dir}/log4j.properties"))): File(format("{params.mahout_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.mahout_user)
def generate_configs(self, env): """ Generates config files and stores them as an archive in tmp_dir based on xml_configs_list and env_configs_list from commandParams """ import params env.set_params(params) xml_configs_list = params.config['commandParams']['xml_configs_list'] env_configs_list = params.config['commandParams']['env_configs_list'] conf_tmp_dir = tempfile.mkdtemp() output_filename = os.path.join( self.get_tmp_dir(), params.config['commandParams']['output_file']) Directory(self.get_tmp_dir(), recursive=True) for file_dict in xml_configs_list: for filename, dict in file_dict.iteritems(): XmlConfig( filename, conf_dir=conf_tmp_dir, configurations=params.config['configurations'][dict], configuration_attributes=params. config['configuration_attributes'][dict], ) for file_dict in env_configs_list: for filename, dict in file_dict.iteritems(): File(os.path.join(conf_tmp_dir, filename), content=InlineTemplate( params.config['configurations'][dict]['content'])) with closing(tarfile.open(output_filename, "w:gz")) as tar: tar.add(conf_tmp_dir, arcname=os.path.basename(".")) tar.close() Directory(conf_tmp_dir, action="delete")
def generate_configs(self, env): """ Generates config files and stores them as an archive in tmp_dir based on xml_configs_list and env_configs_list from commandParams """ import params env.set_params(params) config = self.get_config() xml_configs_list = config['commandParams']['xml_configs_list'] env_configs_list = config['commandParams']['env_configs_list'] properties_configs_list = config['commandParams'][ 'properties_configs_list'] Directory(self.get_tmp_dir(), create_parents=True) conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir()) os.chmod(conf_tmp_dir, 0700) output_filename = os.path.join(self.get_tmp_dir(), config['commandParams']['output_file']) try: for file_dict in xml_configs_list: for filename, dict in file_dict.iteritems(): XmlConfig(filename, conf_dir=conf_tmp_dir, mode=0600, **self.generate_configs_get_xml_file_content( filename, dict)) for file_dict in env_configs_list: for filename, dicts in file_dict.iteritems(): File(os.path.join(conf_tmp_dir, filename), mode=0600, content=InlineTemplate( self.generate_configs_get_template_file_content( filename, dicts))) for file_dict in properties_configs_list: for filename, dict in file_dict.iteritems(): PropertiesFile( os.path.join(conf_tmp_dir, filename), mode=0600, properties=self.generate_configs_get_xml_file_dict( filename, dict)) with closing(tarfile.open(output_filename, "w:gz")) as tar: os.chmod(output_filename, 0600) try: tar.add(conf_tmp_dir, arcname=os.path.basename(".")) finally: tar.close() finally: Directory(conf_tmp_dir, action="delete")
def hbase(name=None): import params XmlConfig("hbase-site.xml", conf_dir=params.hbase_conf_dir, configurations=params.config['configurations']['hbase-site'], configuration_attributes=params.config['configurationAttributes'] ['hbase-site']) if params.service_map.has_key(name): # Manually overriding service logon user & password set by the installation package service_name = params.service_map[name] ServiceConfig(service_name, action="change_user", username=params.hbase_user, password=Script.get_password(params.hbase_user))
def generate_configs(self, env): """ Generates config files and stores them as an archive in tmp_dir based on xml_configs_list and env_configs_list from commandParams """ import params env.set_params(params) config = self.get_config() xml_configs_list = config['commandParams']['xml_configs_list'] env_configs_list = config['commandParams']['env_configs_list'] properties_configs_list = config['commandParams'][ 'properties_configs_list'] Directory(self.get_tmp_dir(), recursive=True) conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir()) output_filename = os.path.join(self.get_tmp_dir(), config['commandParams']['output_file']) try: for file_dict in xml_configs_list: for filename, dict in file_dict.iteritems(): XmlConfig(filename, conf_dir=conf_tmp_dir, **self.generate_configs_get_xml_file_content( filename, dict)) for file_dict in env_configs_list: for filename, dicts in file_dict.iteritems(): File(os.path.join(conf_tmp_dir, filename), content=InlineTemplate( self.generate_configs_get_template_file_content( filename, dicts))) for file_dict in properties_configs_list: for filename, dict in file_dict.iteritems(): PropertiesFile( os.path.join(conf_tmp_dir, filename), properties=self.generate_configs_get_xml_file_dict( filename, dict)) archive_dir(output_filename, conf_tmp_dir) finally: Directory(conf_tmp_dir, action="delete")
def hbase(name=None): import params # ensure that matching LZO libraries are installed for HBase lzo_utils.install_lzo_if_needed() Directory(params.etc_prefix_dir, mode=0755) Directory(params.hbase_conf_dir, owner=params.hbase_user, group=params.user_group, create_parents=True) Directory(params.java_io_tmpdir, create_parents=True, mode=0777) # If a file location is specified in ioengine parameter, # ensure that directory exists. Otherwise create the # directory with permissions assigned to hbase:hadoop. ioengine_input = params.ioengine_param if ioengine_input != None: if ioengine_input.startswith("file:/"): ioengine_fullpath = ioengine_input[5:] ioengine_dir = os.path.dirname(ioengine_fullpath) Directory(ioengine_dir, owner=params.hbase_user, group=params.user_group, create_parents=True, mode=0755) parent_dir = os.path.dirname(params.tmp_dir) # In case if we have several placeholders in path while ("${" in parent_dir): parent_dir = os.path.dirname(parent_dir) if parent_dir != os.path.abspath(os.sep): Directory( parent_dir, create_parents=True, cd_access="a", ) Execute(("chmod", "1777", parent_dir), sudo=True) XmlConfig("hbase-site.xml", conf_dir=params.hbase_conf_dir, configurations=params.config['configurations']['hbase-site'], configuration_attributes=params.config['configurationAttributes'] ['hbase-site'], owner=params.hbase_user, group=params.user_group) if check_stack_feature(StackFeature.PHOENIX_CORE_HDFS_SITE_REQUIRED, params.version_for_stack_feature_checks): XmlConfig( "core-site.xml", conf_dir=params.hbase_conf_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configurationAttributes'] ['core-site'], owner=params.hbase_user, group=params.user_group) if 'hdfs-site' in params.config['configurations']: XmlConfig( "hdfs-site.xml", conf_dir=params.hbase_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params. config['configurationAttributes']['hdfs-site'], owner=params.hbase_user, group=params.user_group) else: File(format("{params.hbase_conf_dir}/hdfs-site.xml"), action="delete") File(format("{params.hbase_conf_dir}/core-site.xml"), action="delete") if 'hbase-policy' in params.config['configurations']: XmlConfig( "hbase-policy.xml", conf_dir=params.hbase_conf_dir, configurations=params.config['configurations']['hbase-policy'], configuration_attributes=params.config['configurationAttributes'] ['hbase-policy'], owner=params.hbase_user, group=params.user_group) # Manually overriding ownership of file installed by hadoop package else: File(format("{params.hbase_conf_dir}/hbase-policy.xml"), owner=params.hbase_user, group=params.user_group) File( format("{hbase_conf_dir}/hbase-env.sh"), owner=params.hbase_user, content=InlineTemplate(params.hbase_env_sh_template), group=params.user_group, ) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'hbase.conf'), owner='root', group='root', mode=0644, content=Template("hbase.conf.j2")) hbase_TemplateConfig( params.metric_prop_file_name, tag='GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS') hbase_TemplateConfig('regionservers') if params.security_enabled: hbase_TemplateConfig(format("hbase_{name}_jaas.conf")) if name != "client": Directory( params.pid_dir, owner=params.hbase_user, create_parents=True, cd_access="a", mode=0755, ) Directory( params.log_dir, owner=params.hbase_user, create_parents=True, cd_access="a", mode=0755, ) if (params.log4j_props != None): File(format("{params.hbase_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.hbase_user, content=InlineTemplate(params.log4j_props)) elif (os.path.exists(format("{params.hbase_conf_dir}/log4j.properties"))): File(format("{params.hbase_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.hbase_user) if name == "master" and params.default_fs: if not params.hbase_hdfs_root_dir_protocol or params.hbase_hdfs_root_dir_protocol == urlparse( params.default_fs).scheme: params.HdfsResource(params.hbase_hdfs_root_dir, type="directory", action="create_on_execute", owner=params.hbase_user) params.HdfsResource(params.hbase_staging_dir, type="directory", action="create_on_execute", owner=params.hbase_user, mode=0711) if params.create_hbase_home_directory: params.HdfsResource(params.hbase_home_directory, type="directory", action="create_on_execute", owner=params.hbase_user, mode=0755) params.HdfsResource(None, action="execute") if name in ('master', 'regionserver') and not params.default_fs: Directory( params.hbase_staging_dir, owner=params.hbase_user, create_parents=True, cd_access="a", mode=0711, ) if params.phoenix_enabled: Package(params.phoenix_package, retry_on_repo_unavailability=params. agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count)
def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None): """ :param env: Python environment :param type: Spark component type :param upgrade_type: If in a stack upgrade, either UPGRADE_TYPE_ROLLING or UPGRADE_TYPE_NON_ROLLING :param action: Action to perform, such as generate configs :param config_dir: Optional config directory to write configs to. """ import params # ensure that matching LZO libraries are installed for Spark lzo_utils.install_lzo_if_needed() if config_dir is None: config_dir = params.spark_conf Directory([params.spark_pid_dir, params.spark_log_dir], owner=params.spark_user, group=params.user_group, mode=0775, create_parents = True, cd_access = 'a', ) if type == 'server' and action == 'config': params.HdfsResource(params.spark_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.spark_user, mode=0775 ) params.HdfsResource(None, action="execute") PropertiesFile(os.path.join(config_dir, "spark-defaults.conf"), properties = params.config['configurations']['spark-defaults'], key_value_delimiter = " ", owner=params.spark_user, group=params.spark_group, mode=0644 ) # create spark-env.sh in etc/conf dir File(os.path.join(config_dir, 'spark-env.sh'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_env_sh), mode=0644, ) #create log4j.properties in etc/conf dir File(os.path.join(config_dir, 'log4j.properties'), owner=params.spark_user, group=params.spark_group, content=params.spark_log4j_properties, mode=0644, ) #create metrics.properties in etc/conf dir File(os.path.join(config_dir, 'metrics.properties'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_metrics_properties), mode=0644 ) Directory(params.spark_logs_dir, owner=params.spark_user, group=params.spark_group, mode=0755, ) if params.is_hive_installed: XmlConfig("hive-site.xml", conf_dir=config_dir, configurations=params.spark_hive_properties, owner=params.spark_user, group=params.spark_group, mode=0644) if params.has_spark_thriftserver: PropertiesFile(params.spark_thrift_server_conf_file, properties = params.config['configurations']['spark-thrift-sparkconf'], owner = params.hive_user, group = params.user_group, key_value_delimiter = " ", mode=0644 ) effective_version = params.version if upgrade_type is not None else params.version_for_stack_feature_checks if effective_version: effective_version = format_stack_version(effective_version) if check_stack_feature(StackFeature.SPARK_JAVA_OPTS_SUPPORT, effective_version): File(os.path.join(params.spark_conf, 'java-opts'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_javaopts_properties), mode=0644 ) else: File(os.path.join(params.spark_conf, 'java-opts'), action="delete" ) if params.spark_thrift_fairscheduler_content and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): # create spark-thrift-fairscheduler.xml File(os.path.join(config_dir,"spark-thrift-fairscheduler.xml"), owner=params.spark_user, group=params.spark_group, mode=0755, content=InlineTemplate(params.spark_thrift_fairscheduler_content) )
def setup_spark(env, type, upgrade_type = None, action = None): import params # ensure that matching LZO libraries are installed for Spark lzo_utils.install_lzo_if_needed() Directory([params.spark_pid_dir, params.spark_log_dir], owner=params.spark_user, group=params.user_group, mode=0775, create_parents = True, cd_access = 'a', ) if type == 'server' and action == 'config': params.HdfsResource(params.spark_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.spark_user, mode=0775 ) params.HdfsResource(None, action="execute") PropertiesFile(format("{spark_conf}/spark-defaults.conf"), properties = params.config['configurations']['spark-defaults'], key_value_delimiter = " ", owner=params.spark_user, group=params.spark_group, mode=0644 ) # create spark-env.sh in etc/conf dir File(os.path.join(params.spark_conf, 'spark-env.sh'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_env_sh), mode=0644, ) #create log4j.properties in etc/conf dir File(os.path.join(params.spark_conf, 'log4j.properties'), owner=params.spark_user, group=params.spark_group, content=params.spark_log4j_properties, mode=0644, ) #create metrics.properties in etc/conf dir File(os.path.join(params.spark_conf, 'metrics.properties'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_metrics_properties), mode=0644 ) if params.is_hive_installed: XmlConfig("hive-site.xml", conf_dir=params.spark_conf, configurations=params.spark_hive_properties, owner=params.spark_user, group=params.spark_group, mode=0644) if params.has_spark_thriftserver: PropertiesFile(params.spark_thrift_server_conf_file, properties = params.config['configurations']['spark-thrift-sparkconf'], owner = params.hive_user, group = params.user_group, key_value_delimiter = " ", mode=0644 ) effective_version = params.version if upgrade_type is not None else params.stack_version_formatted if effective_version: effective_version = format_stack_version(effective_version) if params.spark_thrift_fairscheduler_content and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): # create spark-thrift-fairscheduler.xml File(os.path.join(params.spark_conf,"spark-thrift-fairscheduler.xml"), owner=params.spark_user, group=params.spark_group, mode=0755, content=InlineTemplate(params.spark_thrift_fairscheduler_content) )
def druid(upgrade_type=None, nodeType=None): import params ensure_base_directories() # Environment Variables File(format("{params.druid_conf_dir}/druid-env.sh"), owner=params.druid_user, content=InlineTemplate(params.druid_env_sh_template) ) # common config druid_common_config = mutable_config_dict(params.config['configurations']['druid-common']) # User cannot override below configs druid_common_config['druid.host'] = params.hostname druid_common_config['druid.extensions.directory'] = params.druid_extensions_dir druid_common_config['druid.extensions.hadoopDependenciesDir'] = params.druid_hadoop_dependencies_dir druid_common_config['druid.selectors.indexing.serviceName'] = params.config['configurations']['druid-overlord'][ 'druid.service'] druid_common_config['druid.selectors.coordinator.serviceName'] = \ params.config['configurations']['druid-coordinator']['druid.service'] # delete the password and user if empty otherwiswe derby will fail. if 'derby' == druid_common_config['druid.metadata.storage.type']: del druid_common_config['druid.metadata.storage.connector.user'] del druid_common_config['druid.metadata.storage.connector.password'] druid_env_config = mutable_config_dict(params.config['configurations']['druid-env']) PropertiesFile("common.runtime.properties", dir=params.druid_common_conf_dir, properties=druid_common_config, owner=params.druid_user, group=params.user_group, ) Logger.info("Created common.runtime.properties") File(format("{params.druid_common_conf_dir}/druid-log4j.xml"), mode=0644, owner=params.druid_user, group=params.user_group, content=InlineTemplate(params.log4j_props) ) Logger.info("Created log4j file") File("/etc/logrotate.d/druid", mode=0644, owner='root', group='root', content=InlineTemplate(params.logrotate_props) ) Logger.info("Created log rotate file") # Write Hadoop Configs if configured if 'core-site' in params.config['configurations']: XmlConfig("core-site.xml", conf_dir=params.druid_common_conf_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configuration_attributes']['core-site'], owner=params.druid_user, group=params.user_group ) if 'mapred-site' in params.config['configurations']: XmlConfig("mapred-site.xml", conf_dir=params.druid_common_conf_dir, configurations=params.config['configurations']['mapred-site'], configuration_attributes=params.config['configuration_attributes']['mapred-site'], owner=params.druid_user, group=params.user_group ) if 'yarn-site' in params.config['configurations']: XmlConfig("yarn-site.xml", conf_dir=params.druid_common_conf_dir, configurations=params.config['configurations']['yarn-site'], configuration_attributes=params.config['configuration_attributes']['yarn-site'], owner=params.druid_user, group=params.user_group ) if 'hdfs-site' in params.config['configurations']: XmlConfig("hdfs-site.xml", conf_dir=params.druid_common_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configuration_attributes']['hdfs-site'], owner=params.druid_user, group=params.user_group ) # node specific configs for node_type in ['coordinator', 'overlord', 'historical', 'broker', 'middleManager', 'router']: node_config_dir = format('{params.druid_conf_dir}/{node_type}') node_type_lowercase = node_type.lower() # Write runtime.properties file node_config = mutable_config_dict(params.config['configurations'][format('druid-{node_type_lowercase}')]) PropertiesFile("runtime.properties", dir=node_config_dir, properties=node_config, owner=params.druid_user, group=params.user_group, ) Logger.info(format("Created druid-{node_type_lowercase} runtime.properties")) # Write jvm configs File(format('{node_config_dir}/jvm.config'), owner=params.druid_user, group=params.user_group, content=InlineTemplate( "-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}", node_heap_memory=druid_env_config[format('druid.{node_type_lowercase}.jvm.heap.memory')], log4j_config_file=format("{params.druid_common_conf_dir}/druid-log4j.xml"), node_direct_memory=druid_env_config[ format('druid.{node_type_lowercase}.jvm.direct.memory')], node_jvm_opts=druid_env_config[format('druid.{node_type_lowercase}.jvm.opts')]) ) Logger.info(format("Created druid-{node_type_lowercase} jvm.config")) # All druid nodes have dependency on hdfs_client ensure_hadoop_directories() # Pull all required dependencies pulldeps()