def storm(name=None): import params import os Directory( params.log_dir, owner=params.storm_user, group=params.user_group, mode=0777, create_parents=True, cd_access="a", ) Directory( [params.pid_dir, params.local_dir], owner=params.storm_user, group=params.user_group, create_parents=True, cd_access="a", mode=0755, ) Directory( params.conf_dir, group=params.user_group, create_parents=True, cd_access="a", ) File( format("{limits_conf_dir}/storm.conf"), owner='root', group='root', mode=0644, content=Template("storm.conf.j2")) File( format("{conf_dir}/config.yaml"), content=Template("config.yaml.j2"), owner=params.storm_user, group=params.user_group) configurations = params.config['configurations']['storm-site'] File( format("{conf_dir}/storm.yaml"), content=yaml_config_template(configurations), owner=params.storm_user, group=params.user_group) File( format("{conf_dir}/storm-env.sh"), owner=params.storm_user, content=InlineTemplate(params.storm_env_sh_template)) generate_logfeeder_input_config( 'storm', Template( "input.config-storm.json.j2", extra_imports=[default])) # Generate atlas-application.properties.xml file and symlink the hook jars if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.conf_dir, params.atlas_hook_filename) setup_atlas_hook( SERVICE.STORM, params.storm_atlas_application_properties, atlas_hook_filepath, params.storm_user, params.user_group) if params.storm_logs_supported: Directory( params.log4j_dir, owner=params.storm_user, group=params.user_group, mode=0755, create_parents=True) File( format("{log4j_dir}/cluster.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_cluster_log4j_content)) File( format("{log4j_dir}/worker.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_worker_log4j_content)) if params.security_enabled: TemplateConfig( format("{conf_dir}/storm_jaas.conf"), owner=params.storm_user, mode=0644) TemplateConfig( format("{conf_dir}/client_jaas.conf"), owner=params.storm_user, mode=0644) minRuid = configurations['_storm.min.ruid'] if configurations.has_key( '_storm.min.ruid') else '' min_user_ruid = int( minRuid) if minRuid.isdigit() else _find_real_user_min_uid() File( format("{conf_dir}/worker-launcher.cfg"), content=Template( "worker-launcher.cfg.j2", min_user_ruid=min_user_ruid), owner='root', group=params.user_group) else: File(format("{conf_dir}/storm_jaas.conf"), action="delete") File(format("{conf_dir}/client_jaas.conf"), action="delete")
def storm(name=None): import params import os Directory( params.log_dir, owner=params.storm_user, group=params.user_group, mode=0777, create_parents=True, cd_access="a", ) Directory( [params.pid_dir, params.local_dir], owner=params.storm_user, group=params.user_group, create_parents=True, cd_access="a", mode=0755, ) Directory( params.conf_dir, group=params.user_group, create_parents=True, cd_access="a", ) File(format("{conf_dir}/config.yaml"), content=Template("config.yaml.j2"), owner=params.storm_user, group=params.user_group) File(params.conf_dir + "/jmxetric-conf.xml", content=StaticFile("jmxetric-conf.xml"), owner=params.storm_user) File(params.storm_lib_dir + "/gmetric4j-1.0.3.jar", content=StaticFile("gmetric4j-1.0.3.jar"), owner=params.storm_user) File(params.storm_lib_dir + "/jmxetric-1.0.4.jar", content=StaticFile("jmxetric-1.0.4.jar"), owner=params.storm_user) File(params.storm_lib_dir + "/oncrpc-1.0.7.jar", content=StaticFile("oncrpc-1.0.7.jar"), owner=params.storm_user) configurations = params.config['configurations']['storm-site'] File(format("{conf_dir}/storm.yaml"), content=yaml_config_template(configurations), owner=params.storm_user, group=params.user_group) File(format("{conf_dir}/storm-env.sh"), owner=params.storm_user, content=InlineTemplate(params.storm_env_sh_template)) # Generate atlas-application.properties.xml file and symlink the hook jars if params.enable_atlas_hook: script_path = os.path.realpath(__file__).split( '/services')[0] + '/hooks/before-INSTALL/scripts/atlas' sys.path.append(script_path) from setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook, setup_atlas_jar_symlinks atlas_hook_filepath = os.path.join(params.conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.STORM, params.storm_atlas_application_properties, atlas_hook_filepath, params.storm_user, params.user_group) storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib") setup_atlas_jar_symlinks("storm", storm_extlib_dir) if params.has_metric_collector: File(format("{conf_dir}/storm-metrics2.properties"), owner=params.storm_user, group=params.user_group, content=Template("storm-metrics2.properties.j2")) # Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2 Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"), action="delete") # On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar", action="delete") sink_jar = params.sink_jar Execute(format( "{sudo} ln -s {sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar" ), not_if=format( "ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"), only_if=format("ls {sink_jar}")) if params.storm_logs_supported: Directory(params.log4j_dir, owner=params.storm_user, group=params.user_group, mode=0755, create_parents=True) File(format("{log4j_dir}/cluster.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_cluster_log4j_content)) File(format("{log4j_dir}/worker.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_worker_log4j_content)) if params.security_enabled: TemplateConfig(format("{conf_dir}/storm_jaas.conf"), owner=params.storm_user) TemplateConfig(format("{conf_dir}/client_jaas.conf"), owner=params.storm_user) minRuid = configurations['_storm.min.ruid'] if configurations.has_key( '_storm.min.ruid') else '' min_user_ruid = int( minRuid) if minRuid.isdigit() else _find_real_user_min_uid() File(format("{conf_dir}/worker-launcher.cfg"), content=Template("worker-launcher.cfg.j2", min_user_ruid=min_user_ruid), owner='root', group=params.user_group)
def kafka(upgrade_type=None): import params ensure_base_directories() kafka_server_config = mutable_config_dict( params.config['configurations']['kafka-broker']) # This still has an issue of hostnames being alphabetically out-of-order for broker.id in HDP-2.2. # Starting in HDP 2.3, Kafka handles the generation of broker.id so Ambari doesn't have to. effective_version = params.hdp_stack_version if upgrade_type is None else format_hdp_stack_version( params.version) Logger.info(format("Effective stack version: {effective_version}")) if effective_version is not None and effective_version != "" and compare_versions( effective_version, '2.2.0.0') >= 0 and compare_versions( effective_version, '2.3.0.0') < 0: if len(params.kafka_hosts ) > 0 and params.hostname in params.kafka_hosts: brokerid = str(sorted(params.kafka_hosts).index(params.hostname)) kafka_server_config['broker.id'] = brokerid Logger.info(format("Calculating broker.id as {brokerid}")) # listeners and advertised.listeners are only added in 2.3.0.0 onwards. if effective_version is not None and effective_version != "" and compare_versions( effective_version, '2.3.0.0') >= 0: listeners = kafka_server_config['listeners'].replace( "localhost", params.hostname) Logger.info(format("Kafka listeners: {listeners}")) if params.security_enabled and params.kafka_kerberos_enabled: Logger.info("Kafka kerberos security is enabled.") if "SASL" not in listeners: listeners = listeners.replace("PLAINTEXT", "PLAINTEXTSASL") kafka_server_config['listeners'] = listeners kafka_server_config['advertised.listeners'] = listeners Logger.info(format("Kafka advertised listeners: {listeners}")) else: kafka_server_config['listeners'] = listeners if 'advertised.listeners' in kafka_server_config: advertised_listeners = kafka_server_config[ 'advertised.listeners'].replace("localhost", params.hostname) kafka_server_config[ 'advertised.listeners'] = advertised_listeners Logger.info( format( "Kafka advertised listeners: {advertised_listeners}")) else: kafka_server_config['host.name'] = params.hostname if params.has_metric_collector: kafka_server_config[ 'kafka.timeline.metrics.host'] = params.metric_collector_host kafka_server_config[ 'kafka.timeline.metrics.port'] = params.metric_collector_port kafka_server_config[ 'kafka.timeline.metrics.protocol'] = params.metric_collector_protocol kafka_server_config[ 'kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path kafka_server_config[ 'kafka.timeline.metrics.truststore.type'] = params.metric_truststore_type kafka_server_config[ 'kafka.timeline.metrics.truststore.password'] = params.metric_truststore_password kafka_data_dir = kafka_server_config['log.dirs'] kafka_data_dirs = filter(None, kafka_data_dir.split(",")) Directory( kafka_data_dirs, mode=0755, cd_access='a', owner=params.kafka_user, group=params.user_group, create_parents=True, recursive_ownership=True, ) PropertiesFile( "server.properties", dir=params.conf_dir, properties=kafka_server_config, owner=params.kafka_user, group=params.user_group, ) File(format("{conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template)) if (params.log4j_props != None): File(format("{conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.kafka_user, content=params.log4j_props) if params.security_enabled and params.kafka_kerberos_enabled: TemplateConfig(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user) TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'kafka.conf'), owner='root', group='root', mode=0644, content=Template("kafka.conf.j2")) setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir) setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
def storm(name=None): import params import os Directory( params.log_dir, owner=params.storm_user, group=params.user_group, mode=0777, create_parents=True, cd_access="a", ) Directory( [params.pid_dir, params.local_dir], owner=params.storm_user, group=params.user_group, create_parents=True, cd_access="a", mode=0755, ) Directory( params.conf_dir, group=params.user_group, create_parents=True, cd_access="a", ) File(format("{conf_dir}/config.yaml"), content=Template("config.yaml.j2"), owner=params.storm_user, group=params.user_group) configurations = params.config['configurations']['storm-site'] File(format("{conf_dir}/storm.yaml"), content=yaml_config_template(configurations), owner=params.storm_user, group=params.user_group) File(format("{conf_dir}/storm-env.sh"), owner=params.storm_user, content=InlineTemplate(params.storm_env_sh_template)) # Generate atlas-application.properties.xml file and symlink the hook jars if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.STORM, params.storm_atlas_application_properties, atlas_hook_filepath, params.storm_user, params.user_group) storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib") setup_atlas_jar_symlinks("storm", storm_extlib_dir) if params.has_metric_collector: File(format("{conf_dir}/storm-metrics2.properties"), owner=params.storm_user, group=params.user_group, content=Template("storm-metrics2.properties.j2")) # Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2 Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"), action="delete") # On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar", action="delete") if check_stack_feature(StackFeature.STORM_METRICS_APACHE_CLASSES, params.version_for_stack_feature_checks): sink_jar = params.metric_collector_sink_jar else: sink_jar = params.metric_collector_legacy_sink_jar Execute(format( "{sudo} ln -s {sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar" ), not_if=format( "ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"), only_if=format("ls {sink_jar}")) if params.storm_logs_supported: Directory(params.log4j_dir, owner=params.storm_user, group=params.user_group, mode=0755, create_parents=True) File(format("{log4j_dir}/cluster.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_cluster_log4j_content)) File(format("{log4j_dir}/worker.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_worker_log4j_content)) if params.security_enabled: TemplateConfig(format("{conf_dir}/storm_jaas.conf"), owner=params.storm_user) if params.stack_version_formatted and check_stack_feature( StackFeature.ROLLING_UPGRADE, params.stack_version_formatted): TemplateConfig(format("{conf_dir}/client_jaas.conf"), owner=params.storm_user) minRuid = configurations[ '_storm.min.ruid'] if configurations.has_key( '_storm.min.ruid') else '' min_user_ruid = int( minRuid) if minRuid.isdigit() else _find_real_user_min_uid() File(format("{conf_dir}/worker-launcher.cfg"), content=Template("worker-launcher.cfg.j2", min_user_ruid=min_user_ruid), owner='root', group=params.user_group)
def storm(): import params Directory(params.log_dir, owner=params.storm_user, group=params.user_group, mode=0775, recursive=True) Directory( [params.pid_dir, params.local_dir, params.conf_dir], owner=params.storm_user, group=params.user_group, recursive=True, cd_access="a", ) File(format("{conf_dir}/config.yaml"), content=Template("config.yaml.j2"), owner=params.storm_user, group=params.user_group) configurations = params.config['configurations']['storm-site'] File(format("{conf_dir}/storm.yaml"), content=Template("storm.yaml.j2", extra_imports=[escape_yaml_propetry], configurations=configurations), owner=params.storm_user, group=params.user_group) if params.has_metric_collector: File(format("{conf_dir}/storm-metrics2.properties"), owner=params.storm_user, group=params.user_group, content=Template("storm-metrics2.properties.j2")) Execute(format( "sudo ln -s {metric_collector_sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar" ), not_if=format( "ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"), only_if=format("ls {metric_collector_sink_jar}")) File(format("{conf_dir}/storm-env.sh"), owner=params.storm_user, content=InlineTemplate(params.storm_env_sh_template)) if params.security_enabled: TemplateConfig(format("{conf_dir}/storm_jaas.conf"), owner=params.storm_user) if params.hdp_stack_version != "" and compare_versions( params.hdp_stack_version, '2.2') >= 0: TemplateConfig(format("{conf_dir}/client_jaas.conf"), owner=params.storm_user) minRuid = configurations[ '_storm.min.ruid'] if configurations.has_key( '_storm.min.ruid') else '' min_user_ruid = int( minRuid) if minRuid.isdigit() else _find_real_user_min_uid() File(format("{conf_dir}/worker-launcher.cfg"), content=Template("worker-launcher.cfg.j2", min_user_ruid=min_user_ruid), owner='root', group=params.user_group)
def ams(name=None, action=None): import params if name == 'collector': Directory( params.ams_collector_conf_dir, owner=params.ams_user, group=params.user_group, create_parents=True, recursive_ownership=True, ) Directory(params.ams_checkpoint_dir, owner=params.ams_user, group=params.user_group, cd_access="a", create_parents=True, recursive_ownership=True) new_ams_site = {} new_ams_site.update(params.config['configurations']['ams-site']) if params.clusterHostInfoDict: master_components = [] slave_components = [] components = dict(params.clusterHostInfoDict).keys() known_slave_components = [ "nodemanager", "metrics_monitor", "datanode", "hbase_regionserver" ] for component in components: if component and component.endswith("_hosts"): component_name = component[:-6] elif component and component.endswith("_host"): component_name = component[:-5] else: continue if component_name in known_slave_components: slave_components.append(component_name) else: master_components.append(component_name) if slave_components: new_ams_site[ 'timeline.metrics.initial.configured.slave.components'] = ",".join( slave_components) if master_components: if 'ambari_server' not in master_components: master_components.append('ambari_server') new_ams_site[ 'timeline.metrics.initial.configured.master.components'] = ",".join( master_components) hbase_total_heapsize_with_trailing_m = params.hbase_heapsize hbase_total_heapsize = int( hbase_total_heapsize_with_trailing_m[:-1]) * 1024 * 1024 new_ams_site['hbase_total_heapsize'] = hbase_total_heapsize XmlConfig( "ams-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=new_ams_site, configuration_attributes=params.config['configurationAttributes'] ['ams-site'], owner=params.ams_user, group=params.user_group) XmlConfig( "ssl-server.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['ams-ssl-server'], configuration_attributes=params.config['configurationAttributes'] ['ams-ssl-server'], owner=params.ams_user, group=params.user_group) merged_ams_hbase_site = {} merged_ams_hbase_site.update( params.config['configurations']['ams-hbase-site']) if params.security_enabled: merged_ams_hbase_site.update( params.config['configurations']['ams-hbase-security-site']) # Add phoenix client side overrides merged_ams_hbase_site['phoenix.query.maxGlobalMemoryPercentage'] = str( params.phoenix_max_global_mem_percent) merged_ams_hbase_site[ 'phoenix.spool.directory'] = params.phoenix_client_spool_dir XmlConfig( "hbase-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=merged_ams_hbase_site, configuration_attributes=params.config['configurationAttributes'] ['ams-hbase-site'], owner=params.ams_user, group=params.user_group) if params.security_enabled: TemplateConfig(os.path.join(params.hbase_conf_dir, "ams_collector_jaas.conf"), owner=params.ams_user, template_tag=None) if (params.log4j_props != None): File(format("{params.ams_collector_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.ams_user, content=InlineTemplate(params.log4j_props)) File(format("{ams_collector_conf_dir}/ams-env.sh"), owner=params.ams_user, content=InlineTemplate(params.ams_env_sh_template)) Directory( params.ams_collector_log_dir, owner=params.ams_user, group=params.user_group, cd_access="a", create_parents=True, mode=0755, ) Directory( params.ams_collector_pid_dir, owner=params.ams_user, group=params.user_group, cd_access="a", create_parents=True, mode=0755, ) # Hack to allow native HBase libs to be included for embedded hbase File(os.path.join(params.ams_hbase_home_dir, "bin", "hadoop"), owner=params.ams_user, mode=0755) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') # Setting up security limits File(os.path.join(params.limits_conf_dir, 'ams.conf'), owner='root', group='root', mode=0644, content=Template("ams.conf.j2")) # Phoenix spool file dir if not /tmp if not os.path.exists(params.phoenix_client_spool_dir): Directory(params.phoenix_client_spool_dir, owner=params.ams_user, mode=0755, group=params.user_group, cd_access="a", create_parents=True) pass if not params.is_local_fs_rootdir and params.is_ams_distributed: # Configuration needed to support NN HA XmlConfig( "hdfs-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params. config['configurationAttributes']['hdfs-site'], owner=params.ams_user, group=params.user_group, mode=0644) XmlConfig( "hdfs-site.xml", conf_dir=params.hbase_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params. config['configurationAttributes']['hdfs-site'], owner=params.ams_user, group=params.user_group, mode=0644) # Remove spnego configs from core-site if platform does not have python-kerberos library truncated_core_site = {} truncated_core_site.update( params.config['configurations']['core-site']) if is_spnego_enabled( params) and is_redhat_centos_6_plus() == False: truncated_core_site.pop('hadoop.http.authentication.type') truncated_core_site.pop('hadoop.http.filter.initializers') XmlConfig("core-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=truncated_core_site, configuration_attributes=params. config['configurationAttributes']['core-site'], owner=params.ams_user, group=params.user_group, mode=0644) XmlConfig("core-site.xml", conf_dir=params.hbase_conf_dir, configurations=truncated_core_site, configuration_attributes=params. config['configurationAttributes']['core-site'], owner=params.ams_user, group=params.user_group, mode=0644) if params.metric_collector_https_enabled: export_ca_certs(params.ams_collector_conf_dir) pass elif name == 'monitor': if is_spnego_enabled(params) and is_redhat_centos_6_plus(): try: import kerberos except ImportError: raise ImportError( "python-kerberos package need to be installed to run AMS in SPNEGO mode" ) Directory(params.ams_monitor_conf_dir, owner=params.ams_user, group=params.user_group, create_parents=True) Directory(params.ams_monitor_log_dir, owner=params.ams_user, group=params.user_group, mode=0755, create_parents=True) if params.host_in_memory_aggregation and params.log4j_props is not None: File(format("{params.ams_monitor_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.ams_user, content=InlineTemplate(params.log4j_props)) XmlConfig( "ams-site.xml", conf_dir=params.ams_monitor_conf_dir, configurations=params.config['configurations']['ams-site'], configuration_attributes=params. config['configurationAttributes']['ams-site'], owner=params.ams_user, group=params.user_group) XmlConfig("ssl-server.xml", conf_dir=params.ams_monitor_conf_dir, configurations=params.config['configurations'] ['ams-ssl-server'], configuration_attributes=params. config['configurationAttributes']['ams-ssl-server'], owner=params.ams_user, group=params.user_group) pass Execute( format( "{sudo} chown -R {ams_user}:{user_group} {ams_monitor_log_dir}" )) Directory(params.ams_monitor_pid_dir, owner=params.ams_user, group=params.user_group, cd_access="a", mode=0755, create_parents=True) Directory(format("{ams_monitor_dir}/psutil/build"), owner=params.ams_user, group=params.user_group, cd_access="a", create_parents=True) Execute( format( "{sudo} chown -R {ams_user}:{user_group} {ams_monitor_dir}")) TemplateConfig(format("{ams_monitor_conf_dir}/metric_monitor.ini"), owner=params.ams_user, group=params.user_group, template_tag=None) TemplateConfig(format("{ams_monitor_conf_dir}/metric_groups.conf"), owner=params.ams_user, group=params.user_group, template_tag=None) File(format("{ams_monitor_conf_dir}/ams-env.sh"), owner=params.ams_user, content=InlineTemplate(params.ams_env_sh_template)) if params.metric_collector_https_enabled or params.is_aggregation_https_enabled: export_ca_certs(params.ams_monitor_conf_dir) pass elif name == 'grafana': ams_grafana_directories = [ params.ams_grafana_conf_dir, params.ams_grafana_log_dir, params.ams_grafana_data_dir, params.ams_grafana_pid_dir ] for ams_grafana_directory in ams_grafana_directories: Directory(ams_grafana_directory, owner=params.ams_user, group=params.user_group, mode=0755, create_parents=True, recursive_ownership=True) File(format("{ams_grafana_conf_dir}/ams-grafana-env.sh"), owner=params.ams_user, group=params.user_group, content=InlineTemplate(params.ams_grafana_env_sh_template)) File(format("{ams_grafana_conf_dir}/ams-grafana.ini"), owner=params.ams_user, group=params.user_group, content=InlineTemplate(params.ams_grafana_ini_template), mode=0600) if action != 'stop': for dir in ams_grafana_directories: Execute(('chown', '-R', params.ams_user, dir), sudo=True) if params.metric_collector_https_enabled: export_ca_certs(params.ams_grafana_conf_dir) pass
def storm(name=None): import params Directory(params.log_dir, owner=params.storm_user, group=params.user_group, mode=0777, recursive=True ) Directory([params.pid_dir, params.local_dir], owner=params.storm_user, group=params.user_group, recursive=True, cd_access="a", mode=0755, ) Directory(params.conf_dir, group=params.user_group, recursive=True, cd_access="a", ) File(format("{conf_dir}/config.yaml"), content=Template("config.yaml.j2"), owner=params.storm_user, group=params.user_group ) configurations = params.config['configurations']['storm-site'] File(format("{conf_dir}/storm.yaml"), content=yaml_config_template(configurations), owner=params.storm_user, group=params.user_group ) if params.has_metric_collector: File(format("{conf_dir}/storm-metrics2.properties"), owner=params.storm_user, group=params.user_group, content=Template("storm-metrics2.properties.j2") ) # Remove symlink. It can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2 Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar", action="delete") Execute(format("{sudo} ln -s {metric_collector_sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"), not_if=format("ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"), only_if=format("ls {metric_collector_sink_jar}") ) File(format("{conf_dir}/storm-env.sh"), owner=params.storm_user, content=InlineTemplate(params.storm_env_sh_template) ) if params.storm_logs_supported: Directory(params.log4j_dir, owner=params.storm_user, group=params.user_group, mode=0755, recursive=True ) File(format("{log4j_dir}/cluster.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_cluster_log4j_content) ) File(format("{log4j_dir}/worker.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_worker_log4j_content) ) if params.security_enabled: TemplateConfig(format("{conf_dir}/storm_jaas.conf"), owner=params.storm_user ) if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0: TemplateConfig(format("{conf_dir}/client_jaas.conf"), owner=params.storm_user ) minRuid = configurations['_storm.min.ruid'] if configurations.has_key('_storm.min.ruid') else '' min_user_ruid = int(minRuid) if minRuid.isdigit() else _find_real_user_min_uid() File(format("{conf_dir}/worker-launcher.cfg"), content=Template("worker-launcher.cfg.j2", min_user_ruid = min_user_ruid), owner='root', group=params.user_group )
def streamline(env, upgrade_type=None): import params ensure_base_directories() # Logger.info(format("Effective stack version: {effective_version}")) File(format("{conf_dir}/streamline-env.sh"), owner=params.streamline_user, content=InlineTemplate(params.streamline_env_sh_template)) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') Directory( [params.jar_storage], owner=params.streamline_user, group=params.user_group, create_parents=True, cd_access="a", mode=0755, ) # this is hard-coded as we are not accepting # the registry local-jars # should be removed from future releases Directory("/tmp/schema-registry/local-jars", owner=params.streamline_user, group=params.user_group, create_parents=True, cd_access="a", mode=0755) Directory( [params.topology_test_results], owner=params.streamline_user, group=params.user_group, create_parents=True, cd_access="a", mode=0755, ) File(os.path.join(params.limits_conf_dir, 'streamline.conf'), owner='root', group='root', mode=0644, content=Template("streamline.conf.j2")) File(format("{conf_dir}/streamline.yaml"), content=Template("streamline.yaml.j2"), owner=params.streamline_user, group=params.user_group, mode=0644) if params.security_enabled: if params.streamline_jaas_conf_template: File(format("{conf_dir}/streamline_jaas.conf"), owner=params.streamline_user, content=InlineTemplate(params.streamline_jaas_conf_template)) else: TemplateConfig(format("{conf_dir}/streamline_jaas.conf"), owner=params.streamline_user) if not os.path.islink(params.streamline_managed_log_dir): Link(params.streamline_managed_log_dir, to=params.streamline_log_dir)
def kafka(upgrade_type=None): import params ensure_base_directories() kafka_server_config = mutable_config_dict( params.config['configurations']['kafka-broker']) # This still has an issue of hostnames being alphabetically out-of-order for broker.id in HDP-2.2. # Starting in HDP 2.3, Kafka handles the generation of broker.id so Ambari doesn't have to. effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version( params.version) Logger.info(format("Effective stack version: {effective_version}")) kafka_server_config['host.name'] = params.hostname if params.has_metric_collector: kafka_server_config[ 'kafka.timeline.metrics.hosts'] = params.ams_collector_hosts kafka_server_config[ 'kafka.timeline.metrics.port'] = params.metric_collector_port kafka_server_config[ 'kafka.timeline.metrics.protocol'] = params.metric_collector_protocol kafka_server_config[ 'kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path kafka_server_config[ 'kafka.timeline.metrics.truststore.type'] = params.metric_truststore_type kafka_server_config[ 'kafka.timeline.metrics.truststore.password'] = params.metric_truststore_password kafka_data_dir = kafka_server_config['log.dirs'] kafka_data_dirs = filter(None, kafka_data_dir.split(",")) Directory( kafka_data_dirs, mode=0755, cd_access='a', owner=params.kafka_user, group=params.user_group, create_parents=True, recursive_ownership=True, ) PropertiesFile( "server.properties", dir=params.conf_dir, properties=kafka_server_config, owner=params.kafka_user, group=params.user_group, ) File(format("{conf_dir}/kafka-env.sh"), mode=0755, owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template)) File(format("{kafka_bin_dir}/kafka-run-class.sh"), mode=0755, owner=params.kafka_user, content=InlineTemplate(params.kafka_run_class_content_template)) if (params.log4j_props != None): File(format("{conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.kafka_user, content=InlineTemplate(params.log4j_props)) if params.security_enabled and params.kafka_kerberos_enabled: if params.kafka_jaas_conf_template: File(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user, content=InlineTemplate(params.kafka_jaas_conf_template)) else: TemplateConfig(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user) if params.kafka_client_jaas_conf_template: File(format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user, content=InlineTemplate( params.kafka_client_jaas_conf_template)) else: TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'kafka.conf'), owner='root', group='root', mode=0644, content=Template("kafka.conf.j2")) File(os.path.join(params.conf_dir, 'tools-log4j.properties'), owner='root', group='root', mode=0644, content=Template("tools-log4j.properties.j2")) setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir) setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
def metadata(): import params Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) File(format("{expanded_war_dir}/atlas.war"), content=StaticFile(format('{metadata_home}/server/webapp/atlas.war'))) PropertiesFile(format('{conf_dir}/{conf_file}'), properties=params.application_properties, mode=0644, owner=params.metadata_user, group=params.user_group) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content)) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=StaticFile('atlas-log4j.xml')) if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user)
def hbase_TemplateConfig(name, tag=None): import params TemplateConfig(format("{hbase_conf_dir}/{name}"), owner=params.hbase_user, template_tag=tag)
def metadata(type='server'): import params # Needed by both Server and Client Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) if type == "server": Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(format('{conf_dir}/solr'), mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) File(format("{expanded_war_dir}/atlas.war"), content=StaticFile( format('{metadata_home}/server/webapp/atlas.war'))) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_log4j_content)) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content)) File(format("{conf_dir}/solr/solrconfig.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_solrconfig_content)) # Needed by both Server and Client PropertiesFile(format('{conf_dir}/{conf_file}'), properties=params.application_properties, mode=0644, owner=params.metadata_user, group=params.user_group) if type == 'server' and params.search_backend_solr and params.has_logsearch_solr: solr_cloud_util.setup_solr_client(params.config) random_num = random.random() upload_conf_set('basic_configs', random_num) create_collection('vertex_index', 'basic_configs') create_collection('edge_index', 'basic_configs') create_collection('fulltext_index', 'basic_configs') if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user)
def knox(): import params Directory( [ params.knox_data_dir, params.knox_data_backup_dir, params.knox_logs_dir, params.knox_pid_dir, params.knox_conf_dir, os.path.join(params.knox_conf_dir, "topologies"), params.knox_descriptors_dir, params.knox_shared_providers_dir ], owner=params.knox_user, group=params.knox_group, create_parents=True, cd_access="a", mode=0755, recursive_ownership=True, ) XmlConfig( "gateway-site.xml", conf_dir=params.knox_conf_dir, configurations=params.config['configurations']['gateway-site'], configuration_attributes=params.config['configurationAttributes'] ['gateway-site'], owner=params.knox_user, group=params.knox_group, ) File(format("{params.knox_conf_dir}/gateway-log4j.properties"), mode=0644, group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.gateway_log4j)) File(format("{params.knox_conf_dir}/topologies/default.xml"), mode=0600, group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.topology_template)) if params.admin_topology_template: File(format("{params.knox_conf_dir}/topologies/admin.xml"), mode=0600, group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.admin_topology_template)) knoxsso_topology_template_content = get_config("knoxsso-topology") if knoxsso_topology_template_content: File(os.path.join(params.knox_conf_dir, "topologies", "knoxsso.xml"), mode=0600, group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.knoxsso_topology_template)) if params.security_enabled: TemplateConfig(format("{knox_conf_dir}/krb5JAASLogin.conf"), owner=params.knox_user, template_tag=None) generate_logfeeder_input_config( 'knox', Template("input.config-knox.json.j2", extra_imports=[default])) cmd = format( '{knox_client_bin} create-master --master {knox_master_secret!p}') master_secret_exist = as_user(format('test -f {knox_master_secret_path}'), params.knox_user) Execute( cmd, user=params.knox_user, environment={'JAVA_HOME': params.java_home}, not_if=master_secret_exist, ) cmd = format( '{knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}') cert_store_exist = as_user(format('test -f {knox_cert_store_path}'), params.knox_user) Execute( cmd, user=params.knox_user, environment={'JAVA_HOME': params.java_home}, not_if=cert_store_exist, )
def metadata(type='server'): import params Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) File(format("{expanded_war_dir}/atlas.war"), content = StaticFile(format('{metadata_home}/server/webapp/atlas.war')) ) PropertiesFile(format('{conf_dir}/{conf_file}'), properties = params.application_properties, mode=0644, owner=params.metadata_user, group=params.user_group ) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content) ) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_log4j_content) ) File(format("{conf_dir}/users-credentials.properties"), mode=0644, owner=params.metadata_user, group=params.user_group, content=StaticFile('users-credentials.properties') ) File(format("{conf_dir}/policy-store.txt"), mode=0644, owner=params.metadata_user, group=params.user_group, content=StaticFile('policy-store.txt') ) if type == 'server': random_num = random.random() upload_conf_set('basic_configs', random_num) create_collection('vertex_index', 'basic_configs') create_collection('edge_index', 'basic_configs') create_collection('fulltext_index', 'basic_configs') if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user)
def kafka(): import params Directory([params.kafka_log_dir, params.kafka_pid_dir, params.conf_dir], mode=0755, cd_access='a', owner=params.kafka_user, group=params.user_group, recursive=True ) brokerid = str(sorted(params.kafka_hosts).index(params.hostname)) kafka_server_config = mutable_config_dict(params.config['configurations']['kafka-broker']) kafka_server_config['broker.id'] = brokerid #listeners and advertised.listeners are only added in 2.3.0.0 onwards. if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.3.0.0') >= 0: if params.security_enabled and params.kafka_kerberos_enabled: listeners = kafka_server_config['listeners'].replace("localhost", params.hostname).replace("PLAINTEXT", "PLAINTEXTSASL") kafka_server_config['listeners'] = listeners kafka_server_config['advertised.listeners'] = listeners else: listeners = kafka_server_config['listeners'].replace("localhost", params.hostname) kafka_server_config['listeners'] = listeners if 'advertised.listeners' in kafka_server_config: advertised_listeners = kafka_server_config['advertised.listeners'].replace("localhost", params.hostname) kafka_server_config['advertised.listeners'] = advertised_listeners else: kafka_server_config['host.name'] = params.hostname kafka_server_config['kafka.metrics.reporters'] = params.kafka_metrics_reporters if(params.has_metric_collector): kafka_server_config['kafka.timeline.metrics.host'] = params.metric_collector_host kafka_server_config['kafka.timeline.metrics.port'] = params.metric_collector_port kafka_data_dir = kafka_server_config['log.dirs'] Directory(filter(None,kafka_data_dir.split(",")), mode=0755, cd_access='a', owner=params.kafka_user, group=params.user_group, recursive=True) conf_dir = params.conf_dir properties_config("server.properties", conf_dir=params.conf_dir, configurations=kafka_server_config, owner=params.kafka_user, group=params.user_group, brokerid=brokerid) File(format("{conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template) ) if (params.log4j_props != None): File(format("{conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.kafka_user, content=params.log4j_props ) if params.security_enabled and params.kafka_kerberos_enabled: TemplateConfig(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user) setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir) setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
def shell_file(name): import params File(params.ganglia_shell_cmds_dir + os.sep + name, content=StaticFile(name), mode=0755) def ganglia_TemplateConfig(name, mode=0755, tag=None): import params TemplateConfig(format("{params.ganglia_shell_cmds_dir}/{name}"), owner="root", group="root", template_tag=tag, mode=mode) def generate_daemon(ganglia_service, name=None, role=None, owner=None, group=None): import params cmd = "" if ganglia_service == "gmond": if role == "server": cmd = "{params.ganglia_shell_cmds_dir}/setupGanglia.sh -c {name} -m -o {owner} -g {group}"
def metadata(type='server'): import params # Needed by both Server and Client Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) if type == "server": Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(format('{conf_dir}/solr'), mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True, recursive_ownership=True) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) File(format("{expanded_war_dir}/atlas.war"), content=StaticFile( format('{metadata_home}/server/webapp/atlas.war'))) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_log4j_content)) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content)) if params.metadata_solrconfig_content: File(format("{conf_dir}/solr/solrconfig.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_solrconfig_content)) # Needed by both Server and Client PropertiesFile(format('{conf_dir}/{conf_file}'), properties=params.application_properties, mode=0644, owner=params.metadata_user, group=params.user_group) if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user) if type == 'server' and params.search_backend_solr and params.has_infra_solr: solr_cloud_util.setup_solr_client(params.config) check_znode() jaasFile = params.atlas_jaas_file if params.security_enabled else None upload_conf_set('atlas_configs', jaasFile) create_collection('vertex_index', 'atlas_configs', jaasFile) create_collection('edge_index', 'atlas_configs', jaasFile) create_collection('fulltext_index', 'atlas_configs', jaasFile) File(params.atlas_hbase_setup, group=params.user_group, owner=params.hbase_user, content=Template("atlas_hbase_setup.rb.j2")) if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, get_stack_feature_version(params.config)) and\ params.security_enabled and not params.host_with_kafka: File(params.atlas_kafka_setup, group=params.user_group, owner=params.kafka_user, content=Template("atlas_kafka_acl.sh.j2")) File(format("{kafka_conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template)) File(format("{kafka_conf_dir}/kafka_jaas.conf"), group=params.user_group, owner=params.kafka_user, content=Template("kafka_jaas.conf.j2"))
def kafka(): import params Directory([params.kafka_log_dir, params.kafka_pid_dir, params.conf_dir], mode=0755, cd_access='a', owner=params.kafka_user, group=params.user_group, recursive=True) kafka_server_config = mutable_config_dict( params.config['configurations']['kafka-broker']) # This still has an issue of out of alphabetical order of hostnames can get out of order broker.id assigned to them for HDP-2.2. # Since from HDP-2.3 kafka is handling the generation of broker.id ambari doesn't need to generate one. if params.hdp_stack_version != "" and compare_versions( params.hdp_stack_version, '2.2.0.0') >= 0 and compare_versions( params.hdp_stack_version, '2.3.0.0') < 0: brokerid = str(sorted(params.kafka_hosts).index(params.hostname)) kafka_server_config['broker.id'] = brokerid #listeners and advertised.listeners are only added in 2.3.0.0 onwards. if params.hdp_stack_version != "" and compare_versions( params.hdp_stack_version, '2.3.0.0') >= 0: if params.security_enabled and params.kafka_kerberos_enabled: listeners = kafka_server_config['listeners'].replace( "localhost", params.hostname) if "SASL" not in listeners: listeners = listeners.replace("PLAINTEXT", "PLAINTEXTSASL") kafka_server_config['listeners'] = listeners kafka_server_config['advertised.listeners'] = listeners else: listeners = kafka_server_config['listeners'].replace( "localhost", params.hostname) kafka_server_config['listeners'] = listeners if 'advertised.listeners' in kafka_server_config: advertised_listeners = kafka_server_config[ 'advertised.listeners'].replace("localhost", params.hostname) kafka_server_config[ 'advertised.listeners'] = advertised_listeners else: kafka_server_config['host.name'] = params.hostname if (params.has_metric_collector): kafka_server_config[ 'kafka.timeline.metrics.host'] = params.metric_collector_host kafka_server_config[ 'kafka.timeline.metrics.port'] = params.metric_collector_port kafka_data_dir = kafka_server_config['log.dirs'] Directory(filter(None, kafka_data_dir.split(",")), mode=0755, cd_access='a', owner=params.kafka_user, group=params.user_group, recursive=True) PropertiesFile( "server.properties", dir=params.conf_dir, properties=kafka_server_config, owner=params.kafka_user, group=params.user_group, ) File(format("{conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template)) if (params.log4j_props != None): File(format("{conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.kafka_user, content=params.log4j_props) if params.security_enabled and params.kafka_kerberos_enabled: TemplateConfig(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user) TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, recursive=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'kafka.conf'), owner='root', group='root', mode=0644, content=Template("kafka.conf.j2")) setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir) setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
def ams(name=None): import params if name == 'collector': if not check_windows_service_exists( params.ams_collector_win_service_name): Execute( format( "cmd /C cd {ams_collector_home_dir} & ambari-metrics-collector.cmd setup" )) Directory(params.ams_collector_conf_dir, owner=params.ams_user, create_parents=True) Directory(params.ams_checkpoint_dir, owner=params.ams_user, create_parents=True) XmlConfig( "ams-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['ams-site'], configuration_attributes=params.config['configurationAttributes'] ['ams-site'], owner=params.ams_user, ) merged_ams_hbase_site = {} merged_ams_hbase_site.update( params.config['configurations']['ams-hbase-site']) if params.security_enabled: merged_ams_hbase_site.update( params.config['configurations']['ams-hbase-security-site']) XmlConfig( "hbase-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=merged_ams_hbase_site, configuration_attributes=params.config['configurationAttributes'] ['ams-hbase-site'], owner=params.ams_user, ) if (params.log4j_props != None): File(os.path.join(params.ams_collector_conf_dir, "log4j.properties"), owner=params.ams_user, content=params.log4j_props) File(os.path.join(params.ams_collector_conf_dir, "ams-env.cmd"), owner=params.ams_user, content=InlineTemplate(params.ams_env_sh_template)) ServiceConfig(params.ams_collector_win_service_name, action="change_user", username=params.ams_user, password=Script.get_password(params.ams_user)) if not params.is_local_fs_rootdir: # Configuration needed to support NN HA XmlConfig( "hdfs-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params. config['configurationAttributes']['hdfs-site'], owner=params.ams_user, group=params.user_group, mode=0644) XmlConfig( "hdfs-site.xml", conf_dir=params.hbase_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params. config['configurationAttributes']['hdfs-site'], owner=params.ams_user, group=params.user_group, mode=0644) XmlConfig( "core-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params. config['configurationAttributes']['core-site'], owner=params.ams_user, group=params.user_group, mode=0644) XmlConfig( "core-site.xml", conf_dir=params.hbase_conf_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params. config['configurationAttributes']['core-site'], owner=params.ams_user, group=params.user_group, mode=0644) else: ServiceConfig(params.ams_embedded_hbase_win_service_name, action="change_user", username=params.ams_user, password=Script.get_password(params.ams_user)) # creating symbolic links on ams jars to make them available to services links_pairs = [ ("%COLLECTOR_HOME%\\hbase\\lib\\ambari-metrics-hadoop-sink-with-common.jar", "%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar" ), ] for link_pair in links_pairs: link, target = link_pair real_link = os.path.expandvars(link) target = compress_backslashes( glob.glob(os.path.expandvars(target))[0]) if not os.path.exists(real_link): #TODO check the symlink destination too. Broken in Python 2.x on Windows. Execute('cmd /c mklink "{0}" "{1}"'.format( real_link, target)) pass elif name == 'monitor': if not check_windows_service_exists( params.ams_monitor_win_service_name): Execute( format( "cmd /C cd {ams_monitor_home_dir} & ambari-metrics-monitor.cmd setup" )) # creating symbolic links on ams jars to make them available to services links_pairs = [ ("%HADOOP_HOME%\\share\\hadoop\\common\\lib\\ambari-metrics-hadoop-sink-with-common.jar", "%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar" ), ("%HBASE_HOME%\\lib\\ambari-metrics-hadoop-sink-with-common.jar", "%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar" ), ] for link_pair in links_pairs: link, target = link_pair real_link = os.path.expandvars(link) target = compress_backslashes( glob.glob(os.path.expandvars(target))[0]) if not os.path.exists(real_link): #TODO check the symlink destination too. Broken in Python 2.x on Windows. Execute('cmd /c mklink "{0}" "{1}"'.format(real_link, target)) Directory(params.ams_monitor_conf_dir, owner=params.ams_user, create_parents=True) if params.host_in_memory_aggregation: if params.log4j_props is not None: File(os.path.join(params.ams_monitor_conf_dir, "log4j.properties"), owner=params.ams_user, content=params.log4j_props) pass XmlConfig( "ams-site.xml", conf_dir=params.ams_monitor_conf_dir, configurations=params.config['configurations']['ams-site'], configuration_attributes=params. config['configurationAttributes']['ams-site'], owner=params.ams_user, group=params.user_group) XmlConfig("ssl-server.xml", conf_dir=params.ams_monitor_conf_dir, configurations=params.config['configurations'] ['ams-ssl-server'], configuration_attributes=params. config['configurationAttributes']['ams-ssl-server'], owner=params.ams_user, group=params.user_group) pass TemplateConfig(os.path.join(params.ams_monitor_conf_dir, "metric_monitor.ini"), owner=params.ams_user, template_tag=None) TemplateConfig(os.path.join(params.ams_monitor_conf_dir, "metric_groups.conf"), owner=params.ams_user, template_tag=None) ServiceConfig(params.ams_monitor_win_service_name, action="change_user", username=params.ams_user, password=Script.get_password(params.ams_user))
def metadata(type='server'): import params # Needed by both Server and Client Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) if type == "server": Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(format('{conf_dir}/solr'), mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True, recursive_ownership=True) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) File(format("{expanded_war_dir}/atlas.war"), content=StaticFile( format('{metadata_home}/server/webapp/atlas.war'))) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_log4j_content)) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content)) files_to_chown = [ format("{conf_dir}/policy-store.txt"), format("{conf_dir}/users-credentials.properties") ] for file in files_to_chown: if os.path.exists(file): Execute( ('chown', format('{metadata_user}:{user_group}'), file), sudo=True) Execute(('chmod', '644', file), sudo=True) if params.metadata_solrconfig_content: File(format("{conf_dir}/solr/solrconfig.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_solrconfig_content)) # Needed by both Server and Client PropertiesFile(format('{conf_dir}/{conf_file}'), properties=params.application_properties, mode=0644, owner=params.metadata_user, group=params.user_group) if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user) if type == 'server' and params.search_backend_solr and params.has_infra_solr: solr_cloud_util.setup_solr_client(params.config) check_znode() jaasFile = params.atlas_jaas_file if params.security_enabled else None upload_conf_set('atlas_configs', jaasFile) create_collection('vertex_index', 'atlas_configs', jaasFile) create_collection('edge_index', 'atlas_configs', jaasFile) create_collection('fulltext_index', 'atlas_configs', jaasFile) File(params.atlas_hbase_setup, group=params.user_group, owner=params.hbase_user, content=Template("atlas_hbase_setup.rb.j2")) is_atlas_upgrade_support = check_stack_feature( StackFeature.ATLAS_UPGRADE_SUPPORT, get_stack_feature_version(params.config)) if is_atlas_upgrade_support and params.security_enabled: File(params.atlas_kafka_setup, group=params.user_group, owner=params.kafka_user, content=Template("atlas_kafka_acl.sh.j2")) # files required only in case if kafka broker is not present on the host as configured component if not params.host_with_kafka: File(format("{kafka_conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template)) File(format("{kafka_conf_dir}/kafka_jaas.conf"), group=params.user_group, owner=params.kafka_user, content=Template("kafka_jaas.conf.j2")) if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len( params.namenode_host) > 1: XmlConfig( "hdfs-site.xml", conf_dir=params.conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configuration_attributes'] ['hdfs-site'], owner=params.metadata_user, group=params.user_group, mode=0644) else: File(format('{conf_dir}/hdfs-site.xml'), action="delete")
def metadata(type='server'): import params # Needed by both Server and Client Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) if type == "server": Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(format('{conf_dir}/solr'), mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True, recursive_ownership=True) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) File(format("{expanded_war_dir}/atlas.war"), content=StaticFile( format('{metadata_home}/server/webapp/atlas.war'))) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_log4j_content)) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content)) if not is_empty(params.atlas_admin_username) and not is_empty( params.atlas_admin_password): psswd_output = hashlib.sha256( params.atlas_admin_password).hexdigest() ModifyPropertiesFile( format("{conf_dir}/users-credentials.properties"), properties={ format('{atlas_admin_username}'): format('ROLE_ADMIN::{psswd_output}') }, owner=params.metadata_user) files_to_chown = [ format("{conf_dir}/policy-store.txt"), format("{conf_dir}/users-credentials.properties") ] for file in files_to_chown: if os.path.exists(file): Execute( ('chown', format('{metadata_user}:{user_group}'), file), sudo=True) Execute(('chmod', '644', file), sudo=True) if params.metadata_solrconfig_content: File(format("{conf_dir}/solr/solrconfig.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_solrconfig_content)) # Needed by both Server and Client PropertiesFile(format('{conf_dir}/{conf_file}'), properties=params.application_properties, mode=0600, owner=params.metadata_user, group=params.user_group) if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user) if type == 'server' and params.search_backend_solr and params.has_infra_solr: solr_cloud_util.setup_solr_client(params.config) check_znode() jaasFile = params.atlas_jaas_file if params.security_enabled else None upload_conf_set('atlas_configs', jaasFile) if params.security_enabled: # update permissions before creating the collections solr_cloud_util.add_solr_roles( params.config, roles=[ params.infra_solr_role_atlas, params.infra_solr_role_ranger_audit, params.infra_solr_role_dev ], new_service_principals=[params.atlas_jaas_principal]) create_collection('vertex_index', 'atlas_configs', jaasFile) create_collection('edge_index', 'atlas_configs', jaasFile) create_collection('fulltext_index', 'atlas_configs', jaasFile) if params.security_enabled: secure_znode(format('{infra_solr_znode}/configs/atlas_configs'), jaasFile) secure_znode(format('{infra_solr_znode}/collections/vertex_index'), jaasFile) secure_znode(format('{infra_solr_znode}/collections/edge_index'), jaasFile) secure_znode( format('{infra_solr_znode}/collections/fulltext_index'), jaasFile) File(params.atlas_hbase_setup, group=params.user_group, owner=params.hbase_user, content=Template("atlas_hbase_setup.rb.j2")) is_atlas_upgrade_support = check_stack_feature( StackFeature.ATLAS_UPGRADE_SUPPORT, get_stack_feature_version(params.config)) if is_atlas_upgrade_support and params.security_enabled: File(params.atlas_kafka_setup, group=params.user_group, owner=params.kafka_user, content=Template("atlas_kafka_acl.sh.j2")) # files required only in case if kafka broker is not present on the host as configured component if not params.host_with_kafka: File(format("{kafka_conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template)) File(format("{kafka_conf_dir}/kafka_jaas.conf"), group=params.user_group, owner=params.kafka_user, content=Template("kafka_jaas.conf.j2")) if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len( params.namenode_host) > 1: XmlConfig( "hdfs-site.xml", conf_dir=params.conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configurationAttributes'] ['hdfs-site'], owner=params.metadata_user, group=params.user_group, mode=0644) else: File(format('{conf_dir}/hdfs-site.xml'), action="delete") ''' Atlas requires hadoop core-site.xml to resolve users/groups synced in HadoopUGI for authentication and authorization process. Earlier the core-site.xml was available in Hbase conf directory which is a part of Atlas class-path, from stack 2.6 onwards, core-site.xml is no more available in Hbase conf directory. Hence need to create core-site.xml in Atlas conf directory. ''' if params.stack_supports_atlas_core_site and params.has_namenode: XmlConfig( "core-site.xml", conf_dir=params.conf_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configurationAttributes'] ['core-site'], owner=params.metadata_user, group=params.user_group, mode=0644) Directory( format('{metadata_home}/'), owner=params.metadata_user, group=params.user_group, recursive_ownership=True, )
def hbase_TemplateConfig(name, tag=None, user=None): import params TemplateConfig(os.path.join(params.hbase_conf_dir, name), owner=user, template_tag=tag)
def kafka(upgrade_type=None): import params ensure_base_directories() kafka_server_config = mutable_config_dict( params.config['configurations']['kafka-broker']) # This still has an issue of hostnames being alphabetically out-of-order for broker.id in HDP-2.2. # Starting in HDP 2.3, Kafka handles the generation of broker.id so Ambari doesn't have to. effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version( params.version) Logger.info(format("Effective stack version: {effective_version}")) # In HDP-2.2 (Apache Kafka 0.8.1.1) we used to generate broker.ids based on hosts and add them to # kafka's server.properties. In future version brokers can generate their own ids based on zookeeper seq # We need to preserve the broker.id when user is upgrading from HDP-2.2 to any higher version. # Once its preserved it will be written to kafka.log.dirs/meta.properties and it will be used from there on # similarly we need preserve port as well during the upgrade if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and \ check_stack_feature(StackFeature.CREATE_KAFKA_BROKER_ID, params.current_version) and \ check_stack_feature(StackFeature.KAFKA_LISTENERS, params.version): if len(params.kafka_hosts ) > 0 and params.hostname in params.kafka_hosts: brokerid = str(sorted(params.kafka_hosts).index(params.hostname)) kafka_server_config['broker.id'] = brokerid Logger.info(format("Calculating broker.id as {brokerid}")) if 'port' in kafka_server_config: port = kafka_server_config['port'] Logger.info(format("Port config from previous verson: {port}")) listeners = kafka_server_config['listeners'] kafka_server_config['listeners'] = listeners.replace("6667", port) Logger.info( format("Kafka listeners after the port update: {listeners}")) del kafka_server_config['port'] if effective_version is not None and effective_version != "" and \ check_stack_feature(StackFeature.CREATE_KAFKA_BROKER_ID, effective_version): if len(params.kafka_hosts ) > 0 and params.hostname in params.kafka_hosts: brokerid = str(sorted(params.kafka_hosts).index(params.hostname)) kafka_server_config['broker.id'] = brokerid Logger.info(format("Calculating broker.id as {brokerid}")) # listeners and advertised.listeners are only added in 2.3.0.0 onwards. if effective_version is not None and effective_version != "" and \ check_stack_feature(StackFeature.KAFKA_LISTENERS, effective_version): listeners = kafka_server_config['listeners'].replace( "localhost", params.hostname) Logger.info(format("Kafka listeners: {listeners}")) kafka_server_config['listeners'] = listeners if params.security_enabled and params.kafka_kerberos_enabled: Logger.info("Kafka kerberos security is enabled.") kafka_server_config['advertised.listeners'] = listeners Logger.info(format("Kafka advertised listeners: {listeners}")) elif 'advertised.listeners' in kafka_server_config: advertised_listeners = kafka_server_config[ 'advertised.listeners'].replace("localhost", params.hostname) kafka_server_config['advertised.listeners'] = advertised_listeners Logger.info( format("Kafka advertised listeners: {advertised_listeners}")) else: kafka_server_config['host.name'] = params.hostname if params.has_metric_collector: kafka_server_config[ 'kafka.timeline.metrics.hosts'] = params.ams_collector_hosts kafka_server_config[ 'kafka.timeline.metrics.port'] = params.metric_collector_port kafka_server_config[ 'kafka.timeline.metrics.protocol'] = params.metric_collector_protocol kafka_server_config[ 'kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path kafka_server_config[ 'kafka.timeline.metrics.truststore.type'] = params.metric_truststore_type kafka_server_config[ 'kafka.timeline.metrics.truststore.password'] = params.metric_truststore_password kafka_data_dir = kafka_server_config['log.dirs'] kafka_data_dirs = filter(None, kafka_data_dir.split(",")) Directory( kafka_data_dirs, mode=0755, cd_access='a', owner=params.kafka_user, group=params.user_group, create_parents=True, recursive_ownership=True, ) PropertiesFile( "server.properties", dir=params.conf_dir, properties=kafka_server_config, owner=params.kafka_user, group=params.user_group, ) File(format("{conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template)) if (params.log4j_props != None): File(format("{conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.kafka_user, content=params.log4j_props) if params.security_enabled and params.kafka_kerberos_enabled: if params.kafka_jaas_conf_template: File(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user, content=InlineTemplate(params.kafka_jaas_conf_template)) else: TemplateConfig(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user) if params.kafka_client_jaas_conf_template: File(format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user, content=InlineTemplate( params.kafka_client_jaas_conf_template)) else: TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'kafka.conf'), owner='root', group='root', mode=0644, content=Template("kafka.conf.j2")) File(os.path.join(params.conf_dir, 'tools-log4j.properties'), owner='root', group='root', mode=0644, content=Template("tools-log4j.properties.j2")) setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir) setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
def kafka(upgrade_type=None): import params ensure_base_directories() kafka_server_config = mutable_config_dict(params.config['configurations']['kafka-broker']) # This still has an issue of hostnames being alphabetically out-of-order for broker.id in HDP-2.2. # Starting in HDP 2.3, Kafka handles the generation of broker.id so Ambari doesn't have to. effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(params.version) Logger.info(format("Effective stack version: {effective_version}")) # listeners and advertised.listeners are only added in 2.3.0.0 onwards. if effective_version is not None and effective_version != "" and \ check_stack_feature(StackFeature.KAFKA_LISTENERS, effective_version): listeners = kafka_server_config['listeners'].replace("localhost", params.hostname) Logger.info(format("Kafka listeners: {listeners}")) kafka_server_config['listeners'] = listeners if params.security_enabled and params.kafka_kerberos_enabled: Logger.info("Kafka kerberos security is enabled.") kafka_server_config['advertised.listeners'] = listeners Logger.info(format("Kafka advertised listeners: {listeners}")) elif 'advertised.listeners' in kafka_server_config: advertised_listeners = kafka_server_config['advertised.listeners'].replace("localhost", params.hostname) kafka_server_config['advertised.listeners'] = advertised_listeners Logger.info(format("Kafka advertised listeners: {advertised_listeners}")) else: kafka_server_config['host.name'] = params.hostname if params.has_metric_collector: kafka_server_config['kafka.timeline.metrics.hosts'] = params.ams_collector_hosts kafka_server_config['kafka.timeline.metrics.port'] = params.metric_collector_port kafka_server_config['kafka.timeline.metrics.protocol'] = params.metric_collector_protocol kafka_server_config['kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path kafka_server_config['kafka.timeline.metrics.truststore.type'] = params.metric_truststore_type kafka_server_config['kafka.timeline.metrics.truststore.password'] = params.metric_truststore_password kafka_data_dir = kafka_server_config['log.dirs'] kafka_data_dirs = filter(None, kafka_data_dir.split(",")) rack="/default-rack" i=0 if len(params.all_racks) > 0: for host in params.all_hosts: if host == params.hostname: rack=params.all_racks[i] break i=i+1 kafka_server_config['broker.rack']=rack Directory(kafka_data_dirs, mode=0755, cd_access='a', owner=params.kafka_user, group=params.user_group, create_parents = True, recursive_ownership = True, ) PropertiesFile("server.properties", dir=params.conf_dir, properties=kafka_server_config, owner=params.kafka_user, group=params.user_group, ) File(format("{conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template) ) if (params.log4j_props != None): File(format("{conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.kafka_user, content=InlineTemplate(params.log4j_props) ) if params.security_enabled and params.kafka_kerberos_enabled: if params.kafka_jaas_conf_template: File(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user, content=InlineTemplate(params.kafka_jaas_conf_template) ) else: TemplateConfig(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user) if params.kafka_client_jaas_conf_template: File(format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user, content=InlineTemplate(params.kafka_client_jaas_conf_template) ) else: TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents = True, owner='root', group='root' ) File(os.path.join(params.limits_conf_dir, 'kafka.conf'), owner='root', group='root', mode=0644, content=Template("kafka.conf.j2") ) File(os.path.join(params.conf_dir, 'tools-log4j.properties'), owner='root', group='root', mode=0644, content=Template("tools-log4j.properties.j2") ) setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir) setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
def knox(): import params Directory( [ params.knox_data_dir, params.knox_logs_dir, params.knox_pid_dir, params.knox_conf_dir, os.path.join(params.knox_conf_dir, "topologies") ], owner=params.knox_user, group=params.knox_group, create_parents=True, cd_access="a", mode=0755, recursive_ownership=True, ) XmlConfig( "gateway-site.xml", conf_dir=params.knox_conf_dir, configurations=params.config['configurations']['gateway-site'], configuration_attributes=params.config['configuration_attributes'] ['gateway-site'], owner=params.knox_user, group=params.knox_group, ) File(format("{params.knox_conf_dir}/gateway-log4j.properties"), mode=0644, group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.gateway_log4j)) File(format("{params.knox_conf_dir}/topologies/default.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.topology_template)) File(format("{params.knox_conf_dir}/topologies/admin.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.admin_topology_template)) if params.version_formatted and check_stack_feature( StackFeature.KNOX_SSO_TOPOLOGY, params.version_formatted): File(os.path.join(params.knox_conf_dir, "topologies", "knoxsso.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.knoxsso_topology_template)) if params.security_enabled: TemplateConfig(format("{knox_conf_dir}/krb5JAASLogin.conf"), owner=params.knox_user, template_tag=None) cmd = format( '{knox_client_bin} create-master --master {knox_master_secret!p}') master_secret_exist = as_user(format('test -f {knox_master_secret_path}'), params.knox_user) Execute( cmd, user=params.knox_user, environment={'JAVA_HOME': params.java_home}, not_if=master_secret_exist, ) cmd = format( '{knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}') cert_store_exist = as_user(format('test -f {knox_cert_store_path}'), params.knox_user) Execute( cmd, user=params.knox_user, environment={'JAVA_HOME': params.java_home}, not_if=cert_store_exist, )
def druid(upgrade_type=None, nodeType=None): import params ensure_base_directories() # Environment Variables File(format("{params.druid_conf_dir}/druid-env.sh"), owner=params.druid_user, content=InlineTemplate(params.druid_env_sh_template), mode=0700) # common config druid_common_config = mutable_config_dict( params.config['configurations']['druid-common']) # User cannot override below configs druid_common_config['druid.host'] = params.hostname druid_common_config[ 'druid.extensions.directory'] = params.druid_extensions_dir druid_common_config[ 'druid.extensions.hadoopDependenciesDir'] = params.druid_hadoop_dependencies_dir druid_common_config[ 'druid.selectors.indexing.serviceName'] = params.config[ 'configurations']['druid-overlord']['druid.service'] druid_common_config['druid.selectors.coordinator.serviceName'] = \ params.config['configurations']['druid-coordinator']['druid.service'] druid_common_config['druid.extensions.loadList'] = json.dumps( eval(params.druid_extensions_load_list) + eval(params.druid_security_extensions_load_list)) # delete the password and user if empty otherwiswe derby will fail. if 'derby' == druid_common_config['druid.metadata.storage.type']: del druid_common_config['druid.metadata.storage.connector.user'] del druid_common_config['druid.metadata.storage.connector.password'] druid_env_config = mutable_config_dict( params.config['configurations']['druid-env']) PropertiesFile("common.runtime.properties", dir=params.druid_common_conf_dir, properties=druid_common_config, owner=params.druid_user, group=params.user_group, mode=0600) Logger.info("Created common.runtime.properties") File(format("{params.druid_common_conf_dir}/druid-log4j.xml"), mode=0644, owner=params.druid_user, group=params.user_group, content=InlineTemplate(params.log4j_props)) Logger.info("Created log4j file") File("/etc/logrotate.d/druid", mode=0644, owner='root', group='root', content=InlineTemplate(params.logrotate_props)) Logger.info("Created log rotate file") if params.security_enabled: TemplateConfig(params.druid_jaas_file, owner=params.druid_user, group=params.user_group, mode=0644) # node specific configs for node_type in [ 'coordinator', 'overlord', 'historical', 'broker', 'middleManager', 'router' ]: node_config_dir = format('{params.druid_conf_dir}/{node_type}') node_type_lowercase = node_type.lower() # Write runtime.properties file node_config = mutable_config_dict(params.config['configurations'][ format('druid-{node_type_lowercase}')]) if (node_type == 'middleManager'): # Replace correct values for stack_version and druid_jaas_file node_config['druid.indexer.runner.javaOpts'] = format( node_config['druid.indexer.runner.javaOpts']) PropertiesFile("runtime.properties", dir=node_config_dir, properties=node_config, owner=params.druid_user, group=params.user_group, mode=0600) Logger.info( format("Created druid-{node_type_lowercase} runtime.properties")) # Write jvm configs File( format('{node_config_dir}/jvm.config'), owner=params.druid_user, group=params.user_group, content=InlineTemplate( "-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}", node_heap_memory=druid_env_config[format( 'druid.{node_type_lowercase}.jvm.heap.memory')], log4j_config_file=format( "{params.druid_common_conf_dir}/druid-log4j.xml"), node_direct_memory=druid_env_config[format( 'druid.{node_type_lowercase}.jvm.direct.memory')], node_jvm_opts=format(druid_env_config[format( 'druid.{node_type_lowercase}.jvm.opts')]))) Logger.info(format("Created druid-{node_type_lowercase} jvm.config")) # Handling hadoop Lzo jars if enable and node type is hadoop related eg Overlords and MMs if ['middleManager', 'overlord' ].__contains__(node_type_lowercase) and params.lzo_enabled: try: Logger.info( format( "Copying hadoop lzo jars from {hadoop_lib_home} to {druid_hadoop_dependencies_dir}/hadoop-client/*/" )) Execute( format( '{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {druid_hadoop_dependencies_dir}/hadoop-client/*/' )) except Fail as ex: Logger.info( format( "No Hadoop LZO found at {hadoop_lib_home}/hadoop-lzo*.jar" )) generate_logfeeder_input_config( 'druid', Template("input.config-druid.json.j2", extra_imports=[default])) # All druid nodes have dependency on hdfs_client ensure_hadoop_directories() download_database_connector_if_needed() # Pull all required dependencies pulldeps()
def knox(): import params directories = [ params.knox_data_dir, params.knox_logs_dir, params.knox_pid_dir, params.knox_conf_dir, os.path.join(params.knox_conf_dir, "topologies") ] for directory in directories: Directory( directory, owner=params.knox_user, group=params.knox_group, recursive=True, cd_access="a", mode=0755, ) XmlConfig( "gateway-site.xml", conf_dir=params.knox_conf_dir, configurations=params.config['configurations']['gateway-site'], configuration_attributes=params.config['configuration_attributes'] ['gateway-site'], owner=params.knox_user, group=params.knox_group, ) File(format("{params.knox_conf_dir}/gateway-log4j.properties"), mode=0644, group=params.knox_group, owner=params.knox_user, content=params.gateway_log4j) File(format("{params.knox_conf_dir}/topologies/default.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.topology_template)) if params.security_enabled: TemplateConfig(format("{knox_conf_dir}/krb5JAASLogin.conf"), owner=params.knox_user, template_tag=None) dirs_to_chown = tuple(directories) cmd = ('chown', '-R', format('{knox_user}:{knox_group}')) + dirs_to_chown Execute( cmd, sudo=True, ) cmd = format( '{knox_client_bin} create-master --master {knox_master_secret!p}') master_secret_exist = as_user(format('test -f {knox_master_secret_path}'), params.knox_user) Execute( cmd, user=params.knox_user, environment={'JAVA_HOME': params.java_home}, not_if=master_secret_exist, ) cmd = format( '{knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}') cert_store_exist = as_user(format('test -f {knox_cert_store_path}'), params.knox_user) Execute( cmd, user=params.knox_user, environment={'JAVA_HOME': params.java_home}, not_if=cert_store_exist, )
def metadata(type='server'): import params # Needed by both Server and Client Directory( params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, ) if type == "server": Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group) Directory( format('{conf_dir}/solr'), mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, ) Execute(("chown", "-R", params.metadata_user + ":" + params.user_group, format('{conf_dir}/solr')), sudo=True) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group) File(format("{expanded_war_dir}/atlas.war"), content=StaticFile( format('{metadata_home}/server/webapp/atlas.war'))) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_log4j_content)) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content)) files_to_chown = [ format("{conf_dir}/policy-store.txt"), format("{conf_dir}/users-credentials.properties") ] for file in files_to_chown: if os.path.exists(file): Execute( ('chown', format('{metadata_user}:{user_group}'), file), sudo=True) Execute(('chmod', '644', file), sudo=True) if params.metadata_solrconfig_content: File(format("{conf_dir}/solr/solrconfig.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_solrconfig_content)) # Needed by both Server and Client PropertiesFile(format('{conf_dir}/{conf_file}'), properties=params.application_properties, mode=0644, owner=params.metadata_user, group=params.user_group) if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user) if type == 'server' and params.search_backend_solr: create_collection('vertex_index') create_collection('edge_index') create_collection('fulltext_index') File(params.atlas_hbase_setup, group=params.user_group, owner=params.hbase_user, content=Template("atlas_hbase_setup.rb.j2"))
def ams(name=None, action=None): import params if name == 'collector': Directory(params.ams_collector_conf_dir, owner=params.ams_user, group=params.user_group, create_parents = True, recursive_ownership = True, ) Directory(params.ams_checkpoint_dir, owner=params.ams_user, group=params.user_group, cd_access="a", create_parents = True, recursive_ownership = True ) XmlConfig("ams-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['ams-site'], configuration_attributes=params.config['configuration_attributes']['ams-site'], owner=params.ams_user, group=params.user_group ) XmlConfig("ssl-server.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['ams-ssl-server'], configuration_attributes=params.config['configuration_attributes']['ams-ssl-server'], owner=params.ams_user, group=params.user_group ) merged_ams_hbase_site = {} merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site']) if params.security_enabled: merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-security-site']) # Add phoenix client side overrides merged_ams_hbase_site['phoenix.query.maxGlobalMemoryPercentage'] = str(params.phoenix_max_global_mem_percent) merged_ams_hbase_site['phoenix.spool.directory'] = params.phoenix_client_spool_dir XmlConfig( "hbase-site.xml", conf_dir = params.ams_collector_conf_dir, configurations = merged_ams_hbase_site, configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'], owner = params.ams_user, group = params.user_group ) if params.security_enabled: TemplateConfig(os.path.join(params.hbase_conf_dir, "ams_collector_jaas.conf"), owner = params.ams_user, template_tag = None) if (params.log4j_props != None): File(format("{params.ams_collector_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.ams_user, content=InlineTemplate(params.log4j_props) ) File(format("{ams_collector_conf_dir}/ams-env.sh"), owner=params.ams_user, content=InlineTemplate(params.ams_env_sh_template) ) Directory(params.ams_collector_log_dir, owner=params.ams_user, group=params.user_group, cd_access="a", create_parents = True, mode=0755, ) Directory(params.ams_collector_pid_dir, owner=params.ams_user, group=params.user_group, cd_access="a", create_parents = True, mode=0755, ) # Hack to allow native HBase libs to be included for embedded hbase File(os.path.join(params.ams_hbase_home_dir, "bin", "hadoop"), owner=params.ams_user, mode=0755 ) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents = True, owner='root', group='root' ) # Setting up security limits File(os.path.join(params.limits_conf_dir, 'ams.conf'), owner='root', group='root', mode=0644, content=Template("ams.conf.j2") ) # Phoenix spool file dir if not /tmp if not os.path.exists(params.phoenix_client_spool_dir): Directory(params.phoenix_client_spool_dir, owner=params.ams_user, mode = 0755, group=params.user_group, cd_access="a", create_parents = True ) pass if not params.is_local_fs_rootdir and params.is_ams_distributed: # Configuration needed to support NN HA XmlConfig("hdfs-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configuration_attributes']['hdfs-site'], owner=params.ams_user, group=params.user_group, mode=0644 ) XmlConfig("hdfs-site.xml", conf_dir=params.hbase_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configuration_attributes']['hdfs-site'], owner=params.ams_user, group=params.user_group, mode=0644 ) # Remove spnego configs from core-site, since AMS does not support spnego (AMBARI-14384) truncated_core_site = {} truncated_core_site.update(params.config['configurations']['core-site']) if 'core-site' in params.config['configurations']: if 'hadoop.http.authentication.type' in params.config['configurations']['core-site']: truncated_core_site.pop('hadoop.http.authentication.type') if 'hadoop.http.filter.initializers' in params.config['configurations']['core-site']: truncated_core_site.pop('hadoop.http.filter.initializers') XmlConfig("core-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=truncated_core_site, configuration_attributes=params.config['configuration_attributes']['core-site'], owner=params.ams_user, group=params.user_group, mode=0644 ) XmlConfig("core-site.xml", conf_dir=params.hbase_conf_dir, configurations=truncated_core_site, configuration_attributes=params.config['configuration_attributes']['core-site'], owner=params.ams_user, group=params.user_group, mode=0644 ) if params.metric_collector_https_enabled: export_ca_certs(params.ams_collector_conf_dir) pass elif name == 'monitor': Directory(params.ams_monitor_conf_dir, owner=params.ams_user, group=params.user_group, create_parents = True ) Directory(params.ams_monitor_log_dir, owner=params.ams_user, group=params.user_group, mode=0755, create_parents = True ) Execute(format("{sudo} chown -R {ams_user}:{user_group} {ams_monitor_log_dir}") ) Directory(params.ams_monitor_pid_dir, owner=params.ams_user, group=params.user_group, cd_access="a", mode=0755, create_parents = True ) Directory(format("{ams_monitor_dir}/psutil/build"), owner=params.ams_user, group=params.user_group, cd_access="a", create_parents = True) Execute(format("{sudo} chown -R {ams_user}:{user_group} {ams_monitor_dir}") ) TemplateConfig( format("{ams_monitor_conf_dir}/metric_monitor.ini"), owner=params.ams_user, group=params.user_group, template_tag=None ) TemplateConfig( format("{ams_monitor_conf_dir}/metric_groups.conf"), owner=params.ams_user, group=params.user_group, template_tag=None ) File(format("{ams_monitor_conf_dir}/ams-env.sh"), owner=params.ams_user, content=InlineTemplate(params.ams_env_sh_template) ) if params.metric_collector_https_enabled: export_ca_certs(params.ams_monitor_conf_dir) pass elif name == 'grafana': ams_grafana_directories = [ params.ams_grafana_conf_dir, params.ams_grafana_log_dir, params.ams_grafana_data_dir, params.ams_grafana_pid_dir ] for ams_grafana_directory in ams_grafana_directories: Directory(ams_grafana_directory, owner=params.ams_user, group=params.user_group, mode=0755, create_parents = True, recursive_ownership = True ) File(format("{ams_grafana_conf_dir}/ams-grafana-env.sh"), owner=params.ams_user, group=params.user_group, content=InlineTemplate(params.ams_grafana_env_sh_template) ) File(format("{ams_grafana_conf_dir}/ams-grafana.ini"), owner=params.ams_user, group=params.user_group, content=InlineTemplate(params.ams_grafana_ini_template), mode=0600 ) if action != 'stop': for dir in ams_grafana_directories: Execute(('chown', '-R', params.ams_user, dir), sudo=True ) if params.metric_collector_https_enabled: export_ca_certs(params.ams_grafana_conf_dir) pass
def kafka(upgrade_type=None): import params ensure_base_directories() File( format("{kafka_bin}"), owner=params.kafka_user, mode=0755, content=InlineTemplate(params.kafka_init_content)) kafka_server_config = mutable_config_dict( params.config['configurations']['kafka-broker']) # listeners and advertised. listeners = kafka_server_config['listeners'].replace( "localhost", params.hostname) Logger.info(format("Kafka listeners: {listeners}")) kafka_server_config['listeners'] = listeners if params.kerberos_security_enabled and params.kafka_kerberos_enabled: Logger.info("Kafka kerberos security is enabled.") if "SASL" not in listeners: listeners = kafka_server_config['listeners'] listeners = re.sub(r"(^|\b)PLAINTEXT://", "SASL_PLAINTEXT://", listeners) listeners = re.sub(r"(^|\b)PLAINTEXTSASL://", "SASL_PLAINTEXT://", listeners) listeners = re.sub(r"(^|\b)SSL://", "SASL_SSL://", listeners) kafka_server_config['listeners'] = listeners kafka_server_config['advertised.listeners'] = listeners Logger.info(format("Kafka advertised listeners: {listeners}")) elif 'advertised.listeners' in kafka_server_config: advertised_listeners = kafka_server_config[ 'advertised.listeners'].replace("localhost", params.hostname) kafka_server_config['advertised.listeners'] = advertised_listeners Logger.info( format("Kafka advertised listeners: {advertised_listeners}")) #kafka_server_config['host.name'] = params.hostname if params.has_metric_collector: kafka_server_config[ 'kafka.timeline.metrics.hosts'] = params.ams_collector_hosts kafka_server_config[ 'kafka.timeline.metrics.port'] = params.metric_collector_port kafka_server_config[ 'kafka.timeline.metrics.protocol'] = params.metric_collector_protocol kafka_server_config[ 'kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path kafka_server_config[ 'kafka.timeline.metrics.truststore.type'] = params.metric_truststore_type kafka_server_config[ 'kafka.timeline.metrics.truststore.password'] = params.metric_truststore_password rack = "/default-rack" i = 0 if len(params.all_racks) > 0: for host in params.all_hosts: if host == params.hostname: rack = params.all_racks[i] break i = i + 1 Directory( params.kafka_data_dirs, mode=0755, cd_access='a', owner=params.kafka_user, group=params.user_group, create_parents=True, recursive_ownership=True, ) PropertiesFile( "server.properties", mode=0640, dir=params.conf_dir, properties=kafka_server_config, owner=params.kafka_user, group=params.user_group, ) File( format("{conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template)) if (params.log4j_props != None): File( format("{conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.kafka_user, content=InlineTemplate(params.log4j_props)) if (params.kerberos_security_enabled and params.kafka_kerberos_enabled) or params.kafka_other_sasl_enabled: if params.kafka_jaas_conf_template: File( format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user, content=InlineTemplate(params.kafka_jaas_conf_template)) else: TemplateConfig( format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user) if params.kafka_client_jaas_conf_template: File( format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user, content=InlineTemplate(params.kafka_client_jaas_conf_template)) else: TemplateConfig( format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user) # On some OS this folder could be not exists, so we will create it before pushing there files Directory( params.limits_conf_dir, create_parents=True, owner='root', group='root') File( os.path.join(params.limits_conf_dir, 'kafka.conf'), owner='root', group='root', mode=0644, content=Template("kafka.conf.j2")) File( os.path.join(params.conf_dir, 'tools-log4j.properties'), owner='root', group='root', mode=0644, content=Template("tools-log4j.properties.j2")) generate_logfeeder_input_config( 'kafka', Template( "input.config-kafka.json.j2", extra_imports=[default])) setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir) setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)