def hive(name=None): import params Directory(params.hive_etc_dir_prefix, mode=0755 ) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) XmlConfig("hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes']['hive-site'], owner=params.hive_user, group=params.user_group, mode=0644) # Generate atlas-application.properties.xml file if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, content=InlineTemplate(params.hive_env_sh_template) ) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents = True, owner='root', group='root' ) File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2") ) File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), mode = 0644, ) if name != "client": setup_non_client() if name == 'hiveserver2': setup_hiveserver2() if name == 'metastore': setup_metastore() # schematool work
def hcat(): import params Directory( params.hive_conf_dir, create_parents=True, owner=params.webhcat_user, group=params.user_group, ) Directory( params.hcat_conf_dir, create_parents=True, owner=params.webhcat_user, group=params.user_group, ) Directory(params.hcat_pid_dir, owner=params.webhcat_user, create_parents=True) XmlConfig( "hive-site.xml", conf_dir=params.hive_client_conf_dir, configurations=params.config['configurations']['hive-site'], configuration_attributes=params.config['configuration_attributes'] ['hive-site'], owner=params.hive_user, group=params.user_group, mode=0644) File(format("{hcat_conf_dir}/hcat-env.sh"), owner=params.webhcat_user, group=params.user_group, content=InlineTemplate(params.hcat_env_sh_template)) # Generate atlas-application.properties.xml file if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)
def sqoop(type=None): import params Link(params.sqoop_lib + "/mysql-connector-java.jar", to='/usr/share/java/mysql-connector-java.jar') jdbc_connector() Directory(params.sqoop_conf_dir, owner=params.sqoop_user, group=params.user_group, create_parents=True) configs = {} sqoop_site_config = get_config('sqoop-site') if sqoop_site_config: configs.update(sqoop_site_config) XmlConfig( "sqoop-site.xml", conf_dir=params.sqoop_conf_dir, configurations=configs, configuration_attributes=params.config['configurationAttributes'] ['sqoop-site'], owner=params.sqoop_user, group=params.user_group) # Generate atlas-application.properties.xml file and symlink the hook jars if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.sqoop_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.SQOOP, params.sqoop_atlas_application_properties, atlas_hook_filepath, params.sqoop_user, params.user_group) setup_atlas_jar_symlinks("sqoop", params.sqoop_lib) File(format("{sqoop_conf_dir}/sqoop-env.sh"), owner=params.sqoop_user, group=params.user_group, content=InlineTemplate(params.sqoop_env_sh_template)) update_config_permissions( ["sqoop-env-template.sh", "sqoop-site-template.xml", "sqoop-site.xml"]) pass
def start(self, env, upgrade_type=None): import params env.set_params(params) self.configure(env) # for security setup_ranger_hbase(upgrade_type=upgrade_type, service_name="hbase-master") if params.enable_hbase_atlas_hook: Logger.info( "Hbase Atlas hook is enabled, configuring Atlas HBase Hook.") hbase_atlas_hook_file_path = os.path.join( params.hbase_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HBASE, params.hbase_atlas_hook_properties, hbase_atlas_hook_file_path, params.hbase_user, params.user_group) else: Logger.info( "Hbase Atlas hook is disabled, skippking Atlas configurations." ) hbase_service('master', action='start')
def oozie_server_specific(upgrade_type): import params no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user) File(params.pid_file, action="delete", not_if=no_op_test ) oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir] Directory( oozie_server_directories, owner = params.oozie_user, group = params.user_group, mode = 0755, create_parents = True, cd_access="a", ) Directory(params.oozie_libext_dir, create_parents = True, ) hashcode_file = format("{oozie_home}/.hashcode") skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share") untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home) Execute( untar_sharelib, # time-expensive not_if = format("{no_op_test} || {skip_recreate_sharelib}"), sudo = True, ) configure_cmds = [] # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params) # Copy the first oozie ext-2.2.zip file that is found. # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP. if source_ext_zip_paths is not None: for source_ext_zip_path in source_ext_zip_paths: if os.path.isfile(source_ext_zip_path): configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir)) configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) Execute(configure_cmds, not_if=no_op_test, sudo=True, ) break Directory(params.oozie_webapps_conf_dir, owner = params.oozie_user, group = params.user_group, recursive_ownership = True, recursion_follow_links = True, ) # download the database JAR download_database_library_if_needed() #falcon el extension if params.has_falcon_host: Execute(format('{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'), not_if = no_op_test) Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), not_if = no_op_test) if params.lzo_enabled and len(params.all_lzo_packages) > 0: Package(params.all_lzo_packages, retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), not_if = no_op_test, ) prepare_war(params) File(hashcode_file, mode = 0644, ) if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted): # Create hive-site and tez-site configs for oozie Directory(params.hive_conf_dir, create_parents = True, owner = params.oozie_user, group = params.user_group ) if 'hive-site' in params.config['configurations']: hive_site_config = update_credential_provider_path(params.config['configurations']['hive-site'], 'hive-site', os.path.join(params.hive_conf_dir, 'hive-site.jceks'), params.oozie_user, params.user_group ) XmlConfig("hive-site.xml", conf_dir=params.hive_conf_dir, configurations=hive_site_config, configuration_attributes=params.config['configuration_attributes']['hive-site'], owner=params.oozie_user, group=params.user_group, mode=0644 ) if 'tez-site' in params.config['configurations']: XmlConfig( "tez-site.xml", conf_dir = params.hive_conf_dir, configurations = params.config['configurations']['tez-site'], configuration_attributes=params.config['configuration_attributes']['tez-site'], owner = params.oozie_user, group = params.user_group, mode = 0664 ) # If Atlas is also installed, need to generate Atlas Hive hook (hive-atlas-application.properties file) in directory # {stack_root}/{current_version}/atlas/hook/hive/ # Because this is a .properties file instead of an xml file, it will not be read automatically by Oozie. # However, should still save the file on this host so that can upload it to the Oozie Sharelib in DFS. if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.hive_conf_dir, params.atlas_hook_filename) Logger.info("Has atlas in cluster, will save Atlas Hive hook into location %s" % str(atlas_hook_filepath)) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.oozie_user, params.user_group) Directory(params.oozie_server_dir, owner = params.oozie_user, group = params.user_group, recursive_ownership = True, ) if params.security_enabled: File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'), owner=params.oozie_user, group=params.user_group, content=Template("zkmigrator_jaas.conf.j2") )
def hive(name=None): import params hive_client_conf_path = format("{stack_root}/current/{component_directory}/conf") # Permissions 644 for conf dir (client) files, and 600 for conf.server mode_identified = 0644 if params.hive_config_dir == hive_client_conf_path else 0600 Directory(params.hive_etc_dir_prefix, mode=0755 ) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) params.hive_site_config = update_credential_provider_path(params.hive_site_config, 'hive-site', os.path.join(params.hive_conf_dir, 'hive-site.jceks'), params.hive_user, params.user_group ) XmlConfig("hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes']['hive-site'], owner=params.hive_user, group=params.user_group, mode=mode_identified) # Generate atlas-application.properties.xml file if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, content=InlineTemplate(params.hive_env_sh_template), mode=mode_identified ) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents = True, owner='root', group='root' ) File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2") ) if params.security_enabled: File(os.path.join(params.hive_config_dir, 'zkmigrator_jaas.conf'), owner=params.hive_user, group=params.user_group, content=Template("zkmigrator_jaas.conf.j2") ) File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), mode = 0644, ) if name != "client": setup_non_client() if name == 'hiveserver2': setup_hiveserver2() if name == 'metastore': setup_metastore()
def hive_interactive(name=None): import params MB_TO_BYTES = 1048576 # if warehouse directory is in DFS if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse( params.default_fs).scheme: # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=params.hive_apps_whs_mode) else: Logger.info( format( "Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS." )) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode) # list of properties that should be excluded from the config # this approach is a compromise against adding a dedicated config # type for hive_server_interactive or needed config groups on a # per component basis exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting'] # List of configs to be excluded from hive2 client, but present in Hive2 server. exclude_list_for_hive2_client = [ 'javax.jdo.option.ConnectionPassword', 'hadoop.security.credential.provider.path' ] # Copy Tarballs in HDFS. if params.stack_version_formatted_major and check_stack_feature( StackFeature.ROLLING_UPGRADE, params.stack_version_formatted_major): resource_created = copy_to_hdfs( "tez_hive2", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, skip=params.sysprep_skip_copy_tarballs_hdfs) if resource_created: params.HdfsResource(None, action="execute") Directory(params.hive_interactive_etc_dir_prefix, mode=0755) Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) ''' As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml, we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml. ''' merged_hive_interactive_site = {} merged_hive_interactive_site.update( params.config['configurations']['hive-site']) merged_hive_interactive_site.update( params.config['configurations']['hive-interactive-site']) for item in exclude_list: if item in merged_hive_interactive_site.keys(): del merged_hive_interactive_site[item] ''' Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to convert it to bytes before we write it down to config file. ''' if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys(): hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get( "hive.llap.io.memory.size") hive_llap_io_mem_size_in_bytes = long( hive_llap_io_mem_size_in_mb) * MB_TO_BYTES merged_hive_interactive_site[ 'hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes Logger.info( "Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing " "it to config file.".format(hive_llap_io_mem_size_in_mb, hive_llap_io_mem_size_in_bytes)) ''' Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook', which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed. ''' # Generate atlas-application.properties.xml file if params.enable_atlas_hook and params.stack_supports_atlas_hook_for_hive_interactive: Logger.info("Setup for Atlas Hive2 Hook started.") atlas_hook_filepath = os.path.join( params.hive_server_interactive_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) Logger.info("Setup for Atlas Hive2 Hook done.") else: # Required for HDP 2.5 stacks Logger.info( "Skipping setup for Atlas Hook, as it is disabled/ not supported.") remove_atlas_hook_if_exists(merged_hive_interactive_site) ''' As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml, we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml. ''' merged_tez_interactive_site = {} if 'tez-site' in params.config['configurations']: merged_tez_interactive_site.update( params.config['configurations']['tez-site']) Logger.info( "Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'." ) else: Logger.error( "Tez's 'tez-site' couldn't be retrieved from passed-in configurations." ) merged_tez_interactive_site.update( params.config['configurations']['tez-interactive-site']) XmlConfig("tez-site.xml", conf_dir=params.tez_interactive_config_dir, configurations=merged_tez_interactive_site, configuration_attributes=params.config['configurationAttributes'] ['tez-interactive-site'], owner=params.tez_interactive_user, group=params.user_group, mode=0664) ''' Merge properties from hiveserver2-interactive-site into hiveserver2-site ''' merged_hiveserver2_interactive_site = {} if 'hiveserver2-site' in params.config['configurations']: merged_hiveserver2_interactive_site.update( params.config['configurations']['hiveserver2-site']) Logger.info( "Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'." ) else: Logger.error( "'hiveserver2-site' couldn't be retrieved from passed-in configurations." ) merged_hiveserver2_interactive_site.update( params.config['configurations']['hiveserver2-interactive-site']) # Create config files under /etc/hive2/conf and /etc/hive2/conf/conf.server: # hive-site.xml # hive-env.sh # llap-daemon-log4j2.properties # llap-cli-log4j2.properties # hive-log4j2.properties # hive-exec-log4j2.properties # beeline-log4j2.properties hive2_conf_dirs_list = params.hive_conf_dirs_list hive2_client_conf_path = format( "{stack_root}/current/{component_directory}/conf") # Making copy of 'merged_hive_interactive_site' in 'merged_hive_interactive_site_copy', and deleting 'javax.jdo.option.ConnectionPassword' # config from there, as Hive2 client shouldn't have that config. merged_hive_interactive_site_copy = merged_hive_interactive_site.copy() for item in exclude_list_for_hive2_client: if item in merged_hive_interactive_site.keys(): del merged_hive_interactive_site_copy[item] for conf_dir in hive2_conf_dirs_list: mode_identified = 0644 if conf_dir == hive2_client_conf_path else 0600 if conf_dir == hive2_client_conf_path: XmlConfig( "hive-site.xml", conf_dir=conf_dir, configurations=merged_hive_interactive_site_copy, configuration_attributes=params. config['configurationAttributes']['hive-interactive-site'], owner=params.hive_user, group=params.user_group, mode=0644) else: merged_hive_interactive_site = update_credential_provider_path( merged_hive_interactive_site, 'hive-site', os.path.join(conf_dir, 'hive-site.jceks'), params.hive_user, params.user_group) XmlConfig( "hive-site.xml", conf_dir=conf_dir, configurations=merged_hive_interactive_site, configuration_attributes=params. config['configurationAttributes']['hive-interactive-site'], owner=params.hive_user, group=params.user_group, mode=0600) XmlConfig( "hiveserver2-site.xml", conf_dir=conf_dir, configurations=merged_hiveserver2_interactive_site, configuration_attributes=params.config['configurationAttributes'] ['hiveserver2-interactive-site'], owner=params.hive_user, group=params.user_group, mode=mode_identified) hive_server_interactive_conf_dir = conf_dir File(format("{hive_server_interactive_conf_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, mode=mode_identified, content=InlineTemplate(params.hive_interactive_env_sh_template)) llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties' File(format( "{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.llap_daemon_log4j)) llap_cli_log4j2_filename = 'llap-cli-log4j2.properties' File(format( "{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.llap_cli_log4j2)) hive_log4j2_filename = 'hive-log4j2.properties' File(format( "{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.hive_log4j2)) hive_exec_log4j2_filename = 'hive-exec-log4j2.properties' File(format( "{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.hive_exec_log4j2)) beeline_log4j2_filename = 'beeline-log4j2.properties' File(format( "{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.beeline_log4j2)) File(os.path.join(hive_server_interactive_conf_dir, "hadoop-metrics2-hiveserver2.properties"), owner=params.hive_user, group=params.user_group, mode=mode_identified, content=Template("hadoop-metrics2-hiveserver2.properties.j2")) File(format( "{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties" ), owner=params.hive_user, group=params.user_group, mode=mode_identified, content=Template("hadoop-metrics2-llapdaemon.j2")) File(format( "{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties" ), owner=params.hive_user, group=params.user_group, mode=mode_identified, content=Template("hadoop-metrics2-llaptaskscheduler.j2")) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2")) if not os.path.exists(params.target_hive_interactive): jdbc_connector(params.target_hive_interactive, params.hive_intaractive_previous_jdbc_jar) File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}{check_db_connection_jar_name}")), mode=0644) File(params.start_hiveserver2_interactive_path, mode=0755, content=Template(format('{start_hiveserver2_interactive_script}'))) Directory(params.hive_pid_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_log_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_interactive_var_lib, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755)
def storm(name=None): import params import os Directory( params.log_dir, owner=params.storm_user, group=params.user_group, mode=0777, create_parents=True, cd_access="a", ) Directory( [params.pid_dir, params.local_dir], owner=params.storm_user, group=params.user_group, create_parents=True, cd_access="a", mode=0755, ) Directory( params.conf_dir, group=params.user_group, create_parents=True, cd_access="a", ) File(format("{conf_dir}/config.yaml"), content=Template("config.yaml.j2"), owner=params.storm_user, group=params.user_group) configurations = params.config['configurations']['storm-site'] File(format("{conf_dir}/storm.yaml"), content=yaml_config_template(configurations), owner=params.storm_user, group=params.user_group) File(format("{conf_dir}/storm-env.sh"), owner=params.storm_user, content=InlineTemplate(params.storm_env_sh_template)) # Generate atlas-application.properties.xml file and symlink the hook jars if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.STORM, params.storm_atlas_application_properties, atlas_hook_filepath, params.storm_user, params.user_group) storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib") setup_atlas_jar_symlinks("storm", storm_extlib_dir) if params.has_metric_collector: File(format("{conf_dir}/storm-metrics2.properties"), owner=params.storm_user, group=params.user_group, content=Template("storm-metrics2.properties.j2")) # Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2 Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"), action="delete") # On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar", action="delete") if check_stack_feature(StackFeature.STORM_METRICS_APACHE_CLASSES, params.version_for_stack_feature_checks): sink_jar = params.metric_collector_sink_jar else: sink_jar = params.metric_collector_legacy_sink_jar Execute(format( "{sudo} ln -s {sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar" ), not_if=format( "ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"), only_if=format("ls {sink_jar}")) if params.storm_logs_supported: Directory(params.log4j_dir, owner=params.storm_user, group=params.user_group, mode=0755, create_parents=True) File(format("{log4j_dir}/cluster.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_cluster_log4j_content)) File(format("{log4j_dir}/worker.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_worker_log4j_content)) if params.security_enabled: TemplateConfig(format("{conf_dir}/storm_jaas.conf"), owner=params.storm_user) if params.stack_version_formatted and check_stack_feature( StackFeature.ROLLING_UPGRADE, params.stack_version_formatted): TemplateConfig(format("{conf_dir}/client_jaas.conf"), owner=params.storm_user) minRuid = configurations[ '_storm.min.ruid'] if configurations.has_key( '_storm.min.ruid') else '' min_user_ruid = int( minRuid) if minRuid.isdigit() else _find_real_user_min_uid() File(format("{conf_dir}/worker-launcher.cfg"), content=Template("worker-launcher.cfg.j2", min_user_ruid=min_user_ruid), owner='root', group=params.user_group)
def hive(name=None): import params if name == 'hiveserver2': # HDP 2.1.* or lower params.HdfsResource(params.webhcat_apps_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=0755) # Create webhcat dirs. if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hcat_user, mode=params.hcat_hdfs_user_mode) params.HdfsResource(params.webhcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.webhcat_hdfs_user_mode) # HDP 2.2 or higher, copy mapreduce.tar.gz to HDFS # copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) # copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. # This can use a different source and dest location to account for both HDP 2.1 and 2.2 copy_to_hdfs("pig", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.pig_tar_source, custom_dest_file=params.pig_tar_dest_file, skip=params.host_sys_prepped) copy_to_hdfs("hive", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.hive_tar_source, custom_dest_file=params.hive_tar_dest_file, skip=params.host_sys_prepped) wildcard_tarballs = ["sqoop", "hadoop_streaming"] for tarball_name in wildcard_tarballs: source_file_pattern = eval("params." + tarball_name + "_tar_source") dest_dir = eval("params." + tarball_name + "_tar_dest_dir") if source_file_pattern is None or dest_dir is None: continue source_files = glob.glob( source_file_pattern) if "*" in source_file_pattern else [ source_file_pattern ] for source_file in source_files: src_filename = os.path.basename(source_file) dest_file = os.path.join(dest_dir, src_filename) copy_to_hdfs(tarball_name, params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=source_file, custom_dest_file=dest_file, skip=params.host_sys_prepped) # ******* End Copy Tarballs ******* # ********************************* # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=0777) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode) if not is_empty(params.hive_exec_scratchdir) and not urlparse( params.hive_exec_scratchdir).path.startswith("/tmp"): params.HdfsResource( params.hive_exec_scratchdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.hdfs_user, mode=0777 ) # Hive expects this dir to be writeable by everyone as it is used as a temp dir params.HdfsResource(None, action="execute") Directory(params.hive_etc_dir_prefix, mode=0755) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) XmlConfig( "hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes'] ['hive-site'], owner=params.hive_user, group=params.user_group, mode=0644) # Generate atlas-application.properties.xml file if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) if name == 'hiveserver2': XmlConfig( "hiveserver2-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hiveserver2-site'], configuration_attributes=params.config['configuration_attributes'] ['hiveserver2-site'], owner=params.hive_user, group=params.user_group, mode=0644) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, content=InlineTemplate(params.hive_env_sh_template)) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2")) if (name == 'metastore' or name == 'hiveserver2') and not os.path.exists(params.target): jdbc_connector() File( format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}{check_db_connection_jar_name}")), mode=0644, ) if name == 'metastore': File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) if params.init_metastore_schema: create_schema_cmd = format( "export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}") check_schema_created_cmd = as_user( format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}"), params.hive_user) # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. # Fixing it with the hack below: quoted_hive_metastore_user_passwd = quote_bash_args( quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[ 1:-1] Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr( check_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Execute(create_schema_cmd, not_if=check_schema_created_cmd, user=params.hive_user) elif name == 'hiveserver2': File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}'))) if name != "client": crt_directory(params.hive_pid_dir) crt_directory(params.hive_log_dir) crt_directory(params.hive_var_lib)
def falcon(type, action=None, upgrade_type=None): import params if action == 'config': Directory( params.falcon_pid_dir, owner=params.falcon_user, create_parents=True, mode=0755, cd_access="a", ) Directory( params.falcon_log_dir, owner=params.falcon_user, create_parents=True, mode=0755, cd_access="a", ) Directory(params.falcon_webapp_dir, owner=params.falcon_user, create_parents=True) Directory(params.falcon_home, owner=params.falcon_user, create_parents=True) Directory(params.etc_prefix_dir, mode=0755, create_parents=True) Directory(params.falcon_conf_dir, owner=params.falcon_user, create_parents=True) File( params.falcon_conf_dir + '/falcon-env.sh', content=InlineTemplate(params.falcon_env_sh_template), owner=params.falcon_user, group=params.user_group, ) PropertiesFile(params.falcon_conf_dir + '/client.properties', properties=params.falcon_client_properties, mode=0644, owner=params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/runtime.properties', properties=params.falcon_runtime_properties, mode=0644, owner=params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/startup.properties', properties=params.falcon_startup_properties, mode=0644, owner=params.falcon_user) if params.falcon_graph_storage_directory: Directory(params.falcon_graph_storage_directory, owner=params.falcon_user, group=params.user_group, mode=0775, create_parents=True, cd_access="a") if params.falcon_graph_serialize_path: Directory(params.falcon_graph_serialize_path, owner=params.falcon_user, group=params.user_group, mode=0775, create_parents=True, cd_access="a") # Generate atlas-application.properties.xml file if params.falcon_atlas_support: # If Atlas is added later than Falcon, this package will be absent. install_atlas_hook_packages( params.atlas_plugin_package, params.atlas_ubuntu_plugin_package, params.host_sys_prepped, params.agent_stack_retry_on_unavailability, params.agent_stack_retry_count) atlas_hook_filepath = os.path.join(params.falcon_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.FALCON, params.falcon_atlas_application_properties, atlas_hook_filepath, params.falcon_user, params.user_group) # Falcon 0.10 uses FALCON_EXTRA_CLASS_PATH. # Setup symlinks for older versions. if params.current_version_formatted and check_stack_feature( StackFeature.FALCON_ATLAS_SUPPORT_2_3, params.current_version_formatted): setup_atlas_jar_symlinks("falcon", params.falcon_webinf_lib) if type == 'server': if action == 'config': if params.store_uri[0:4] == "hdfs": params.HdfsResource(params.store_uri, type="directory", action="create_on_execute", owner=params.falcon_user, mode=0755) elif params.store_uri[0:4] == "file": Directory(params.store_uri[7:], owner=params.falcon_user, create_parents=True) # TODO change to proper mode params.HdfsResource(params.falcon_apps_dir, type="directory", action="create_on_execute", owner=params.falcon_user, mode=0777) # In HDP 2.4 and earlier, the data-mirroring directory was copied to HDFS. if params.supports_data_mirroring: params.HdfsResource(params.dfs_data_mirroring_dir, type="directory", action="create_on_execute", owner=params.falcon_user, group=params.proxyuser_group, recursive_chown=True, recursive_chmod=True, mode=0770, source=params.local_data_mirroring_dir) # Falcon Extensions were supported in HDP 2.5 and higher. effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version( params.version) supports_falcon_extensions = effective_version and check_stack_feature( StackFeature.FALCON_EXTENSIONS, effective_version) if supports_falcon_extensions: params.HdfsResource(params.falcon_extensions_dest_dir, type="directory", action="create_on_execute", owner=params.falcon_user, group=params.proxyuser_group, recursive_chown=True, recursive_chmod=True, mode=0755, source=params.falcon_extensions_source_dir) # Create the extensons HiveDR store params.HdfsResource(os.path.join( params.falcon_extensions_dest_dir, "mirroring"), type="directory", action="create_on_execute", owner=params.falcon_user, group=params.proxyuser_group, mode=0770) # At least one HDFS Dir should be created, so execute the change now. params.HdfsResource(None, action="execute") Directory(params.falcon_local_dir, owner=params.falcon_user, create_parents=True, cd_access="a") if params.falcon_embeddedmq_enabled == True: Directory(os.path.abspath( os.path.join(params.falcon_embeddedmq_data, "..")), owner=params.falcon_user, create_parents=True) Directory(params.falcon_embeddedmq_data, owner=params.falcon_user, create_parents=True) # although Falcon's falcon-config.sh will use 'which hadoop' to figure # this out, in an upgraded cluster, it's possible that 'which hadoop' # still points to older binaries; it's safer to just pass in the # hadoop home directory to use environment_dictionary = {"HADOOP_HOME": params.hadoop_home_dir} pid = get_user_call_output.get_user_call_output( format("cat {server_pid_file}"), user=params.falcon_user, is_checked_call=False)[1] process_exists = format("ls {server_pid_file} && ps -p {pid}") if action == 'start': try: Execute( format('{falcon_home}/bin/falcon-config.sh server falcon'), user=params.falcon_user, path=params.hadoop_bin_dir, environment=environment_dictionary, not_if=process_exists, ) except: show_logs(params.falcon_log_dir, params.falcon_user) raise if not os.path.exists(params.target_jar_file): try: File(params.target_jar_file, content=DownloadSource(params.bdb_resource_name), mode=0755) except: exc_msg = traceback.format_exc() exception_message = format( "Caught Exception while downloading {bdb_resource_name}:\n{exc_msg}" ) Logger.error(exception_message) if not os.path.isfile(params.target_jar_file): error_message = """ If you are using bdb as the Falcon graph db store, please run ambari-server setup --jdbc-db=bdb --jdbc-driver=<path to je5.0.73.jar> on the ambari server host. Otherwise falcon startup will fail. Otherwise please configure Falcon to use HBase as the backend as described in the Falcon documentation. """ Logger.error(error_message) try: Execute( format( '{falcon_home}/bin/falcon-start -port {falcon_port}'), user=params.falcon_user, path=params.hadoop_bin_dir, environment=environment_dictionary, not_if=process_exists, ) except: show_logs(params.falcon_log_dir, params.falcon_user) raise if action == 'stop': try: Execute(format('{falcon_home}/bin/falcon-stop'), user=params.falcon_user, path=params.hadoop_bin_dir, environment=environment_dictionary) except: show_logs(params.falcon_log_dir, params.falcon_user) raise File(params.server_pid_file, action='delete')
def hive(name=None): import params hive_client_conf_path = format("/etc/hive/conf") # Permissions 644 for conf dir (client) files, and 600 for conf.server mode_identified = 0644 if params.hive_config_dir == hive_client_conf_path else 0600 if name == 'hiveserver2': if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hcat_user, mode=params.hcat_hdfs_user_mode) params.HdfsResource(params.webhcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.webhcat_hdfs_user_mode) # ****** Begin Copy Tarballs ****** # ********************************* # if copy tarball to HDFS feature supported copy mapreduce.tar.gz and tez.tar.gz to HDFS copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. # This can use a different source and dest location to account copy_to_hdfs("pig", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.pig_tar_source, custom_dest_file=params.pig_tar_dest_file, skip=params.sysprep_skip_copy_tarballs_hdfs) copy_to_hdfs("hive", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.hive_tar_source, custom_dest_file=params.hive_tar_dest_file, skip=params.sysprep_skip_copy_tarballs_hdfs) wildcard_tarballs = ["sqoop", "hadoop_streaming"] for tarball_name in wildcard_tarballs: source_file_pattern = eval("params." + tarball_name + "_tar_source") dest_dir = eval("params." + tarball_name + "_tar_dest_dir") if source_file_pattern is None or dest_dir is None: continue source_files = glob.glob( source_file_pattern) if "*" in source_file_pattern else [ source_file_pattern ] for source_file in source_files: src_filename = os.path.basename(source_file) dest_file = os.path.join(dest_dir, src_filename) copy_to_hdfs(tarball_name, params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=source_file, custom_dest_file=dest_file, skip=params.sysprep_skip_copy_tarballs_hdfs) # ******* End Copy Tarballs ******* # ********************************* # if warehouse directory is in DFS if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse( params.default_fs).scheme: # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=0777) else: Logger.info( format( "Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS." )) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode) if not is_empty(params.hive_exec_scratchdir) and not urlparse( params.hive_exec_scratchdir).path.startswith("/tmp"): params.HdfsResource( params.hive_exec_scratchdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.hdfs_user, mode=0777 ) # Hive expects this dir to be writeable by everyone as it is used as a temp dir params.HdfsResource(None, action="execute") Directory(params.hive_etc_dir_prefix, mode=0755) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) params.hive_site_config = update_credential_provider_path( params.hive_site_config, 'hive-site', os.path.join(params.hive_conf_dir, 'hive-site.jceks'), params.hive_user, params.user_group) XmlConfig( "hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes'] ['hive-site'], owner=params.hive_user, group=params.user_group, mode=mode_identified) # Generate atlas-application.properties.xml file if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) if name == 'hiveserver2': XmlConfig( "hiveserver2-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hiveserver2-site'], configuration_attributes=params.config['configuration_attributes'] ['hiveserver2-site'], owner=params.hive_user, group=params.user_group, mode=0600) if params.hive_metastore_site_supported and name == 'metastore': XmlConfig( "hivemetastore-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations'] ['hivemetastore-site'], configuration_attributes=params.config['configuration_attributes'] ['hivemetastore-site'], owner=params.hive_user, group=params.user_group, mode=0600) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, mode=mode_identified, content=InlineTemplate(params.hive_env_sh_template)) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2")) if params.security_enabled: File(os.path.join(params.hive_config_dir, 'zkmigrator_jaas.conf'), owner=params.hive_user, group=params.user_group, content=Template("zkmigrator_jaas.conf.j2")) if name == 'metastore' or name == 'hiveserver2': if params.hive_jdbc_target is not None and not os.path.exists( params.hive_jdbc_target): jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar) if params.hive2_jdbc_target is not None and not os.path.exists( params.hive2_jdbc_target): jdbc_connector(params.hive2_jdbc_target, params.hive2_previous_jdbc_jar) File( format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}{check_db_connection_jar_name}")), mode=0644, ) if name == 'metastore': File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, mode=0600, content=Template("hadoop-metrics2-hivemetastore.properties.j2")) File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) elif name == 'hiveserver2': File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}'))) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hiveserver2.properties"), owner=params.hive_user, group=params.user_group, mode=0600, content=Template("hadoop-metrics2-hiveserver2.properties.j2")) if name != "client": Directory(params.hive_pid_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_log_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_var_lib, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755)
def webhcat(): import params Directory(params.templeton_pid_dir, owner=params.webhcat_user, mode=0755, group=params.user_group, create_parents=True) Directory(params.templeton_log_dir, owner=params.webhcat_user, mode=0755, group=params.user_group, create_parents=True) Directory(params.config_dir, create_parents=True, owner=params.webhcat_user, group=params.user_group, cd_access="a") # Replace _HOST with hostname in relevant principal-related properties webhcat_site = params.config['configurations']['webhcat-site'].copy() for prop_name in [ 'templeton.hive.properties', 'templeton.kerberos.principal' ]: if prop_name in webhcat_site: webhcat_site[prop_name] = webhcat_site[prop_name].replace( "_HOST", params.hostname) XmlConfig( "webhcat-site.xml", conf_dir=params.config_dir, configurations=webhcat_site, configuration_attributes=params.config['configuration_attributes'] ['webhcat-site'], owner=params.webhcat_user, group=params.user_group, ) # if we're in an upgrade of a secure cluster, make sure hive-site and yarn-site are created if check_stack_feature(StackFeature.CONFIG_VERSIONING, params.stack_version_formatted_major) and \ params.version and params.stack_root: XmlConfig( "hive-site.xml", conf_dir=format("{stack_root}/{version}/hive/conf"), configurations=params.config['configurations']['hive-site'], configuration_attributes=params.config['configuration_attributes'] ['hive-site'], owner=params.hive_user, group=params.user_group, ) XmlConfig( "yarn-site.xml", conf_dir=format("{stack_root}/{version}/hadoop/conf"), configurations=params.config['configurations']['yarn-site'], configuration_attributes=params.config['configuration_attributes'] ['yarn-site'], owner=params.yarn_user, group=params.user_group, ) File(format("{config_dir}/webhcat-env.sh"), owner=params.webhcat_user, group=params.user_group, content=InlineTemplate(params.webhcat_env_sh_template)) Directory(params.webhcat_conf_dir, cd_access='a', create_parents=True) log4j_webhcat_filename = 'webhcat-log4j.properties' if (params.log4j_webhcat_props != None): File(format("{config_dir}/{log4j_webhcat_filename}"), mode=0644, group=params.user_group, owner=params.webhcat_user, content=InlineTemplate(params.log4j_webhcat_props)) elif (os.path.exists("{config_dir}/{log4j_webhcat_filename}.template")): File(format("{config_dir}/{log4j_webhcat_filename}"), mode=0644, group=params.user_group, owner=params.webhcat_user, content=StaticFile( format("{config_dir}/{log4j_webhcat_filename}.template"))) # Generate atlas-application.properties.xml file if params.enable_atlas_hook: # WebHCat uses a different config dir than the rest of the daemons in Hive. atlas_hook_filepath = os.path.join(params.config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)
def webhcat(): import params Directory(params.templeton_pid_dir, owner=params.webhcat_user, mode=0755, group=params.user_group, create_parents=True) Directory(params.templeton_log_dir, owner=params.webhcat_user, mode=0755, group=params.user_group, create_parents=True) Directory(params.config_dir, create_parents=True, owner=params.webhcat_user, group=params.user_group, cd_access="a") if params.security_enabled: kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};") else: kinit_if_needed = "" if kinit_if_needed: Execute(kinit_if_needed, user=params.webhcat_user, path='/bin' ) # Replace _HOST with hostname in relevant principal-related properties webhcat_site = params.config['configurations']['webhcat-site'].copy() for prop_name in ['templeton.hive.properties', 'templeton.kerberos.principal']: if prop_name in webhcat_site: webhcat_site[prop_name] = webhcat_site[prop_name].replace("_HOST", params.hostname) XmlConfig("webhcat-site.xml", conf_dir=params.config_dir, configurations=webhcat_site, configuration_attributes=params.config['configuration_attributes']['webhcat-site'], owner=params.webhcat_user, group=params.user_group, ) File(format("{config_dir}/webhcat-env.sh"), owner=params.webhcat_user, group=params.user_group, content=InlineTemplate(params.webhcat_env_sh_template) ) Directory(params.webhcat_conf_dir, cd_access='a', create_parents=True ) log4j_webhcat_filename = 'webhcat-log4j.properties' if (params.log4j_webhcat_props != None): File(format("{config_dir}/{log4j_webhcat_filename}"), mode=0644, group=params.user_group, owner=params.webhcat_user, content=params.log4j_webhcat_props ) elif (os.path.exists("{config_dir}/{log4j_webhcat_filename}.template")): File(format("{config_dir}/{log4j_webhcat_filename}"), mode=0644, group=params.user_group, owner=params.webhcat_user, content=StaticFile(format("{config_dir}/{log4j_webhcat_filename}.template")) ) # Generate atlas-application.properties.xml file if params.enable_atlas_hook: # WebHCat uses a different config dir than the rest of the daemons in Hive. atlas_hook_filepath = os.path.join(params.config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)