def hive(name=None): import params Directory(params.hive_etc_dir_prefix, mode=0755 ) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) XmlConfig("hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes']['hive-site'], owner=params.hive_user, group=params.user_group, mode=0644) # Generate atlas-application.properties.xml file if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, content=InlineTemplate(params.hive_env_sh_template) ) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents = True, owner='root', group='root' ) File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2") ) File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), mode = 0644, ) if name != "client": setup_non_client() if name == 'hiveserver2': setup_hiveserver2() if name == 'metastore': setup_metastore() # schematool work
def hcat(): import params Directory( params.hive_conf_dir, create_parents=True, owner=params.hcat_user, group=params.user_group, ) Directory( params.hcat_conf_dir, create_parents=True, owner=params.hcat_user, group=params.user_group, ) Directory(params.hcat_pid_dir, owner=params.webhcat_user, create_parents=True) XmlConfig( "hive-site.xml", conf_dir=params.hive_client_conf_dir, configurations=params.config['configurations']['hive-site'], configuration_attributes=params.config['configuration_attributes'] ['hive-site'], owner=params.hive_user, group=params.user_group, mode=0644) File(format("{hcat_conf_dir}/hcat-env.sh"), owner=params.hcat_user, group=params.user_group, content=InlineTemplate(params.hcat_env_sh_template)) # Generate atlas-application.properties.xml file if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)
def sqoop(type=None): import params Link(params.sqoop_lib + "/mysql-connector-java.jar", to='/usr/share/java/mysql-connector-java.jar') jdbc_connector() Directory(params.sqoop_conf_dir, owner=params.sqoop_user, group=params.user_group, create_parents=True) configs = {} configs.update(params.config['configurations']['sqoop-site']) XmlConfig( "sqoop-site.xml", conf_dir=params.sqoop_conf_dir, configurations=configs, configuration_attributes=params.config['configuration_attributes'] ['sqoop-site'], owner=params.sqoop_user, group=params.user_group) # Generate atlas-application.properties.xml file and symlink the hook jars if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.sqoop_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.SQOOP, params.sqoop_atlas_application_properties, atlas_hook_filepath, params.sqoop_user, params.user_group) setup_atlas_jar_symlinks("sqoop", params.sqoop_lib) File(format("{sqoop_conf_dir}/sqoop-env.sh"), owner=params.sqoop_user, group=params.user_group, content=InlineTemplate(params.sqoop_env_sh_template)) update_config_permissions( ["sqoop-env-template.sh", "sqoop-site-template.xml", "sqoop-site.xml"]) pass
hdfs_principal_name = config['configurations']['hadoop-env'][ 'hdfs_principal_name'] hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] dfs_type = default("/commandParams/dfs_type", "") ######################################################## ############# Atlas related params ##################### ######################################################## #region Atlas Hooks needed by Hive on Oozie hive_atlas_application_properties = default( '/configurations/hive-atlas-application.properties', {}) if has_atlas_in_cluster(): atlas_hook_filename = default( '/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') #endregion import functools #create partial functions with common arguments for every HdfsResource call #to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file= "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled=security_enabled, keytab=hdfs_user_keytab,
def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=None): """ If the Atlas Hive Hook direcotry is present, Atlas is installed, and this is the first Oozie Server, then copy the entire contents of that directory to the Oozie Sharelib in DFS, e.g., /usr/$stack/$current_version/atlas/hook/hive/ -> hdfs:///user/oozie/share/lib/lib_$timetamp/hive :param upgrade_type: If in the middle of a stack upgrade, the type as UPGRADE_TYPE_ROLLING or UPGRADE_TYPE_NON_ROLLING :param upgrade_direction: If in the middle of a stack upgrade, the direction as Direction.UPGRADE or Direction.DOWNGRADE. """ import params # Calculate the effective version since this code can also be called during EU/RU in the upgrade direction. effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(params.version) if not check_stack_feature(StackFeature.ATLAS_HOOK_SUPPORT, effective_version): return # Important that oozie_server_hostnames is sorted by name so that this only runs on a single Oozie server. if not (len(params.oozie_server_hostnames) > 0 and params.hostname == params.oozie_server_hostnames[0]): Logger.debug("Will not attempt to copy Atlas Hive hook to DFS since this is not the first Oozie Server " "sorted by hostname.") return if not has_atlas_in_cluster(): Logger.debug("Will not attempt to copy Atlas Hve hook to DFS since Atlas is not installed on the cluster.") return if upgrade_type is not None and upgrade_direction == Direction.DOWNGRADE: Logger.debug("Will not attempt to copy Atlas Hve hook to DFS since in the middle of Rolling/Express upgrade " "and performing a Downgrade.") return current_version = get_current_version() atlas_hive_hook_dir = format("{stack_root}/{current_version}/atlas/hook/hive/") if not os.path.exists(atlas_hive_hook_dir): Logger.error(format("ERROR. Atlas is installed in cluster but this Oozie server doesn't " "contain directory {atlas_hive_hook_dir}")) return atlas_hive_hook_impl_dir = os.path.join(atlas_hive_hook_dir, "atlas-hive-plugin-impl") num_files = len([name for name in os.listdir(atlas_hive_hook_impl_dir) if os.path.exists(os.path.join(atlas_hive_hook_impl_dir, name))]) Logger.info("Found %d files/directories inside Atlas Hive hook impl directory %s"% (num_files, atlas_hive_hook_impl_dir)) # This can return over 100 files, so take the first 5 lines after "Available ShareLib" # Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env does not export OOZIE_URL command = format(r'source {conf_dir}/oozie-env.sh ; oozie admin -oozie {oozie_base_url} -shareliblist hive | grep "\[Available ShareLib\]" -A 5') code, out = checked_call(command, user=params.oozie_user, tries=10, try_sleep=5, logoutput=True) hive_sharelib_dir = __parse_sharelib_from_output(out) if hive_sharelib_dir is None: raise Fail("Could not parse Hive sharelib from output.") Logger.info(format("Parsed Hive sharelib = {hive_sharelib_dir} and will attempt to copy/replace {num_files} files to it from {atlas_hive_hook_impl_dir}")) params.HdfsResource(hive_sharelib_dir, type="directory", action="create_on_execute", source=atlas_hive_hook_impl_dir, user=params.hdfs_user, owner=params.oozie_user, group=params.hdfs_user, mode=0755, recursive_chown=True, recursive_chmod=True, replace_existing_files=True ) Logger.info("Copying Atlas Hive hook properties file to Oozie Sharelib in DFS.") atlas_hook_filepath_source = os.path.join(params.hive_conf_dir, params.atlas_hook_filename) atlas_hook_file_path_dest_in_dfs = os.path.join(hive_sharelib_dir, params.atlas_hook_filename) params.HdfsResource(atlas_hook_file_path_dest_in_dfs, type="file", source=atlas_hook_filepath_source, action="create_on_execute", owner=params.oozie_user, group=params.hdfs_user, mode=0755, replace_existing_files=True ) params.HdfsResource(None, action="execute") # Update the sharelib after making any changes # Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env does not export OOZIE_URL Execute(format("source {conf_dir}/oozie-env.sh ; oozie admin -oozie {oozie_base_url} -sharelibupdate"), user=params.oozie_user, tries=5, try_sleep=5, logoutput=True, )
def oozie_server_specific(upgrade_type): import params no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user) File(params.pid_file, action="delete", not_if=no_op_test ) oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir] Directory( oozie_server_directories, owner = params.oozie_user, group = params.user_group, mode = 0755, create_parents = True, cd_access="a", ) Directory(params.oozie_libext_dir, create_parents = True, ) hashcode_file = format("{oozie_home}/.hashcode") skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share") untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home) Execute( untar_sharelib, # time-expensive not_if = format("{no_op_test} || {skip_recreate_sharelib}"), sudo = True, ) configure_cmds = [] # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params) # Copy the first oozie ext-2.2.zip file that is found. # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP. if source_ext_zip_paths is not None: for source_ext_zip_path in source_ext_zip_paths: if os.path.isfile(source_ext_zip_path): configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir)) configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) Execute(configure_cmds, not_if=no_op_test, sudo=True, ) break Directory(params.oozie_webapps_conf_dir, owner = params.oozie_user, group = params.user_group, recursive_ownership = True, recursion_follow_links = True, ) # download the database JAR download_database_library_if_needed() #falcon el extension if params.has_falcon_host: Execute(format('{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'), not_if = no_op_test) Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), not_if = no_op_test) if params.lzo_enabled and len(params.all_lzo_packages) > 0: Package(params.all_lzo_packages, retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), not_if = no_op_test, ) prepare_war(params) File(hashcode_file, mode = 0644, ) if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted): # Create hive-site and tez-site configs for oozie Directory(params.hive_conf_dir, create_parents = True, owner = params.oozie_user, group = params.user_group ) if 'hive-site' in params.config['configurations']: hive_site_config = update_credential_provider_path(params.config['configurations']['hive-site'], 'hive-site', os.path.join(params.hive_conf_dir, 'hive-site.jceks'), params.oozie_user, params.user_group ) XmlConfig("hive-site.xml", conf_dir=params.hive_conf_dir, configurations=hive_site_config, configuration_attributes=params.config['configuration_attributes']['hive-site'], owner=params.oozie_user, group=params.user_group, mode=0644 ) if 'tez-site' in params.config['configurations']: XmlConfig( "tez-site.xml", conf_dir = params.hive_conf_dir, configurations = params.config['configurations']['tez-site'], configuration_attributes=params.config['configuration_attributes']['tez-site'], owner = params.oozie_user, group = params.user_group, mode = 0664 ) # If Atlas is also installed, need to generate Atlas Hive hook (hive-atlas-application.properties file) in directory # {stack_root}/{current_version}/atlas/hook/hive/ # Because this is a .properties file instead of an xml file, it will not be read automatically by Oozie. # However, should still save the file on this host so that can upload it to the Oozie Sharelib in DFS. if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.hive_conf_dir, params.atlas_hook_filename) Logger.info("Has atlas in cluster, will save Atlas Hive hook into location %s" % str(atlas_hook_filepath)) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.oozie_user, params.user_group) Directory(params.oozie_server_dir, owner = params.oozie_user, group = params.user_group, recursive_ownership = True, ) if params.security_enabled: File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'), owner=params.oozie_user, group=params.user_group, content=Template("zkmigrator_jaas.conf.j2") )
def storm(name=None): import params import os Directory( params.log_dir, owner=params.storm_user, group=params.user_group, mode=0777, create_parents=True, cd_access="a", ) Directory( [params.pid_dir, params.local_dir], owner=params.storm_user, group=params.user_group, create_parents=True, cd_access="a", mode=0755, ) Directory( params.conf_dir, group=params.user_group, create_parents=True, cd_access="a", ) File(format("{conf_dir}/config.yaml"), content=Template("config.yaml.j2"), owner=params.storm_user, group=params.user_group) configurations = params.config['configurations']['storm-site'] File(format("{conf_dir}/storm.yaml"), content=yaml_config_template(configurations), owner=params.storm_user, group=params.user_group) File(format("{conf_dir}/storm-env.sh"), owner=params.storm_user, content=InlineTemplate(params.storm_env_sh_template)) # Generate atlas-application.properties.xml file and symlink the hook jars if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.STORM, params.storm_atlas_application_properties, atlas_hook_filepath, params.storm_user, params.user_group) storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib") setup_atlas_jar_symlinks("storm", storm_extlib_dir) if params.has_metric_collector: File(format("{conf_dir}/storm-metrics2.properties"), owner=params.storm_user, group=params.user_group, content=Template("storm-metrics2.properties.j2")) # Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2 Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"), action="delete") # On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar", action="delete") if check_stack_feature(StackFeature.STORM_METRICS_APACHE_CLASSES, params.version_for_stack_feature_checks): sink_jar = params.metric_collector_sink_jar else: sink_jar = params.metric_collector_legacy_sink_jar Execute(format( "{sudo} ln -s {sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar" ), not_if=format( "ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"), only_if=format("ls {sink_jar}")) if params.storm_logs_supported: Directory(params.log4j_dir, owner=params.storm_user, group=params.user_group, mode=0755, create_parents=True) File(format("{log4j_dir}/cluster.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_cluster_log4j_content)) File(format("{log4j_dir}/worker.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_worker_log4j_content)) if params.security_enabled: TemplateConfig(format("{conf_dir}/storm_jaas.conf"), owner=params.storm_user) if params.stack_version_formatted and check_stack_feature( StackFeature.ROLLING_UPGRADE, params.stack_version_formatted): TemplateConfig(format("{conf_dir}/client_jaas.conf"), owner=params.storm_user) minRuid = configurations[ '_storm.min.ruid'] if configurations.has_key( '_storm.min.ruid') else '' min_user_ruid = int( minRuid) if minRuid.isdigit() else _find_real_user_min_uid() File(format("{conf_dir}/worker-launcher.cfg"), content=Template("worker-launcher.cfg.j2", min_user_ruid=min_user_ruid), owner='root', group=params.user_group)
def hive(name=None): import params if name == 'hiveserver2': # copy tarball to HDFS feature not supported if not (params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major)): params.HdfsResource(params.webhcat_apps_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=0755) # Create webhcat dirs. if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.hcat_hdfs_user_mode) params.HdfsResource(params.webhcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.webhcat_hdfs_user_mode) # ****** Begin Copy Tarballs ****** # ********************************* # if copy tarball to HDFS feature supported copy mapreduce.tar.gz and tez.tar.gz to HDFS if params.stack_version_formatted_major and check_stack_feature( StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major): copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. # This can use a different source and dest location to account copy_to_hdfs("pig", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.pig_tar_source, custom_dest_file=params.pig_tar_dest_file, host_sys_prepped=params.host_sys_prepped) copy_to_hdfs("hive", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.hive_tar_source, custom_dest_file=params.hive_tar_dest_file, host_sys_prepped=params.host_sys_prepped) wildcard_tarballs = ["sqoop", "hadoop_streaming"] for tarball_name in wildcard_tarballs: source_file_pattern = eval("params." + tarball_name + "_tar_source") dest_dir = eval("params." + tarball_name + "_tar_dest_dir") if source_file_pattern is None or dest_dir is None: continue source_files = glob.glob( source_file_pattern) if "*" in source_file_pattern else [ source_file_pattern ] for source_file in source_files: src_filename = os.path.basename(source_file) dest_file = os.path.join(dest_dir, src_filename) copy_to_hdfs(tarball_name, params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=source_file, custom_dest_file=dest_file, host_sys_prepped=params.host_sys_prepped) # ******* End Copy Tarballs ******* # ********************************* # if warehouse directory is in DFS if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse( params.default_fs).scheme: # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=0777) else: Logger.info( format( "Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS." )) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode) if not is_empty(params.hive_exec_scratchdir) and not urlparse( params.hive_exec_scratchdir).path.startswith("/tmp"): params.HdfsResource( params.hive_exec_scratchdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.hdfs_user, mode=0777 ) # Hive expects this dir to be writeable by everyone as it is used as a temp dir params.HdfsResource(None, action="execute") Directory(params.hive_etc_dir_prefix, mode=0755) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) XmlConfig( "hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes'] ['hive-site'], owner=params.hive_user, group=params.user_group, mode=0644) # Generate atlas-application.properties.xml file if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) if name == 'hiveserver2': XmlConfig( "hiveserver2-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hiveserver2-site'], configuration_attributes=params.config['configuration_attributes'] ['hiveserver2-site'], owner=params.hive_user, group=params.user_group, mode=0644) if params.hive_metastore_site_supported and name == 'metastore': XmlConfig( "hivemetastore-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations'] ['hivemetastore-site'], configuration_attributes=params.config['configuration_attributes'] ['hivemetastore-site'], owner=params.hive_user, group=params.user_group, mode=0644) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, content=InlineTemplate(params.hive_env_sh_template)) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2")) if name == 'metastore' or name == 'hiveserver2': if params.hive_jdbc_target is not None and not os.path.exists( params.hive_jdbc_target): jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar) if params.hive2_jdbc_target is not None and not os.path.exists( params.hive2_jdbc_target): jdbc_connector(params.hive2_jdbc_target, params.hive2_previous_jdbc_jar) File( format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}{check_db_connection_jar_name}")), mode=0644, ) if name == 'metastore': File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hivemetastore.properties.j2")) File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) if params.init_metastore_schema: create_schema_cmd = format( "export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose") check_schema_created_cmd = as_user( format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. # Fixing it with the hack below: quoted_hive_metastore_user_passwd = quote_bash_args( quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[ 1:-1] Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr( check_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Execute(create_schema_cmd, not_if=check_schema_created_cmd, user=params.hive_user) elif name == 'hiveserver2': File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}'))) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hiveserver2.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hiveserver2.properties.j2")) if name != "client": Directory(params.hive_pid_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_log_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_var_lib, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755)
def webhcat(): import params Directory(params.templeton_pid_dir, owner=params.webhcat_user, mode=0755, group=params.user_group, create_parents=True) Directory(params.templeton_log_dir, owner=params.webhcat_user, mode=0755, group=params.user_group, create_parents=True) Directory(params.config_dir, create_parents=True, owner=params.webhcat_user, group=params.user_group, cd_access="a") if params.security_enabled: kinit_if_needed = format( "{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};") else: kinit_if_needed = "" if kinit_if_needed: Execute(kinit_if_needed, user=params.webhcat_user, path='/bin') # Replace _HOST with hostname in relevant principal-related properties webhcat_site = params.config['configurations']['webhcat-site'].copy() for prop_name in [ 'templeton.hive.properties', 'templeton.kerberos.principal' ]: if prop_name in webhcat_site: webhcat_site[prop_name] = webhcat_site[prop_name].replace( "_HOST", params.hostname) XmlConfig( "webhcat-site.xml", conf_dir=params.config_dir, configurations=webhcat_site, configuration_attributes=params.config['configuration_attributes'] ['webhcat-site'], owner=params.webhcat_user, group=params.user_group, ) # if we're in an upgrade of a secure cluster, make sure hive-site and yarn-site are created if params.stack_version_formatted_major and check_stack_feature(StackFeature.CONFIG_VERSIONING, params.stack_version_formatted_major) and \ params.version and params.stack_root: XmlConfig( "hive-site.xml", conf_dir=format("{stack_root}/{version}/hive/conf"), configurations=params.config['configurations']['hive-site'], configuration_attributes=params.config['configuration_attributes'] ['hive-site'], owner=params.hive_user, group=params.user_group, ) XmlConfig( "yarn-site.xml", conf_dir=format("{stack_root}/{version}/hadoop/conf"), configurations=params.config['configurations']['yarn-site'], configuration_attributes=params.config['configuration_attributes'] ['yarn-site'], owner=params.yarn_user, group=params.user_group, ) File(format("{config_dir}/webhcat-env.sh"), owner=params.webhcat_user, group=params.user_group, content=InlineTemplate(params.webhcat_env_sh_template)) Directory(params.webhcat_conf_dir, cd_access='a', create_parents=True) log4j_webhcat_filename = 'webhcat-log4j.properties' if (params.log4j_webhcat_props != None): File(format("{config_dir}/{log4j_webhcat_filename}"), mode=0644, group=params.user_group, owner=params.webhcat_user, content=params.log4j_webhcat_props) elif (os.path.exists("{config_dir}/{log4j_webhcat_filename}.template")): File(format("{config_dir}/{log4j_webhcat_filename}"), mode=0644, group=params.user_group, owner=params.webhcat_user, content=StaticFile( format("{config_dir}/{log4j_webhcat_filename}.template"))) # Generate atlas-application.properties.xml file if has_atlas_in_cluster(): # WebHCat uses a different config dir than the rest of the daemons in Hive. atlas_hook_filepath = os.path.join(params.config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)
def falcon(type, action = None, upgrade_type=None): import params if action == 'config': Directory(params.falcon_pid_dir, owner = params.falcon_user, create_parents = True, mode = 0755, cd_access = "a", ) Directory(params.falcon_log_dir, owner = params.falcon_user, create_parents = True, mode = 0755, cd_access = "a", ) Directory(params.falcon_webapp_dir, owner = params.falcon_user, create_parents = True) Directory(params.falcon_home, owner = params.falcon_user, create_parents = True) Directory(params.etc_prefix_dir, mode = 0755, create_parents = True) Directory(params.falcon_conf_dir, owner = params.falcon_user, create_parents = True) File(params.falcon_conf_dir + '/falcon-env.sh', content = InlineTemplate(params.falcon_env_sh_template), owner = params.falcon_user, group=params.user_group, ) PropertiesFile(params.falcon_conf_dir + '/client.properties', properties = params.falcon_client_properties, mode = 0644, owner = params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/runtime.properties', properties = params.falcon_runtime_properties, mode = 0644, owner = params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/startup.properties', properties = params.falcon_startup_properties, mode = 0644, owner = params.falcon_user) if params.falcon_graph_storage_directory: Directory(params.falcon_graph_storage_directory, owner = params.falcon_user, group = params.user_group, mode = 0775, create_parents = True, cd_access = "a") if params.falcon_graph_serialize_path: Directory(params.falcon_graph_serialize_path, owner = params.falcon_user, group = params.user_group, mode = 0775, create_parents = True, cd_access = "a") # Generate atlas-application.properties.xml file if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.falcon_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.FALCON, params.falcon_atlas_application_properties, atlas_hook_filepath, params.falcon_user, params.user_group) if type == 'server': if action == 'config': if params.store_uri[0:4] == "hdfs": params.HdfsResource(params.store_uri, type = "directory", action = "create_on_execute", owner = params.falcon_user, mode = 0755) elif params.store_uri[0:4] == "file": Directory(params.store_uri[7:], owner = params.falcon_user, create_parents = True) # TODO change to proper mode params.HdfsResource(params.falcon_apps_dir, type = "directory", action = "create_on_execute", owner = params.falcon_user, mode = 0777) # In HDP 2.4 and earlier, the data-mirroring directory was copied to HDFS. if params.supports_data_mirroring: params.HdfsResource(params.dfs_data_mirroring_dir, type = "directory", action = "create_on_execute", owner = params.falcon_user, group = params.proxyuser_group, recursive_chown = True, recursive_chmod = True, mode = 0770, source = params.local_data_mirroring_dir) if params.supports_falcon_extensions: params.HdfsResource(params.falcon_extensions_dest_dir, type = "directory", action = "create_on_execute", owner = params.falcon_user, group = params.proxyuser_group, recursive_chown = True, recursive_chmod = True, mode = 0755, source = params.falcon_extensions_source_dir) # Create the extensons HiveDR store params.HdfsResource(os.path.join(params.falcon_extensions_dest_dir, "mirroring"), type = "directory", action = "create_on_execute", owner = params.falcon_user, group = params.proxyuser_group, mode = 0770) # At least one HDFS Dir should be created, so execute the change now. params.HdfsResource(None, action = "execute") Directory(params.falcon_local_dir, owner = params.falcon_user, create_parents = True, cd_access = "a") if params.falcon_embeddedmq_enabled == True: Directory( os.path.abspath(os.path.join(params.falcon_embeddedmq_data, "..")), owner = params.falcon_user, create_parents = True) Directory(params.falcon_embeddedmq_data, owner = params.falcon_user, create_parents = True) # although Falcon's falcon-config.sh will use 'which hadoop' to figure # this out, in an upgraded cluster, it's possible that 'which hadoop' # still points to older binaries; it's safer to just pass in the # hadoop home directory to use environment_dictionary = { "HADOOP_HOME" : params.hadoop_home_dir } pid = get_user_call_output.get_user_call_output(format("cat {server_pid_file}"), user=params.falcon_user, is_checked_call=False)[1] process_exists = format("ls {server_pid_file} && ps -p {pid}") if action == 'start': if not os.path.exists(params.target_jar_file): try : File(params.target_jar_file, content = DownloadSource(params.bdb_resource_name)) except : exc_msg = traceback.format_exc() exception_message = format("Caught Exception while downloading {bdb_resource_name}:\n{exc_msg}") Logger.error(exception_message) if not os.path.isfile(params.target_jar_file) : error_message = """ If you are using bdb as the Falcon graph db store, please run ambari-server setup --jdbc-db=bdb --jdbc-driver=<path to je5.0.73.jar on the ambari server host. Otherwise falcon startup will fail. Otherwise please configure Falcon to use HBase as the backend as described in the Falcon documentation. """ Logger.error(error_message) try: Execute(format('{falcon_home}/bin/falcon-start -port {falcon_port}'), user = params.falcon_user, path = params.hadoop_bin_dir, environment=environment_dictionary, not_if = process_exists, ) except: show_logs(params.falcon_log_dir, params.falcon_user) raise if action == 'stop': try: Execute(format('{falcon_home}/bin/falcon-stop'), user = params.falcon_user, path = params.hadoop_bin_dir, environment=environment_dictionary) except: show_logs(params.falcon_log_dir, params.falcon_user) raise File(params.server_pid_file, action = 'delete')