def setup_metastore(): import params if params.hive_metastore_site_supported: hivemetastore_site_config = get_config("hivemetastore-site") if hivemetastore_site_config: XmlConfig("hivemetastore-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations'] ['hivemetastore-site'], configuration_attributes=params. config['configuration_attributes']['hivemetastore-site'], owner=params.hive_user, group=params.user_group, mode=0600) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hivemetastore.properties.j2"), mode=0600) File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) if not is_empty(params.hive_exec_scratchdir): dirPathStr = urlparse(params.hive_exec_scratchdir).path pathComponents = dirPathStr.split("/") if dirPathStr.startswith("/tmp") and len(pathComponents) > 2: Directory(params.hive_exec_scratchdir, owner=params.hive_user, create_parents=True, mode=0777)
def setup_metastore(): import params if params.hive_metastore_site_supported: hivemetastore_site_config = get_config("hivemetastore-site") if hivemetastore_site_config: XmlConfig("hivemetastore-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hivemetastore-site'], configuration_attributes=params.config['configuration_attributes']['hivemetastore-site'], owner=params.hive_user, group=params.user_group, mode=0600) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hivemetastore.properties.j2"), mode=0600 ) File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh') )
def setup_metastore(): import params if params.hive_metastore_site_supported: hivemetastore_site_config = get_config("hivemetastore-site") if hivemetastore_site_config: XmlConfig("hivemetastore-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations'] ['hivemetastore-site'], configuration_attributes=params. config['configuration_attributes']['hivemetastore-site'], owner=params.hive_user, group=params.user_group, mode=0600) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hivemetastore.properties.j2"), mode=0600) File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) if params.init_metastore_schema: create_schema_cmd = format( "export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose") check_schema_created_cmd = as_user( format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. # Fixing it with the hack below: quoted_hive_metastore_user_passwd = quote_bash_args( quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[ 1:-1] Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr( check_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Execute(create_schema_cmd, not_if=check_schema_created_cmd, user=params.hive_user)
def knox(): import params XmlConfig( "gateway-site.xml", conf_dir=params.knox_conf_dir, configurations=params.config['configurations']['gateway-site'], configuration_attributes=params.config['configuration_attributes'] ['gateway-site'], owner=params.knox_user) # Manually overriding service logon user & password set by the installation package ServiceConfig(params.knox_gateway_win_service_name, action="change_user", username=params.knox_user, password=Script.get_password(params.knox_user)) File(os.path.join(params.knox_conf_dir, "gateway-log4j.properties"), owner=params.knox_user, content=params.gateway_log4j) File(os.path.join(params.knox_conf_dir, "topologies", "default.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.topology_template)) if params.admin_topology_template: File(os.path.join(params.knox_conf_dir, "topologies", "admin.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.admin_topology_template)) if params.version_formatted and check_stack_feature( StackFeature.KNOX_SSO_TOPOLOGY, params.version_formatted): knoxsso_topology_template_content = get_config("knoxsso-topology") if knoxsso_topology_template_content: File(os.path.join(params.knox_conf_dir, "topologies", "knoxsso.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.knoxsso_topology_template)) if params.security_enabled: TemplateConfig(os.path.join(params.knox_conf_dir, "krb5JAASLogin.conf"), owner=params.knox_user, template_tag=None) if not os.path.isfile(params.knox_master_secret_path): cmd = format( 'cmd /C {knox_client_bin} create-master --master {knox_master_secret!p}' ) Execute(cmd) cmd = format( 'cmd /C {knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}' ) Execute(cmd)
def setup_metastore(): import params if params.hive_metastore_site_supported: hivemetastore_site_config = get_config("hivemetastore-site") if hivemetastore_site_config: XmlConfig("hivemetastore-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations'] ['hivemetastore-site'], configuration_attributes=params. config['configurationAttributes']['hivemetastore-site'], owner=params.hive_user, group=params.user_group, mode=0600) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hivemetastore.properties.j2"), mode=0600) File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) if params.hive_repl_cmrootdir is not None and params.hive_repl_cmrootdir.strip( ) != "": params.HdfsResource(params.hive_repl_cmrootdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=01777) if params.hive_repl_rootdir is not None and params.hive_repl_rootdir.strip( ) != "": params.HdfsResource(params.hive_repl_rootdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=0700) params.HdfsResource(None, action="execute") generate_logfeeder_input_config( 'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def setup_metastore(): import params if params.hive_metastore_site_supported: hivemetastore_site_config = get_config("hivemetastore-site") if hivemetastore_site_config: XmlConfig("hivemetastore-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations'] ['hivemetastore-site'], configuration_attributes=params. config['configuration_attributes']['hivemetastore-site'], owner=params.hive_user, group=params.user_group, mode=0600) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hivemetastore.properties.j2"), mode=0600) File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) if params.hive_repl_cmrootdir: params.HdfsResource(params.hive_repl_cmrootdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=01777) if params.hive_repl_rootdir: params.HdfsResource(params.hive_repl_rootdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=0700) if params.hive_repl_cmrootdir or params.hive_repl_rootdir: params.HdfsResource(None, action="execute")
def sqoop(type=None): import params Link(params.sqoop_lib + "/mysql-connector-java.jar", to='/usr/share/java/mysql-connector-java.jar') jdbc_connector() Directory(params.sqoop_conf_dir, owner=params.sqoop_user, group=params.user_group, create_parents=True) configs = {} sqoop_site_config = get_config('sqoop-site') if sqoop_site_config: configs.update(sqoop_site_config) XmlConfig( "sqoop-site.xml", conf_dir=params.sqoop_conf_dir, configurations=configs, configuration_attributes=params.config['configurationAttributes'] ['sqoop-site'], owner=params.sqoop_user, group=params.user_group) # Generate atlas-application.properties.xml file and symlink the hook jars if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.sqoop_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.SQOOP, params.sqoop_atlas_application_properties, atlas_hook_filepath, params.sqoop_user, params.user_group) setup_atlas_jar_symlinks("sqoop", params.sqoop_lib) File(format("{sqoop_conf_dir}/sqoop-env.sh"), owner=params.sqoop_user, group=params.user_group, content=InlineTemplate(params.sqoop_env_sh_template)) update_config_permissions( ["sqoop-env-template.sh", "sqoop-site-template.xml", "sqoop-site.xml"]) pass
def knox(): import params Directory( [ params.knox_data_dir, params.knox_logs_dir, params.knox_pid_dir, params.knox_conf_dir, os.path.join(params.knox_conf_dir, "topologies") ], owner=params.knox_user, group=params.knox_group, create_parents=True, cd_access="a", mode=0755, recursive_ownership=True, ) XmlConfig( "gateway-site.xml", conf_dir=params.knox_conf_dir, configurations=params.config['configurations']['gateway-site'], configuration_attributes=params.config['configurationAttributes'] ['gateway-site'], owner=params.knox_user, group=params.knox_group, ) File(format("{params.knox_conf_dir}/gateway-log4j.properties"), mode=0644, group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.gateway_log4j)) File(format("{params.knox_conf_dir}/topologies/default.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.topology_template)) if params.admin_topology_template: File(format("{params.knox_conf_dir}/topologies/admin.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.admin_topology_template)) if params.version_formatted and check_stack_feature( StackFeature.KNOX_SSO_TOPOLOGY, params.version_formatted): knoxsso_topology_template_content = get_config("knoxsso-topology") if knoxsso_topology_template_content: File(os.path.join(params.knox_conf_dir, "topologies", "knoxsso.xml"), group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.knoxsso_topology_template)) if params.security_enabled: TemplateConfig(format("{knox_conf_dir}/krb5JAASLogin.conf"), owner=params.knox_user, template_tag=None) cmd = format( '{knox_client_bin} create-master --master {knox_master_secret!p}') master_secret_exist = as_user(format('test -f {knox_master_secret_path}'), params.knox_user) Execute( cmd, user=params.knox_user, environment={'JAVA_HOME': params.java_home}, not_if=master_secret_exist, ) cmd = format( '{knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}') cert_store_exist = as_user(format('test -f {knox_cert_store_path}'), params.knox_user) Execute( cmd, user=params.knox_user, environment={'JAVA_HOME': params.java_home}, not_if=cert_store_exist, )
def knox(): import params Directory( [ params.knox_data_dir, params.knox_data_backup_dir, params.knox_logs_dir, params.knox_pid_dir, params.knox_conf_dir, os.path.join(params.knox_conf_dir, "topologies"), params.knox_descriptors_dir, params.knox_shared_providers_dir ], owner=params.knox_user, group=params.knox_group, create_parents=True, cd_access="a", mode=0755, recursive_ownership=True, ) XmlConfig( "gateway-site.xml", conf_dir=params.knox_conf_dir, configurations=params.config['configurations']['gateway-site'], configuration_attributes=params.config['configurationAttributes'] ['gateway-site'], owner=params.knox_user, group=params.knox_group, ) File(format("{params.knox_conf_dir}/gateway-log4j.properties"), mode=0644, group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.gateway_log4j)) File(format("{params.knox_conf_dir}/topologies/default.xml"), mode=0600, group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.topology_template)) if params.admin_topology_template: File(format("{params.knox_conf_dir}/topologies/admin.xml"), mode=0600, group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.admin_topology_template)) knoxsso_topology_template_content = get_config("knoxsso-topology") if knoxsso_topology_template_content: File(os.path.join(params.knox_conf_dir, "topologies", "knoxsso.xml"), mode=0600, group=params.knox_group, owner=params.knox_user, content=InlineTemplate(params.knoxsso_topology_template)) if params.security_enabled: TemplateConfig(format("{knox_conf_dir}/krb5JAASLogin.conf"), owner=params.knox_user, template_tag=None) generate_logfeeder_input_config( 'knox', Template("input.config-knox.json.j2", extra_imports=[default])) cmd = format( '{knox_client_bin} create-master --master {knox_master_secret!p}') master_secret_exist = as_user(format('test -f {knox_master_secret_path}'), params.knox_user) Execute( cmd, user=params.knox_user, environment={'JAVA_HOME': params.java_home}, not_if=master_secret_exist, ) cmd = format( '{knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}') cert_store_exist = as_user(format('test -f {knox_cert_store_path}'), params.knox_user) Execute( cmd, user=params.knox_user, environment={'JAVA_HOME': params.java_home}, not_if=cert_store_exist, )