def create_metastore_schema(): import params create_schema_cmd = format( "export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose") check_schema_created_cmd = as_user( format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) quoted_hive_metastore_user_passwd = quote_bash_args( quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[ 1:-1] Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr( check_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Execute(create_schema_cmd, not_if=check_schema_created_cmd, user=params.hive_user)
def create_metastore_schema(): import params create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose") check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. # Fixing it with the hack below: quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1] Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Execute(create_schema_cmd, not_if = check_schema_created_cmd, user = params.hive_user )
def test_attribute_environment_non_root(self, popen_mock, select_mock, os_read_mock): expected_user = '******' subproc_mock = MagicMock() subproc_mock.wait.return_value = MagicMock() subproc_mock.stdout = MagicMock() subproc_mock.returncode = 0 popen_mock.return_value = subproc_mock select_mock.return_value = ([subproc_mock.stdout], None, None) os_read_mock.return_value = None with Environment("/") as env: execute_resource = Execute('echo "1"', user=expected_user, environment={ 'JAVA_HOME': '/test/java/home', 'PATH': "/bin" }) expected_command = [ '/bin/bash', '--login', '--noprofile', '-c', 'ambari-sudo.sh su test_user -l -s /bin/bash -c ' + quote_bash_args('export PATH=' + quote_bash_args(os.environ['PATH'] + ':/bin') + ' JAVA_HOME=/test/java/home ; echo "1"') ] self.assertEqual(popen_mock.call_args_list[0][0][0], expected_command)
def create_hive_metastore_schema(): import params SYS_DB_CREATED_FILE = "/etc/hive/sys.db.created" if os.path.isfile(SYS_DB_CREATED_FILE): Logger.info("Sys DB is already created") return create_hive_schema_cmd = format( "export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -initSchema " "-dbType hive " "-metaDbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} " "-verbose") check_hive_schema_created_cmd = as_user( format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -info " "-dbType hive " "-metaDbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} " "-verbose"), params.hive_user) # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. # Fixing it with the hack below: quoted_hive_metastore_user_passwd = quote_bash_args( quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd.startswith("'") and quoted_hive_metastore_user_passwd.endswith("'") \ or quoted_hive_metastore_user_passwd.startswith('"') and quoted_hive_metastore_user_passwd.endswith('"'): quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[ 1:-1] Logger.sensitive_strings[repr(create_hive_schema_cmd)] = repr( create_hive_schema_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Logger.sensitive_strings[repr(check_hive_schema_created_cmd)] = repr( check_hive_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) try: if params.security_enabled: hive_kinit_cmd = format( "{kinit_path_local} -kt {hive_server2_keytab} {hive_principal}; " ) Execute(hive_kinit_cmd, user=params.hive_user) Execute(create_hive_schema_cmd, not_if=check_hive_schema_created_cmd, user=params.hive_user) Execute("touch " + SYS_DB_CREATED_FILE, user="******") Logger.info("Sys DB is set up") except: Logger.error("Could not create Sys DB.") Logger.error(traceback.format_exc())
def setup_metastore(): import params if params.hive_metastore_site_supported: hivemetastore_site_config = get_config("hivemetastore-site") if hivemetastore_site_config: XmlConfig("hivemetastore-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations'] ['hivemetastore-site'], configuration_attributes=params. config['configuration_attributes']['hivemetastore-site'], owner=params.hive_user, group=params.user_group, mode=0600) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hivemetastore.properties.j2"), mode=0600) File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) if params.init_metastore_schema: create_schema_cmd = format( "export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose") check_schema_created_cmd = as_user( format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. # Fixing it with the hack below: quoted_hive_metastore_user_passwd = quote_bash_args( quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[ 1:-1] Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr( check_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Execute(create_schema_cmd, not_if=check_schema_created_cmd, user=params.hive_user)
def action_run(self): kinit__path_local = self.resource.kinit_path_local keytab = self.resource.keytab conf_dir = self.resource.conf_dir command = self.resource.command principal = self.resource.principal if isinstance(command, (list, tuple)): command = ' '.join(quote_bash_args(x) for x in command) with Environment.get_instance_copy() as env: if self.resource.security_enabled and not self.resource.kinit_override: Execute(format("{kinit__path_local} -kt {keytab} {principal}"), path=['/bin'], user=self.resource.user) Execute( format("hadoop --config {conf_dir} {command}"), user=self.resource.user, tries=self.resource.tries, try_sleep=self.resource.try_sleep, logoutput=self.resource.logoutput, path=self.resource.bin_dir, environment=self.resource.environment, )
def get_full_ambari_classpath_escaped_for_shell(self): class_path = self._get_ambari_classpath() # When classpath is required we should also set native libs os env variable # This is required for some jdbc (ex. sqlAnywhere) self.set_native_libs_path() return quote_bash_args(class_path)
def _convert_field(self, value, conversion, is_protected): if conversion == 'e': return quote_bash_args(unicode(value)) elif conversion == 'h': return utils.PASSWORDS_HIDE_STRING if is_protected else value elif conversion == 'p': return utils.PASSWORDS_HIDE_STRING if is_protected else self._convert_field(value, 'e', is_protected) return super(ConfigurationFormatter, self).convert_field(value, conversion)
def _convert_field(self, value, conversion, is_protected): if conversion == 'e': return quote_bash_args(unicode(value)) elif conversion == 'h': return "[PROTECTED]" if is_protected else value elif conversion == 'p': return "[PROTECTED]" if is_protected else self._convert_field(value, 'e', is_protected) return super(ConfigurationFormatter, self).convert_field(value, conversion)
def getRunSetupWithoutPasswordCommand(self, expected_hostname): setupFile=self.getRemoteName(self.SETUP_SCRIPT_FILENAME) passphrase=os.environ[AMBARI_PASSPHRASE_VAR_NAME] server=self.shared_state.ambari_server user_run_as = self.shared_state.user_run_as version=self.getAmbariVersion() port=self.getAmbariPort() return "sudo python " + str(setupFile) + " " + str(expected_hostname) + \ " " + str(passphrase) + " " + str(server)+ " " + quote_bash_args(str(user_run_as)) + " " + str(version) + \ " " + str(port)
def action_run(self): conf_dir = self.resource.conf_dir command = self.resource.command if isinstance(command, (list, tuple)): command = ' '.join(quote_bash_args(x) for x in command) Execute(format("hdfs --config {conf_dir} {command}"), user=self.resource.user, tries=self.resource.tries, try_sleep=self.resource.try_sleep, logoutput=self.resource.logoutput, path=self.resource.bin_dir, environment=self.resource.environment, )
def createTargetDir(self): # Creating target dir self.host_log.write("==========================\n") self.host_log.write("Creating target directory...") params = self.shared_state user = params.user command = "sudo mkdir -p {0} ; sudo chown -R {1} {0}".format(self.TEMP_FOLDER,quote_bash_args(params.user)) ssh = SSH(params.user, params.sshkey_file, self.host, command, params.bootdir, self.host_log) retcode = ssh.run() self.host_log.write("\n") return retcode
def action_run(self): conf_dir = self.resource.conf_dir command = self.resource.command if isinstance(command, (list, tuple)): command = ' '.join(quote_bash_args(x) for x in command) Execute (format("hadoop --config {conf_dir} {command}"), user = self.resource.user, tries = self.resource.tries, try_sleep = self.resource.try_sleep, logoutput = self.resource.logoutput, path = self.resource.bin_dir, environment = self.resource.environment, )
def createTargetDir(self): # Creating target dir self.host_log.write("==========================\n") self.host_log.write("Creating target directory...") params = self.shared_state user = params.user command = "SUDO=$([ \"$EUID\" -eq 0 ] && echo || echo sudo) ; $SUDO mkdir -p {0} ; $SUDO chown -R {1} {0} ; $SUDO chmod 755 {3} ; $SUDO chmod 755 {2} ; $SUDO chmod 1777 {0}".format( self.TEMP_FOLDER, quote_bash_args(params.user), DEFAULT_AGENT_DATA_FOLDER, DEFAULT_AGENT_LIB_FOLDER) ssh = SSH(params.user, params.sshPort, params.sshkey_file, self.host, command, params.bootdir, self.host_log) retcode = ssh.run() self.host_log.write("\n") return retcode
def action_run(self): kinit__path_local = self.resource.kinit_path_local keytab = self.resource.keytab conf_dir = self.resource.conf_dir command = self.resource.command principal = self.resource.principal if isinstance(command, (list, tuple)): command = " ".join(quote_bash_args(x) for x in command) with Environment.get_instance_copy() as env: if self.resource.security_enabled and not self.resource.kinit_override: Execute(format("{kinit__path_local} -kt {keytab} {principal}"), path=["/bin"], user=self.resource.user) Execute( format("hadoop --config {conf_dir} {command}"), user=self.resource.user, tries=self.resource.tries, try_sleep=self.resource.try_sleep, logoutput=self.resource.logoutput, path=self.resource.bin_dir, environment=self.resource.environment, )
def check_thrift_port_sasl(address, port, hive_auth="NOSASL", key=None, kinitcmd=None, smokeuser='******', hive_user='******', transport_mode="binary", http_endpoint="cliservice", ssl=False, ssl_keystore=None, ssl_password=None, check_command_timeout=30, ldap_username="", ldap_password=""): """ Hive thrift SASL port check """ # check params to be correctly passed, if not - try to cast them if isinstance(port, str): port = int(port) if isinstance(ssl, str): ssl = bool(ssl) # to pass as beeline argument ssl_str = str(ssl).lower() beeline_url = [ 'jdbc:hive2://{address}:{port}/', "transportMode={transport_mode}" ] # append url according to used transport if transport_mode == "http": beeline_url.append('httpPath={http_endpoint}') # append url according to used auth if hive_auth == "NOSASL": beeline_url.append('auth=noSasl') credential_str = "-n {hive_user}" # append username and password for LDAP if hive_auth == "LDAP": # password might contain special characters that need to be escaped quoted_ldap_password = quote_bash_args(ldap_password) credential_str = "-n {ldap_username} -p {quoted_ldap_password!p}" # append url according to ssl configuration if ssl and ssl_keystore is not None and ssl_password is not None: beeline_url.extend([ 'ssl={ssl_str}', 'sslTrustStore={ssl_keystore}', 'trustStorePassword={ssl_password!p}' ]) # append url according to principal and execute kinit if kinitcmd and hive_auth != "LDAP": beeline_url.append('principal={key}') # prevent concurrent kinit kinit_lock = global_lock.get_lock(global_lock.LOCK_TYPE_KERBEROS) kinit_lock.acquire() try: Execute(kinitcmd, user=smokeuser) finally: kinit_lock.release() # -n the user to connect as (ignored when using the hive principal in the URL, can be different from the user running the beeline command) # -e ';' executes a SQL commmand of NOOP cmd = ("! (beeline -u '%s' %s -e ';' 2>&1 | awk '{print}' | grep -vz -i " + \ "-e 'Connected to:' -e 'Transaction isolation:' -e 'inactive HS2 instance; use service discovery')") % \ (format(";".join(beeline_url)), format(credential_str)) Execute( cmd, user=smokeuser, path=["/bin/", "/usr/bin/", "/usr/lib/hive/bin/", "/usr/sbin/"], timeout=check_command_timeout, timeout_kill_strategy=TerminateStrategy.KILL_PROCESS_TREE, )
def hive(name=None): import params if name == 'hiveserver2': # copy tarball to HDFS feature not supported if not (params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major)): params.HdfsResource(params.webhcat_apps_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=0755 ) # Create webhcat dirs. if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hcat_user, mode=params.hcat_hdfs_user_mode ) params.HdfsResource(params.webhcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.webhcat_hdfs_user_mode ) # ****** Begin Copy Tarballs ****** # ********************************* # if copy tarball to HDFS feature supported copy mapreduce.tar.gz and tez.tar.gz to HDFS if params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major): copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. # This can use a different source and dest location to account copy_to_hdfs("pig", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.pig_tar_source, custom_dest_file=params.pig_tar_dest_file, host_sys_prepped=params.host_sys_prepped) copy_to_hdfs("hive", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.hive_tar_source, custom_dest_file=params.hive_tar_dest_file, host_sys_prepped=params.host_sys_prepped) wildcard_tarballs = ["sqoop", "hadoop_streaming"] for tarball_name in wildcard_tarballs: source_file_pattern = eval("params." + tarball_name + "_tar_source") dest_dir = eval("params." + tarball_name + "_tar_dest_dir") if source_file_pattern is None or dest_dir is None: continue source_files = glob.glob(source_file_pattern) if "*" in source_file_pattern else [source_file_pattern] for source_file in source_files: src_filename = os.path.basename(source_file) dest_file = os.path.join(dest_dir, src_filename) copy_to_hdfs(tarball_name, params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=source_file, custom_dest_file=dest_file, host_sys_prepped=params.host_sys_prepped) # ******* End Copy Tarballs ******* # ********************************* # if warehouse directory is in DFS if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(params.default_fs).scheme: # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=0777 ) else: Logger.info(format("Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS.")) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode ) if not is_empty(params.hive_exec_scratchdir) and not urlparse(params.hive_exec_scratchdir).path.startswith("/tmp"): params.HdfsResource(params.hive_exec_scratchdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.hdfs_user, mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir params.HdfsResource(None, action="execute") Directory(params.hive_etc_dir_prefix, mode=0755 ) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) XmlConfig("hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes']['hive-site'], owner=params.hive_user, group=params.user_group, mode=0644) setup_atlas_hive() if params.hive_specific_configs_supported and name == 'hiveserver2': XmlConfig("hiveserver2-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hiveserver2-site'], configuration_attributes=params.config['configuration_attributes']['hiveserver2-site'], owner=params.hive_user, group=params.user_group, mode=0644) if params.hive_metastore_site_supported and name == 'metastore': XmlConfig("hivemetastore-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hivemetastore-site'], configuration_attributes=params.config['configuration_attributes']['hivemetastore-site'], owner=params.hive_user, group=params.user_group, mode=0644) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, content=InlineTemplate(params.hive_env_sh_template) ) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents = True, owner='root', group='root' ) File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2") ) if name == 'metastore' or name == 'hiveserver2': if params.target_hive is not None and not os.path.exists(params.target_hive): jdbc_connector(params.target_hive) if params.target_hive2 is not None and not os.path.exists(params.target_hive2): jdbc_connector(params.target_hive2) File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), mode = 0644, ) if name == 'metastore': File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hivemetastore.properties.j2") ) File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh') ) if params.init_metastore_schema: create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose") check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. # Fixing it with the hack below: quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1] Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Execute(create_schema_cmd, not_if = check_schema_created_cmd, user = params.hive_user ) elif name == 'hiveserver2': File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}')) ) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hiveserver2.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hiveserver2.properties.j2") ) if name != "client": Directory(params.hive_pid_dir, create_parents = True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_log_dir, create_parents = True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_var_lib, create_parents = True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755)
def test_attribute_environment_non_root(self, popen_mock, select_mock, os_read_mock): expected_user = '******' subproc_mock = MagicMock() subproc_mock.wait.return_value = MagicMock() subproc_mock.stdout = MagicMock() subproc_mock.returncode = 0 popen_mock.return_value = subproc_mock select_mock.return_value = ([subproc_mock.stdout], None, None) os_read_mock.return_value = None with Environment("/") as env: execute_resource = Execute('echo "1"', user=expected_user, environment={'JAVA_HOME': '/test/java/home', 'PATH': "/bin"} ) expected_command = ['/bin/bash', '--login', '--noprofile', '-c', 'ambari-sudo.sh su test_user -l -s /bin/bash -c ' + quote_bash_args('export PATH=' + quote_bash_args(os.environ['PATH'] + ':/bin') + ' JAVA_HOME=/test/java/home ; echo "1"')] self.assertEqual(popen_mock.call_args_list[0][0][0], expected_command)
def hive(name=None): import params if name == 'hiveserver2': # HDP 2.1.* or lower if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, "2.2.0.0") < 0: params.HdfsResource(params.webhcat_apps_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=0755 ) # Create webhcat dirs. if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hcat_user, mode=params.hcat_hdfs_user_mode ) params.HdfsResource(params.webhcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.webhcat_hdfs_user_mode ) # ****** Begin Copy Tarballs ****** # ********************************* # HDP 2.2 or higher, copy mapreduce.tar.gz to HDFS if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, '2.2') >= 0: copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. # This can use a different source and dest location to account for both HDP 2.1 and 2.2 copy_to_hdfs("pig", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.pig_tar_source, custom_dest_file=params.pig_tar_dest_file, host_sys_prepped=params.host_sys_prepped) copy_to_hdfs("hive", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.hive_tar_source, custom_dest_file=params.hive_tar_dest_file, host_sys_prepped=params.host_sys_prepped) wildcard_tarballs = ["sqoop", "hadoop_streaming"] for tarball_name in wildcard_tarballs: source_file_pattern = eval("params." + tarball_name + "_tar_source") dest_dir = eval("params." + tarball_name + "_tar_dest_dir") if source_file_pattern is None or dest_dir is None: continue source_files = glob.glob(source_file_pattern) if "*" in source_file_pattern else [source_file_pattern] for source_file in source_files: src_filename = os.path.basename(source_file) dest_file = os.path.join(dest_dir, src_filename) copy_to_hdfs(tarball_name, params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=source_file, custom_dest_file=dest_file, host_sys_prepped=params.host_sys_prepped) # ******* End Copy Tarballs ******* # ********************************* # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=0777 ) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode ) if not is_empty(params.hive_exec_scratchdir) and not urlparse(params.hive_exec_scratchdir).path.startswith("/tmp"): params.HdfsResource(params.hive_exec_scratchdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.hdfs_user, mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir params.HdfsResource(None, action="execute") Directory(params.hive_etc_dir_prefix, mode=0755 ) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) XmlConfig("hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes']['hive-site'], owner=params.hive_user, group=params.user_group, mode=0644) setup_atlas_hive() if params.hive_specific_configs_supported and name == 'hiveserver2': XmlConfig("hiveserver2-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hiveserver2-site'], configuration_attributes=params.config['configuration_attributes']['hiveserver2-site'], owner=params.hive_user, group=params.user_group, mode=0644) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, content=InlineTemplate(params.hive_env_sh_template) ) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, recursive=True, owner='root', group='root' ) File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2") ) if (name == 'metastore' or name == 'hiveserver2') and not os.path.exists(params.target): jdbc_connector() File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), mode = 0644, ) if name == 'metastore': File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh') ) if params.init_metastore_schema: create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}") check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}"), params.hive_user) # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. # Fixing it with the hack below: quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1] Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Execute(create_schema_cmd, not_if = check_schema_created_cmd, user = params.hive_user ) elif name == 'hiveserver2': File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}')) ) if name != "client": crt_directory(params.hive_pid_dir) crt_directory(params.hive_log_dir) crt_directory(params.hive_var_lib)
def hive(name=None): import params if name == 'hiveserver2': # BigInsights 4.0.* or lower if params.stack_version != "" and compare_versions( params.stack_version, "4.1.0.0") < 0: params.HdfsResource(params.webhcat_apps_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=0755) # Create webhcat dirs. if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hcat_user, mode=params.hcat_hdfs_user_mode) params.HdfsResource(params.webhcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.webhcat_hdfs_user_mode) # ****** Begin Copy Tarballs ****** # ********************************* if params.stack_version != "" and compare_versions( params.stack_version, '4.0.0.0') >= 0: copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, skip=params.host_sys_prepped) # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. copy_to_hdfs("pig", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.pig_tar_source, custom_dest_file=params.pig_tar_dest_file, skip=params.host_sys_prepped) copy_to_hdfs("hive", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.hive_tar_source, custom_dest_file=params.hive_tar_dest_file, skip=params.host_sys_prepped) wildcard_tarballs = ["sqoop", "hadoop_streaming"] for tarball_name in wildcard_tarballs: source_file_pattern = eval("params." + tarball_name + "_tar_source") dest_dir = eval("params." + tarball_name + "_tar_dest_dir") if source_file_pattern is None or dest_dir is None: continue source_files = glob.glob( source_file_pattern) if "*" in source_file_pattern else [ source_file_pattern ] for source_file in source_files: src_filename = os.path.basename(source_file) dest_file = os.path.join(dest_dir, src_filename) copy_to_hdfs(tarball_name, params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=source_file, custom_dest_file=dest_file, skip=params.host_sys_prepped) # ******* End Copy Tarballs ******* # ********************************* params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=0770) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode) # hive.exec.scratchdir should be created via hive_user # otherwise, hive.start.cleanup.scratchdir won't work, because ambari services always started by hive_user if not is_empty(params.hive_exec_scratchdir): params.HdfsResource( params.hive_exec_scratchdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.hdfs_user, mode=0777 ) # Hive expects this dir to be writeable by everyone as it is used as a temp dir params.HdfsResource(None, action="execute") Directory(params.hive_etc_dir_prefix, mode=0755) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) hive_site_permission = 0644 if name in ["hiveserver2", "metastore"]: hive_site_permission = 0640 XmlConfig( "hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes'] ['hive-site'], owner=params.hive_user, group=params.user_group, mode=hive_site_permission) if params.hive_specific_configs_supported and name == 'hiveserver2': XmlConfig( "hiveserver2-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hiveserver2-site'], configuration_attributes=params.config['configuration_attributes'] ['hiveserver2-site'], owner=params.hive_user, group=params.user_group, mode=0644) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, content=InlineTemplate(params.hive_env_sh_template)) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2")) if (name == 'metastore' or name == 'hiveserver2') and not os.path.exists(params.hive_jdbc_target): jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar) File( format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}{check_db_connection_jar_name}")), mode=0644, ) if name == 'metastore': File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) if params.init_metastore_schema: create_schema_cmd = format( "export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}") check_schema_created_cmd = as_user( format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}"), params.hive_user) # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. # Fixing it with the hack below: quoted_hive_metastore_user_passwd = quote_bash_args( quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[ 1:-1] Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr( check_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Execute(create_schema_cmd, not_if=check_schema_created_cmd, user=params.hive_user) elif name == 'hiveserver2': File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}'))) if name != "client": crt_directory(params.hive_pid_dir) crt_directory(params.hive_log_dir) crt_directory(params.hive_var_lib)