def install_windows_msi(url_base, save_dir, save_files, hadoop_user, hadoop_password, stack_version): global _working_dir _working_dir = save_dir save_dir = os.path.abspath(save_dir) msi_save_dir = save_dir # system wide lock to prevent simultaneous installations(when first task failed on timeout) install_lock = SystemWideLock("Global\\hdp_msi_lock") try: # try to acquire lock if not install_lock.lock(): Logger.info("Some other task currently installing hdp.msi, waiting for 10 min for finish") if not install_lock.lock(600000): raise Fail("Timeout on acquiring lock") if _validate_msi_install(): Logger.info("hdp.msi already installed") return stack_version_formatted = format_stack_version(stack_version) hdp_22_specific_props = '' if stack_version_formatted != "" and compare_versions(stack_version_formatted, '2.2') >= 0: hdp_22_specific_props = hdp_22.format(data_dir=data_dir) # MSIs cannot be larger than 2GB. HDPWIN 2.3 needed split in order to accommodate this limitation msi_file = '' for save_file in save_files: if save_file.lower().endswith(".msi"): msi_file = save_file file_url = urlparse.urljoin(url_base, save_file) try: download_file(file_url, os.path.join(msi_save_dir, save_file)) except: raise Fail("Failed to download {url}".format(url=file_url)) File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(log_dir=log_dir, data_dir=data_dir, local_host=local_host, db_flavor=db_flavor, hdp_22_specific_props=hdp_22_specific_props)) # install msi msi_path = os_utils.quote_path(os.path.join(save_dir, msi_file)) log_path = os_utils.quote_path(os.path.join(save_dir, msi_file[:-3] + "log")) layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt")) hadoop_password_arg = os_utils.quote_path(hadoop_password) Execute( INSTALL_MSI_CMD.format(msi_path=msi_path, log_path=log_path, layout_path=layout_path, hadoop_user=hadoop_user, hadoop_password_arg=hadoop_password_arg)) reload_windows_env() # create additional services manually due to hdp.msi limitaitons _ensure_services_created(hadoop_user, hadoop_password) _create_symlinks(stack_version) # finalizing install _write_marker() _validate_msi_install() finally: install_lock.unlock()
def install_windows_msi(msi_url, save_dir, save_file, hadoop_password, stack_version): global _working_dir _working_dir = save_dir save_dir = os.path.abspath(save_dir) msi_save_dir = save_dir # system wide lock to prevent simultaneous installations(when first task failed on timeout) install_lock = SystemWideLock("hdp_msi_lock") try: # try to acquire lock if not install_lock.lock(): Logger.info("Some other task currently installing hdp.msi, waiting for 10 min for finish") if not install_lock.lock(600000): raise Fail("Timeout on acquiring lock") if _validate_msi_install(): Logger.info("hdp.msi already installed") return hdp_stack_version = format_hdp_stack_version(stack_version) hdp_22_specific_props = '' if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: hdp_22_specific_props = hdp_22.format(hdp_data_dir=hdp_data_dir) # install msi download_file(msi_url, os.path.join(msi_save_dir, save_file)) File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(hdp_log_dir=hdp_log_dir, hdp_data_dir=hdp_data_dir, local_host=local_host, db_flavor=db_flavor, hdp_22_specific_props=hdp_22_specific_props)) hdp_msi_path = os_utils.quote_path(os.path.join(save_dir, "hdp.msi")) hdp_log_path = os_utils.quote_path(os.path.join(save_dir, "hdp.log")) hdp_layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt")) hadoop_password_arg = os_utils.quote_path(hadoop_password) Execute( INSTALL_MSI_CMD.format(hdp_msi_path=hdp_msi_path, hdp_log_path=hdp_log_path, hdp_layout_path=hdp_layout_path, hadoop_password_arg=hadoop_password_arg)) reload_windows_env() # create additional services manually due to hdp.msi limitaitons _ensure_services_created(hadoop_password) _create_symlinks(stack_version) # finalizing install _write_marker() _validate_msi_install() finally: install_lock.unlock()
def service_check(self, env): import params env.set_params(params) yarn_exe = os_utils.quote_path(os.path.join(params.yarn_home, "bin", "yarn.cmd")) run_yarn_check_cmd = "cmd /C %s node -list" % yarn_exe component_type = "rm" if params.hadoop_ssl_enabled: component_address = params.rm_webui_https_address else: component_address = params.rm_webui_address # temp_dir = os.path.abspath(os.path.join(params.hadoop_home, os.pardir)), "/tmp" temp_dir = os.path.join(os.path.dirname(params.hadoop_home), "temp") validateStatusFileName = "validateYarnComponentStatusWindows.py" validateStatusFilePath = os.path.join(temp_dir, validateStatusFileName) python_executable = sys.executable validateStatusCmd = "%s %s %s -p %s -s %s" % ( python_executable, validateStatusFilePath, component_type, component_address, params.hadoop_ssl_enabled, ) if params.security_enabled: kinit_cmd = "%s -kt %s %s;" % (params.kinit_path_local, params.smoke_user_keytab, params.smokeuser) smoke_cmd = kinit_cmd + " " + validateStatusCmd else: smoke_cmd = validateStatusCmd File(validateStatusFilePath, content=StaticFile(validateStatusFileName)) Execute(smoke_cmd, tries=3, try_sleep=5, logoutput=True) Execute(run_yarn_check_cmd, logoutput=True)
def service_check(self, env): import params env.set_params(params) yarn_exe = os_utils.quote_path( os.path.join(params.yarn_home, "bin", "yarn.cmd")) run_yarn_check_cmd = "cmd /C %s node -list" % yarn_exe component_type = 'rm' if params.hadoop_ssl_enabled: component_address = params.rm_webui_https_address else: component_address = params.rm_webui_address #temp_dir = os.path.abspath(os.path.join(params.hadoop_home, os.pardir)), "/tmp" temp_dir = os.path.join(os.path.dirname(params.hadoop_home), "temp") validateStatusFileName = "validateYarnComponentStatusWindows.py" validateStatusFilePath = os.path.join(temp_dir, validateStatusFileName) python_executable = sys.executable validateStatusCmd = "%s %s %s -p %s -s %s" % ( python_executable, validateStatusFilePath, component_type, component_address, params.hadoop_ssl_enabled) if params.security_enabled: kinit_cmd = "%s -kt %s %s;" % (params.kinit_path_local, params.smoke_user_keytab, params.smokeuser) smoke_cmd = kinit_cmd + ' ' + validateStatusCmd else: smoke_cmd = validateStatusCmd File(validateStatusFilePath, content=StaticFile(validateStatusFileName)) Execute(smoke_cmd, tries=3, try_sleep=5, logoutput=True) Execute(run_yarn_check_cmd, logoutput=True)
def execute_db_connection_check(self, config, tmp_dir): print "DB connection check started." # initialize needed data ambari_server_hostname = config['commandParams']['ambari_server_host'] check_db_connection_jar_name = "DBConnectionVerification.jar" jdk_location = config['commandParams']['jdk_location'] java64_home = config['commandParams']['java_home'] db_name = config['commandParams']['db_name'] if db_name == DB_MYSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL jdbc_driver = JDBC_DRIVER_MYSQL jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL elif db_name == DB_ORACLE: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE jdbc_driver = JDBC_DRIVER_ORACLE jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE elif db_name == DB_POSTGRESQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL jdbc_driver = JDBC_DRIVER_POSTGRESQL jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL elif db_name == DB_MSSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MSSQL jdbc_driver = JDBC_DRIVER_MSSQL jdbc_name = JDBC_DRIVER_SYMLINK_MSSQL db_connection_url = config['commandParams']['db_connection_url'] user_name = config['commandParams']['user_name'] user_passwd = config['commandParams']['user_passwd'] agent_cache_dir = os.path.abspath(config["hostLevelParams"]["agentCacheDir"]) check_db_connection_url = jdk_location + check_db_connection_jar_name jdbc_path = os.path.join(agent_cache_dir, jdbc_name) check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name) java_bin = "java" class_path_delimiter = ":" if OSCheck.is_windows_family(): java_bin = "java.exe" class_path_delimiter = ";" java_exec = os.path.join(java64_home, "bin",java_bin) if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \ "\n" print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output environment = { "no_proxy": format("{ambari_server_hostname}") } # download and install java if it doesn't exists if not os.path.isfile(java_exec): jdk_name = config['commandParams']['jdk_name'] jdk_url = "{0}/{1}".format(jdk_location, jdk_name) jdk_download_target = os.path.join(agent_cache_dir, jdk_name) java_dir = os.path.dirname(java64_home) try: download_file(jdk_url, jdk_download_target) except Exception, e: message = "Error downloading JDK from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output if jdk_name.endswith(".bin"): install_cmd = format("mkdir -p {java_dir} ; chmod +x {jdk_download_target}; cd {java_dir} ; echo A | " \ "{jdk_curl_target} -noregister > /dev/null 2>&1") install_path = ["/bin","/usr/bin/"] elif jdk_name.endswith(".gz"): install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_download_target} > /dev/null 2>&1") install_path = ["/bin","/usr/bin/"] elif jdk_name.endswith(".exe"): install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( os_utils.quote_path(jdk_download_target), os_utils.quote_path(java64_home), ) install_path = [java_dir] try: Execute(install_cmd, path = install_path) except Exception, e: message = "Error installing java.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output
def execute_db_connection_check(self, config, tmp_dir): Logger.info("DB connection check started.") # initialize needed data ambari_server_hostname = config['commandParams']['ambari_server_host'] check_db_connection_jar_name = "DBConnectionVerification.jar" jdk_location = config['commandParams']['jdk_location'] java_home = config['commandParams']['java_home'] db_name = config['commandParams']['db_name'] no_jdbc_error_message = None if db_name == DB_MYSQL: jdbc_driver_mysql_name = default( "/hostLevelParams/custom_mysql_jdbc_name", None) if not jdbc_driver_mysql_name: no_jdbc_error_message = "The MySQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mysql --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = jdk_location + jdbc_driver_mysql_name jdbc_driver_class = JDBC_DRIVER_CLASS_MYSQL jdbc_name = jdbc_driver_mysql_name elif db_name == DB_ORACLE: jdbc_driver_oracle_name = default( "/hostLevelParams/custom_oracle_jdbc_name", None) if not jdbc_driver_oracle_name: no_jdbc_error_message = "The Oracle JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=oracle --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = jdk_location + jdbc_driver_oracle_name jdbc_driver_class = JDBC_DRIVER_CLASS_ORACLE jdbc_name = jdbc_driver_oracle_name elif db_name == DB_POSTGRESQL: jdbc_driver_postgres_name = default( "/hostLevelParams/custom_postgres_jdbc_name", None) if not jdbc_driver_postgres_name: no_jdbc_error_message = "The Postgres JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=postgres --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = jdk_location + jdbc_driver_postgres_name jdbc_driver_class = JDBC_DRIVER_CLASS_POSTGRESQL jdbc_name = jdbc_driver_postgres_name elif db_name == DB_MSSQL: jdbc_driver_mssql_name = default( "/hostLevelParams/custom_mssql_jdbc_name", None) if not jdbc_driver_mssql_name: no_jdbc_error_message = "The MSSQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mssql --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = jdk_location + jdbc_driver_mssql_name jdbc_driver_class = JDBC_DRIVER_CLASS_MSSQL jdbc_name = jdbc_driver_mssql_name elif db_name == DB_SQLA: jdbc_driver_sqla_name = default( "/hostLevelParams/custom_sqlanywhere_jdbc_name", None) if not jdbc_driver_sqla_name: no_jdbc_error_message = "The SQLAnywhere JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=sqlanywhere --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = jdk_location + jdbc_driver_sqla_name jdbc_driver_class = JDBC_DRIVER_CLASS_SQLA jdbc_name = jdbc_driver_sqla_name else: no_jdbc_error_message = format( "'{db_name}' database type not supported.") if no_jdbc_error_message: Logger.warning(no_jdbc_error_message) db_connection_check_structured_output = { "exit_code": 1, "message": no_jdbc_error_message } return db_connection_check_structured_output db_connection_url = config['commandParams']['db_connection_url'] user_name = config['commandParams']['user_name'] user_passwd = config['commandParams']['user_passwd'] agent_cache_dir = os.path.abspath( config["hostLevelParams"]["agentCacheDir"]) check_db_connection_url = jdk_location + check_db_connection_jar_name jdbc_path = os.path.join(agent_cache_dir, jdbc_name) class_path_delimiter = ":" if db_name == DB_SQLA: jdbc_jar_path = agent_cache_dir + JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE java_library_path = agent_cache_dir + JARS_PATH_IN_ARCHIVE_SQLA + class_path_delimiter + agent_cache_dir + \ LIBS_PATH_IN_ARCHIVE_SQLA else: jdbc_jar_path = jdbc_path java_library_path = agent_cache_dir check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name) java_bin = "java" if OSCheck.is_windows_family(): java_bin = "java.exe" class_path_delimiter = ";" java_exec = os.path.join(java_home, "bin", java_bin) if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \ "\n" Logger.warning(message) db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output environment = {"no_proxy": format("{ambari_server_hostname}")} # download and install java if it doesn't exists if not os.path.isfile(java_exec): jdk_name = config['commandParams']['jdk_name'] jdk_url = "{0}/{1}".format(jdk_location, jdk_name) jdk_download_target = os.path.join(agent_cache_dir, jdk_name) java_dir = os.path.dirname(java_home) try: download_file(jdk_url, jdk_download_target) except Exception, e: message = "Error downloading JDK from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) Logger.exception(message) db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output if jdk_name.endswith(".exe"): install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home), ) install_path = [java_dir] try: Execute(install_cmd, path=install_path) except Exception, e: message = "Error installing java.\n" + str(e) Logger.exception(message) db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output
def execute_db_connection_check(self, config, tmp_dir): print "DB connection check started." # initialize needed data ambari_server_hostname = config['commandParams']['ambari_server_host'] check_db_connection_jar_name = "DBConnectionVerification.jar" jdk_location = config['commandParams']['jdk_location'] java_home = config['commandParams']['java_home'] db_name = config['commandParams']['db_name'] if db_name == DB_MYSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL jdbc_driver_class = JDBC_DRIVER_CLASS_MYSQL jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL elif db_name == DB_ORACLE: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE jdbc_driver_class = JDBC_DRIVER_CLASS_ORACLE jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE elif db_name == DB_POSTGRESQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL jdbc_driver_class = JDBC_DRIVER_CLASS_POSTGRESQL jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL elif db_name == DB_MSSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MSSQL jdbc_driver_class = JDBC_DRIVER_CLASS_MSSQL jdbc_name = JDBC_DRIVER_SYMLINK_MSSQL elif db_name == DB_SQLA: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_SQLA jdbc_driver_class = JDBC_DRIVER_CLASS_SQLA jdbc_name = JDBC_DRIVER_SYMLINK_SQLA db_connection_url = config['commandParams']['db_connection_url'] user_name = config['commandParams']['user_name'] user_passwd = config['commandParams']['user_passwd'] agent_cache_dir = os.path.abspath(config["hostLevelParams"]["agentCacheDir"]) check_db_connection_url = jdk_location + check_db_connection_jar_name jdbc_path = os.path.join(agent_cache_dir, jdbc_name) class_path_delimiter = ":" if db_name == DB_SQLA: jdbc_jar_path = agent_cache_dir + JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE java_library_path = agent_cache_dir + JARS_PATH_IN_ARCHIVE_SQLA + class_path_delimiter + agent_cache_dir + \ LIBS_PATH_IN_ARCHIVE_SQLA else: jdbc_jar_path = jdbc_path java_library_path = agent_cache_dir check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name) java_bin = "java" if OSCheck.is_windows_family(): java_bin = "java.exe" class_path_delimiter = ";" java_exec = os.path.join(java_home, "bin",java_bin) if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \ "\n" print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output environment = { "no_proxy": format("{ambari_server_hostname}") } # download and install java if it doesn't exists if not os.path.isfile(java_exec): jdk_name = config['commandParams']['jdk_name'] jdk_url = "{0}/{1}".format(jdk_location, jdk_name) jdk_download_target = os.path.join(agent_cache_dir, jdk_name) java_dir = os.path.dirname(java_home) try: download_file(jdk_url, jdk_download_target) except Exception, e: message = "Error downloading JDK from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output if jdk_name.endswith(".exe"): install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home), ) install_path = [java_dir] try: Execute(install_cmd, path = install_path) except Exception, e: message = "Error installing java.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output
def install_windows_msi(url_base, save_dir, save_files, hadoop_user, hadoop_password, stack_version): global _working_dir _working_dir = save_dir save_dir = os.path.abspath(save_dir) msi_save_dir = save_dir # system wide lock to prevent simultaneous installations(when first task failed on timeout) install_lock = SystemWideLock("Global\\hdp_msi_lock") try: # try to acquire lock if not install_lock.lock(): Logger.info( "Some other task currently installing hdp.msi, waiting for 10 min for finish" ) if not install_lock.lock(600000): raise Fail("Timeout on acquiring lock") if _validate_msi_install(): Logger.info("hdp.msi already installed") return hdp_stack_version = format_hdp_stack_version(stack_version) hdp_22_specific_props = '' if hdp_stack_version != "" and compare_versions( hdp_stack_version, '2.2') >= 0: hdp_22_specific_props = hdp_22.format(hdp_data_dir=hdp_data_dir) # MSIs cannot be larger than 2GB. HDPWIN 2.3 needed split in order to accommodate this limitation hdp_msi_file = '' for save_file in save_files: if save_file.lower().endswith(".msi"): hdp_msi_file = save_file file_url = urlparse.urljoin(url_base, save_file) try: download_file(file_url, os.path.join(msi_save_dir, save_file)) except: raise Fail("Failed to download {url}".format(url=file_url)) File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format( hdp_log_dir=hdp_log_dir, hdp_data_dir=hdp_data_dir, local_host=local_host, db_flavor=db_flavor, hdp_22_specific_props=hdp_22_specific_props)) # install msi hdp_msi_path = os_utils.quote_path(os.path.join( save_dir, hdp_msi_file)) hdp_log_path = os_utils.quote_path( os.path.join(save_dir, hdp_msi_file[:-3] + "log")) hdp_layout_path = os_utils.quote_path( os.path.join(save_dir, "properties.txt")) hadoop_password_arg = os_utils.quote_path(hadoop_password) Execute( INSTALL_MSI_CMD.format(hdp_msi_path=hdp_msi_path, hdp_log_path=hdp_log_path, hdp_layout_path=hdp_layout_path, hadoop_user=hadoop_user, hadoop_password_arg=hadoop_password_arg)) reload_windows_env() # create additional services manually due to hdp.msi limitaitons _ensure_services_created(hadoop_user, hadoop_password) _create_symlinks(stack_version) # finalizing install _write_marker() _validate_msi_install() finally: install_lock.unlock()