def actionexecute(self, env): config = Script.get_config() structured_output = {} try: repo_info_json = config['hostLevelParams']['repo_info'] repo_info_dict = json.loads(repo_info_json) for item in repo_info_dict["repositories"]: base_url = item["base_url"] repo_name = item["repo_name"] repo_id = item["repo_id"] repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else repo_ubuntu ubuntu_components = [repo_name] + self.UBUNTU_REPO_COMPONENTS_POSTFIX Repository(repo_id, action = "create", base_url = base_url, mirror_list = None, repo_file_name = repo_name, repo_template = template, components = ubuntu_components, # ubuntu specific ) structured_output["repo_update"] = {"exit_code" : 0, "message": format("Repository files successfully updated!")} except Exception, exception: Logger.logger.exception("ERROR: There was an unexpected error while updating repositories") raise Fail("Failed to update repo files!")
def set_uid(user, user_dirs): """ user_dirs - comma separated directories """ import params File(format("{tmp_dir}/changeUid.sh"), content=StaticFile("changeToSecureUid.sh"), mode=0555) ignore_groupsusers_create_str = str( params.ignore_groupsusers_create).lower() Execute( format("{tmp_dir}/changeUid.sh {user} {user_dirs}"), not_if=format( "(test $(id -u {user}) -gt 1000) || ({ignore_groupsusers_create_str})" ))
def actionexecute(self, env): config = Script.get_config() structured_output = {} try: repo_info = config['repositoryFile'] for item in repo_info["repositories"]: base_url = item["baseUrl"] repo_name = item["repoName"] repo_id = item["repoId"] distribution = item["distribution"] if "distribution" in item else None components = item["components"] if "components" in item else None repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else repo_ubuntu ubuntu_components = [distribution if distribution else repo_name] + \ [components.replace(",", " ") if components else self.UBUNTU_REPO_COMPONENTS_POSTFIX] Repository(repo_id, action = "create", base_url = base_url, mirror_list = None, repo_file_name = repo_name, repo_template = template, components = ubuntu_components, # ubuntu specific ) structured_output["repo_update"] = {"exit_code" : 0, "message": format("Repository files successfully updated!")} except Exception, exception: Logger.logger.exception("ERROR: There was an unexpected error while updating repositories") raise Fail("Failed to update repo files!")
def setup_livy(env, type, upgrade_type=None, action=None): import params Directory([params.livy2_pid_dir, params.livy2_log_dir], owner=params.livy2_user, group=params.user_group, mode=0775, create_parents=True) if type == 'server' and action == 'config': params.HdfsResource(params.livy2_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.livy2_user, mode=0775) params.HdfsResource(None, action="execute") # create livy-env.sh in etc/conf dir File( os.path.join(params.livy2_conf, 'livy-env.sh'), owner=params.livy2_user, group=params.livy2_group, content=InlineTemplate(params.livy2_env_sh), mode=0644, ) # create livy.conf in etc/conf dir PropertiesFile( format("{livy2_conf}/livy.conf"), properties=params.config['configurations']['livy2-conf'], key_value_delimiter=" ", owner=params.livy2_user, group=params.livy2_group, ) # create log4j.properties in etc/conf dir File( os.path.join(params.livy2_conf, 'log4j.properties'), owner=params.livy2_user, group=params.livy2_group, content=params.livy2_log4j_properties, mode=0644, ) # create spark-blacklist.properties in etc/conf dir File( os.path.join(params.livy2_conf, 'spark-blacklist.conf'), owner=params.livy2_user, group=params.livy2_group, content=params.livy2_spark_blacklist_properties, mode=0644, ) Directory( params.livy2_logs_dir, owner=params.livy2_user, group=params.livy2_group, mode=0755, )
def actionexecute(self, env): config = Script.get_config() structured_output = {} cmd = self.get_clearcache_cmd() Logger.info("Clearing repository cache") code, output = shell.call(cmd, sudo = True) if 0 == code: structured_output["clear_repocache"] = {"exit_code" : 0, "message": format("Repository cache successfully cleared!")} else: structured_output["clear_repocache"] = {"exit_code": code, "message": "Failed to clear repository cache! {0}".format(str(output))} self.put_structured_out(structured_output)
def remove_stack_version(self, structured_output, version): # check simlinks not refer to version for remove self.check_no_symlink_to_version(structured_output, version) packages_to_remove = self.get_packages_to_remove(version) for package in packages_to_remove: Package(package, action="remove") self.remove_stack_folder(structured_output, version) structured_output["remove_previous_stacks"] = { "exit_code": 0, "message": format("Stack version {0} successfully removed!".format(version)) } self.put_structured_out(structured_output)
def install_jce_policy(): import params jcePolicyInfo = JcePolicyInfo(params.java_home) if jcePolicyInfo.is_unlimited_key_jce_policy(): major, minor = java_version() Logger.info( "The unlimited key JCE policy is required, and appears to have been installed." ) java_security_dir = format("{java_home}/jre/lib/security") Logger.debug("Removing existing JCE policy JAR files: {0}.".format( java_security_dir)) File(format("{java_security_dir}/US_export_policy.jar"), action="delete") File(format("{java_security_dir}/local_policy.jar"), action="delete") src_url = jce_zip_target = '' if minor == 7: # jdk7 src_url = 'http://yum.example.com/hadoop/UnlimitedJCEPolicyJDK7.zip' jce_zip_target = '/tmp/UnlimitedJCEPolicyJDK7.zip' elif minor == 8: # jdk8 src_url = 'http://yum.example.com/hadoop/jce_policy-8.zip' jce_zip_target = '/tmp/jce_policy-8.zip' if src_url and jce_zip_target: Execute('wget ' + src_url + ' -O ' + jce_zip_target) Logger.debug( "Unzipping the unlimited key JCE policy files from {0} into {1}." .format(jce_zip_target, java_security_dir)) extract_cmd = ("unzip", "-o", "-j", "-q", jce_zip_target, "-d", java_security_dir) Execute( extract_cmd, only_if=format( "test -e {java_security_dir} && test -f {jce_zip_target}"), path=['/bin/', '/usr/bin'], sudo=True)
def actionexecute(self, env): config = Script.get_config() structured_output = {} try: repo_info_json = config['hostLevelParams']['repo_info'] repo_info_dict = json.loads(repo_info_json) for item in repo_info_dict["repositories"]: base_url = item["base_url"] repo_name = item["repo_name"] repo_id = item["repo_id"] repo_rhel_suse = config['configurations']['cluster-env'][ 'repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env'][ 'repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_suse_family( ) or OSCheck.is_redhat_family() else repo_ubuntu ubuntu_components = [repo_name ] + self.UBUNTU_REPO_COMPONENTS_POSTFIX Repository( repo_id, action="create", base_url=base_url, mirror_list=None, repo_file_name=repo_name, repo_template=template, components=ubuntu_components, # ubuntu specific ) structured_output["repo_update"] = { "exit_code": 0, "message": format("Repository files successfully updated!") } except Exception, exception: Logger.logger.exception( "ERROR: There was an unexpected error while updating repositories" ) raise Fail("Failed to update repo files!")
def modify_rack_awareness(): import params hostname = params.config["hostname"].lower() ip = socket.gethostbyname(socket.gethostname()) try: response = urllib2.urlopen(CONST_RACK_INFO_URL + ip) rack_info = response.read() rack_info = json.loads(rack_info) if isinstance(rack_info, list): topology_item = '/default/default/default' else: datacenter = rack_info.get(ip).get('datacenter') or 'default' switch = rack_info.get(ip).get('switch') or 'default' rack = rack_info.get(ip).get('rack') or 'default' topology_item = '/' + datacenter.replace( '/', '').lower() + '/' + switch.replace( '/', '').lower() + '/' + rack.replace('/', '').lower() ambari_server_host = params.config['clusterHostInfo'][ 'ambari_server_host'][0] ambari_server_port = params.config['clusterHostInfo'][ 'ambari_server_port'][0] ambari_server_use_ssl = params.config['clusterHostInfo'][ 'ambari_server_use_ssl'][0] == 'true' ambari_server_protocol = 'https' if ambari_server_use_ssl else 'http' cluster_name = params.config['clusterName'] ambari_server_auth_host_url = format( '{ambari_server_protocol}://{ambari_server_host}:{ambari_server_port}' ) rack_update_url = ambari_server_auth_host_url + '/api/v1/clusters/' + cluster_name + '/hosts' rack_update = '{"RequestInfo":{"context":"Set Rack","query":"Hosts/host_name.in(' + hostname + ')"},"Body":{"Hosts":{"rack_info":"' + topology_item + '"}}}' Execute( "curl -u 'ambari-qa:example.com' -H 'X-Requested-By:ambari' -i " + rack_update_url + " -X PUT -d '" + rack_update + "'") except urllib2.HTTPError as e: print "can not get %s rack_info from cmdb" % ip
def execute_db_connection_check(self, config, tmp_dir): Logger.info("DB connection check started.") # initialize needed data ambari_server_hostname = config['commandParams']['ambari_server_host'] check_db_connection_jar_name = "DBConnectionVerification.jar" jdk_location = config['commandParams']['jdk_location'] java_home = config['commandParams']['java_home'] db_name = config['commandParams']['db_name'] no_jdbc_error_message = None if db_name == DB_MYSQL: jdbc_driver_mysql_name = default( "/hostLevelParams/custom_mysql_jdbc_name", None) if not jdbc_driver_mysql_name: no_jdbc_error_message = "The MySQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mysql --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = jdk_location + jdbc_driver_mysql_name jdbc_driver_class = JDBC_DRIVER_CLASS_MYSQL jdbc_name = jdbc_driver_mysql_name elif db_name == DB_ORACLE: jdbc_driver_oracle_name = default( "/hostLevelParams/custom_oracle_jdbc_name", None) if not jdbc_driver_oracle_name: no_jdbc_error_message = "The Oracle JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=oracle --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = jdk_location + jdbc_driver_oracle_name jdbc_driver_class = JDBC_DRIVER_CLASS_ORACLE jdbc_name = jdbc_driver_oracle_name elif db_name == DB_POSTGRESQL: jdbc_driver_postgres_name = default( "/hostLevelParams/custom_postgres_jdbc_name", None) if not jdbc_driver_postgres_name: no_jdbc_error_message = "The Postgres JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=postgres --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = jdk_location + jdbc_driver_postgres_name jdbc_driver_class = JDBC_DRIVER_CLASS_POSTGRESQL jdbc_name = jdbc_driver_postgres_name elif db_name == DB_MSSQL: jdbc_driver_mssql_name = default( "/hostLevelParams/custom_mssql_jdbc_name", None) if not jdbc_driver_mssql_name: no_jdbc_error_message = "The MSSQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mssql --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = jdk_location + jdbc_driver_mssql_name jdbc_driver_class = JDBC_DRIVER_CLASS_MSSQL jdbc_name = jdbc_driver_mssql_name elif db_name == DB_SQLA: jdbc_driver_sqla_name = default( "/hostLevelParams/custom_sqlanywhere_jdbc_name", None) if not jdbc_driver_sqla_name: no_jdbc_error_message = "The SQLAnywhere JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=sqlanywhere --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = jdk_location + jdbc_driver_sqla_name jdbc_driver_class = JDBC_DRIVER_CLASS_SQLA jdbc_name = jdbc_driver_sqla_name else: no_jdbc_error_message = format( "'{db_name}' database type not supported.") if no_jdbc_error_message: Logger.warning(no_jdbc_error_message) db_connection_check_structured_output = { "exit_code": 1, "message": no_jdbc_error_message } return db_connection_check_structured_output db_connection_url = config['commandParams']['db_connection_url'] user_name = config['commandParams']['user_name'] user_passwd = config['commandParams']['user_passwd'] agent_cache_dir = os.path.abspath( config["hostLevelParams"]["agentCacheDir"]) check_db_connection_url = jdk_location + check_db_connection_jar_name jdbc_path = os.path.join(agent_cache_dir, jdbc_name) class_path_delimiter = ":" if db_name == DB_SQLA: jdbc_jar_path = agent_cache_dir + JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE java_library_path = agent_cache_dir + JARS_PATH_IN_ARCHIVE_SQLA + class_path_delimiter + agent_cache_dir + \ LIBS_PATH_IN_ARCHIVE_SQLA else: jdbc_jar_path = jdbc_path java_library_path = agent_cache_dir check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name) java_bin = "java" if OSCheck.is_windows_family(): java_bin = "java.exe" class_path_delimiter = ";" java_exec = os.path.join(java_home, "bin", java_bin) if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \ "\n" Logger.warning(message) db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output environment = {"no_proxy": format("{ambari_server_hostname}")} # download and install java if it doesn't exists if not os.path.isfile(java_exec): jdk_name = config['commandParams']['jdk_name'] jdk_url = "{0}/{1}".format(jdk_location, jdk_name) jdk_download_target = os.path.join(agent_cache_dir, jdk_name) java_dir = os.path.dirname(java_home) try: download_file(jdk_url, jdk_download_target) except Exception, e: message = "Error downloading JDK from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) Logger.exception(message) db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output if jdk_name.endswith(".exe"): install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home), ) install_path = [java_dir] try: Execute(install_cmd, path=install_path) except Exception, e: message = "Error installing java.\n" + str(e) Logger.exception(message) db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output
Execute(install_cmd, path=install_path) except Exception, e: message = "Error installing java.\n" + str(e) Logger.exception(message) db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output else: tmp_java_dir = tempfile.mkdtemp(prefix="jdk_tmp_", dir=tmp_dir) sudo = AMBARI_SUDO_BINARY if jdk_name.endswith(".bin"): chmod_cmd = ("chmod", "+x", jdk_download_target) install_cmd = format( "cd {tmp_java_dir} && echo A | {jdk_download_target} -noregister && {sudo} cp -rp {tmp_java_dir}/* {java_dir}" ) elif jdk_name.endswith(".gz"): chmod_cmd = ("chmod", "a+x", java_dir) install_cmd = format( "cd {tmp_java_dir} && tar -xf {jdk_download_target} && {sudo} cp -rp {tmp_java_dir}/* {java_dir}" ) try: Directory(java_dir) Execute(chmod_cmd, not_if=format("test -e {java_exec}"), sudo=True) Execute(install_cmd, not_if=format("test -e {java_exec}")) File(format("{java_home}/bin/java"), mode=0755, cd_access="a")
def execute_db_connection_check(self, config, tmp_dir): print "DB connection check started." # initialize needed data ambari_server_hostname = config['commandParams']['ambari_server_host'] check_db_connection_jar_name = "DBConnectionVerification.jar" jdk_location = config['commandParams']['jdk_location'] java_home = config['commandParams']['java_home'] db_name = config['commandParams']['db_name'] if db_name == DB_MYSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL jdbc_driver_class = JDBC_DRIVER_CLASS_MYSQL jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL elif db_name == DB_ORACLE: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE jdbc_driver_class = JDBC_DRIVER_CLASS_ORACLE jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE elif db_name == DB_POSTGRESQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL jdbc_driver_class = JDBC_DRIVER_CLASS_POSTGRESQL jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL elif db_name == DB_MSSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MSSQL jdbc_driver_class = JDBC_DRIVER_CLASS_MSSQL jdbc_name = JDBC_DRIVER_SYMLINK_MSSQL elif db_name == DB_SQLA: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_SQLA jdbc_driver_class = JDBC_DRIVER_CLASS_SQLA jdbc_name = JDBC_DRIVER_SYMLINK_SQLA db_connection_url = config['commandParams']['db_connection_url'] user_name = config['commandParams']['user_name'] user_passwd = config['commandParams']['user_passwd'] agent_cache_dir = os.path.abspath(config["hostLevelParams"]["agentCacheDir"]) check_db_connection_url = jdk_location + check_db_connection_jar_name jdbc_path = os.path.join(agent_cache_dir, jdbc_name) class_path_delimiter = ":" if db_name == DB_SQLA: jdbc_jar_path = agent_cache_dir + JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE java_library_path = agent_cache_dir + JARS_PATH_IN_ARCHIVE_SQLA + class_path_delimiter + agent_cache_dir + \ LIBS_PATH_IN_ARCHIVE_SQLA else: jdbc_jar_path = jdbc_path java_library_path = agent_cache_dir check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name) java_bin = "java" if OSCheck.is_windows_family(): java_bin = "java.exe" class_path_delimiter = ";" java_exec = os.path.join(java_home, "bin",java_bin) if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \ "\n" print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output environment = { "no_proxy": format("{ambari_server_hostname}") } # download and install java if it doesn't exists if not os.path.isfile(java_exec): jdk_name = config['commandParams']['jdk_name'] jdk_url = "{0}/{1}".format(jdk_location, jdk_name) jdk_download_target = os.path.join(agent_cache_dir, jdk_name) java_dir = os.path.dirname(java_home) try: download_file(jdk_url, jdk_download_target) except Exception, e: message = "Error downloading JDK from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output if jdk_name.endswith(".exe"): install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home), ) install_path = [java_dir] try: Execute(install_cmd, path = install_path) except Exception, e: message = "Error installing java.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output
return db_connection_check_structured_output if jdk_name.endswith(".exe"): install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home), ) install_path = [java_dir] try: Execute(install_cmd, path = install_path) except Exception, e: message = "Error installing java.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output else: tmp_java_dir = format("{tmp_dir}/jdk") sudo = AMBARI_SUDO_BINARY if jdk_name.endswith(".bin"): chmod_cmd = ("chmod", "+x", jdk_download_target) install_cmd = format("mkdir -p {tmp_java_dir} && cd {tmp_java_dir} && echo A | {jdk_download_target} -noregister && {sudo} cp -rp {tmp_java_dir}/* {java_dir}") elif jdk_name.endswith(".gz"): chmod_cmd = ("chmod","a+x", java_dir) install_cmd = format("mkdir -p {tmp_java_dir} && cd {tmp_java_dir} && tar -xf {jdk_download_target} && {sudo} cp -rp {tmp_java_dir}/* {java_dir}") try: Directory(java_dir) Execute(chmod_cmd, not_if = format("test -e {java_exec}"), sudo = True) Execute(install_cmd, not_if = format("test -e {java_exec}")) File(format("{java_home}/bin/java"), mode=0755, cd_access="a") Execute(("chown","-R", getpass.getuser(), java_home), sudo = True) except Exception, e: message = "Error installing java.\n" + str(e)
def setup_java(): """ Installs jdk using specific params, that comes from ambari-server """ import params java_exec = format("{java_home}/bin/java") if not os.path.isfile(java_exec): jdk_curl_target = format("{tmp_dir}/{jdk_name}") java_dir = os.path.dirname(params.java_home) Directory( params.artifact_dir, create_parents=True, ) File(jdk_curl_target, content=DownloadSource(format("{jdk_location}/{jdk_name}")), not_if=format("test -f {jdk_curl_target}")) File( jdk_curl_target, mode=0755, ) tmp_java_dir = tempfile.mkdtemp(prefix="jdk_tmp_", dir=params.tmp_dir) try: if params.jdk_name.endswith(".bin"): chmod_cmd = ("chmod", "+x", jdk_curl_target) install_cmd = format( "cd {tmp_java_dir} && echo A | {jdk_curl_target} -noregister && {sudo} cp -rp {tmp_java_dir}/* {java_dir}" ) elif params.jdk_name.endswith(".gz"): chmod_cmd = ("chmod", "a+x", java_dir) install_cmd = format( "cd {tmp_java_dir} && tar -xf {jdk_curl_target} && {sudo} /bin/cp -rp {tmp_java_dir}/* {java_dir}" ) Directory(java_dir) Execute( chmod_cmd, sudo=True, ) Execute(install_cmd, ) java_version_dir = os.listdir(tmp_java_dir)[0] Execute("ln -s %s %s" % (java_dir + '/' + java_version_dir, params.java_home)) finally: Directory(tmp_java_dir, action="delete") File( format("{java_home}/bin/java"), mode=0755, cd_access="a", ) Execute( ('chmod', '-R', '755', params.java_home), sudo=True, )
message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output # download jdbc driver from ambari-server resources try: download_file(jdbc_url, jdbc_path) if db_name == DB_MSSQL and OSCheck.is_windows_family(): jdbc_auth_path = os.path.join(agent_cache_dir, JDBC_AUTH_SYMLINK_MSSQL) jdbc_auth_url = jdk_location + JDBC_AUTH_SYMLINK_MSSQL download_file(jdbc_auth_url, jdbc_auth_path) except Exception, e: message = format("Error: Ambari Server cannot download the database JDBC driver and is unable to test the " \ "database connection. You must run tbds-server setup --jdbc-db={db_name} " \ "--jdbc-driver=/path/to/your/{db_name}/driver.jar on the Ambari Server host to make the JDBC " \ "driver available for download and to enable testing the database connection.\n") + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output # try to connect to db db_connection_check_command = format("{java_exec} -cp {check_db_connection_path}{class_path_delimiter}" \ "{jdbc_path} -Djava.library.path={agent_cache_dir} org.apache.ambari.server.DBConnectionVerification \"{db_connection_url}\" " \ "{user_name} {user_passwd!p} {jdbc_driver}") print "INFO db_connection_check_command: " + db_connection_check_command process = subprocess.Popen(db_connection_check_command, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE,
def setup_livy(env, type, upgrade_type=None, action=None): import params Directory([params.livy2_pid_dir, params.livy2_log_dir], owner=params.livy2_user, group=params.user_group, mode=0775, cd_access='a', create_parents=True) if type == 'server' and action == 'config': params.HdfsResource(params.livy2_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.livy2_user, mode=0775) params.HdfsResource(None, action="execute") if params.livy2_recovery_store == 'filesystem': params.HdfsResource(params.livy2_recovery_dir, type="directory", action="create_on_execute", owner=params.livy2_user, mode=0700) params.HdfsResource(None, action="execute") generate_logfeeder_input_config( 'spark2', Template("input.config-spark2.json.j2", extra_imports=[default])) # create livy-env.sh in etc/conf dir File( os.path.join(params.livy2_conf, 'livy-env.sh'), owner=params.livy2_user, group=params.livy2_group, content=InlineTemplate(params.livy2_env_sh), mode=0644, ) # create livy-client.conf in etc/conf dir PropertiesFile( format("{livy2_conf}/livy-client.conf"), properties=params.config['configurations']['livy2-client-conf'], key_value_delimiter=" ", owner=params.livy2_user, group=params.livy2_group, ) # create livy.conf in etc/conf dir PropertiesFile( format("{livy2_conf}/livy.conf"), properties=params.config['configurations']['livy2-conf'], key_value_delimiter=" ", owner=params.livy2_user, group=params.livy2_group, ) # create log4j.properties in etc/conf dir File( os.path.join(params.livy2_conf, 'log4j.properties'), owner=params.livy2_user, group=params.livy2_group, content=params.livy2_log4j_properties, mode=0644, ) # create spark-blacklist.properties in etc/conf dir File( os.path.join(params.livy2_conf, 'spark-blacklist.conf'), owner=params.livy2_user, group=params.livy2_group, content=params.livy2_spark_blacklist_properties, mode=0644, ) Directory( params.livy2_logs_dir, owner=params.livy2_user, group=params.livy2_group, mode=0755, )
) try: Execute(install_cmd, path=["/bin", "/usr/bin/"]) except Exception, e: message = "Error installing java.\n" + str(e) print message db_connection_check_structured_output = { "exit_code": "1", "message": message } return db_connection_check_structured_output try: cmd = format( "/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf " "--retry 5 {jdk_location}{check_db_connection_jar_name} " "-o {check_db_connection_jar_name}'") Execute( cmd, not_if=format( "[ -f /usr/lib/ambari-agent/{check_db_connection_jar_name}]" ), environment=environment) except Exception, e: message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = { "exit_code": "1", "message": message }
message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output # download jdbc driver from ambari-server resources try: download_file(jdbc_url, jdbc_path) if db_name == DB_MSSQL and OSCheck.is_windows_family(): jdbc_auth_path = os.path.join(agent_cache_dir, JDBC_AUTH_SYMLINK_MSSQL) jdbc_auth_url = jdk_location + JDBC_AUTH_SYMLINK_MSSQL download_file(jdbc_auth_url, jdbc_auth_path) except Exception, e: message = format("Error: Ambari Server cannot download the database JDBC driver and is unable to test the " \ "database connection. You must run ambari-server setup --jdbc-db={db_name} " \ "--jdbc-driver=/path/to/your/{db_name}/driver.jar on the Ambari Server host to make the JDBC " \ "driver available for download and to enable testing the database connection.\n") + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output # try to connect to db db_connection_check_command = format("{java_exec} -cp {check_db_connection_path}{class_path_delimiter}" \ "{jdbc_path} -Djava.library.path={agent_cache_dir} org.apache.ambari.server.DBConnectionVerification \"{db_connection_url}\" " \ "{user_name} {user_passwd!p} {jdbc_driver}") print "INFO db_connection_check_command: " + db_connection_check_command process = subprocess.Popen(db_connection_check_command, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE,
try: Execute(install_cmd, path=["/bin", "/usr/bin/"]) except Exception, e: message = "Error installing java.\n" + str(e) print message db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output # download DBConnectionVerification.jar from ambari-server resources try: cmd = format( "/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf " "--retry 5 {jdk_location}{check_db_connection_jar_name} " "-o {check_db_connection_jar_name}'") Execute( cmd, not_if=format( "[ -f /usr/lib/ambari-agent/{check_db_connection_jar_name}]" ), environment=environment) except Exception, e: message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = { "exit_code": 1, "message": message }
def execute_db_connection_check(self, config, tmp_dir): print "DB connection check started." # initialize needed data ambari_server_hostname = config['commandParams']['ambari_server_host'] check_db_connection_jar_name = "DBConnectionVerification.jar" jdk_location = config['commandParams']['jdk_location'] java64_home = config['commandParams']['java_home'] db_name = config['commandParams']['db_name'] if db_name == DB_MYSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL jdbc_driver = JDBC_DRIVER_MYSQL jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL elif db_name == DB_ORACLE: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE jdbc_driver = JDBC_DRIVER_ORACLE jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE elif db_name == DB_POSTGRESQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL jdbc_driver = JDBC_DRIVER_POSTGRESQL jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL elif db_name == DB_MSSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MSSQL jdbc_driver = JDBC_DRIVER_MSSQL jdbc_name = JDBC_DRIVER_SYMLINK_MSSQL db_connection_url = config['commandParams']['db_connection_url'] user_name = config['commandParams']['user_name'] user_passwd = config['commandParams']['user_passwd'] agent_cache_dir = os.path.abspath(config["hostLevelParams"]["agentCacheDir"]) check_db_connection_url = jdk_location + check_db_connection_jar_name jdbc_path = os.path.join(agent_cache_dir, jdbc_name) check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name) java_bin = "java" class_path_delimiter = ":" if OSCheck.is_windows_family(): java_bin = "java.exe" class_path_delimiter = ";" java_exec = os.path.join(java64_home, "bin",java_bin) if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \ "\n" print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output environment = { "no_proxy": format("{ambari_server_hostname}") } # download and install java if it doesn't exists if not os.path.isfile(java_exec): jdk_name = config['commandParams']['jdk_name'] jdk_url = "{0}/{1}".format(jdk_location, jdk_name) jdk_download_target = os.path.join(agent_cache_dir, jdk_name) java_dir = os.path.dirname(java64_home) try: download_file(jdk_url, jdk_download_target) except Exception, e: message = "Error downloading JDK from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output if jdk_name.endswith(".bin"): install_cmd = format("mkdir -p {java_dir} ; chmod +x {jdk_download_target}; cd {java_dir} ; echo A | " \ "{jdk_curl_target} -noregister > /dev/null 2>&1") install_path = ["/bin","/usr/bin/"] elif jdk_name.endswith(".gz"): install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_download_target} > /dev/null 2>&1") install_path = ["/bin","/usr/bin/"] elif jdk_name.endswith(".exe"): install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( os_utils.quote_path(jdk_download_target), os_utils.quote_path(java64_home), ) install_path = [java_dir] try: Execute(install_cmd, path = install_path) except Exception, e: message = "Error installing java.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output
"{jdk_curl_target} -noregister > /dev/null 2>&1") elif jdk_name.endswith(".gz"): install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_curl_target} > /dev/null 2>&1") try: Execute(install_cmd, path = ["/bin","/usr/bin/"]) except Exception, e: message = "Error installing java.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output # download DBConnectionVerification.jar from ambari-server resources try: cmd = format("/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf " "--retry 5 {jdk_location}{check_db_connection_jar_name} " "-o {check_db_connection_jar_name}'") Execute(cmd, not_if=format("[ -f /usr/lib/ambari-agent/{check_db_connection_jar_name}]"), environment = environment) except Exception, e: message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output # download jdbc driver from ambari-server resources try: cmd = format("/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf " "--retry 5 {jdbc_url} -o {jdbc_name}'") Execute(cmd, not_if=format("[ -f /usr/lib/ambari-agent/{jdbc_name}]"), environment = environment)
def execute_db_connection_check(self, config): print "DB connection check started." # initialize needed data ambari_server_hostname = config['commandParams']['ambari_server_host'] check_db_connection_jar_name = "DBConnectionVerification.jar" jdk_location = config['commandParams']['jdk_location'] java64_home = config['commandParams']['java_home'] db_name = config['commandParams']['db_name'] if db_name == DB_MYSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL jdbc_driver = JDBC_DRIVER_MYSQL jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL elif db_name == DB_ORACLE: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE jdbc_driver = JDBC_DRIVER_ORACLE jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE elif db_name == DB_POSTGRESQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL jdbc_driver = JDBC_DRIVER_POSTGRESQL jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL db_connection_url = config['commandParams']['db_connection_url'] user_name = config['commandParams']['user_name'] user_passwd = config['commandParams']['user_passwd'] java_exec = os.path.join(java64_home, "bin", "java") if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \ "\n" print message db_connection_check_structured_output = { "exit_code": "1", "message": message } return db_connection_check_structured_output environment = {"no_proxy": format("{ambari_server_hostname}")} artifact_dir = "/tmp/HDP-artifacts/" jdk_name = config['commandParams']['jdk_name'] jdk_curl_target = format("{artifact_dir}/{jdk_name}") java_dir = os.path.dirname(java64_home) # download DBConnectionVerification.jar from ambari-server resources if not os.path.isfile(java_exec): try: Execute(format( "mkdir -p {artifact_dir} ; curl -kf " "--retry 10 {jdk_location}/{jdk_name} -o {jdk_curl_target}" ), path=["/bin", "/usr/bin/"], environment=environment) except Exception, e: message = "Error downloading JDK from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = { "exit_code": "1", "message": message } return db_connection_check_structured_output if jdk_name.endswith(".bin"): install_cmd = format("mkdir -p {java_dir} ; chmod +x {jdk_curl_target}; cd {java_dir} ; echo A | " \ "{jdk_curl_target} -noregister > /dev/null 2>&1") elif jdk_name.endswith(".gz"): install_cmd = format( "mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_curl_target} > /dev/null 2>&1" ) try: Execute(install_cmd, path=["/bin", "/usr/bin/"]) except Exception, e: message = "Error installing java.\n" + str(e) print message db_connection_check_structured_output = { "exit_code": "1", "message": message } return db_connection_check_structured_output
def execute_db_connection_check(self, config, tmp_dir): print "DB connection check started." # initialize needed data ambari_server_hostname = config['commandParams']['ambari_server_host'] check_db_connection_jar_name = "DBConnectionVerification.jar" jdk_location = config['commandParams']['jdk_location'] java64_home = config['commandParams']['java_home'] db_name = config['commandParams']['db_name'] if db_name == DB_MYSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL jdbc_driver = JDBC_DRIVER_MYSQL jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL elif db_name == DB_ORACLE: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE jdbc_driver = JDBC_DRIVER_ORACLE jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE elif db_name == DB_POSTGRESQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL jdbc_driver = JDBC_DRIVER_POSTGRESQL jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL db_connection_url = config['commandParams']['db_connection_url'] user_name = config['commandParams']['user_name'] user_passwd = config['commandParams']['user_passwd'] java_exec = os.path.join(java64_home, "bin","java") if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \ "\n" print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output environment = { "no_proxy": format("{ambari_server_hostname}") } artifact_dir = format("{tmp_dir}/AMBARI-artifacts/") java_dir = os.path.dirname(java64_home) # download and install java if it doesn't exists if not os.path.isfile(java_exec): try: jdk_name = config['commandParams']['jdk_name'] jdk_curl_target = format("{artifact_dir}/{jdk_name}") Execute(format("mkdir -p {artifact_dir} ; curl -kf " "--retry 10 {jdk_location}/{jdk_name} -o {jdk_curl_target}"), path = ["/bin","/usr/bin/"], environment = environment) except Exception, e: message = "Error downloading JDK from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output if jdk_name.endswith(".bin"): install_cmd = format("mkdir -p {java_dir} ; chmod +x {jdk_curl_target}; cd {java_dir} ; echo A | " \ "{jdk_curl_target} -noregister > /dev/null 2>&1") elif jdk_name.endswith(".gz"): install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_curl_target} > /dev/null 2>&1") try: Execute(install_cmd, path = ["/bin","/usr/bin/"]) except Exception, e: message = "Error installing java.\n" + str(e) print message db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output