def is_ubuntu_family(self, get_os_family_mock): get_os_family_mock.return_value = "ubuntu" self.assertEqual(OSCheck.is_ubuntu_family(), True) get_os_family_mock.return_value = "troll_os" self.assertEqual(OSCheck.is_ubuntu_family(), False)
def test_is_redhat_family(self, get_os_family_mock): get_os_family_mock.return_value = "redhat" self.assertEqual(OSCheck.is_redhat_family(), True) get_os_family_mock.return_value = "troll_os" self.assertEqual(OSCheck.is_redhat_family(), False)
def find_repo_files_for_repos(self, repoNames): repoFiles = [] osType = OSCheck.get_os_family() repoNameList = [] for repoName in repoNames: if len(repoName.strip()) > 0: repoNameList.append("[" + repoName + "]") repoNameList.append("name=" + repoName) if repoNameList: # get list of files if OSCheck.is_suse_family(): fileList = self.get_files_in_dir(REPO_PATH_SUSE) elif OSCheck.is_redhat_family(): fileList = self.get_files_in_dir(REPO_PATH_RHEL) elif OSCheck.is_ubuntu_family(): fileList = self.get_files_in_dir(REPO_PATH_UBUNTU) else: logger.warn("Unsupported OS type, cannot get repository location.") return [] if fileList: for filePath in fileList: with open(filePath, 'r') as file: content = file.readline() while (content != "" ): for repoName in repoNameList: if content.find(repoName) == 0 and filePath not in repoFiles: repoFiles.append(filePath) break; content = file.readline() return repoFiles
def get_postgre_hba_dir(OS_FAMILY): """Return postgre hba dir location depends on OS. Also depends on version of postgres creates symlink like postgresql-->postgresql-9.3 1) /etc/rc.d/init.d/postgresql --> /etc/rc.d/init.d/postgresql-9.3 2) /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1 """ if OSCheck.is_ubuntu_family(): # Like: /etc/postgresql/9.1/main/ return os.path.join(get_pg_hba_init_files(), get_ubuntu_pg_version(), "main") elif OSCheck.is_redhat_family() and int(OSCheck.get_os_major_version()) >= 7: return PG_HBA_ROOT_DEFAULT else: if not os.path.isfile(get_pg_hba_init_files()): # Link: /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1 os.symlink(glob.glob(get_pg_hba_init_files() + '*')[0], get_pg_hba_init_files()) # Get postgres_data location (default: /var/lib/pgsql/data) cmd = "alias exit=return; source " + get_pg_hba_init_files() + " status &>/dev/null; echo $PGDATA" p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (PG_HBA_ROOT, err) = p.communicate() if PG_HBA_ROOT and len(PG_HBA_ROOT.strip()) > 0: return PG_HBA_ROOT.strip() else: return PG_HBA_ROOT_DEFAULT
def get_pg_hba_init_files(): if OSCheck.is_ubuntu_family(): return '/etc/postgresql' elif OSCheck.is_redhat_family(): return '/etc/rc.d/init.d/postgresql' elif OSCheck.is_suse_family(): return '/etc/init.d/postgresql' else: raise Exception("Unsupported OS family '{0}'".format(OSCheck.get_os_family()))
def getFirewallObject(self): if OSCheck.is_ubuntu_family(): return UbuntuFirewallChecks() elif self.OS_TYPE == OSConst.OS_FEDORA and int(self.OS_VERSION) >= 18: return Fedora18FirewallChecks() elif OSCheck.is_suse_family(): return SuseFirewallChecks() else: return FirewallChecks()
def get_erase_cmd(): if OSCheck.is_redhat_family(): return "yum erase -y {0}" elif OSCheck.is_suse_family(): return "zypper -n -q remove {0}" elif OSCheck.is_ubuntu_family(): return "/usr/bin/apt-get -y -q remove {0}" else: raise Exception("Unsupported OS family '{0}', cannot remove package. ".format(OSCheck.get_os_family()))
def getRepoDir(self): if OSCheck.is_redhat_family(): return "/etc/yum.repos.d" elif OSCheck.is_suse_family(): return "/etc/zypp/repos.d" elif OSCheck.is_ubuntu_family(): return "/etc/apt/sources.list.d" else: raise Exception("Unsupported OS family '{0}'".format(OSCheck.get_os_family()))
def getAvaliableAgentPackageVersions(): if OSCheck.is_suse_family(): Command = ["bash", "-c", "zypper --no-gpg-checks -q search -s --match-exact ambari-agent | grep ambari-agent | sed -re 's/\s+/ /g' | cut -d '|' -f 4 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"] elif OSCheck.is_ubuntu_family(): Command = ["bash", "-c", "apt-cache -q show ambari-agent|grep 'Version\:'|cut -d ' ' -f 2| tr '\\n' ', '|sed -s 's/[-|~][A-Za-z0-9]*//g'"] else: Command = ["bash", "-c", "yum -q list all ambari-agent | grep -E '^ambari-agent' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"] return execOsCommand(Command)
def installAgent(projectVersion): """ Run install and make sure the agent install alright """ # The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host if OSCheck.is_suse_family(): Command = ["zypper", "--no-gpg-checks", "install", "-y", "ambari-agent-" + projectVersion] elif OSCheck.is_ubuntu_family(): # add * to end of version in case of some test releases Command = ["apt-get", "install", "-y", "--allow-unauthenticated", "ambari-agent=" + projectVersion + "*"] else: Command = ["yum", "-y", "install", "--nogpgcheck", "ambari-agent-" + projectVersion] return execOsCommand(Command, tries=3, try_sleep=10)
def installAgent(projectVersion): """ Run install and make sure the agent install alright """ # The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host if OSCheck.is_suse_family(): Command = ["zypper", "install", "-y", "ambari-agent-" + projectVersion] elif OSCheck.is_debian_family(): # add * to end of version in case of some test releases Command = ["apt-get", "install", "-y", "--force-yes", "ambari-agent=" + projectVersion + "*"] else: Command = ["yum", "-y", "install", "--nogpgcheck", "ambari-agent-" + projectVersion] return execOsCommand(Command)
def test_get_os_major_version(self, mock_linux_distribution): # 1 mock_linux_distribution.return_value = ('', '123.45.67', '') result = OSCheck.get_os_major_version() self.assertEquals(result, '123') # 2 mock_linux_distribution.return_value = ('Suse', '11', '') result = OSCheck.get_os_major_version() self.assertEquals(result, '11')
def test_get_os_major_version(self, mock_linux_distribution): # 1 mock_linux_distribution.return_value = ("", "123.45.67", "") result = OSCheck.get_os_major_version() self.assertEquals(result, "123") # 2 mock_linux_distribution.return_value = ("Suse", "11", "") result = OSCheck.get_os_major_version() self.assertEquals(result, "11")
def isAgentPackageAlreadyInstalled(projectVersion): if OSCheck.is_ubuntu_family(): Command = ["bash", "-c", "dpkg-query -W -f='${Status} ${Version}\n' ambari-agent | grep -v deinstall | grep " + projectVersion] elif OSCheck.is_windows_family(): Command = ["cmd", "/c", "choco list ambari-agent --local-only | findstr ambari-agent | findstr " + projectVersion] else: Command = ["bash", "-c", "rpm -qa | grep ambari-agent-"+projectVersion] ret = execOsCommand(Command) res = False if ret["exitstatus"] == 0 and ret["log"][0].strip() != "": res = True return res
def test_aliases(self, mock_linux_distribution): OSConst.OS_TYPE_ALIASES['qwerty_os123'] = 'aliased_os5' OSConst.OS_FAMILY_COLLECTION.append({ 'name': 'aliased_os_family', 'os_list': ["aliased_os"] }) mock_linux_distribution.return_value = ('qwerty_os', '123.45.67', '') self.assertEquals(OSCheck.get_os_type(), 'aliased_os') self.assertEquals(OSCheck.get_os_major_version(), '5') self.assertEquals(OSCheck.get_os_version(), '5.45.67') self.assertEquals(OSCheck.get_os_family(), 'aliased_os_family')
def get_ambari_repo_file_full_name(): if OSCheck.is_ubuntu_family(): ambari_repo_file = "/etc/apt/sources.list.d/ambari.list" elif OSCheck.is_redhat_family(): ambari_repo_file = "/etc/yum.repos.d/ambari.repo" elif OSCheck.is_suse_family(): ambari_repo_file = "/etc/zypp/repos.d/ambari.repo" elif OSCheck.is_windows_family(): ambari_repo_file = os.path.join(os.environ[ChocolateyConsts.CHOCOLATEY_INSTALL_VAR_NAME], ChocolateyConsts.CHOCOLATEY_CONFIG_DIR, ChocolateyConsts.CHOCOLATEY_CONFIG_FILENAME) else: raise Exception('Ambari repo file path not set for current OS.') return ambari_repo_file
def getTransparentHugePage(self): thp_regex = "\[(.+)\]" file_name = None if OSCheck.is_ubuntu_family(): file_name = self.THP_FILE_UBUNTU elif OSCheck.is_redhat_family(): file_name = self.THP_FILE_REDHAT if file_name and os.path.isfile(file_name): with open(file_name) as f: file_content = f.read() return re.search(thp_regex, file_content).groups()[0] else: return ""
def installAgent(projectVersion, ret=None): """ Run install and make sure the agent install alright """ # The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host if OSCheck.is_suse_family(): Command = ["zypper", "--no-gpg-checks", "install", "-y", "ambari-agent-" + projectVersion] elif OSCheck.is_ubuntu_family(): # add * to end of version in case of some test releases Command = ["apt-get", "install", "-y", "--allow-unauthenticated", "ambari-agent=" + projectVersion + "*"] elif OSCheck.is_windows_family(): packageParams = "/AmbariRoot:" + AMBARI_INSTALL_ROOT Command = ["cmd", "/c", "choco", "install", "-y", "ambari-agent", "--version=" + projectVersion, "--params=\"" + packageParams + "\""] else: Command = ["yum", "-y", "install", "--nogpgcheck", "ambari-agent-" + projectVersion] return execOsCommand(Command, tries=3, try_sleep=10, ret=ret)
def install_packages(self, env): """ List of packages that are required< by service is received from the server as a command parameter. The method installs all packages from this list exclude_packages - list of regexes (possibly raw strings as well), the packages which match the regex won't be installed. NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ? """ config = self.get_config() if 'host_sys_prepped' in config['hostLevelParams']: # do not install anything on sys-prepped host if config['hostLevelParams']['host_sys_prepped'] == True: Logger.info("Node has all packages pre-installed. Skipping.") return pass try: package_list_str = config['hostLevelParams']['package_list'] agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability']) agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count']) if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: if Script.check_package_condition(package): name = self.format_package_name(package['name']) # HACK: On Windows, only install ambari-metrics packages using Choco Package Installer # TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support # <osFamily>any<osFamily> which would cause installation failure on Windows. if OSCheck.is_windows_family(): if "ambari-metrics" in name: Package(name) else: Package(name, retry_on_repo_unavailability=agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count) except KeyError: pass # No reason to worry if OSCheck.is_windows_family(): #TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"] install_windows_msi(config['hostLevelParams']['jdk_location'], config["hostLevelParams"]["agentCacheDir"], ["hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab", "hdp-2.3.0.0-01.cab"], hadoop_user, self.get_password(hadoop_user), str(config['hostLevelParams']['stack_version'])) reload_windows_env()
def perform_prestart_checks(expected_hostname): # Check if current hostname is equal to expected one (got from the server # during bootstrap. global config if expected_hostname is not None: current_hostname = hostname.hostname(config) if current_hostname != expected_hostname: print("Determined hostname does not match expected. Please check agent " "log for details") msg = "Ambari agent machine hostname ({0}) does not match expected ambari " \ "server hostname ({1}). Aborting registration. Please check hostname, " \ "hostname -f and /etc/hosts file to confirm your " \ "hostname is setup correctly".format(current_hostname, expected_hostname) logger.error(msg) sys.exit(1) # Check if there is another instance running if os.path.isfile(ProcessHelper.pidfile) and not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY: print("%s already exists, exiting" % ProcessHelper.pidfile) sys.exit(1) # check if ambari prefix exists elif config.has_option('agent', 'prefix') and not os.path.isdir(os.path.abspath(config.get('agent', 'prefix'))): msg = "Ambari prefix dir %s does not exists, can't continue" \ % config.get("agent", "prefix") logger.error(msg) print(msg) sys.exit(1) elif not config.has_option('agent', 'prefix'): msg = "Ambari prefix dir %s not configured, can't continue" logger.error(msg) print(msg) sys.exit(1)
def checkLiveServices(self, services, result): osType = OSCheck.get_os_family() for service in services: svcCheckResult = {} if isinstance(service, dict): serviceName = service[osType] else: serviceName = service service_check_live = shlex.split(self.SERVICE_STATUS_CMD) service_check_live[1] = serviceName svcCheckResult['name'] = serviceName svcCheckResult['status'] = "UNKNOWN" svcCheckResult['desc'] = "" try: osStat = subprocess.Popen(service_check_live, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = osStat.communicate() if 0 != osStat.returncode: svcCheckResult['status'] = "Unhealthy" svcCheckResult['desc'] = out if len(out) == 0: svcCheckResult['desc'] = err else: svcCheckResult['status'] = "Healthy" except Exception, e: svcCheckResult['status'] = "Unhealthy" svcCheckResult['desc'] = repr(e) result.append(svcCheckResult)
def findNearestAgentPackageVersion(projectVersion): if projectVersion == "": projectVersion = " " if OSCheck.is_suse_family(): Command = ["bash", "-c", "zypper --no-gpg-checks -q search -s --match-exact ambari-agent | grep '" + projectVersion + "' | cut -d '|' -f 4 | head -n1 | sed -e 's/-\w[^:]*//1' "] elif OSCheck.is_ubuntu_family(): if projectVersion == " ": Command = ["bash", "-c", "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"] else: Command = ["bash", "-c", "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|grep '" + projectVersion + "'|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"] else: Command = ["bash", "-c", "yum -q list all ambari-agent | grep '" + projectVersion + "' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | head -n1 | sed -e 's/-\w[^:]*//1' "] return execOsCommand(Command)
def allAvailablePackages(allAvailablePackages): osType = OSCheck.get_os_family() if OSCheck.is_suse_family(): return _lookUpZypperPackages( ["sudo", "zypper", "search", "--uninstalled-only", "--details"], allAvailablePackages) elif OSCheck.is_redhat_family(): return _lookUpYumPackages( ["sudo", "yum", "list", "available"], 'Available Packages', allAvailablePackages) elif OSCheck.is_ubuntu_family(): return _lookUpAptPackages( LIST_AVAILABLE_PACKAGES_UBUNTU, allAvailablePackages)
def test_get_os_version(self, mock_linux_distribution): # 1 - Any system mock_linux_distribution.return_value = ('', '123.45', '') result = OSCheck.get_os_version() self.assertEquals(result, '123.45') # 2 - Negative case mock_linux_distribution.return_value = ('ssss', '', 'ddddd') try: result = OSCheck.get_os_version() self.fail("Should throw exception in OSCheck.get_os_version()") except Exception as e: # Expected self.assertEquals("Cannot detect os version. Exiting...", str(e)) pass
def test_get_os_release_name(self, mock_linux_distribution): # 1 - Any system mock_linux_distribution.return_value = ('', '', 'MY_NEW_RELEASE') result = OSCheck.get_os_release_name() self.assertEquals(result, 'my_new_release') # 2 - Negative case mock_linux_distribution.return_value = ('aaaa', 'bbbb', '') try: result = OSCheck.get_os_release_name() self.fail("Should throw exception in OSCheck.get_os_release_name()") except Exception as e: # Expected self.assertEquals("Cannot detect os release name. Exiting...", str(e)) pass
def action_create(self): with Environment.get_instance_copy() as env: repo_file_name = self.resource.repo_file_name repo_dir = get_repo_dir() new_content = InlineTemplate(self.resource.repo_template, repo_id=self.resource.repo_id, repo_file_name=self.resource.repo_file_name, base_url=self.resource.base_url, mirror_list=self.resource.mirror_list) repo_file_path = format("{repo_dir}/{repo_file_name}.repo") if os.path.isfile(repo_file_path): existing_content_str = sudo.read_file(repo_file_path) new_content_str = new_content.get_content() if existing_content_str != new_content_str and OSCheck.is_suse_family(): # We need to reset package manager's cache when we replace base urls # at existing repo. That is a case at least under SLES Logger.info("Flushing package manager cache since repo file content is about to change") checked_call(self.update_cmd, sudo=True) if self.resource.append_to_file: content = existing_content_str + '\n' + new_content_str else: content = new_content_str else: # If repo file does not exist yet content = new_content File(repo_file_path, content=content )
def os_major_version(self): """ Example return value: "6" for "Centos 6.3" In case cannot detect --> Fail """ return OSCheck.get_os_major_version()
def os_family(self): """ Return values: redhat, ubuntu, suse In case cannot detect raises exception """ return OSCheck.get_os_family()
def main(argv=None): # Same logic that was in "os_type_check.sh" if len(sys.argv) != 2: print "Usage: <cluster_os>" raise Exception("Error in number of arguments. Usage: <cluster_os>") pass cluster_os = sys.argv[1] current_os = OSCheck.get_os_family() + OSCheck.get_os_major_version() # If agent/server have the same {"family","main_version"} - then ok. print "Cluster primary/cluster OS type is %s and local/current OS type is %s" % ( cluster_os, current_os) if current_os == cluster_os: sys.exit(0) else: raise Exception("Local OS is not compatible with cluster primary OS. Please perform manual bootstrap on this host.")
def get_postgre_running_status(OS_FAMILY): """Return postgre running status indicator""" if OS_FAMILY == OSConst.UBUNTU_FAMILY: return os.path.join(get_ubuntu_pg_version(), "main") elif OSCheck.is_redhat7(): return "active" else: return PG_STATUS_RUNNING_DEFAULT
def main(heartbeat_stop_callback=None): global config parser = OptionParser() parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="verbose log output", default=False) parser.add_option( "-e", "--expected-hostname", dest="expected_hostname", action="store", help= "expected hostname of current host. If hostname differs, agent will fail", default=None) (options, args) = parser.parse_args() expected_hostname = options.expected_hostname logging_level = logging.DEBUG if options.verbose else logging.INFO setup_logging(logger, AmbariConfig.AmbariConfig.getLogFile(), logging_level) global is_logger_setup is_logger_setup = True setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level) Logger.initialize_logger('resource_management', logging_level=logging_level) default_cfg = {'agent': {'prefix': '/home/ambari'}} config.load(default_cfg) if (len(sys.argv) > 1) and sys.argv[1] == 'stop': stop_agent() if (len(sys.argv) > 2) and sys.argv[1] == 'reset': reset_agent(sys.argv) # Check for ambari configuration file. resolve_ambari_config() # Add syslog hanlder based on ambari config file add_syslog_handler(logger) # Starting data cleanup daemon data_cleaner = None if config.has_option('agent', 'data_cleanup_interval') and int( config.get('agent', 'data_cleanup_interval')) > 0: data_cleaner = DataCleaner(config) data_cleaner.start() perform_prestart_checks(expected_hostname) # Starting ping port listener try: #This acts as a single process machine-wide lock (albeit incomplete, since # we still need an extra file to track the Agent PID) ping_port_listener = PingPortListener(config) except Exception as ex: err_message = "Failed to start ping port listener of: " + str(ex) logger.error(err_message) sys.stderr.write(err_message) sys.exit(1) ping_port_listener.start() update_log_level(config) if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY: daemonize() # # Iterate through the list of server hostnames and connect to the first active server # active_server = None server_hostnames = hostname.server_hostnames(config) connected = False stopped = False # Keep trying to connect to a server or bail out if ambari-agent was stopped while not connected and not stopped: for server_hostname in server_hostnames: try: server_ip = socket.gethostbyname(server_hostname) server_url = config.get_api_url(server_hostname) logger.info('Connecting to Ambari server at %s (%s)', server_url, server_ip) except socket.error: logger.warn( "Unable to determine the IP address of the Ambari server '%s'", server_hostname) # Wait until MAX_RETRIES to see if server is reachable netutil = NetUtil(heartbeat_stop_callback) (retries, connected, stopped) = netutil.try_to_connect(server_url, MAX_RETRIES, logger) # if connected, launch controller if connected: logger.info('Connected to Ambari server %s', server_hostname) # Set the active server active_server = server_hostname # Launch Controller communication controller = Controller(config, server_hostname, heartbeat_stop_callback) controller.start() controller.join() # # If Ambari Agent connected to the server or # Ambari Agent was stopped using stop event # Clean up if not Windows OS # if connected or stopped: if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY: ExitHelper().execute_cleanup() stop_agent() logger.info("finished") break pass # for server_hostname in server_hostnames pass # while not (connected or stopped) return active_server
def execute(self): """ Sets up logging; Parses command parameters and executes method relevant to command type """ parser = OptionParser() parser.add_option("-o", "--out-files-logging", dest="log_out_files", action="store_true", help="use this option to enable outputting *.out files of the service pre-start") (self.options, args) = parser.parse_args() self.log_out_files = self.options.log_out_files # parse arguments if len(args) < 6: print "Script expects at least 6 arguments" print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout sys.exit(1) self.command_name = str.lower(sys.argv[1]) self.command_data_file = sys.argv[2] self.basedir = sys.argv[3] self.stroutfile = sys.argv[4] self.load_structured_out() self.logging_level = sys.argv[5] Script.tmp_dir = sys.argv[6] # optional script arguments for forcing https protocol and ca_certs file if len(sys.argv) >= 8: Script.force_https_protocol = sys.argv[7] if len(sys.argv) >= 9: Script.ca_cert_file_path = sys.argv[8] logging_level_str = logging._levelNames[self.logging_level] Logger.initialize_logger(__name__, logging_level=logging_level_str) # on windows we need to reload some of env variables manually because there is no default paths for configs(like # /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated # in agent, so other Script executions will not be able to access to new env variables if OSCheck.is_windows_family(): reload_windows_env() # !!! status commands re-use structured output files; if the status command doesn't update the # the file (because it doesn't have to) then we must ensure that the file is reset to prevent # old, stale structured output from a prior status command from being used if self.command_name == "status": Script.structuredOut = {} self.put_structured_out({}) # make sure that script has forced https protocol and ca_certs file passed from agent ensure_ssl_using_protocol(Script.get_force_https_protocol_name(), Script.get_ca_cert_file_path()) try: with open(self.command_data_file) as f: pass Script.config = ConfigDictionary(json.load(f)) Script.execution_command = ExecutionCommand(Script.config) Script.module_configs = Script.execution_command.get_module_configs() Script.cluster_settings = Script.execution_command.get_cluster_settings() Script.stack_settings = Script.execution_command.get_stack_settings() # load passwords here(used on windows to impersonate different users) Script.passwords = {} for k, v in _PASSWORD_MAP.iteritems(): if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config): Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config) except IOError: Logger.logger.exception("Can not read json file with command parameters: ") sys.exit(1) Script.repository_util = RepositoryUtil(Script.config) # Run class method depending on a command type try: method = self.choose_method_to_execute(self.command_name) with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env: env.config.download_path = Script.tmp_dir if not self.is_hook(): self.execute_prefix_function(self.command_name, 'pre', env) method(env) if not self.is_hook(): self.execute_prefix_function(self.command_name, 'post', env) except Fail as ex: ex.pre_raise() raise finally: if self.should_expose_component_version(self.command_name): self.save_component_version_to_structured_out(self.command_name)
def execute(parameters=None, host_name=None): """ Returns a tuple containing the result code and a pre-formatted result label Keyword arguments: parameters (dictionary): a mapping of parameter key to value host_name (string): the name of this host where the alert is running """ result_code = RESULT_CODE_UNKNOWN if parameters is None: return (result_code, ['There were no parameters supplied to the script.']) scheme = 'http' http_uri = None https_uri = None http_policy = 'HTTP_ONLY' if NODEMANAGER_HTTP_ADDRESS_KEY in parameters: http_uri = parameters[NODEMANAGER_HTTP_ADDRESS_KEY] if NODEMANAGER_HTTPS_ADDRESS_KEY in parameters: https_uri = parameters[NODEMANAGER_HTTPS_ADDRESS_KEY] if YARN_HTTP_POLICY_KEY in parameters: http_policy = parameters[YARN_HTTP_POLICY_KEY] # determine the right URI and whether to use SSL uri = http_uri if http_policy == 'HTTPS_ONLY': scheme = 'https' if https_uri is not None: uri = https_uri label = '' url_response = None node_healthy = 'false' total_time = 0 # some yarn-site structures don't have the web ui address if uri is None: if host_name is None: host_name = socket.getfqdn() uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT) if OSCheck.is_windows_family(): uri_host, uri_port = uri.split(':') # on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1 uri_host = resolve_address(uri_host) uri = '{0}:{1}'.format(uri_host, uri_port) query = "{0}://{1}/ws/v1/node/info".format(scheme, uri) try: # execute the query for the JSON that includes templeton status url_response = urllib2.urlopen(query) except urllib2.HTTPError, httpError: label = CRITICAL_HTTP_STATUS_MESSAGE.format(str(httpError.code), query, str(httpError)) return (RESULT_CODE_CRITICAL, [label])
# a map of the Ambari role to the component name # for use with /usr/hdp/current/<component> SERVER_ROLE_DIRECTORY_MAP = { 'HIVE_METASTORE': 'hive-metastore', 'HIVE_SERVER': 'hive-server2', 'WEBHCAT_SERVER': 'hive-webhcat', 'HIVE_CLIENT': 'hive-client', 'HCAT': 'hive-client' } component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "HIVE_CLIENT") config = Script.get_config() if OSCheck.is_windows_family(): hive_metastore_win_service_name = "metastore" hive_client_win_service_name = "hwi" hive_server_win_service_name = "hiveserver2" webhcat_server_win_service_name = "templeton" else: hive_pid_dir = config['configurations']['hive-env']['hive_pid_dir'] hive_pid = 'hive-server.pid' hive_metastore_pid = 'hive.pid' hcat_pid_dir = config['configurations']['hive-env'][ 'hcat_pid_dir'] #hcat_pid_dir webhcat_pid_file = format('{hcat_pid_dir}/webhcat.pid') process_name = 'mysqld'
def __init__(self): # OS info self.OS_VERSION = OSCheck().get_os_major_version() self.OS_TYPE = OSCheck.get_os_type() self.OS_FAMILY = OSCheck.get_os_family()
def get_ntp_service(): if OSCheck.is_redhat_family(): return "ntpd" elif OSCheck.is_suse_family() or OSCheck.is_ubuntu_family(): return "ntp"
def main(initializer_module, heartbeat_stop_callback=None): global config global home_dir parser = OptionParser() parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="verbose log output", default=False) parser.add_option( "-e", "--expected-hostname", dest="expected_hostname", action="store", help= "expected hostname of current host. If hostname differs, agent will fail", default=None) parser.add_option("--home", dest="home_dir", action="store", help="Home directory", default="") (options, args) = parser.parse_args() expected_hostname = options.expected_hostname home_dir = options.home_dir logging_level = logging.DEBUG if options.verbose else logging.INFO setup_logging(logger, AmbariConfig.AmbariConfig.getLogFile(), logging_level) global is_logger_setup is_logger_setup = True setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level) setup_logging(alerts_logger_2, AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level) setup_logging(alerts_logger_global, AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level) setup_logging(apscheduler_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level) setup_logging(apscheduler_logger_global, AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level) Logger.initialize_logger('resource_management', logging_level=logging_level) #with Environment() as env: # File("/abc") # init data, once loggers are setup to see exceptions/errors of initialization. initializer_module.init() if home_dir != "": # When running multiple Ambari Agents on this host for simulation, each one will use a unique home directory. Logger.info("Agent is using Home Dir: %s" % str(home_dir)) # use the host's locale for numeric formatting try: locale.setlocale(locale.LC_ALL, '') except locale.Error as ex: logger.warning( "Cannot set locale for ambari-agent. Please check your systemwide locale settings. Failed due to: {0}." .format(str(ex))) default_cfg = {'agent': {'prefix': '/home/ambari'}} config.load(default_cfg) if (len(sys.argv) > 1) and sys.argv[1] == 'stop': stop_agent() if (len(sys.argv) > 2) and sys.argv[1] == 'reset': reset_agent(sys.argv) # Check for ambari configuration file. resolve_ambari_config() # Add syslog hanlder based on ambari config file add_syslog_handler(logger) # Starting data cleanup daemon data_cleaner = None if config.has_option('agent', 'data_cleanup_interval') and int( config.get('agent', 'data_cleanup_interval')) > 0: data_cleaner = DataCleaner(config) data_cleaner.start() perform_prestart_checks(expected_hostname) # Starting ping port listener try: #This acts as a single process machine-wide lock (albeit incomplete, since # we still need an extra file to track the Agent PID) ping_port_listener = PingPortListener(config) except Exception as ex: err_message = "Failed to start ping port listener of: " + str(ex) logger.error(err_message) sys.stderr.write(err_message) sys.exit(1) ping_port_listener.start() update_log_level(config) update_open_files_ulimit(config) if not config.use_system_proxy_setting(): logger.info('Agent is configured to ignore system proxy settings') #reconfigure_urllib2_opener(ignore_system_proxy=True) if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY: daemonize() # # Iterate through the list of server hostnames and connect to the first active server # active_server = None server_hostnames = hostname.server_hostnames(config) connected = False stopped = False # Keep trying to connect to a server or bail out if ambari-agent was stopped while not connected and not stopped: for server_hostname in server_hostnames: server_url = config.get_api_url(server_hostname) try: server_ip = socket.gethostbyname(server_hostname) logger.info('Connecting to Ambari server at %s (%s)', server_url, server_ip) except socket.error: logger.warn( "Unable to determine the IP address of the Ambari server '%s'", server_hostname) # Wait until MAX_RETRIES to see if server is reachable netutil = NetUtil(config, initializer_module.stop_event) (retries, connected, stopped) = netutil.try_to_connect(server_url, MAX_RETRIES, logger) # if connected, launch controller if connected: logger.info('Connected to Ambari server %s', server_hostname) # Set the active server active_server = server_hostname # Launch Controller communication run_threads(initializer_module) # # If Ambari Agent connected to the server or # Ambari Agent was stopped using stop event # Clean up if not Windows OS # if connected or stopped: ExitHelper().exit() logger.info("finished") break pass # for server_hostname in server_hostnames pass # while not (connected or stopped) return active_server
def getOperatingSystem(self): return OSCheck.get_os_type()
def execute(configurations={}, parameters={}, host_name=None): """ Returns a tuple containing the result code and a pre-formatted result label Keyword arguments: configurations (dictionary): a mapping of configuration key to value parameters (dictionary): a mapping of script parameter key to value host_name (string): the name of this host where the alert is running """ result_code = RESULT_CODE_UNKNOWN if configurations is None: return (result_code, ['There were no configurations supplied to the script.']) scheme = 'http' http_uri = None https_uri = None http_policy = 'HTTP_ONLY' if SMOKEUSER_KEY in configurations: smokeuser = configurations[SMOKEUSER_KEY] executable_paths = None if EXECUTABLE_SEARCH_PATHS in configurations: executable_paths = configurations[EXECUTABLE_SEARCH_PATHS] security_enabled = False if SECURITY_ENABLED_KEY in configurations: security_enabled = str(configurations[SECURITY_ENABLED_KEY]).upper() == 'TRUE' kerberos_keytab = None if KERBEROS_KEYTAB in configurations: kerberos_keytab = configurations[KERBEROS_KEYTAB] kerberos_principal = None if KERBEROS_PRINCIPAL in configurations: kerberos_principal = configurations[KERBEROS_PRINCIPAL] kerberos_principal = kerberos_principal.replace('_HOST', host_name) if NODEMANAGER_HTTP_ADDRESS_KEY in configurations: http_uri = configurations[NODEMANAGER_HTTP_ADDRESS_KEY] if NODEMANAGER_HTTPS_ADDRESS_KEY in configurations: https_uri = configurations[NODEMANAGER_HTTPS_ADDRESS_KEY] if YARN_HTTP_POLICY_KEY in configurations: http_policy = configurations[YARN_HTTP_POLICY_KEY] # parse script arguments connection_timeout = CONNECTION_TIMEOUT_DEFAULT if CONNECTION_TIMEOUT_KEY in parameters: connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY]) # determine the right URI and whether to use SSL uri = http_uri if http_policy == 'HTTPS_ONLY': scheme = 'https' if https_uri is not None: uri = https_uri label = '' url_response = None node_healthy = 'false' total_time = 0 # some yarn-site structures don't have the web ui address if uri is None: if host_name is None: host_name = socket.getfqdn() uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT) if OSCheck.is_windows_family(): uri_host, uri_port = uri.split(':') # on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1 uri_host = resolve_address(uri_host) uri = '{0}:{1}'.format(uri_host, uri_port) query = "{0}://{1}/ws/v1/node/info".format(scheme,uri) try: if kerberos_principal is not None and kerberos_keytab is not None and security_enabled: env = Environment.get_instance() # curl requires an integer timeout curl_connection_timeout = int(connection_timeout) url_response, error_msg, time_millis = curl_krb_request(env.tmp_dir, kerberos_keytab, kerberos_principal, query, "nm_health_alert", executable_paths, False, "NodeManager Health", smokeuser, connection_timeout=curl_connection_timeout) json_response = json.loads(url_response) else: # execute the query for the JSON that includes templeton status url_response = urllib2.urlopen(query, timeout=connection_timeout) json_response = json.loads(url_response.read()) except urllib2.HTTPError, httpError: label = CRITICAL_HTTP_STATUS_MESSAGE.format(str(httpError.code), query, str(httpError), traceback.format_exc()) return (RESULT_CODE_CRITICAL, [label])
def test_main(self, exithelper_exit_mock, cleanup_mock, ping_port_init_mock, ping_port_start_mock, data_clean_init_mock, data_clean_start_mock, parse_args_mock, start_mock, Controller_is_alive_mock, Controller_init_mock, try_to_connect_mock, update_log_level_mock, daemonize_mock, perform_prestart_checks_mock, ambari_config_mock, stop_mock, bind_signal_handlers_mock, setup_logging_mock, socket_mock): data_clean_init_mock.return_value = None Controller_init_mock.return_value = None Controller_is_alive_mock.return_value = False ping_port_init_mock.return_value = None options = MagicMock() parse_args_mock.return_value = (options, MagicMock) try_to_connect_mock.return_value = (0, True, False ) # (retries, connected, stopped) # use default unix config ambari_config_mock.return_value = self.init_ambari_config_mock() #testing call without command-line arguments main.main() self.assertTrue(setup_logging_mock.called) self.assertTrue(perform_prestart_checks_mock.called) if OSCheck.get_os_family() != OSConst.WINSRV_FAMILY: self.assertTrue(daemonize_mock.called) self.assertTrue(update_log_level_mock.called) try_to_connect_mock.assert_called_once_with(ANY, main.MAX_RETRIES, ANY) self.assertTrue(start_mock.called) self.assertTrue(data_clean_init_mock.called) self.assertTrue(data_clean_start_mock.called) self.assertTrue(ping_port_init_mock.called) self.assertTrue(ping_port_start_mock.called) self.assertTrue(exithelper_exit_mock.called) perform_prestart_checks_mock.reset_mock() # Testing call with --expected-hostname parameter options.expected_hostname = "test.hst" main.main() perform_prestart_checks_mock.assert_called_once_with( options.expected_hostname) # Test with multiple server hostnames default_server_hostnames = hostname.cached_server_hostnames hostname.cached_server_hostnames = ['host1', 'host2', 'host3'] def try_to_connect_impl(*args, **kwargs): for server_hostname in hostname.cached_server_hostnames: if (args[0].find(server_hostname) != -1): if server_hostname == 'host1': return 0, False, False elif server_hostname == 'host2': return 0, False, False elif server_hostname == 'host3': return 0, True, False else: return 0, True, False pass try_to_connect_mock.reset_mock() try_to_connect_mock.side_effect = try_to_connect_impl active_server = main.main() self.assertEquals(active_server, 'host3') hostname.cached_server_hostnames = default_server_hostnames pass
def os_release_name(self): """ For Ubuntu 12.04: precise """ return OSCheck.get_os_release_name()
def getOsFamily(self): return OSCheck.get_os_family()
def main(heartbeat_stop_callback=None): global config parser = OptionParser() parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="verbose log output", default=False) parser.add_option( "-e", "--expected-hostname", dest="expected_hostname", action="store", help= "expected hostname of current host. If hostname differs, agent will fail", default=None) (options, args) = parser.parse_args() expected_hostname = options.expected_hostname current_user = getpass.getuser() logging_level = logging.DEBUG if options.verbose else logging.INFO setup_logging(logger, AmbariConfig.AmbariConfig.getLogFile(), logging_level) setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level) Logger.initialize_logger('resource_management', logging_level=logging_level) default_cfg = {'agent': {'prefix': '/home/ambari'}} config.load(default_cfg) if (len(sys.argv) > 1) and sys.argv[1] == 'stop': stop_agent() if (len(sys.argv) > 2) and sys.argv[1] == 'reset': reset_agent(sys.argv) # Check for ambari configuration file. resolve_ambari_config() # Add syslog hanlder based on ambari config file add_syslog_handler(logger) # Starting data cleanup daemon data_cleaner = None if config.has_option('agent', 'data_cleanup_interval') and int( config.get('agent', 'data_cleanup_interval')) > 0: data_cleaner = DataCleaner(config) data_cleaner.start() perform_prestart_checks(expected_hostname) # Starting ping port listener try: #This acts as a single process machine-wide lock (albeit incomplete, since # we still need an extra file to track the Agent PID) ping_port_listener = PingPortListener(config) except Exception as ex: err_message = "Failed to start ping port listener of: " + str(ex) logger.error(err_message) sys.stderr.write(err_message) sys.exit(1) ping_port_listener.start() update_log_level(config) server_hostname = hostname.server_hostname(config) server_url = config.get_api_url() if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY: daemonize() try: server_ip = socket.gethostbyname(server_hostname) logger.info('Connecting to Ambari server at %s (%s)', server_url, server_ip) except socket.error: logger.warn( "Unable to determine the IP address of the Ambari server '%s'", server_hostname) # Wait until server is reachable netutil = NetUtil(heartbeat_stop_callback) retries, connected = netutil.try_to_connect(server_url, -1, logger) # Ambari Agent was stopped using stop event if connected: # Launch Controller communication controller = Controller(config, heartbeat_stop_callback) controller.start() # controller.join() is not appropriate here as it blocks the signal handlers for stop etc. while controller.is_alive(): time.sleep(0.1) ExitHelper().exit(0)
def execute(self): """ Sets up logging; Parses command parameters and executes method relevant to command type """ parser = OptionParser() parser.add_option( "-o", "--out-files-logging", dest="log_out_files", action="store_true", help= "use this option to enable outputting *.out files of the service pre-start" ) (self.options, args) = parser.parse_args() self.log_out_files = self.options.log_out_files # parse arguments if len(args) < 6: print "Script expects at least 6 arguments" print USAGE.format(os.path.basename( sys.argv[0])) # print to stdout sys.exit(1) self.command_name = str.lower(sys.argv[1]) self.command_data_file = sys.argv[2] self.basedir = sys.argv[3] self.stroutfile = sys.argv[4] self.load_structured_out() self.logging_level = sys.argv[5] Script.tmp_dir = sys.argv[6] logging_level_str = logging._levelNames[self.logging_level] Logger.initialize_logger(__name__, logging_level=logging_level_str) # on windows we need to reload some of env variables manually because there is no default paths for configs(like # /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated # in agent, so other Script executions will not be able to access to new env variables if OSCheck.is_windows_family(): reload_windows_env() try: with open(self.command_data_file) as f: pass Script.config = ConfigDictionary(json.load(f)) # load passwords here(used on windows to impersonate different users) Script.passwords = {} for k, v in _PASSWORD_MAP.iteritems(): if get_path_from_configuration( k, Script.config) and get_path_from_configuration( v, Script.config): Script.passwords[get_path_from_configuration( k, Script.config)] = get_path_from_configuration( v, Script.config) except IOError: Logger.logger.exception( "Can not read json file with command parameters: ") sys.exit(1) # Run class method depending on a command type try: method = self.choose_method_to_execute(self.command_name) with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env: env.config.download_path = Script.tmp_dir if self.command_name == "start" and not self.is_hook(): self.pre_start() method(env) if self.command_name == "start" and not self.is_hook(): self.post_start() except Fail as ex: ex.pre_raise() raise finally: if self.should_expose_component_version(self.command_name): self.save_component_version_to_structured_out()
def _collect(self): # can be parameterized or static # if not parameterized, this will return the static value uri_value = self._get_configuration_value(self.uri) if uri_value is None: uri_value = self.host_name logger.debug("[Alert][{0}] Setting the URI to this host since it wasn't specified".format( self.get_name())) # in some cases, a single property is a comma-separated list like # host1:8080,host2:8081,host3:8083 uri_value_array = uri_value.split(',') if len(uri_value_array) > 1: for item in uri_value_array: if self.host_name in item: uri_value = item if logger.isEnabledFor(logging.DEBUG): logger.debug("[Alert][{0}] Extracted {1} as the host name while parsing the CSV URI {2}".format( self.get_name(), uri_value, str(uri_value_array))) break host = BaseAlert.get_host_from_url(uri_value) if host is None or host == "localhost" or host == "0.0.0.0": host = self.host_name try: port = int(get_port_from_url(uri_value)) except: if self.default_port is None: label = 'Unable to determine port from URI {0}'.format(uri_value) return (self.RESULT_UNKNOWN, [label]) port = self.default_port if logger.isEnabledFor(logging.DEBUG): logger.debug("[Alert][{0}] Checking {1} on port {2}".format( self.get_name(), host, str(port))) s = None try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(self.critical_timeout) if OSCheck.is_windows_family(): # on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1 host = resolve_address(host) start_time = time.time() s.connect((host, port)) if self.socket_command is not None: s.sendall(self.socket_command) data = s.recv(1024) if self.socket_command_response is not None and data != self.socket_command_response: raise Exception("Expected response {0}, Actual response {1}".format( self.socket_command_response, data)) end_time = time.time() milliseconds = end_time - start_time seconds = milliseconds / 1000.0 # not sure why this happens sometimes, but we don't always get a # socket exception if the connect() is > than the critical threshold if seconds >= self.critical_timeout: return (self.RESULT_CRITICAL, ['Socket Timeout', host, port]) result = self.RESULT_OK if seconds >= self.warning_timeout: result = self.RESULT_WARNING return (result, [seconds, port]) except Exception as e: return (self.RESULT_CRITICAL, [str(e), host, port]) finally: if s is not None: try: s.close() except: # no need to log a close failure pass
def install_packages(self, env): """ List of packages that are required< by service is received from the server as a command parameter. The method installs all packages from this list exclude_packages - list of regexes (possibly raw strings as well), the packages which match the regex won't be installed. NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ? """ config = self.get_config() if 'host_sys_prepped' in config['hostLevelParams']: # do not install anything on sys-prepped host if config['hostLevelParams']['host_sys_prepped'] == True: Logger.info("Node has all packages pre-installed. Skipping.") return pass try: package_list_str = config['hostLevelParams']['package_list'] agent_stack_retry_on_unavailability = bool( config['hostLevelParams'] ['agent_stack_retry_on_unavailability']) agent_stack_retry_count = int( config['hostLevelParams']['agent_stack_retry_count']) try: available_packages_in_repos = packages_analyzer.get_available_packages_in_repos( config['repositoryFile']['repositories']) except Exception as err: available_packages_in_repos = [] if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: if self.check_package_condition(package): name = self.get_package_from_available( package['name'], available_packages_in_repos) # HACK: On Windows, only install ambari-metrics packages using Choco Package Installer # TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support # <osFamily>any<osFamily> which would cause installation failure on Windows. if OSCheck.is_windows_family(): if "ambari-metrics" in name: Package(name) else: Package(name, retry_on_repo_unavailability= agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count) except KeyError: pass # No reason to worry if OSCheck.is_windows_family(): #TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented hadoop_user = config["configurations"]["cluster-env"][ "hadoop.user.name"] install_windows_msi( config['hostLevelParams']['jdk_location'], config["hostLevelParams"]["agentCacheDir"], [ "hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab", "hdp-2.3.0.0-01.cab" ], hadoop_user, self.get_password(hadoop_user), str(config['hostLevelParams']['stack_version'])) reload_windows_env()
def copyNeededFiles(self): # get the params params = self.shared_state # Copying the files fileToCopy = self.getRepoFile() target = self.getRemoteName(self.AMBARI_REPO_FILENAME) if (os.path.exists(fileToCopy)): self.host_log.write("==========================\n") self.host_log.write("Copying repo file to 'tmp' folder...") scp = SCP(params.user, params.sshkey_file, self.host, fileToCopy, target, params.bootdir, self.host_log) retcode1 = scp.run() self.host_log.write("\n") # Move file to repo dir self.host_log.write("==========================\n") self.host_log.write("Moving file to repo dir...") targetDir = self.getRepoDir() command = self.getMoveRepoFileCommand(targetDir) ssh = SSH(params.user, params.sshkey_file, self.host, command, params.bootdir, self.host_log) retcode2 = ssh.run() self.host_log.write("\n") # Change permissions on ambari.repo self.host_log.write("==========================\n") self.host_log.write("Changing permissions for ambari.repo...") command = self.getRepoFileChmodCommand() ssh = SSH(params.user, params.sshkey_file, self.host, command, params.bootdir, self.host_log) retcode4 = ssh.run() self.host_log.write("\n") # Update repo cache for ubuntu OS if OSCheck.is_ubuntu_family(): self.host_log.write("==========================\n") self.host_log.write("Update apt cache of repository...") command = self.getAptUpdateCommand() ssh = SSH(params.user, params.sshkey_file, self.host, command, params.bootdir, self.host_log) retcode2 = ssh.run() self.host_log.write("\n") retcode = max(retcode1["exitstatus"], retcode2["exitstatus"], retcode4["exitstatus"]) else: self.host_log.write("==========================\n") self.host_log.write("Copying required files...") self.host_log.write("Ambari repo file not found: {0}".format(self.getRepoFile())) retcode = -1 pass # copy the setup script file self.host_log.write("==========================\n") self.host_log.write("Copying setup script file...") fileToCopy = params.setup_agent_file target = self.getRemoteName(self.SETUP_SCRIPT_FILENAME) scp = SCP(params.user, params.sshkey_file, self.host, fileToCopy, target, params.bootdir, self.host_log) retcode3 = scp.run() self.host_log.write("\n") return max(retcode, retcode3["exitstatus"])
def get_postgre_running_status(): """Return postgre running status indicator""" if OSCheck.is_ubuntu_family(): return os.path.join(get_ubuntu_pg_version(), "main") else: return PG_STATUS_RUNNING_DEFAULT
def getOperatingSystemRelease(self): return OSCheck.get_os_version()
from resource_management.core.logger import Logger from resource_management.libraries.script.script import Script from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries import functions from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format from resource_management.libraries.functions.substitute_vars import substitute_vars from resource_management.libraries.resources.hdfs_resource import HdfsResource from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames import status_params from ambari_commons import OSCheck import ConfigParser import os if OSCheck.is_windows_family(): from params_windows import * else: from params_linux import * # server configurations config = Script.get_config() exec_tmp_dir = Script.get_tmp_dir() def get_combined_memory_mb(value1, value2): try: part1 = int(value1.strip()[:-1]) if value1.lower().strip()[-1:] == 'm' else int(value1) part2 = int(value2.strip()[:-1]) if value2.lower().strip()[-1:] == 'm' else int(value2) return str(part1 + part2) + 'm' except: return None pass
def format_package_name(self, name, repo_version=None): from resource_management.libraries.functions.default import default """ This function replaces ${stack_version} placeholder with actual version. If the package version is passed from the server, use that as an absolute truth. :param name name of the package :param repo_version actual version of the repo currently installing """ stack_version_package_formatted = "" if not repo_version: repo_version = self.get_stack_version_before_packages_installed() package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_' # repositoryFile is the truth # package_version should be made to the form W_X_Y_Z_nnnn package_version = default("repositoryFile/repoVersion", None) if package_version is not None: package_version = package_version.replace( '.', package_delimiter).replace('-', package_delimiter) # TODO remove legacy checks if package_version is None: package_version = default("roleParams/package_version", None) # TODO remove legacy checks if package_version is None: package_version = default("hostLevelParams/package_version", None) # The cluster effective version comes down when the version is known after the initial # install. In that case we should not be guessing which version when invoking INSTALL, but # use the supplied version to build the package_version effective_version = default("commandParams/version", None) role_command = default("roleCommand", None) if (package_version is None or '*' in package_version) \ and effective_version is not None and 'INSTALL' == role_command: package_version = effective_version.replace( '.', package_delimiter).replace('-', package_delimiter) Logger.info( "Version {0} was provided as effective cluster version. Using package version {1}" .format(effective_version, package_version)) if package_version: stack_version_package_formatted = package_version if OSCheck.is_ubuntu_family(): stack_version_package_formatted = package_version.replace( '_', package_delimiter) # Wildcards cause a lot of troubles with installing packages, if the version contains wildcards we try to specify it. if not package_version or '*' in package_version: stack_version_package_formatted = repo_version.replace( '.', package_delimiter).replace( '-', package_delimiter ) if STACK_VERSION_PLACEHOLDER in name else name package_name = name.replace(STACK_VERSION_PLACEHOLDER, stack_version_package_formatted) return package_name