コード例 #1
0
ファイル: TestOSCheck.py プロジェクト: biggeng/ambari
  def is_ubuntu_family(self, get_os_family_mock):

    get_os_family_mock.return_value = "ubuntu"
    self.assertEqual(OSCheck.is_ubuntu_family(), True)

    get_os_family_mock.return_value = "troll_os"
    self.assertEqual(OSCheck.is_ubuntu_family(), False)
コード例 #2
0
ファイル: utils.py プロジェクト: zouzhberk/ambaridemo
def get_postgre_hba_dir(OS_FAMILY):
  """Return postgre hba dir location depends on OS.
  Also depends on version of postgres creates symlink like postgresql-->postgresql-9.3
  1) /etc/rc.d/init.d/postgresql --> /etc/rc.d/init.d/postgresql-9.3
  2) /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1
  """
  if OSCheck.is_ubuntu_family():
    # Like: /etc/postgresql/9.1/main/
    return os.path.join(get_pg_hba_init_files(), get_ubuntu_pg_version(),
                        "main")
  elif OSCheck.is_redhat_family() and int(OSCheck.get_os_major_version()) >= 7:
    return PG_HBA_ROOT_DEFAULT
  else:
    if not os.path.isfile(get_pg_hba_init_files()):
      # Link: /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1
      os.symlink(glob.glob(get_pg_hba_init_files() + '*')[0],
                 get_pg_hba_init_files())

    # Get postgres_data location (default: /var/lib/pgsql/data)
    cmd = "alias exit=return; source " + get_pg_hba_init_files() + " status &>/dev/null; echo $PGDATA"
    p = subprocess.Popen(cmd,
                         stdout=subprocess.PIPE,
                         stdin=subprocess.PIPE,
                         stderr=subprocess.PIPE,
                         shell=True)
    (PG_HBA_ROOT, err) = p.communicate()

    if PG_HBA_ROOT and len(PG_HBA_ROOT.strip()) > 0:
      return PG_HBA_ROOT.strip()
    else:
      return PG_HBA_ROOT_DEFAULT
コード例 #3
0
  def find_repo_files_for_repos(self, repoNames):
    repoFiles = []
    osType = OSCheck.get_os_family()
    repoNameList = []
    for repoName in repoNames:
      if len(repoName.strip()) > 0:
        repoNameList.append("[" + repoName + "]")
        repoNameList.append("name=" + repoName)
    if repoNameList:
      # get list of files
      if OSCheck.is_suse_family():
        fileList = self.get_files_in_dir(REPO_PATH_SUSE)
      elif OSCheck.is_redhat_family():
        fileList = self.get_files_in_dir(REPO_PATH_RHEL)
      elif OSCheck.is_ubuntu_family():
        fileList = self.get_files_in_dir(REPO_PATH_UBUNTU)
      else:
        logger.warn("Unsupported OS type, cannot get repository location.")
        return []

      if fileList:
        for filePath in fileList:
          with open(filePath, 'r') as file:
            content = file.readline()
            while (content != "" ):
              for repoName in repoNameList:
                if content.find(repoName) == 0 and filePath not in repoFiles:
                  repoFiles.append(filePath)
                  break;
              content = file.readline()

    return repoFiles
コード例 #4
0
 def getFirewallObject(self):
   if OSCheck.is_ubuntu_family():
     return UbuntuFirewallChecks()
   elif self.OS_TYPE == OSConst.OS_FEDORA and int(self.OS_VERSION) >= 18:
     return Fedora18FirewallChecks()
   elif OSCheck.is_suse_family():
     return SuseFirewallChecks()
   else:
     return FirewallChecks()
コード例 #5
0
ファイル: utils.py プロジェクト: zouzhberk/ambaridemo
def get_pg_hba_init_files():
  if OSCheck.is_ubuntu_family():
    return '/etc/postgresql'
  elif OSCheck.is_redhat_family():
    return '/etc/rc.d/init.d/postgresql'
  elif OSCheck.is_suse_family():
    return '/etc/init.d/postgresql'
  else:
    raise Exception("Unsupported OS family '{0}'".format(OSCheck.get_os_family()))
コード例 #6
0
def get_erase_cmd():
  if OSCheck.is_redhat_family():
    return "yum erase -y {0}"
  elif OSCheck.is_suse_family():
    return "zypper -n -q remove {0}"
  elif OSCheck.is_ubuntu_family():
    return "/usr/bin/apt-get -y -q remove {0}"
  else:
    raise Exception("Unsupported OS family '{0}', cannot remove package. ".format(OSCheck.get_os_family()))
コード例 #7
0
ファイル: bootstrap.py プロジェクト: zouzhberk/ambaridemo
 def getRepoDir(self):
   if OSCheck.is_redhat_family():
     return "/etc/yum.repos.d"
   elif OSCheck.is_suse_family():
     return "/etc/zypp/repos.d"
   elif OSCheck.is_ubuntu_family():
     return "/etc/apt/sources.list.d"
   else:
     raise Exception("Unsupported OS family '{0}'".format(OSCheck.get_os_family()))
コード例 #8
0
ファイル: setupAgent.py プロジェクト: duxia/ambari
def isAgentPackageAlreadyInstalled(projectVersion):
    if OSCheck.is_ubuntu_family():
      Command = ["bash", "-c", "dpkg-query -W -f='${Status} ${Version}\n' ambari-agent | grep -v deinstall | grep " + projectVersion]
    else:
      Command = ["bash", "-c", "rpm -qa | grep ambari-agent-"+projectVersion]
    ret = execOsCommand(Command)
    res = False
    if ret["exitstatus"] == 0 and ret["log"][0].strip() != "":
        res = True
    return res
コード例 #9
0
ファイル: setupAgent.py プロジェクト: duxia/ambari
def installAgent(projectVersion):
  """ Run install and make sure the agent install alright """
  # The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host
  if OSCheck.is_suse_family():
    Command = ["zypper", "--no-gpg-checks", "install", "-y", "ambari-agent-" + projectVersion]
  elif OSCheck.is_ubuntu_family():
    # add * to end of version in case of some test releases
    Command = ["apt-get", "install", "-y", "--allow-unauthenticated", "ambari-agent=" + projectVersion + "*"]
  else:
    Command = ["yum", "-y", "install", "--nogpgcheck", "ambari-agent-" + projectVersion]
  return execOsCommand(Command, tries=3, try_sleep=10)
コード例 #10
0
ファイル: setupAgent.py プロジェクト: duxia/ambari
def getAvaliableAgentPackageVersions():
  if OSCheck.is_suse_family():
    Command = ["bash", "-c",
        "zypper --no-gpg-checks -q search -s --match-exact ambari-agent | grep ambari-agent | sed -re 's/\s+/ /g' | cut -d '|' -f 4 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"]
  elif OSCheck.is_ubuntu_family():
    Command = ["bash", "-c",
        "apt-cache -q show ambari-agent|grep 'Version\:'|cut -d ' ' -f 2| tr '\\n' ', '|sed -s 's/[-|~][A-Za-z0-9]*//g'"]
  else:
    Command = ["bash", "-c",
        "yum -q list all ambari-agent | grep -E '^ambari-agent' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"]
  return execOsCommand(Command)
コード例 #11
0
  def copyNeededFiles(self):
    # Copying the files
    fileToCopy = self.getRepoFile()
    target = self.getRemoteName(self.AMBARI_REPO_FILENAME)

    self.host_log.write("==========================\n")
    self.host_log.write("Copying repo file to 'tmp' folder...")
    params = self.shared_state
    scp = SCP(params.user, params.sshkey_file, self.host, fileToCopy,
              target, params.bootdir, self.host_log)
    retcode1 = scp.run()
    self.host_log.write("\n")

    # Move file to repo dir
    self.host_log.write("==========================\n")
    self.host_log.write("Moving file to repo dir...")
    targetDir = self.getRepoDir()
    command = self.getMoveRepoFileCommand(targetDir)
    ssh = SSH(params.user, params.sshkey_file, self.host, command,
              params.bootdir, self.host_log)
    retcode2 = ssh.run()
    self.host_log.write("\n")
    
    # Change permissions on ambari.repo
    self.host_log.write("==========================\n")
    self.host_log.write("Changing permissions for ambari.repo...")
    command = self.getRepoFileChmodCommand()
    ssh = SSH(params.user, params.sshkey_file, self.host, command,
              params.bootdir, self.host_log)
    retcode4 = ssh.run()
    self.host_log.write("\n")

    # Update repo cache for ubuntu OS
    if OSCheck.is_ubuntu_family():
      self.host_log.write("==========================\n")
      self.host_log.write("Update apt cache of repository...")
      command = self.getAptUpdateCommand()
      ssh = SSH(params.user, params.sshkey_file, self.host, command,
                params.bootdir, self.host_log)
      retcode2 = ssh.run()
      self.host_log.write("\n")

    self.host_log.write("==========================\n")
    self.host_log.write("Copying setup script file...")
    fileToCopy = params.setup_agent_file
    target = self.getRemoteName(self.SETUP_SCRIPT_FILENAME)
    scp = SCP(params.user, params.sshkey_file, self.host, fileToCopy,
              target, params.bootdir, self.host_log)
    retcode3 = scp.run()
    self.host_log.write("\n")

    return max(retcode1["exitstatus"], retcode2["exitstatus"], retcode3["exitstatus"], retcode4["exitstatus"])
コード例 #12
0
ファイル: script.py プロジェクト: maduhu/HDP2.5-ambari
  def format_package_name(self, name):
    from resource_management.libraries.functions.default import default
    """
    This function replaces ${stack_version} placeholder into actual version.  If the package
    version is passed from the server, use that as an absolute truth.
    """

    # two different command types put things in different objects.  WHY.
    # package_version is the form W_X_Y_Z_nnnn
    package_version = default("roleParams/package_version", None)
    if not package_version:
      package_version = default("hostLevelParams/package_version", None)

    package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_'

    # The cluster effective version comes down when the version is known after the initial
    # install.  In that case we should not be guessing which version when invoking INSTALL, but
    # use the supplied version to build the package_version
    effective_version = default("commandParams/version", None)
    role_command = default("roleCommand", None)

    if (package_version is None or '*' in package_version) \
        and effective_version is not None and 'INSTALL' == role_command:
      package_version = effective_version.replace('.', package_delimiter).replace('-', package_delimiter)
      Logger.info("Version {0} was provided as effective cluster version.  Using package version {1}".format(effective_version, package_version))

    if package_version:
      stack_version_package_formatted = package_version
      if OSCheck.is_ubuntu_family():
        stack_version_package_formatted = package_version.replace('_', package_delimiter)

    # Wildcards cause a lot of troubles with installing packages, if the version contains wildcards we try to specify it.
    if not package_version or '*' in package_version:
      stack_version_package_formatted = self.get_stack_version_before_packages_installed().replace('.', package_delimiter).replace('-', package_delimiter) if STACK_VERSION_PLACEHOLDER in name else name

    package_name = name.replace(STACK_VERSION_PLACEHOLDER, stack_version_package_formatted)
    
    return package_name
コード例 #13
0
def installAgent(projectVersion, ret=None):
  """ Run install and make sure the agent install alright """
  # The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host
  if OSCheck.is_suse_family():
    Command = ["zypper", "--no-gpg-checks", "install", "-y", "ambari-agent-" + projectVersion]
  elif OSCheck.is_ubuntu_family():
    # add * to end of version in case of some test releases
    Command = ["apt-get", "install", "-y", "--allow-unauthenticated", "ambari-agent=" + projectVersion + "*"]
  elif OSCheck.is_windows_family():
    packageParams = "/AmbariRoot:" + AMBARI_INSTALL_ROOT
    Command = ["cmd", "/c", "choco", "install", "-y", "ambari-agent", "--version=" + projectVersion, "--params=\"" + packageParams + "\""]
  else:
    Command = ["yum", "-y", "install", "--nogpgcheck", "ambari-agent-" + projectVersion]
  return execOsCommand(Command, tries=3, try_sleep=10, ret=ret)
コード例 #14
0
ファイル: os_utils.py プロジェクト: maduhu/HDP2.5-ambari
def get_ambari_repo_file_full_name():
  if OSCheck.is_ubuntu_family():
    ambari_repo_file = "/etc/apt/sources.list.d/ambari.list"
  elif OSCheck.is_redhat_family():
    ambari_repo_file = "/etc/yum.repos.d/ambari.repo"
  elif OSCheck.is_suse_family():
    ambari_repo_file = "/etc/zypp/repos.d/ambari.repo"
  elif OSCheck.is_windows_family():
    ambari_repo_file = os.path.join(os.environ[ChocolateyConsts.CHOCOLATEY_INSTALL_VAR_NAME],
                                    ChocolateyConsts.CHOCOLATEY_CONFIG_DIR, ChocolateyConsts.CHOCOLATEY_CONFIG_FILENAME)
  else:
    raise Exception('Ambari repo file path not set for current OS.')

  return ambari_repo_file
コード例 #15
0
ファイル: HostInfo.py プロジェクト: maduhu/HDP2.5-ambari
  def getTransparentHugePage(self):
    thp_regex = "\[(.+)\]"
    file_name = None
    if OSCheck.is_ubuntu_family():
      file_name = self.THP_FILE_UBUNTU
    elif OSCheck.is_redhat_family():
      file_name = self.THP_FILE_REDHAT

    if file_name and os.path.isfile(file_name):
      with open(file_name) as f:
        file_content = f.read()
        return re.search(thp_regex, file_content).groups()[0]
    else:
      return ""
コード例 #16
0
def setup_atlas_hive(configuration_directory=None):
  import params

  if params.has_atlas:
    if configuration_directory is None:
      configuration_directory = format("{hive_config_dir}")

    if not params.host_sys_prepped:
      Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, # FIXME HACK: install the package during RESTART/START when install_packages is not triggered.
      )

    PropertiesFile(format('{configuration_directory}/client.properties'),
                   properties = params.atlas_client_props,
                   owner = params.hive_user,
                   group = params.user_group,
                   mode = 0644)
コード例 #17
0
ファイル: bootstrap.py プロジェクト: zouzhberk/ambaridemo
  def __init__(self, host, shared_state):
    threading.Thread.__init__(self)
    self.host = host
    self.shared_state = shared_state
    self.status = {
      "start_time": None,
      "return_code": None,
    }
    log_file = os.path.join(self.shared_state.bootdir, self.host + ".log")
    self.host_log = HostLog(log_file)
    self.daemon = True

    if OSCheck.is_ubuntu_family():
      self.AMBARI_REPO_FILENAME = self.AMBARI_REPO_FILENAME + ".list"
    else:
      self.AMBARI_REPO_FILENAME = self.AMBARI_REPO_FILENAME + ".repo"
コード例 #18
0
ファイル: bootstrap.py プロジェクト: zouzhberk/ambaridemo
 def checkSudoPackage(self):
   """ Checking 'sudo' package on remote host """
   self.host_log.write("==========================\n")
   self.host_log.write("Checking 'sudo' package on remote host...")
   params = self.shared_state
   if OSCheck.is_ubuntu_family():
     command = "dpkg --get-selections|grep -e '^sudo\s*install'"
   else:
     command = "rpm -qa | grep -e '^sudo\-'"
   ssh = SSH(params.user, params.sshkey_file, self.host, command,
             params.bootdir, self.host_log,
             errorMessage="Error: Sudo command is not available. "
                          "Please install the sudo command.")
   retcode = ssh.run()
   self.host_log.write("\n")
   return retcode
コード例 #19
0
def allAvailablePackages(allAvailablePackages):
  osType = OSCheck.get_os_family()

  if OSCheck.is_suse_family():
    return _lookUpZypperPackages(
      ["sudo", "zypper", "search", "--uninstalled-only", "--details"],
      allAvailablePackages)
  elif OSCheck.is_redhat_family():
    return _lookUpYumPackages(
      ["sudo", "yum", "list", "available"],
      'Available Packages',
      allAvailablePackages)
  elif OSCheck.is_ubuntu_family():
     return _lookUpAptPackages(
      LIST_AVAILABLE_PACKAGES_UBUNTU,
      allAvailablePackages)
コード例 #20
0
ファイル: setupAgent.py プロジェクト: duxia/ambari
def findNearestAgentPackageVersion(projectVersion):
  if projectVersion == "":
    projectVersion = "  "
  if OSCheck.is_suse_family():
    Command = ["bash", "-c", "zypper --no-gpg-checks -q search -s --match-exact ambari-agent | grep '" + projectVersion +
                                 "' | cut -d '|' -f 4 | head -n1 | sed -e 's/-\w[^:]*//1' "]
  elif OSCheck.is_ubuntu_family():
    if projectVersion == "  ":
      Command = ["bash", "-c", "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"]
    else:
      Command = ["bash", "-c", "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|grep '" +
               projectVersion + "'|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"]
  else:
    Command = ["bash", "-c", "yum -q list all ambari-agent | grep '" + projectVersion +
                              "' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | head -n1 | sed -e 's/-\w[^:]*//1' "]
  return execOsCommand(Command)
コード例 #21
0
def allInstalledPackages(allInstalledPackages):
  """
  All installed packages in system
  """
  osType = OSCheck.get_os_family()

  if OSCheck.is_suse_family():
    return _lookUpZypperPackages(
      ["sudo", "zypper", "search", "--installed-only", "--details"],
      allInstalledPackages)
  elif OSCheck.is_redhat_family():
    return _lookUpYumPackages(
      ["sudo", "yum", "list", "installed"],
      'Installed Packages',
      allInstalledPackages)
  elif OSCheck.is_ubuntu_family():
     return _lookUpAptPackages(
      LIST_INSTALLED_PACKAGES_UBUNTU,
      allInstalledPackages)
コード例 #22
0
 def _check_postgre_up():
   pg_status, retcode, out, err = PGConfig._get_postgre_status()
   if pg_status == PGConfig.PG_STATUS_RUNNING:
     print_info_msg("PostgreSQL is running")
     return pg_status, 0, out, err
   else:
     # run initdb only on non ubuntu systems as ubuntu does not have initdb cmd.
     if not OSCheck.is_ubuntu_family():
       print "Running initdb: This may take upto a minute."
       retcode, out, err = run_os_command(PGConfig.PG_INITDB_CMD)
       if retcode == 0:
         print out
     print "About to start PostgreSQL"
     try:
       process = subprocess.Popen(PGConfig.PG_START_CMD.split(' '),
                                  stdout=subprocess.PIPE,
                                  stdin=subprocess.PIPE,
                                  stderr=subprocess.PIPE
       )
       if OSCheck.is_suse_family():
         time.sleep(20)
         result = process.poll()
         print_info_msg("Result of postgres start cmd: " + str(result))
         if result is None:
           process.kill()
           pg_status, retcode, out, err = PGConfig._get_postgre_status()
         else:
           retcode = result
       else:
         out, err = process.communicate()
         retcode = process.returncode
         pg_status, retcode, out, err = PGConfig._get_postgre_status()
       if pg_status == PGConfig.PG_STATUS_RUNNING:
         print_info_msg("Postgres process is running. Returning...")
         return pg_status, 0, out, err
     except (Exception), e:
       pg_status, retcode, out, err = PGConfig._get_postgre_status()
       if pg_status == PGConfig.PG_STATUS_RUNNING:
         return pg_status, 0, out, err
       else:
         print_error_msg("Postgres start failed. " + str(e))
     return pg_status, retcode, out, err
コード例 #23
0
ファイル: setupAgent.py プロジェクト: p4hm/ambari
def installAgent(projectVersion):
    """ Run install and make sure the agent install alright """
    # The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host
    if OSCheck.is_suse_family():
        Command = [
            "zypper", "--no-gpg-checks", "install", "-y",
            "ambari-agent-" + projectVersion
        ]
    elif OSCheck.is_ubuntu_family():
        # add * to end of version in case of some test releases
        Command = [
            "apt-get", "install", "-y", "--allow-unauthenticated",
            "ambari-agent=" + projectVersion + "*"
        ]
    else:
        Command = [
            "yum", "-y", "install", "--nogpgcheck",
            "ambari-agent-" + projectVersion
        ]
    return execOsCommand(Command, tries=3, try_sleep=10)
コード例 #24
0
 def checkSudoPackage(self):
     """ Checking 'sudo' package on remote host """
     self.host_log.write("==========================\n")
     self.host_log.write("Checking 'sudo' package on remote host...")
     params = self.shared_state
     if OSCheck.is_ubuntu_family():
         command = "dpkg --get-selections|grep -e '^sudo\s*install'"
     else:
         command = "rpm -qa | grep -e '^sudo\-'"
     ssh = SSH(params.user,
               params.sshkey_file,
               self.host,
               command,
               params.bootdir,
               self.host_log,
               errorMessage="Error: Sudo command is not available. "
               "Please install the sudo command.")
     retcode = ssh.run()
     self.host_log.write("\n")
     return retcode
コード例 #25
0
def findNearestAgentPackageVersion(projectVersion):
  if projectVersion == "":
    projectVersion = "  "
  if OSCheck.is_suse_family():
    Command = ["bash", "-c", "zypper --no-gpg-checks -q search -s --match-exact ambari-agent | grep '" + projectVersion +
                                 "' | cut -d '|' -f 4 | head -n1 | sed -e 's/-\w[^:]*//1' "]
  elif OSCheck.is_windows_family():
    listPackagesCommand = ["cmd", "/c", "choco list ambari-agent --pre --all | findstr " + projectVersion + " > agentPackages.list"]
    execOsCommand(listPackagesCommand)
    Command = ["cmd", "/c", "powershell",  "get-content agentPackages.list | select-object -last 1 | foreach-object {$_ -replace 'ambari-agent ', ''}"]
  elif OSCheck.is_ubuntu_family():
    if projectVersion == "  ":
      Command = ["bash", "-c", "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"]
    else:
      Command = ["bash", "-c", "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|grep '" +
               projectVersion + "'|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"]
  else:
    Command = ["bash", "-c", "yum -q list all ambari-agent | grep '" + projectVersion +
                              "' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | head -n1 | sed -e 's/-\w[^:]*//1' "]
  return execOsCommand(Command)
コード例 #26
0
def setup_atlas_hive(configuration_directory=None):
    import params

    if params.has_atlas:
        if configuration_directory is None:
            configuration_directory = format("{hive_config_dir}")

        if not params.host_sys_prepped:
            Package(
                params.atlas_ubuntu_plugin_package
                if OSCheck.is_ubuntu_family() else params.
                atlas_plugin_package,  # FIXME HACK: install the package during RESTART/START when install_packages is not triggered.
                retry_on_repo_unavailability=params.
                agent_stack_retry_on_unavailability,
                retry_count=params.agent_stack_retry_count)

        PropertiesFile(format('{configuration_directory}/{atlas_conf_file}'),
                       properties=params.atlas_props,
                       owner=params.hive_user,
                       group=params.user_group,
                       mode=0644)
コード例 #27
0
ファイル: setupAgent.py プロジェクト: soener/ambari
def isAgentPackageAlreadyInstalled(projectVersion):
    if OSCheck.is_ubuntu_family():
        Command = [
            "bash", "-c",
            "dpkg-query -W -f='${Status} ${Version}\n' ambari-agent | grep -v deinstall | grep "
            + projectVersion
        ]
    elif OSCheck.is_windows_family():
        Command = [
            "cmd", "/c",
            "choco list ambari-agent --local-only | findstr ambari-agent"
        ]
    else:
        Command = [
            "bash", "-c", "rpm -qa | grep ambari-agent-" + projectVersion
        ]
    ret = execOsCommand(Command)
    res = False
    if ret["exitstatus"] == 0 and ret["log"][0].strip() != "":
        res = True
    return res
コード例 #28
0
ファイル: setupAgent.py プロジェクト: zll405361679/ambari
def findNearestAgentPackageVersion(projectVersion):
    if projectVersion == "":
        projectVersion = "  "
    if OSCheck.is_suse_family():
        Command = [
            "bash", "-c",
            "zypper --no-gpg-checks --non-interactive -q search -s --match-exact ambari-agent | grep '"
            + projectVersion +
            "' | cut -d '|' -f 4 | head -n1 | sed -e 's/-\w[^:]*//1' "
        ]
    elif OSCheck.is_windows_family():
        listPackagesCommand = [
            "cmd", "/c", "choco list ambari-agent --pre --all | findstr " +
            projectVersion + " > agentPackages.list"
        ]
        execOsCommand(listPackagesCommand)
        Command = [
            "cmd", "/c", "powershell",
            "get-content agentPackages.list | select-object -last 1 | foreach-object {$_ -replace 'ambari-agent ', ''}"
        ]
    elif OSCheck.is_ubuntu_family():
        if projectVersion == "  ":
            Command = [
                "bash", "-c",
                "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"
            ]
        else:
            Command = [
                "bash", "-c",
                "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|grep '"
                + projectVersion +
                "'|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"
            ]
    else:
        Command = [
            "bash", "-c",
            "yum -q list all ambari-agent | grep '" + projectVersion +
            "' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | head -n1 | sed -e 's/-\w[^:]*//1' "
        ]
    return execOsCommand(Command)
コード例 #29
0
ファイル: utils.py プロジェクト: Liujinan001/ambari-2.7.5
def get_postgre_hba_dir(OS_FAMILY):
    """Return postgre hba dir location depends on OS.
  Also depends on version of postgres creates symlink like postgresql-->postgresql-9.3
  1) /etc/rc.d/init.d/postgresql --> /etc/rc.d/init.d/postgresql-9.3
  2) /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1
  """
    if OSCheck.is_ubuntu_family():
        # Like: /etc/postgresql/9.1/main/
        return os.path.join(get_pg_hba_init_files(), get_ubuntu_pg_version(),
                            "main")
    elif glob.glob(
            get_pg_hba_init_files() + '*'
    ):  # this happens when the service file is of old format (not like /usr/lib/systemd/system/postgresql.service)
        if not os.path.isfile(get_pg_hba_init_files()):
            # Link: /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1
            os.symlink(
                glob.glob(get_pg_hba_init_files() + '*')[0],
                get_pg_hba_init_files())

        pg_hba_init_basename = os.path.basename(get_pg_hba_init_files())
        # Get postgres_data location (default: /var/lib/pgsql/data)
        cmd = "alias basename='echo {0}; true' ; alias exit=return; source {1} status &>/dev/null; echo $PGDATA".format(
            pg_hba_init_basename, get_pg_hba_init_files())
        p = subprocess32.Popen(cmd,
                               stdout=subprocess32.PIPE,
                               stdin=subprocess32.PIPE,
                               stderr=subprocess32.PIPE,
                               shell=True)
        (PG_HBA_ROOT, err) = p.communicate()

        if PG_HBA_ROOT and len(PG_HBA_ROOT.strip()) > 0:
            return PG_HBA_ROOT.strip()

    if not os.path.exists(PG_HBA_ROOT_DEFAULT):
        versioned_dirs = glob.glob(PG_HBA_ROOT_DEFAULT_VERSIONED)
        if versioned_dirs:
            return versioned_dirs[0]

    return PG_HBA_ROOT_DEFAULT
コード例 #30
0
ファイル: setupAgent.py プロジェクト: zll405361679/ambari
def getAvailableAgentPackageVersions():
    if OSCheck.is_suse_family():
        Command = [
            "bash", "-c",
            "zypper --no-gpg-checks --non-interactive -q search -s --match-exact ambari-agent | grep ambari-agent | sed -re 's/\s+/ /g' | cut -d '|' -f 4 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"
        ]
    elif OSCheck.is_windows_family():
        Command = [
            "cmd", "/c",
            "choco list ambari-agent --pre --all | findstr ambari-agent"
        ]
    elif OSCheck.is_ubuntu_family():
        Command = [
            "bash", "-c",
            "apt-cache -q show ambari-agent|grep 'Version\:'|cut -d ' ' -f 2| tr '\\n' ', '|sed -s 's/[-|~][A-Za-z0-9]*//g'"
        ]
    else:
        Command = [
            "bash", "-c",
            "yum -q list all ambari-agent | grep -E '^ambari-agent' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"
        ]
    return execOsCommand(Command)
コード例 #31
0
def verifyDependencies():
    """
  Verify that we have no dependency issues in package manager. Dependency issues could appear because of aborted or terminated
  package installation process or invalid packages state after manual modification of packages list on the host
   :return True if no dependency issues found, False if dependency issue present
  :rtype bool
  """
    check_str = None
    cmd = None

    if OSCheck.is_redhat_family():
        cmd = ['/usr/bin/yum', '-d', '0', '-e', '0', 'check', 'dependencies']
        check_str = "has missing requires|Error:"
    elif OSCheck.is_suse_family():
        cmd = [
            '/usr/bin/zypper', '--quiet', '--non-interactive', 'verify',
            '--dry-run'
        ]
        check_str = "\d+ new package(s)? to install"
    elif OSCheck.is_ubuntu_family():
        cmd = ['/usr/bin/apt-get', '-qq', 'check']
        check_str = "has missing dependency|E:"

    if check_str is None or cmd is None:
        raise Fail("Unsupported OSFamily on the Agent Host")

    code, out = rmf_shell.checked_call(cmd, sudo=True)

    output_regex = re.compile(check_str)

    if code or (out and output_regex.search(out)):
        err_msg = Logger.filter_text(
            "Failed to verify package dependencies. Execution of '%s' returned %s. %s"
            % (cmd, code, out))
        Logger.error(err_msg)
        return False

    return True
コード例 #32
0
def get_available_packages_in_repos(repositories):
  """
  Gets all (both installed and available) packages that are available at given repositories.
  :param repositories: from command configs like config['repositoryFile']['repositories']
  :return: installed and available packages from these repositories
  """

  available_packages = []
  installed_packages = []
  available_packages_in_repos = []
  repo_ids = [repository['repoId'] for repository in repositories]
  if OSCheck.is_ubuntu_family():
    allInstalledPackages(installed_packages)
    repo_urls = [repository['baseUrl'] for repository in repositories]
    repo_urls = [repo_url.replace("http://","") for repo_url in repo_urls]
    repo_urls = [repo_url.replace("/","_") for repo_url in repo_urls]
    for url in repo_urls:
      _lookUpAptPackages(
        LIST_AVAILABLE_PACKAGES_UBUNTU % url,
        available_packages)
      for package in installed_packages:
        if url in package[2]:
          available_packages_in_repos.append(package[0])
    for package in available_packages:
      available_packages_in_repos.append(package[0])
  elif OSCheck.is_suse_family():
    for repo in repo_ids:
      _lookUpZypperPackages(["sudo", "zypper", "--no-gpg-checks", "search", "--details", "--repo", repo],
                            available_packages)
    available_packages_in_repos += [package[0] for package in available_packages]
  elif OSCheck.is_redhat_family():
    for repo in repo_ids:
      _lookUpYumPackages(["sudo", "yum", "list", "available", "--disablerepo=*", "--enablerepo=" + repo],
                         'Available Packages', available_packages)
      _lookUpYumPackages(["sudo", "yum", "list", "installed", "--disablerepo=*", "--enablerepo=" + repo],
                         'Installed Packages', installed_packages)
    available_packages_in_repos += [package[0] for package in available_packages + installed_packages]
  return available_packages_in_repos
コード例 #33
0
def setup_atlas_sqoop():
  import params

  if params.has_atlas:

    if not params.host_sys_prepped:
      Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
              retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count)

    PropertiesFile(format('{sqoop_conf_dir}/{atlas_conf_file}'),
                     properties = params.atlas_props,
                     owner = params.sqoop_user,
                     group = params.user_group,
                     mode = 0644)

    atlas_sqoop_hook_dir = os.path.join(params.atlas_home_dir, "hook", "sqoop")
    if os.path.exists(atlas_sqoop_hook_dir):

      src_files = os.listdir(atlas_sqoop_hook_dir)
      for file_name in src_files:
        atlas_sqoop_hook_file_name = os.path.join(atlas_sqoop_hook_dir, file_name)
        sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name)
        if (os.path.isfile(atlas_sqoop_hook_file_name)):
          Link(sqoop_lib_file_name, to = atlas_sqoop_hook_file_name)
コード例 #34
0
ファイル: status_params.py プロジェクト: fkpp/ambari
    hive_metastore_win_service_name = "metastore"
    hive_client_win_service_name = "hwi"
    hive_server_win_service_name = "hiveserver2"
    webhcat_server_win_service_name = "templeton"
else:
    hive_pid_dir = config['configurations']['hive-env']['hive_pid_dir']
    hive_pid = 'hive-server.pid'
    hive_interactive_pid = 'hive-interactive.pid'
    hive_metastore_pid = 'hive.pid'

    hcat_pid_dir = config['configurations']['hive-env'][
        'hcat_pid_dir']  #hcat_pid_dir
    webhcat_pid_file = format('{hcat_pid_dir}/webhcat.pid')

    process_name = 'mysqld'
    if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
        daemon_name = 'mysql'
    else:
        daemon_name = 'mysqld'

    # Security related/required params
    hostname = config['hostname']
    security_enabled = config['configurations']['cluster-env'][
        'security_enabled']
    kinit_path_local = get_kinit_path(
        default('/configurations/kerberos-env/executable_search_paths', None))
    tmp_dir = Script.get_tmp_dir()
    hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
    hive_user = config['configurations']['hive-env']['hive_user']
    webhcat_user = config['configurations']['hive-env']['webhcat_user']
コード例 #35
0
def get_ntp_service():
    if OSCheck.is_redhat_family():
        return "ntpd"
    elif OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
        return "ntp"
コード例 #36
0
def install_atlas_hook_packages(atlas_plugin_package, atlas_ubuntu_plugin_package, host_sys_prepped,
                                agent_stack_retry_on_unavailability, agent_stack_retry_count):
  if not host_sys_prepped:
    Package(atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else atlas_plugin_package,
            retry_on_repo_unavailability=agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count)
コード例 #37
0
  def copyNeededFiles(self):
    # get the params
    params = self.shared_state

    # Copying the files
    fileToCopy = self.getRepoFile()
    target = self.getRemoteName(self.AMBARI_REPO_FILENAME)

    if (os.path.exists(fileToCopy)):
      self.host_log.write("==========================\n")
      self.host_log.write("Copying repo file to 'tmp' folder...")
      scp = SCP(params.user, params.sshkey_file, self.host, fileToCopy,
                target, params.bootdir, self.host_log)
      retcode1 = scp.run()
      self.host_log.write("\n")

      # Move file to repo dir
      self.host_log.write("==========================\n")
      self.host_log.write("Moving file to repo dir...")
      targetDir = self.getRepoDir()
      command = self.getMoveRepoFileCommand(targetDir)
      ssh = SSH(params.user, params.sshkey_file, self.host, command,
                params.bootdir, self.host_log)
      retcode2 = ssh.run()
      self.host_log.write("\n")

      # Change permissions on ambari.repo
      self.host_log.write("==========================\n")
      self.host_log.write("Changing permissions for ambari.repo...")
      command = self.getRepoFileChmodCommand()
      ssh = SSH(params.user, params.sshkey_file, self.host, command,
                params.bootdir, self.host_log)
      retcode4 = ssh.run()
      self.host_log.write("\n")

      # Update repo cache for ubuntu OS
      if OSCheck.is_ubuntu_family():
        self.host_log.write("==========================\n")
        self.host_log.write("Update apt cache of repository...")
        command = self.getAptUpdateCommand()
        ssh = SSH(params.user, params.sshkey_file, self.host, command,
                  params.bootdir, self.host_log)
        retcode2 = ssh.run()
        self.host_log.write("\n")

      retcode = max(retcode1["exitstatus"], retcode2["exitstatus"], retcode4["exitstatus"])
    else:
      self.host_log.write("==========================\n")
      self.host_log.write("Copying required files...")
      self.host_log.write("Ambari repo file not found: {0}".format(self.getRepoFile()))
      retcode = -1
      pass

    # copy the setup script file
    self.host_log.write("==========================\n")
    self.host_log.write("Copying setup script file...")
    fileToCopy = params.setup_agent_file
    target = self.getRemoteName(self.SETUP_SCRIPT_FILENAME)
    scp = SCP(params.user, params.sshkey_file, self.host, fileToCopy,
              target, params.bootdir, self.host_log)
    retcode3 = scp.run()
    self.host_log.write("\n")

    return max(retcode, retcode3["exitstatus"])
コード例 #38
0
ファイル: HostInfo.py プロジェクト: maduhu/HDP2.5-ambari
def get_ntp_service():
  if OSCheck.is_redhat_family():
    return "ntpd"
  elif OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
    return "ntp"
コード例 #39
0
ファイル: utils.py プロジェクト: zouzhberk/ambaridemo
def get_postgre_running_status():
  """Return postgre running status indicator"""
  if OSCheck.is_ubuntu_family():
    return os.path.join(get_ubuntu_pg_version(), "main")
  else:
    return PG_STATUS_RUNNING_DEFAULT
コード例 #40
0
ファイル: script.py プロジェクト: zll405361679/ambari
  def format_package_name(self, name, repo_version=None):
    from resource_management.libraries.functions.default import default
    """
    This function replaces ${stack_version} placeholder with actual version.  If the package
    version is passed from the server, use that as an absolute truth.

    :param name name of the package
    :param repo_version actual version of the repo currently installing
    """
    if not STACK_VERSION_PLACEHOLDER in name:
      return name

    stack_version_package_formatted = ""

    package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_'

    # repositoryFile is the truth
    # package_version should be made to the form W_X_Y_Z_nnnn
    package_version = default("repositoryFile/repoVersion", None)

    # TODO remove legacy checks
    if package_version is None:
      package_version = default("roleParams/package_version", None)

    # TODO remove legacy checks
    if package_version is None:
      package_version = default("hostLevelParams/package_version", None)

    package_version = None
    if (package_version is None or '-' not in package_version) and default('/repositoryFile', None):
      self.load_available_packages()
      package_name = self.get_package_from_available(name, self.available_packages_in_repos)
      if package_name is None:
        raise Fail("Cannot match package for regexp name {0}. Available packages: {1}".format(name, self.available_packages_in_repos))
      return package_name

    if package_version is not None:
      package_version = package_version.replace('.', package_delimiter).replace('-', package_delimiter)

    # The cluster effective version comes down when the version is known after the initial
    # install.  In that case we should not be guessing which version when invoking INSTALL, but
    # use the supplied version to build the package_version
    effective_version = default("commandParams/version", None)
    role_command = default("roleCommand", None)

    if (package_version is None or '*' in package_version) \
        and effective_version is not None and 'INSTALL' == role_command:
      package_version = effective_version.replace('.', package_delimiter).replace('-', package_delimiter)
      Logger.info("Version {0} was provided as effective cluster version.  Using package version {1}".format(effective_version, package_version))

    if package_version:
      stack_version_package_formatted = package_version
      if OSCheck.is_ubuntu_family():
        stack_version_package_formatted = package_version.replace('_', package_delimiter)

    # Wildcards cause a lot of troubles with installing packages, if the version contains wildcards we try to specify it.
    if not package_version or '*' in package_version:
      repo_version = self.get_stack_version_before_packages_installed()
      stack_version_package_formatted = repo_version.replace('.', package_delimiter).replace('-', package_delimiter) if STACK_VERSION_PLACEHOLDER in name else name

    package_name = name.replace(STACK_VERSION_PLACEHOLDER, stack_version_package_formatted)

    return package_name
コード例 #41
0
ファイル: utils.py プロジェクト: Liujinan001/ambari-2.7.5
def get_postgre_running_status():
    """Return postgre running status indicator"""
    if OSCheck.is_ubuntu_family():
        return os.path.join(get_ubuntu_pg_version(), "main")
    else:
        return PG_STATUS_RUNNING_DEFAULT
コード例 #42
0
if OSCheck.is_windows_family():
  hive_metastore_win_service_name = "metastore"
  hive_client_win_service_name = "hwi"
  hive_server_win_service_name = "hiveserver2"
  webhcat_server_win_service_name = "templeton"
else:
  hive_pid_dir = config['configurations']['hive-env']['hive_pid_dir']
  hive_pid = 'hive-server.pid'

  hive_metastore_pid = 'hive.pid'

  hcat_pid_dir = config['configurations']['hive-env']['hcat_pid_dir'] #hcat_pid_dir
  webhcat_pid_file = format('{hcat_pid_dir}/webhcat.pid')

  process_name = 'mysqld'
  if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
    daemon_name = 'mysql'
  else:
    daemon_name = 'mysqld'

  # Security related/required params
  hostname = config['hostname']
  security_enabled = config['configurations']['cluster-env']['security_enabled']
  kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
  tmp_dir = Script.get_tmp_dir()
  hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
  hive_user = config['configurations']['hive-env']['hive_user']
  webhcat_user = config['configurations']['hive-env']['webhcat_user']

  # default configuration directories
  hadoop_conf_dir = conf_select.get_hadoop_conf_dir()