def install_windows_msi(url_base, save_dir, save_files, hadoop_user, hadoop_password, stack_version):
  global _working_dir
  _working_dir = save_dir
  save_dir = os.path.abspath(save_dir)
  msi_save_dir = save_dir
  # system wide lock to prevent simultaneous installations(when first task failed on timeout)
  install_lock = SystemWideLock("Global\\hdp_msi_lock")
  try:
    # try to acquire lock
    if not install_lock.lock():
      Logger.info("Some other task currently installing hdp.msi, waiting for 10 min for finish")
      if not install_lock.lock(600000):
        raise Fail("Timeout on acquiring lock")
    if _validate_msi_install():
      Logger.info("hdp.msi already installed")
      return

    stack_version_formatted = format_stack_version(stack_version)
    hdp_22_specific_props = ''
    if stack_version_formatted != "" and compare_versions(stack_version_formatted, '2.2') >= 0:
      hdp_22_specific_props = hdp_22.format(data_dir=data_dir)

    # MSIs cannot be larger than 2GB. HDPWIN 2.3 needed split in order to accommodate this limitation
    msi_file = ''
    for save_file in save_files:
      if save_file.lower().endswith(".msi"):
        msi_file = save_file
      file_url = urlparse.urljoin(url_base, save_file)
      try:
        download_file(file_url, os.path.join(msi_save_dir, save_file))
      except:
        raise Fail("Failed to download {url}".format(url=file_url))

    File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(log_dir=log_dir,
                                                                                         data_dir=data_dir,
                                                                                         local_host=local_host,
                                                                                         db_flavor=db_flavor,
                                                                                         hdp_22_specific_props=hdp_22_specific_props))

    # install msi
    msi_path = os_utils.quote_path(os.path.join(save_dir, msi_file))
    log_path = os_utils.quote_path(os.path.join(save_dir, msi_file[:-3] + "log"))
    layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt"))
    hadoop_password_arg = os_utils.quote_path(hadoop_password)

    Execute(
      INSTALL_MSI_CMD.format(msi_path=msi_path, log_path=log_path, layout_path=layout_path,
                             hadoop_user=hadoop_user, hadoop_password_arg=hadoop_password_arg))
    reload_windows_env()
    # create additional services manually due to hdp.msi limitaitons
    _ensure_services_created(hadoop_user, hadoop_password)
    _create_symlinks(stack_version)
    # finalizing install
    _write_marker()
    _validate_msi_install()
  finally:
    install_lock.unlock()
Example #2
0
def install_windows_msi(msi_url, save_dir, save_file, hadoop_password, stack_version):
  global _working_dir
  _working_dir = save_dir
  save_dir = os.path.abspath(save_dir)
  msi_save_dir = save_dir
  # system wide lock to prevent simultaneous installations(when first task failed on timeout)
  install_lock = SystemWideLock("hdp_msi_lock")
  try:
    # try to acquire lock
    if not install_lock.lock():
      Logger.info("Some other task currently installing hdp.msi, waiting for 10 min for finish")
      if not install_lock.lock(600000):
        raise Fail("Timeout on acquiring lock")
    if _validate_msi_install():
      Logger.info("hdp.msi already installed")
      return

    hdp_stack_version = format_hdp_stack_version(stack_version)
    hdp_22_specific_props = ''
    if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
      hdp_22_specific_props = hdp_22.format(hdp_data_dir=hdp_data_dir)

    # install msi
    download_file(msi_url, os.path.join(msi_save_dir, save_file))
    File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(hdp_log_dir=hdp_log_dir,
                                                                                         hdp_data_dir=hdp_data_dir,
                                                                                         local_host=local_host,
                                                                                         db_flavor=db_flavor,
                                                                                         hdp_22_specific_props=hdp_22_specific_props))
    hdp_msi_path = os_utils.quote_path(os.path.join(save_dir, "hdp.msi"))
    hdp_log_path = os_utils.quote_path(os.path.join(save_dir, "hdp.log"))
    hdp_layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt"))
    hadoop_password_arg = os_utils.quote_path(hadoop_password)

    Execute(
      INSTALL_MSI_CMD.format(hdp_msi_path=hdp_msi_path, hdp_log_path=hdp_log_path, hdp_layout_path=hdp_layout_path,
                             hadoop_password_arg=hadoop_password_arg))
    reload_windows_env()
    # create additional services manually due to hdp.msi limitaitons
    _ensure_services_created(hadoop_password)
    _create_symlinks(stack_version)
    # finalizing install
    _write_marker()
    _validate_msi_install()
  finally:
    install_lock.unlock()
    def service_check(self, env):
        import params

        env.set_params(params)

        yarn_exe = os_utils.quote_path(os.path.join(params.yarn_home, "bin", "yarn.cmd"))

        run_yarn_check_cmd = "cmd /C %s node -list" % yarn_exe

        component_type = "rm"
        if params.hadoop_ssl_enabled:
            component_address = params.rm_webui_https_address
        else:
            component_address = params.rm_webui_address

        # temp_dir = os.path.abspath(os.path.join(params.hadoop_home, os.pardir)), "/tmp"
        temp_dir = os.path.join(os.path.dirname(params.hadoop_home), "temp")
        validateStatusFileName = "validateYarnComponentStatusWindows.py"
        validateStatusFilePath = os.path.join(temp_dir, validateStatusFileName)
        python_executable = sys.executable
        validateStatusCmd = "%s %s %s -p %s -s %s" % (
            python_executable,
            validateStatusFilePath,
            component_type,
            component_address,
            params.hadoop_ssl_enabled,
        )

        if params.security_enabled:
            kinit_cmd = "%s -kt %s %s;" % (params.kinit_path_local, params.smoke_user_keytab, params.smokeuser)
            smoke_cmd = kinit_cmd + " " + validateStatusCmd
        else:
            smoke_cmd = validateStatusCmd

        File(validateStatusFilePath, content=StaticFile(validateStatusFileName))

        Execute(smoke_cmd, tries=3, try_sleep=5, logoutput=True)

        Execute(run_yarn_check_cmd, logoutput=True)
  def execute_db_connection_check(self, config, tmp_dir):
    print "DB connection check started."
  
    # initialize needed data
  
    ambari_server_hostname = config['commandParams']['ambari_server_host']
    check_db_connection_jar_name = "DBConnectionVerification.jar"
    jdk_location = config['commandParams']['jdk_location']
    java_home = config['commandParams']['java_home']
    db_name = config['commandParams']['db_name']

    if db_name == DB_MYSQL:
      jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL
      jdbc_driver_class = JDBC_DRIVER_CLASS_MYSQL
      jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL
    elif db_name == DB_ORACLE:
      jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE
      jdbc_driver_class = JDBC_DRIVER_CLASS_ORACLE
      jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE
    elif db_name == DB_POSTGRESQL:
      jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL
      jdbc_driver_class = JDBC_DRIVER_CLASS_POSTGRESQL
      jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL
    elif db_name == DB_MSSQL:
      jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MSSQL
      jdbc_driver_class = JDBC_DRIVER_CLASS_MSSQL
      jdbc_name = JDBC_DRIVER_SYMLINK_MSSQL
    elif db_name == DB_SQLA:
      jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_SQLA
      jdbc_driver_class = JDBC_DRIVER_CLASS_SQLA
      jdbc_name = JDBC_DRIVER_SYMLINK_SQLA
  
    db_connection_url = config['commandParams']['db_connection_url']
    user_name = config['commandParams']['user_name']
    user_passwd = config['commandParams']['user_passwd']
    agent_cache_dir = os.path.abspath(config["hostLevelParams"]["agentCacheDir"])
    check_db_connection_url = jdk_location + check_db_connection_jar_name
    jdbc_path = os.path.join(agent_cache_dir, jdbc_name)
    class_path_delimiter = ":"
    if db_name == DB_SQLA:
      jdbc_jar_path = agent_cache_dir + JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE
      java_library_path = agent_cache_dir + JARS_PATH_IN_ARCHIVE_SQLA + class_path_delimiter + agent_cache_dir + \
                          LIBS_PATH_IN_ARCHIVE_SQLA
    else:
      jdbc_jar_path = jdbc_path
      java_library_path = agent_cache_dir

    check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name)

    java_bin = "java"
    if OSCheck.is_windows_family():
      java_bin = "java.exe"
      class_path_delimiter = ";"

    java_exec = os.path.join(java_home, "bin",java_bin)

    if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
        or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
      message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \
                "\n"
      print message
      db_connection_check_structured_output = {"exit_code" : 1, "message": message}
      return db_connection_check_structured_output

    environment = { "no_proxy": format("{ambari_server_hostname}") }
    # download and install java if it doesn't exists
    if not os.path.isfile(java_exec):
      jdk_name = config['commandParams']['jdk_name']
      jdk_url = "{0}/{1}".format(jdk_location, jdk_name)
      jdk_download_target = os.path.join(agent_cache_dir, jdk_name)
      java_dir = os.path.dirname(java_home)
      try:
        download_file(jdk_url, jdk_download_target)
      except Exception, e:
        message = "Error downloading JDK from Ambari Server resources. Check network access to " \
                  "Ambari Server.\n" + str(e)
        print message
        db_connection_check_structured_output = {"exit_code" : 1, "message": message}
        return db_connection_check_structured_output

      if jdk_name.endswith(".exe"):
        install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
        os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home),
        )
        install_path = [java_dir]
        try:
          Execute(install_cmd, path = install_path)
        except Exception, e:
          message = "Error installing java.\n" + str(e)
          print message
          db_connection_check_structured_output = {"exit_code" : 1, "message": message}
          return db_connection_check_structured_output