Example #1
0
    def adjust_jce_permissions(self, jdk_path):
        ambari_user = read_ambari_user()
        cmds = []
        if ambari_user:
            cmds.append(
                self.SET_JCE_PERMISSIONS.format(
                    ambari_user, jdk_path, configDefaults.JDK_SECURITY_DIR))
        cmds.append(
            self.SET_JCE_FILE_MODE.format(jdk_path,
                                          configDefaults.JDK_SECURITY_DIR,
                                          "*"))
        cmds.append(
            self.SET_JCE_JAR_MODE.format(jdk_path,
                                         configDefaults.JDK_SECURITY_DIR,
                                         "*.jar"))

        cmd = " && ".join(cmds)

        process = subprocess.Popen(cmd,
                                   stdout=subprocess.PIPE,
                                   stdin=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   shell=True)
        (stdoutdata, stderrdata) = process.communicate()

        if process.returncode != 0:
            print_warning_msg(
                "Failed to change jce permissions. {0}\n{1}".format(
                    stderrdata, stdoutdata))
Example #2
0
def run_schema_upgrade():
  db_title = get_db_type(get_ambari_properties()).title
  confirm = get_YN_input("Ambari Server configured for %s. Confirm "
                        "you have made a backup of the Ambari Server database [y/n] (y)? " % db_title, True)

  if not confirm:
    print_error_msg("Database backup is not confirmed")
    return 1

  jdk_path = get_java_exe_path()
  if jdk_path is None:
    print_error_msg("No JDK found, please run the \"setup\" "
                    "command to install a JDK automatically or install any "
                    "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
    return 1

  print 'Upgrading database schema'

  command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath())

  ambari_user = read_ambari_user()
  current_user = ensure_can_start_under_current_user(ambari_user)
  environ = generate_env(ambari_user, current_user)

  (retcode, stdout, stderr) = run_os_command(command, env=environ)
  print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
  if retcode > 0:
    print_error_msg("Error executing schema upgrade, please check the server logs.")
  else:
    print_info_msg('Schema upgrade completed')
  return retcode
Example #3
0
  def do_checks(self):
    try:
      user = read_ambari_user()
      create_user = False
      update_user_setting = False
      if user is not None:
        create_user = get_YN_input(self.NR_USER_CHANGE_PROMPT.format(user), False)
        update_user_setting = create_user  # Only if we will create another user
      else:  # user is not configured yet
        update_user_setting = True  # Write configuration anyway
        create_user = get_YN_input(self.NR_USER_CUSTOMIZE_PROMPT, False)
        if not create_user:
          user = self.NR_DEFAULT_USER

      if create_user:
        (retcode, user) = self._create_custom_user()
        if retcode != 0:
          return retcode

      if update_user_setting:
        write_property(NR_USER_PROPERTY, user)

      adjust_directory_permissions(user)
    except OSError as e:
      print_error_msg("Failed: %s" % str(e))
      return 4
    except Exception as e:
      print_error_msg("Unexpected error %s" % str(e))
      return 1
    return 0
Example #4
0
  def do_checks(self):
    try:
      user = read_ambari_user()
      create_user = False
      update_user_setting = False
      if user is not None:
        create_user = get_YN_input_optional(self.NR_USER_CHANGE_PROMPT.format(user), False,SETUP_USE_DEFAULT)
        update_user_setting = create_user  # Only if we will create another user
      else:  # user is not configured yet
        update_user_setting = True  # Write configuration anyway
        create_user = get_YN_input_optional(self.NR_USER_CUSTOMIZE_PROMPT, False,SETUP_USE_DEFAULT)
        if not create_user:
          user = self.NR_DEFAULT_USER

      if create_user:
        (retcode, user) = self._create_custom_user()
        if retcode != 0:
          return retcode

      if update_user_setting:
        write_property(NR_USER_PROPERTY, user)

      adjust_directory_permissions(user)
    except OSError as e:
      print_error_msg("Failed: %s" % str(e))
      return 4
    except Exception as e:
      print_error_msg("Unexpected error %s" % str(e))
      return 1
    return 0
Example #5
0
  def do_checks(self):
    try:
      user = read_ambari_user()
      if not user:
        user = self.NR_DEFAULT_USER

      if self.user is not None:   #Command-line parameter is the default
        update_user_setting = True
        prompt_msg = self.NR_USER_CUSTOMIZE_PROMPT.format('y')
      else:
        update_user_setting = False
        if user != self.NR_DEFAULT_USER:
          prompt_msg = self.NR_USER_CHANGE_PROMPT.format(user, 'n')
        else:
          prompt_msg = self.NR_USER_CUSTOMIZE_PROMPT.format('n')
        self.user = user if user else self.NR_DEFAULT_USER

      self.register_service = get_YN_input(prompt_msg, update_user_setting)
      if self.register_service:
        retcode = self._create_custom_user()
        if retcode != 0:
          return retcode

      adjust_directory_permissions(self.user)
    except OSError as e:
      print_error_msg("Failed: %s" % str(e))
      return 4
    except Exception as e:
      print_error_msg("Unexpected error %s" % str(e))
      return 1
    return 0
Example #6
0
def run_mpack_install_checker(options, mpack_stacks):
    """
  Run MpackInstallChecker to validate that there is no cluster deployed with a stack that is not included in the management pack
  :param options: Options passed
  :param mpack_stacks: List of stacks included in the management pack
  :return: Output of MpackInstallChecker
  """
    properties = get_ambari_properties()
    database_type = properties[JDBC_DATABASE_PROPERTY]
    jdk_path = get_java_exe_path()

    if not jdk_path or not database_type:
        # Ambari Server has not been setup, so no cluster would be present
        return (0, "", "")

    parse_properties_file(options)
    options.database_index = LINUX_DBMS_KEYS_LIST.index(
        properties[JDBC_DATABASE_PROPERTY])
    ensure_jdbc_driver_is_installed(options, properties)

    serverClassPath = ServerClassPath(properties, options)
    class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()

    command = MPACK_INSTALL_CHECKER_CMD.format(jdk_path, class_path,
                                               ",".join(mpack_stacks))

    ambari_user = read_ambari_user()
    current_user = ensure_can_start_under_current_user(ambari_user)
    environ = generate_env(options, ambari_user, current_user)

    return run_os_command(command, env=environ)
Example #7
0
def run_schema_upgrade(args):
    db_title = get_db_type(get_ambari_properties()).title
    confirm = get_YN_input(
        "Ambari Server configured for %s. Confirm "
        "you have made a backup of the Ambari Server database [y/n] (y)? " %
        db_title, True)

    if not confirm:
        print_error_msg("Database backup is not confirmed")
        return 1

    jdk_path = get_java_exe_path()
    if jdk_path is None:
        print_error_msg(
            "No JDK found, please run the \"setup\" "
            "command to install a JDK automatically or install any "
            "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
        return 1

    ensure_jdbc_driver_is_installed(args, get_ambari_properties())

    print_info_msg('Upgrading database schema', True)

    serverClassPath = ServerClassPath(get_ambari_properties(), args)
    class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell(
        validate_classpath=True)

    set_debug_mode_from_options(args)
    debug_mode = get_debug_mode()
    debug_start = (debug_mode & 1) or SCHEMA_UPGRADE_DEBUG
    suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
    suspend_mode = 'y' if suspend_start else 'n'
    command = SCHEMA_UPGRADE_HELPER_CMD_DEBUG.format(
        jdk_path, class_path,
        suspend_mode) if debug_start else SCHEMA_UPGRADE_HELPER_CMD.format(
            jdk_path, class_path)

    ambari_user = read_ambari_user()
    current_user = ensure_can_start_under_current_user(ambari_user)
    environ = generate_env(args, ambari_user, current_user)

    (retcode, stdout, stderr) = run_os_command(command, env=environ)
    print_info_msg(
        "Return code from schema upgrade command, retcode = {0}".format(
            str(retcode)), True)
    if stdout:
        print_info_msg("Console output from schema upgrade command:", True)
        print_info_msg(stdout, True)
        print
    if retcode > 0:
        print_error_msg(
            "Error executing schema upgrade, please check the server logs.")
        if stderr:
            print_error_msg("Error output from schema upgrade command:")
            print_error_msg(stderr)
            print
    else:
        print_info_msg('Schema upgrade completed', True)
    return retcode
Example #8
0
def run_db_cleanup(options):

    if validate_args(options):
        return 1

    db_title = get_db_type(get_ambari_properties()).title

    confirmBackup = get_YN_input("Ambari Server configured for {0}. Confirm you have made a backup of the Ambari Server database [y/n]".format(
            db_title), True)
    if not confirmBackup:
        print_info_msg("Ambari Server Database cleanup aborted")
        return 0

    status, stateDesc = is_server_runing()
    if status:
        print_error_msg("The database cleanup cannot proceed while Ambari Server is running. Please shut down Ambari first.")
        return 1

    confirm = get_YN_input(
        "Ambari server is using db type {0}. Cleanable database entries older than {1} will be cleaned up. Proceed [y/n]".format(
            db_title, options.cleanup_from_date), True)
    if not confirm:
        print_info_msg("Ambari Server Database cleanup aborted")
        return 0

    jdk_path = get_java_exe_path()
    if jdk_path is None:
        print_error_msg("No JDK found, please run the \"setup\" command to install a JDK automatically or install any "
                        "JDK manually to {0}".format(configDefaults.JDK_INSTALL_DIR));
        return 1

    ensure_jdbc_driver_is_installed(options, get_ambari_properties())

    serverClassPath = ServerClassPath(get_ambari_properties(), options)
    class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()

    ambari_user = read_ambari_user()
    current_user = ensure_can_start_under_current_user(ambari_user)
    environ = generate_env(options, ambari_user, current_user)

    print "Cleaning up the database ..."
    command = DB_CLEANUP_CMD.format(jdk_path, class_path, options.cluster_name, options.cleanup_from_date)
    (retcode, stdout, stderr) = run_os_command(command, env=environ)

    print_info_msg("Return code from database cleanup command, retcode = " + str(retcode))

    if stdout:
        print "Console output from database cleanup command:"
        print stdout
        print
    if stderr:
        print "Error output from database cleanup command:"
        print stderr
        print
    if retcode > 0:
        print_error_msg("Error wncountered while cleaning up the Ambari Server Database. Check the ambari-server.log for details.")
    else:
        print "Cleanup completed. Check the ambari-server.log for details."
    return retcode
Example #9
0
def setup_https(args):
  if not is_root():
        err = 'ambari-server setup-https should be run with ' \
              'root-level privileges'
        raise FatalException(4, err)
  args.exit_message = None
  if not get_silent():
    properties = get_ambari_properties()
    try:
      security_server_keys_dir = properties.get_property(SSL_KEY_DIR)
      client_api_ssl_port = DEFAULT_SSL_API_PORT if properties.get_property(SSL_API_PORT) in ("") \
            else properties.get_property(SSL_API_PORT)
      api_ssl = properties.get_property(SSL_API) in ['true']
      client_api_ssl_port_old_value = properties.get_property(SSL_API_PORT)
      api_ssl_old_value = properties.get_property(SSL_API)
      cert_was_imported = False
      cert_must_import = True
      if api_ssl:
        if get_YN_input("Do you want to disable HTTPS [y/n] (n)? ", False):
          properties.process_pair(SSL_API, "false")
          cert_must_import=False
        else:
          properties.process_pair(SSL_API_PORT, \
                                  get_validated_string_input( \
                                      "SSL port ["+str(client_api_ssl_port)+"] ? ", \
                                      str(client_api_ssl_port), \
                                      "^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
          cert_was_imported = import_cert_and_key_action(security_server_keys_dir, properties)
      else:
        if get_YN_input("Do you want to configure HTTPS [y/n] (y)? ", True):
          properties.process_pair(SSL_API_PORT, \
                                  get_validated_string_input("SSL port ["+str(client_api_ssl_port)+"] ? ", \
                                                               str(client_api_ssl_port), "^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
          cert_was_imported = import_cert_and_key_action(security_server_keys_dir, properties)
        else:
          return False

      if cert_must_import and not cert_was_imported:
        print 'Setup of HTTPS failed. Exiting.'
        return False

      conf_file = find_properties_file()
      f = open(conf_file, 'w')
      properties.store(f, "Changed by 'ambari-server setup-https' command")

      if api_ssl_old_value != properties.get_property(SSL_API) \
          or client_api_ssl_port_old_value != properties.get_property(SSL_API_PORT):
        print "Ambari server URL changed. To make use of the Tez View in Ambari " \
              "please update the property tez.tez-ui.history-url.base in tez-site"

      ambari_user = read_ambari_user()
      if ambari_user:
        adjust_directory_permissions(ambari_user)
      return True
    except (KeyError), e:
        err = 'Property ' + str(e) + ' is not defined'
        raise FatalException(1, err)
Example #10
0
 def adjust_jce_permissions(self, jdk_path):
   ambari_user = read_ambari_user()
   cmd = self.SET_JCE_PERMISSIONS.format(ambari_user, jdk_path,configDefaults.JDK_SECURITY_DIR)
   process = subprocess.Popen(cmd,
                          stdout=subprocess.PIPE,
                          stdin=subprocess.PIPE,
                          stderr=subprocess.PIPE,
                          shell=True
                          )
   (stdoutdata, stderrdata) = process.communicate()
Example #11
0
def setup_https(args):
    if not is_root():
        warn = 'ambari-server setup-https is run as ' \
              'non-root user, some sudo privileges might be required'
        print warn
    args.exit_message = None
    if not get_silent():
        properties = get_ambari_properties()
        try:
            security_server_keys_dir = properties.get_property(SSL_KEY_DIR)
            client_api_ssl_port = DEFAULT_SSL_API_PORT if properties.get_property(SSL_API_PORT) in ("") \
                  else properties.get_property(SSL_API_PORT)
            api_ssl = properties.get_property(SSL_API) in ['true']
            cert_was_imported = False
            cert_must_import = True
            if api_ssl:
                if get_YN_input("Do you want to disable HTTPS [y/n] (n)? ",
                                False):
                    properties.process_pair(SSL_API, "false")
                    cert_must_import = False
                else:
                    properties.process_pair(SSL_API_PORT, \
                                            get_validated_string_input( \
                                                "SSL port ["+str(client_api_ssl_port)+"] ? ", \
                                                str(client_api_ssl_port), \
                                                "^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
                    cert_was_imported = import_cert_and_key_action(
                        security_server_keys_dir, properties)
            else:
                if get_YN_input("Do you want to configure HTTPS [y/n] (y)? ",
                                True):
                    properties.process_pair(SSL_API_PORT, \
                                            get_validated_string_input("SSL port ["+str(client_api_ssl_port)+"] ? ", \
                                                                         str(client_api_ssl_port), "^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
                    cert_was_imported = import_cert_and_key_action(
                        security_server_keys_dir, properties)
                else:
                    return False

            if cert_must_import and not cert_was_imported:
                print 'Setup of HTTPS failed. Exiting.'
                return False

            conf_file = find_properties_file()
            f = open(conf_file, 'w')
            properties.store(f,
                             "Changed by 'ambari-server setup-https' command")

            ambari_user = read_ambari_user()
            if ambari_user:
                adjust_directory_permissions(ambari_user)
            return True
        except (KeyError), e:
            err = 'Property ' + str(e) + ' is not defined'
            raise FatalException(1, err)
Example #12
0
def migrate_ldap_pam(args):
  properties = get_ambari_properties()

  if get_value_from_properties(properties,CLIENT_SECURITY,"") != 'pam':
    err = "PAM is not configured. Please configure PAM authentication first."
    raise FatalException(1, err)

  db_title = get_db_type(properties).title
  confirm = get_YN_input("Ambari Server configured for %s. Confirm "
                        "you have made a backup of the Ambari Server database [y/n] (y)? " % db_title, True)

  if not confirm:
    print_error_msg("Database backup is not confirmed")
    return 1

  jdk_path = get_java_exe_path()
  if jdk_path is None:
    print_error_msg("No JDK found, please run the \"setup\" "
                    "command to install a JDK automatically or install any "
                    "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
    return 1

  # At this point, the args does not have the ambari database information.
  # Augment the args with the correct ambari database information
  parse_properties_file(args)

  ensure_jdbc_driver_is_installed(args, properties)

  print 'Migrating LDAP Users & Groups to PAM'

  serverClassPath = ServerClassPath(properties, args)
  class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()

  command = LDAP_TO_PAM_MIGRATION_HELPER_CMD.format(jdk_path, class_path)

  ambari_user = read_ambari_user()
  current_user = ensure_can_start_under_current_user(ambari_user)
  environ = generate_env(args, ambari_user, current_user)

  (retcode, stdout, stderr) = run_os_command(command, env=environ)
  print_info_msg("Return code from LDAP to PAM migration command, retcode = " + str(retcode))
  if stdout:
    print "Console output from LDAP to PAM migration command:"
    print stdout
    print
  if stderr:
    print "Error output from LDAP to PAM migration command:"
    print stderr
    print
  if retcode > 0:
    print_error_msg("Error executing LDAP to PAM migration, please check the server logs.")
  else:
    print_info_msg('LDAP to PAM migration completed')
  return retcode
Example #13
0
def check_database(options):
    logger.info("Check database consistency.")
    jdk_path = serverConfiguration.get_java_exe_path()

    if jdk_path is None:
        print_error_msg(
            "No JDK found, please run the \"setup\" "
            "command to install a JDK automatically or install any "
            "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
        sys.exit(1)

    properties = serverConfiguration.get_ambari_properties()
    serverConfiguration.parse_properties_file(options)

    database_type = properties[JDBC_DATABASE_PROPERTY]
    if not database_type:
        print_error_msg("Please run \"ambari-server setup\" command"
                        " to initialize ambari db properties.")
        sys.exit(1)

    options.database_index = LINUX_DBMS_KEYS_LIST.index(
        properties[JDBC_DATABASE_PROPERTY])

    dbConfiguration.ensure_jdbc_driver_is_installed(
        options, serverConfiguration.get_ambari_properties())

    serverClassPath = ServerClassPath(
        serverConfiguration.get_ambari_properties(), options)
    class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()

    command = CHECK_DATABASE_HELPER_CMD.format(jdk_path, class_path)

    ambari_user = serverConfiguration.read_ambari_user()
    current_user = setupSecurity.ensure_can_start_under_current_user(
        ambari_user)
    environ = setupSecurity.generate_env(options, ambari_user, current_user)

    (retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ)

    if retcode > 0:
        print str(stdout)
        raise FatalException(
            1, 'Database check failed to complete. Please check ' +
            configDefaults.SERVER_LOG_FILE + ' and ' +
            configDefaults.DB_CHECK_LOG + ' for more information.')
    else:
        print str(stdout)
        if not stdout.startswith("No errors"):
            print "Ambari Server 'check-database' completed"
            sys.exit(1)
Example #14
0
def setup_https(args):
  if not is_root():
        err = 'tbds-server setup-https should be run with ' \
              'root-level privileges'
        raise FatalException(4, err)
  args.exit_message = None
  if not get_silent():
    properties = get_ambari_properties()
    try:
      security_server_keys_dir = properties.get_property(SSL_KEY_DIR)
      client_api_ssl_port = DEFAULT_SSL_API_PORT if properties.get_property(SSL_API_PORT) in ("") \
            else properties.get_property(SSL_API_PORT)
      api_ssl = properties.get_property(SSL_API) in ['true']
      cert_was_imported = False
      cert_must_import = True
      if api_ssl:
        if get_YN_input("Do you want to disable HTTPS [y/n] (n)? ", False):
          properties.process_pair(SSL_API, "false")
          cert_must_import=False
        else:
          properties.process_pair(SSL_API_PORT, \
                                  get_validated_string_input( \
                                      "SSL port ["+str(client_api_ssl_port)+"] ? ", \
                                      str(client_api_ssl_port), \
                                      "^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
          cert_was_imported = import_cert_and_key_action(security_server_keys_dir, properties)
      else:
        if get_YN_input("Do you want to configure HTTPS [y/n] (y)? ", True):
          properties.process_pair(SSL_API_PORT, \
                                  get_validated_string_input("SSL port ["+str(client_api_ssl_port)+"] ? ", \
                                                               str(client_api_ssl_port), "^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
          cert_was_imported = import_cert_and_key_action(security_server_keys_dir, properties)
        else:
          return False

      if cert_must_import and not cert_was_imported:
        print 'Setup of HTTPS failed. Exiting.'
        return False

      conf_file = find_properties_file()
      f = open(conf_file, 'w')
      properties.store(f, "Changed by 'tbds-server setup-https' command")

      ambari_user = read_ambari_user()
      if ambari_user:
        adjust_directory_permissions(ambari_user)
      return True
    except (KeyError), e:
        err = 'Property ' + str(e) + ' is not defined'
        raise FatalException(1, err)
def run_schema_upgrade(args):
  db_title = get_db_type(get_ambari_properties()).title
  confirm = get_YN_input("Ambari Server configured for %s. Confirm "
                        "you have made a backup of the Ambari Server database [y/n] (y)? " % db_title, True)

  if not confirm:
    print_error_msg("Database backup is not confirmed")
    return 1

  jdk_path = get_java_exe_path()
  if jdk_path is None:
    print_error_msg("No JDK found, please run the \"setup\" "
                    "command to install a JDK automatically or install any "
                    "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
    return 1

  ensure_jdbc_driver_is_installed(args, get_ambari_properties())

  print 'Upgrading database schema'

  serverClassPath = ServerClassPath(get_ambari_properties(), args)
  class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()

  command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, class_path)

  ambari_user = read_ambari_user()
  current_user = ensure_can_start_under_current_user(ambari_user)
  environ = generate_env(args, ambari_user, current_user)

  (retcode, stdout, stderr) = run_os_command(command, env=environ)
  print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
  if stdout:
    print "Console output from schema upgrade command:"
    print stdout
    print
  if stderr:
    print "Error output from schema upgrade command:"
    print stderr
    print
  if retcode > 0:
    print_error_msg("Error executing schema upgrade, please check the server logs.")
  else:
    print_info_msg('Schema upgrade completed')
  return retcode
Example #16
0
def check_database(options):

    jdk_path = serverConfiguration.get_java_exe_path()

    if jdk_path is None:
        print_error_msg(
            "No JDK found, please run the \"setup\" "
            "command to install a JDK automatically or install any "
            "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
        sys.exit(1)

    properties = serverConfiguration.get_ambari_properties()
    serverConfiguration.parse_properties_file(options)
    options.database_index = LINUX_DBMS_KEYS_LIST.index(
        properties[JDBC_DATABASE_PROPERTY])

    dbConfiguration.ensure_jdbc_driver_is_installed(
        options, serverConfiguration.get_ambari_properties())

    serverClassPath = ServerClassPath(
        serverConfiguration.get_ambari_properties(), options)
    class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()

    command = CHECK_DATABASE_HELPER_CMD.format(jdk_path, class_path)

    ambari_user = serverConfiguration.read_ambari_user()
    current_user = setupSecurity.ensure_can_start_under_current_user(
        ambari_user)
    environ = setupSecurity.generate_env(options, ambari_user, current_user)

    (retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ)
    print_info_msg("Return code from check database command, retcode = " +
                   str(retcode))

    if retcode > 0:
        print_error_msg(
            "Database check failed to complete. Please check ambari-server.log and ambari-server-check-database.log for problem."
        )
        raise FatalException(1, 'Database check failed.')
    else:
        print str(stdout)
Example #17
0
def run_schema_upgrade():
  jdk_path = get_java_exe_path()
  if jdk_path is None:
    print_error_msg("No JDK found, please run the \"setup\" "
                    "command to install a JDK automatically or install any "
                    "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
    return 1

  print 'Upgrading database schema'

  command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath())

  ambari_user = read_ambari_user()
  current_user = ensure_can_start_under_current_user(ambari_user)
  environ = generate_env(ambari_user, current_user)

  (retcode, stdout, stderr) = run_os_command(command, env=environ)
  print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode))
  if retcode > 0:
    print_error_msg("Error executing schema upgrade, please check the server logs.")
  else:
    print_info_msg('Schema upgrade completed')
  return retcode
Example #18
0
def upgrade(args):
    print_info_msg("Upgrade Ambari Server", True)
    if not is_root():
        err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
        raise FatalException(4, err)
    print_info_msg(
        'Updating Ambari Server properties in {0} ...'.format(
            AMBARI_PROPERTIES_FILE), True)
    retcode = update_ambari_properties()
    if not retcode == 0:
        err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
        raise FatalException(retcode, err)

    print_info_msg(
        'Updating Ambari Server properties in {0} ...'.format(AMBARI_ENV_FILE),
        True)
    retcode = update_ambari_env()
    if not retcode == 0:
        err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting'
        raise FatalException(retcode, err)

    retcode = update_krb_jaas_login_properties()
    if retcode == -2:
        pass  # no changes done, let's be silent
    elif retcode == 0:
        print_info_msg("File {0} updated.".format(AMBARI_KRB_JAAS_LOGIN_FILE),
                       True)
    elif not retcode == 0:
        err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting'
        raise FatalException(retcode, err)

    restore_custom_services()
    replay_mpack_logs()
    try:
        update_database_name_property(upgrade=True)
    except FatalException:
        return -1

    # Ignore the server version & database options passed via command-line arguments
    parse_properties_file(args)

    #TODO check database version
    change_objects_owner(args)

    retcode = run_schema_upgrade(args)
    if not retcode == 0:
        print_error_msg(
            "Ambari server upgrade failed. Please look at {0}, for more details."
            .format(configDefaults.SERVER_LOG_FILE))
        raise FatalException(11, 'Schema upgrade failed.')

    user = read_ambari_user()
    if user is None:
        warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
        print_warning_msg(warn)
    else:
        adjust_directory_permissions(user)

    # local repo
    upgrade_local_repo(args)

    # create jdbc symlinks if jdbc drivers are available in resources
    check_jdbc_drivers(args)

    properties = get_ambari_properties()
    if properties == -1:
        err = "Error getting ambari properties"
        print_error_msg(err)
        raise FatalException(-1, err)

    # Move *.py files from custom_actions to custom_actions/scripts
    # This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
    ambari_version = get_ambari_version(properties)
    if ambari_version is None:
        args.warnings.append(
            "*.py files were not moved from custom_actions to custom_actions/scripts."
        )
    elif compare_versions(ambari_version, "2.0.0") == 0:
        move_user_custom_actions()

    # Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
    admin_views_dirs = get_admin_views_dir(properties)
    for admin_views_dir in admin_views_dirs:
        shutil.rmtree(admin_views_dir)

    # Modify timestamp of views jars to current time
    views_jars = get_views_jars(properties)
    for views_jar in views_jars:
        os.utime(views_jar, None)

    # check if ambari has obsolete LDAP configuration
    if properties.get_property(
            LDAP_PRIMARY_URL_PROPERTY
    ) and not properties.get_property(IS_LDAP_CONFIGURED):
        args.warnings.append(
            "Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration."
        )

    # adding custom jdbc name and previous custom jdbc properties
    # we need that to support new dynamic jdbc names for upgraded ambari
    add_jdbc_properties(properties)
Example #19
0
      remove_password_file(LDAP_MGR_PASSWORD_FILENAME)
  pass

  if ts_password and not is_alias_string(ts_password):
    retCode = save_passwd_for_alias(SSL_TRUSTSTORE_PASSWORD_ALIAS, ts_password, masterKey)
    if retCode != 0:
      print 'Failed to save secure TrustStore password.'
    else:
      propertyMap[SSL_TRUSTSTORE_PASSWORD_PROPERTY] = get_alias_string(SSL_TRUSTSTORE_PASSWORD_ALIAS)
  pass

  update_properties_2(properties, propertyMap)

  # Since files for store and master are created we need to ensure correct
  # permissions
  ambari_user = read_ambari_user()
  if ambari_user:
    adjust_directory_permissions(ambari_user)

  return 0

def setup_ambari_krb5_jaas():
  jaas_conf_file = search_file(SECURITY_KERBEROS_JASS_FILENAME, get_conf_dir())
  if os.path.exists(jaas_conf_file):
    print 'Setting up Ambari kerberos JAAS configuration to access ' + \
          'secured Hadoop daemons...'
    principal = get_validated_string_input('Enter ambari server\'s kerberos '
                                           'principal name ([email protected]): ', '*****@*****.**', '.*', '', False,
                                           False)
    keytab = get_validated_string_input('Enter keytab path for ambari '
                                        'server\'s kerberos principal: ',
def server_process_main(options, scmStatus=None):
  # debug mode, including stop Java process at startup
  try:
    set_debug_mode_from_options(options)
  except AttributeError:
    pass

  if not check_reverse_lookup():
    print_warning_msg("The hostname was not found in the reverse DNS lookup. "
                      "This may result in incorrect behavior. "
                      "Please check the DNS setup and fix the issue.")

  check_database_name_property()
  parse_properties_file(options)

  ambari_user = read_ambari_user()
  current_user = ensure_can_start_under_current_user(ambari_user)

  print_info_msg("TBDS Server is not running...")

  jdk_path = find_jdk()
  if jdk_path is None:
    err = "No JDK found, please run the \"tbds-server setup\" " \
          "command to install a JDK automatically or install any " \
          "JDK manually to " + configDefaults.JDK_INSTALL_DIR
    raise FatalException(1, err)

  properties = get_ambari_properties()

  # Preparations
  if is_root():
    print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT

  ensure_jdbc_driver_is_installed(options, properties)

  ensure_dbms_is_running(options, properties, scmStatus)

  if scmStatus is not None:
    scmStatus.reportStartPending()

  refresh_stack_hash(properties)

  if scmStatus is not None:
    scmStatus.reportStartPending()

  ensure_server_security_is_configured()

  if scmStatus is not None:
    scmStatus.reportStartPending()

  java_exe = get_java_exe_path()

  class_path = get_conf_dir()
  class_path = os.path.abspath(class_path) + os.pathsep + get_ambari_classpath()

  debug_mode = get_debug_mode()
  debug_start = (debug_mode & 1) or SERVER_START_DEBUG
  suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
  suspend_mode = 'y' if suspend_start else 'n'

  (param_list, environ) = generate_child_process_param_list(ambari_user, current_user,
                                                 java_exe, class_path, debug_start, suspend_mode)

  if not os.path.exists(configDefaults.PID_DIR):
    os.makedirs(configDefaults.PID_DIR, 0755)

  print_info_msg("Running server: " + str(param_list))
  procJava = subprocess.Popen(param_list, env=environ)

  pidJava = procJava.pid
  if pidJava <= 0:
    procJava.terminate()
    exitcode = procJava.returncode
    exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME)
    save_pid(exitcode, exitfile)

    if scmStatus is not None:
      scmStatus.reportStopPending()

    raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
  else:
    pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
    save_pid(pidJava, pidfile)
    print "Server PID at: "+pidfile
    print "Server out at: "+configDefaults.SERVER_OUT_FILE
    print "Server log at: "+configDefaults.SERVER_LOG_FILE

    wait_for_server_start(pidfile, scmStatus)

  if scmStatus is not None:
    scmStatus.reportStarted()

  return procJava
Example #21
0
def import_file_to_keystore(source, destination):
    shutil.copy(source, destination)
    set_file_permissions(destination, "660", read_ambari_user(), False)
Example #22
0
def server_process_main(options, scmStatus=None):
    # debug mode, including stop Java process at startup
    try:
        set_debug_mode_from_options(options)
    except AttributeError:
        pass

    if not check_reverse_lookup():
        print_warning_msg(
            "The hostname was not found in the reverse DNS lookup. "
            "This may result in incorrect behavior. "
            "Please check the DNS setup and fix the issue.")

    check_database_name_property()
    parse_properties_file(options)

    ambari_user = read_ambari_user()
    current_user = ensure_can_start_under_current_user(ambari_user)

    print_info_msg("Ambari Server is not running...")

    jdk_path = find_jdk()
    if jdk_path is None:
        err = "No JDK found, please run the \"ambari-server setup\" " \
              "command to install a JDK automatically or install any " \
              "JDK manually to " + configDefaults.JDK_INSTALL_DIR
        raise FatalException(1, err)

    properties = get_ambari_properties()

    # Preparations
    if is_root():
        print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT

    ensure_jdbc_driver_is_installed(options, properties)

    ensure_dbms_is_running(options, properties, scmStatus)

    if scmStatus is not None:
        scmStatus.reportStartPending()

    refresh_stack_hash(properties)

    if scmStatus is not None:
        scmStatus.reportStartPending()

    ensure_server_security_is_configured()

    if scmStatus is not None:
        scmStatus.reportStartPending()

    java_exe = get_java_exe_path()

    class_path = get_conf_dir()
    class_path = os.path.abspath(
        class_path) + os.pathsep + get_ambari_classpath()

    debug_mode = get_debug_mode()
    debug_start = (debug_mode & 1) or SERVER_START_DEBUG
    suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
    suspend_mode = 'y' if suspend_start else 'n'

    (param_list,
     environ) = generate_child_process_param_list(ambari_user, current_user,
                                                  java_exe, class_path,
                                                  debug_start, suspend_mode)

    if not os.path.exists(configDefaults.PID_DIR):
        os.makedirs(configDefaults.PID_DIR, 0755)

    print_info_msg("Running server: " + str(param_list))
    procJava = subprocess.Popen(param_list, env=environ)

    pidJava = procJava.pid
    if pidJava <= 0:
        procJava.terminate()
        exitcode = procJava.returncode
        exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME)
        save_pid(exitcode, exitfile)

        if scmStatus is not None:
            scmStatus.reportStopPending()

        raise FatalException(
            -1,
            AMBARI_SERVER_DIE_MSG.format(exitcode,
                                         configDefaults.SERVER_OUT_FILE))
    else:
        pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
        save_pid(pidJava, pidfile)
        print "Server PID at: " + pidfile
        print "Server out at: " + configDefaults.SERVER_OUT_FILE
        print "Server log at: " + configDefaults.SERVER_LOG_FILE

        wait_for_server_start(pidfile, scmStatus)

    if scmStatus is not None:
        scmStatus.reportStarted()

    return procJava
def upgrade(args):
  if not is_root():
    err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
    raise FatalException(4, err)
  print 'Updating properties in ' + AMBARI_PROPERTIES_FILE + ' ...'
  retcode = update_ambari_properties()
  if not retcode == 0:
    err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
    raise FatalException(retcode, err)

  retcode = update_ambari_env()
  if not retcode == 0:
    err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting'
    raise FatalException(retcode, err)

  retcode = update_krb_jaas_login_properties()
  if retcode == -2:
    pass  # no changes done, let's be silent
  elif retcode == 0:
    print 'File ' + AMBARI_KRB_JAAS_LOGIN_FILE + ' updated.'
  elif not retcode == 0:
    err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting'
    raise FatalException(retcode, err)

  restore_custom_services()
  try:
    update_database_name_property(upgrade=True)
  except FatalException:
    return -1

  # Ignore the server version & database options passed via command-line arguments
  parse_properties_file(args)

  #TODO check database version
  change_objects_owner(args)

  retcode = run_schema_upgrade(args)
  if not retcode == 0:
    print_error_msg("Ambari server upgrade failed. Please look at {0}, for more details.".format(configDefaults.SERVER_LOG_FILE))
    raise FatalException(11, 'Schema upgrade failed.')

  user = read_ambari_user()
  if user is None:
    warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
    print_warning_msg(warn)
  else:
    adjust_directory_permissions(user)

  # local repo
  upgrade_local_repo(args)

  # create jdbc symlinks if jdbc drivers are available in resources
  check_jdbc_drivers(args)

  properties = get_ambari_properties()
  if properties == -1:
    err = "Error getting ambari properties"
    print_error_msg(err)
    raise FatalException(-1, err)

  # Move *.py files from custom_actions to custom_actions/scripts
  # This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
  ambari_version = get_ambari_version(properties)
  if ambari_version is None:
    args.warnings.append("*.py files were not moved from custom_actions to custom_actions/scripts.")
  elif compare_versions(ambari_version, "2.0.0") == 0:
    move_user_custom_actions()

  # Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
  admin_views_dirs = get_admin_views_dir(properties)
  for admin_views_dir in admin_views_dirs:
    shutil.rmtree(admin_views_dir)

  # Remove ambari views directory for the rest of the jars, at the time of upgrade. At restart all jars present in Ambari will be extracted into work directory
  views_dir =  get_views_dir(properties)
  for views in views_dir:
    shutil.rmtree(views)

  # check if ambari has obsolete LDAP configuration
  if properties.get_property(LDAP_PRIMARY_URL_PROPERTY) and not properties.get_property(IS_LDAP_CONFIGURED):
    args.warnings.append("Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration.")
Example #24
0
def update_host_names(args, options):
    services_stopped = userInput.get_YN_input(
        "Please, confirm Ambari services are stopped [y/n] (n)? ", False)
    if not services_stopped:
        print 'Exiting...'
        sys.exit(1)

    pending_commands = userInput.get_YN_input(
        "Please, confirm there are no pending commands on cluster [y/n] (n)? ",
        False)
    if not pending_commands:
        print 'Exiting...'
        sys.exit(1)

    db_backup_done = userInput.get_YN_input(
        "Please, confirm you have made backup of the Ambari db [y/n] (n)? ",
        False)
    if not db_backup_done:
        print 'Exiting...'
        sys.exit(1)

    status, pid = serverUtils.is_server_runing()
    if status:
        raise FatalException(1, "Ambari Server should be stopped")

    try:
        host_mapping_file_path = args[1]
    except IndexError:
        #host_mapping file is mandatory
        raise FatalException(
            1,
            "Invalid number of host update arguments. Probably, you forgot to add json file with "
            "host changes.")

    if not os.path.isfile(host_mapping_file_path):
        raise FatalException(1, "Invalid file path or file doesn't exist")

    if not os.access(host_mapping_file_path, os.R_OK):
        raise FatalException(1, "File is not readable")

    jdk_path = serverConfiguration.get_java_exe_path()

    if jdk_path is None:
        print_error_msg(
            "No JDK found, please run the \"setup\" "
            "command to install a JDK automatically or install any "
            "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
        sys.exit(1)

    properties = serverConfiguration.get_ambari_properties()
    serverConfiguration.parse_properties_file(options)
    options.database_index = LINUX_DBMS_KEYS_LIST.index(
        properties[JDBC_DATABASE_PROPERTY])

    dbConfiguration.ensure_jdbc_driver_is_installed(
        options, serverConfiguration.get_ambari_properties())

    serverClassPath = ServerClassPath(
        serverConfiguration.get_ambari_properties(), options)
    class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()

    command = HOST_UPDATE_HELPER_CMD.format(jdk_path, class_path,
                                            host_mapping_file_path)

    ambari_user = serverConfiguration.read_ambari_user()
    current_user = setupSecurity.ensure_can_start_under_current_user(
        ambari_user)
    environ = setupSecurity.generate_env(options, ambari_user, current_user)

    (retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ)
    print_info_msg("Return code from update host names command, retcode = " +
                   str(retcode))

    if retcode > 0:
        print_error_msg(
            "Error executing update host names, please check the server logs.")
        raise FatalException(1, 'Host names update failed.')
    else:
        print_info_msg('Host names update completed successfully')
Example #25
0
def setup(options):
  retcode = verify_setup_allowed()
  if not retcode == 0:
    raise FatalException(1, None)

  if not is_root():
    err = configDefaults.MESSAGE_ERROR_SETUP_NOT_ROOT
    raise FatalException(4, err)

  # proceed jdbc properties if they were set
  if _check_jdbc_options(options):
    proceedJDBCProperties(options)
    return

  (retcode, err) = disable_security_enhancements()
  if not retcode == 0:
    raise FatalException(retcode, err)

  #Create ambari user, if needed
  retcode = check_ambari_user()
  if not retcode == 0:
    err = 'Failed to create user. Exiting.'
    raise FatalException(retcode, err)

  print configDefaults.MESSAGE_CHECK_FIREWALL
  check_firewall()

  # proceed jdbc properties if they were set
  if _check_jdbc_options(options):
    proceedJDBCProperties(options)

  print 'Checking JDK...'
  try:
    download_and_install_jdk(options)
  except FatalException as e:
    err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e)
    raise FatalException(e.code, err)

  print 'Completing setup...'
  retcode = configure_os_settings()
  if not retcode == 0:
    err = 'Configure of OS settings in ambari.properties failed. Exiting.'
    raise FatalException(retcode, err)

  print 'Configuring database...'
  prompt_db_properties(options)

  #DB setup should be done last after doing any setup.

  _setup_database(options)

  check_jdbc_drivers(options)

  print 'Extracting system views...'
  retcode = extract_views()
  if not retcode == 0:
    err = 'Error while extracting system views. Exiting'
    raise FatalException(retcode, err)

  # we've already done this, but new files were created so run it one time.
  adjust_directory_permissions(read_ambari_user())
Example #26
0
def import_file_to_keystore(source, destination):
  shutil.copy(source, destination)
  set_file_permissions(destination, "660", read_ambari_user(), False)
def server_process_main(options, scmStatus=None):
    # debug mode, including stop Java process at startup
    try:
        set_debug_mode_from_options(options)
    except AttributeError:
        pass

    if not check_reverse_lookup():
        print_warning_msg(
            "The hostname was not found in the reverse DNS lookup. "
            "This may result in incorrect behavior. "
            "Please check the DNS setup and fix the issue.")

    check_database_name_property()
    parse_properties_file(options)

    is_active_instance = get_is_active_instance()
    if not is_active_instance:
        print_warning_msg(
            "This instance of ambari server is not designated as active. Cannot start ambari server."
        )
        err = "This is not an active instance. Shutting down..."
        raise FatalException(1, err)

    ambari_user = read_ambari_user()
    current_user = ensure_can_start_under_current_user(ambari_user)

    print_info_msg("Ambari Server is not running...")

    jdk_path = find_jdk()
    if jdk_path is None:
        err = "No JDK found, please run the \"ambari-server setup\" " \
              "command to install a JDK automatically or install any " \
              "JDK manually to " + configDefaults.JDK_INSTALL_DIR
        raise FatalException(1, err)

    properties = get_ambari_properties()

    if not options.skip_properties_validation:
        missing_properties = get_missing_properties(properties)
        if missing_properties:
            err = "Required properties are not found: " + str(missing_properties) + ". To skip properties validation " \
                  "use \"--skip-properties-validation\""
            raise FatalException(1, err)

    # Preparations
    if is_root():
        print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT

    ensure_jdbc_driver_is_installed(options, properties)

    ensure_dbms_is_running(options, properties, scmStatus)

    if scmStatus is not None:
        scmStatus.reportStartPending()

    refresh_stack_hash(properties)

    if scmStatus is not None:
        scmStatus.reportStartPending()

    ensure_server_security_is_configured()

    if scmStatus is not None:
        scmStatus.reportStartPending()

    java_exe = get_java_exe_path()

    serverClassPath = ServerClassPath(properties, options)

    debug_mode = get_debug_mode()
    debug_start = (debug_mode & 1) or SERVER_START_DEBUG
    suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
    suspend_mode = 'y' if suspend_start else 'n'

    if options.skip_database_validation:
        global jvm_args
        jvm_args += " -DskipDatabaseConsistencyValidation"

    param_list = generate_child_process_param_list(
        ambari_user, java_exe,
        serverClassPath.get_full_ambari_classpath_escaped_for_shell(
            validate_classpath=True), debug_start, suspend_mode)
    environ = generate_env(options, ambari_user, current_user)

    if not os.path.exists(configDefaults.PID_DIR):
        os.makedirs(configDefaults.PID_DIR, 0755)

    # The launched shell process and sub-processes should have a group id that
    # is different from the parent.
    def make_process_independent():
        if IS_FOREGROUND:  # upstart script is not able to track process from different pgid.
            return

        processId = os.getpid()
        if processId > 0:
            try:
                os.setpgid(processId, processId)
            except OSError, e:
                print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(
                    pidJava, str(e)))
                pass
Example #28
0
def upgrade(args):
    if not is_root():
        err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
        raise FatalException(4, err)

    print 'Updating properties in ' + AMBARI_PROPERTIES_FILE + ' ...'
    retcode = update_ambari_properties()
    if not retcode == 0:
        err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
        raise FatalException(retcode, err)

    try:
        update_database_name_property(upgrade=True)
    except FatalException:
        return -1

    # Ignore the server version & database options passed via command-line arguments
    parse_properties_file(args)

    #TODO check database version
    change_objects_owner(args)

    retcode = run_schema_upgrade()
    if not retcode == 0:
        print_error_msg(
            "Ambari server upgrade failed. Please look at {0}, for more details."
            .format(configDefaults.SERVER_LOG_FILE))
        raise FatalException(11, 'Schema upgrade failed.')

    user = read_ambari_user()
    if user is None:
        warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
        print_warning_msg(warn)
    else:
        adjust_directory_permissions(user)

    # local repo
    upgrade_local_repo(args)

    # create jdbc symlinks if jdbc drivers are available in resources
    check_jdbc_drivers(args)

    properties = get_ambari_properties()
    if properties == -1:
        err = "Error getting ambari properties"
        print_error_msg(err)
        raise FatalException(-1, err)

    # Move *.py files from custom_actions to custom_actions/scripts
    # This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
    ambari_version = get_ambari_version(properties)
    if ambari_version is None:
        args.warnings.append(
            "*.py files were not moved from custom_actions to custom_actions/scripts."
        )
    elif compare_versions(ambari_version, "2.0.0") == 0:
        move_user_custom_actions()

    # Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
    admin_views_dirs = get_admin_views_dir(properties)
    for admin_views_dir in admin_views_dirs:
        shutil.rmtree(admin_views_dir)

    # check if ambari has obsolete LDAP configuration
    if properties.get_property(
            LDAP_PRIMARY_URL_PROPERTY
    ) and not properties.get_property(IS_LDAP_CONFIGURED):
        args.warnings.append(
            "Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration."
        )
Example #29
0
def server_process_main(options, scmStatus=None):
  # debug mode, including stop Java process at startup
  try:
    set_debug_mode_from_options(options)
  except AttributeError:
    pass

  if not check_reverse_lookup():
    print_warning_msg("The hostname was not found in the reverse DNS lookup. "
                      "This may result in incorrect behavior. "
                      "Please check the DNS setup and fix the issue.")

  check_database_name_property()
  parse_properties_file(options)

  is_active_instance = get_is_active_instance()
  if not is_active_instance:
      print_warning_msg("This instance of ambari server is not designated as active. Cannot start ambari server.")
      err = "This is not an active instance. Shutting down..."
      raise FatalException(1, err)

  ambari_user = read_ambari_user()
  current_user = ensure_can_start_under_current_user(ambari_user)

  print_info_msg("Ambari Server is not running...")

  jdk_path = find_jdk()
  if jdk_path is None:
    err = "No JDK found, please run the \"ambari-server setup\" " \
          "command to install a JDK automatically or install any " \
          "JDK manually to " + configDefaults.JDK_INSTALL_DIR
    raise FatalException(1, err)

  properties = get_ambari_properties()

  # Preparations
  if is_root():
    print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT

  ensure_jdbc_driver_is_installed(options, properties)

  ensure_dbms_is_running(options, properties, scmStatus)

  if scmStatus is not None:
    scmStatus.reportStartPending()

  refresh_stack_hash(properties)

  if scmStatus is not None:
    scmStatus.reportStartPending()

  ensure_server_security_is_configured()

  if scmStatus is not None:
    scmStatus.reportStartPending()

  java_exe = get_java_exe_path()

  class_path = get_conf_dir()
  class_path = os.path.abspath(class_path) + os.pathsep + get_ambari_classpath()
  jdbc_driver_path = get_jdbc_driver_path(options, properties)
  if jdbc_driver_path not in class_path:
    class_path = class_path + os.pathsep + jdbc_driver_path

  if SERVER_CLASSPATH_KEY in os.environ:
      class_path =  os.environ[SERVER_CLASSPATH_KEY] + os.pathsep + class_path

  native_libs_path = get_native_libs_path(options, properties)
  if native_libs_path is not None:
    if LIBRARY_PATH_KEY in os.environ:
      native_libs_path = os.environ[LIBRARY_PATH_KEY] + os.pathsep + native_libs_path
    os.environ[LIBRARY_PATH_KEY] = native_libs_path

  debug_mode = get_debug_mode()
  debug_start = (debug_mode & 1) or SERVER_START_DEBUG
  suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
  suspend_mode = 'y' if suspend_start else 'n'

  param_list = generate_child_process_param_list(ambari_user, java_exe,
                                                 class_path, debug_start,
                                                 suspend_mode)
  environ = generate_env(ambari_user, current_user)

  if not os.path.exists(configDefaults.PID_DIR):
    os.makedirs(configDefaults.PID_DIR, 0755)

  print_info_msg("Running server: " + str(param_list))
  procJava = subprocess.Popen(param_list, env=environ)

  pidJava = procJava.pid
  if pidJava <= 0:
    procJava.terminate()
    exitcode = procJava.returncode
    exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME)
    save_pid(exitcode, exitfile)

    if scmStatus is not None:
      scmStatus.reportStopPending()

    raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
  else:
    # Change the group id to the process id of the parent so that the launched
    # process and sub-processes have a group id that is different from the parent.
    try:
      os.setpgid(pidJava, 0)
    except OSError, e:
      print_warning_msg('setpgid({0}, 0) failed - {1}'.format(pidJava, str(e)))
      pass
    pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
    save_pid(pidJava, pidfile)
    print "Server PID at: "+pidfile
    print "Server out at: "+configDefaults.SERVER_OUT_FILE
    print "Server log at: "+configDefaults.SERVER_LOG_FILE

    wait_for_server_start(pidfile, scmStatus)
Example #30
0
def server_process_main(options, scmStatus=None):
    properties = get_ambari_properties()
    if properties == -1:
        err = "Error getting ambari properties"
        raise FatalException(-1, err)

    properties_for_print = []
    logger.info("Ambari server properties config:")
    for key, value in properties.getPropertyDict().items():
        if "passwd" not in key and "password" not in key:
            properties_for_print.append(key + "=" + value)

    logger.info(properties_for_print)

    # debug mode, including stop Java process at startup
    try:
        set_debug_mode_from_options(options)
    except AttributeError:
        pass

    if not check_reverse_lookup():
        print_warning_msg(
            "The hostname was not found in the reverse DNS lookup. "
            "This may result in incorrect behavior. "
            "Please check the DNS setup and fix the issue.")

    check_database_name_property()
    parse_properties_file(options)

    is_active_instance = get_is_active_instance()
    if not is_active_instance:
        print_warning_msg(
            "This instance of ambari server is not designated as active. Cannot start ambari server."
        )
        err = "This is not an active instance. Shutting down..."
        raise FatalException(1, err)

    ambari_user = read_ambari_user()
    current_user = ensure_can_start_under_current_user(ambari_user)

    print_info_msg("Ambari Server is not running...")

    jdk_path = find_jdk()
    if jdk_path is None:
        err = "No JDK found, please run the \"ambari-server setup\" " \
              "command to install a JDK automatically or install any " \
              "JDK manually to " + configDefaults.JDK_INSTALL_DIR
        raise FatalException(1, err)

    if not options.skip_properties_validation:
        missing_properties = get_missing_properties(properties)
        if missing_properties:
            err = "Required properties are not found: " + str(missing_properties) + ". To skip properties validation " \
                  "use \"--skip-properties-validation\""
            raise FatalException(1, err)

    # Preparations
    if is_root():
        print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT

    ensure_jdbc_driver_is_installed(options, properties)

    ensure_dbms_is_running(options, properties, scmStatus)

    if scmStatus is not None:
        scmStatus.reportStartPending()

    refresh_stack_hash(properties)

    if scmStatus is not None:
        scmStatus.reportStartPending()

    ensure_server_security_is_configured()

    if scmStatus is not None:
        scmStatus.reportStartPending()

    java_exe = get_java_exe_path()

    serverClassPath = ServerClassPath(properties, options)

    debug_mode = get_debug_mode()
    debug_start = (debug_mode & 1) or SERVER_START_DEBUG
    suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
    suspend_mode = 'y' if suspend_start else 'n'

    environ = generate_env(options, ambari_user, current_user)
    class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell(
        validate_classpath=True)

    if options.skip_database_check:
        global jvm_args
        jvm_args += " -DskipDatabaseConsistencyCheck"
        print "Ambari Server is starting with the database consistency check skipped. Do not make any changes to your cluster " \
              "topology or perform a cluster upgrade until you correct the database consistency issues. See \"" \
              + configDefaults.DB_CHECK_LOG + "\" for more details on the consistency issues."
        properties.process_pair(CHECK_DATABASE_SKIPPED_PROPERTY, "true")
    else:
        print "Ambari database consistency check started..."
        if options.fix_database_consistency:
            jvm_args += " -DfixDatabaseConsistency"
        properties.process_pair(CHECK_DATABASE_SKIPPED_PROPERTY, "false")

    update_properties(properties)
    param_list = generate_child_process_param_list(ambari_user, java_exe,
                                                   class_path, debug_start,
                                                   suspend_mode)

    # The launched shell process and sub-processes should have a group id that
    # is different from the parent.
    def make_process_independent():
        if IS_FOREGROUND:  # upstart script is not able to track process from different pgid.
            return

        processId = os.getpid()
        if processId > 0:
            try:
                os.setpgid(processId, processId)
            except OSError, e:
                print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(
                    pidJava, str(e)))
                pass
Example #31
0
def import_cert_and_key(security_server_keys_dir):
    import_cert_path = get_validated_filepath_input( \
        "Enter path to Certificate: ", \
        "Certificate not found")
    import_key_path  =  get_validated_filepath_input( \
        "Enter path to Private Key: ", "Private Key not found")
    pem_password = get_validated_string_input(
        "Please enter password for Private Key: ", "", None, None, True)

    certInfoDict = get_cert_info(import_cert_path)

    if not certInfoDict:
        print_warning_msg('Unable to get Certificate information')
    else:
        #Validate common name of certificate
        if not is_valid_cert_host(certInfoDict):
            print_warning_msg('Unable to validate Certificate hostname')

        #Validate issue and expirations dates of certificate
        if not is_valid_cert_exp(certInfoDict):
            print_warning_msg(
                'Unable to validate Certificate issue and expiration dates')

    #jetty requires private key files with non-empty key passwords
    retcode = 0
    err = ''
    if not pem_password:
        print 'Generating random password for HTTPS keystore...done.'
        pem_password = generate_random_string()
        retcode, out, err = run_os_command(
            CHANGE_KEY_PWD_CND.format(import_key_path, pem_password))
        import_key_path += '.secured'

    if retcode == 0:
        keystoreFilePath = os.path.join(security_server_keys_dir, \
                                        SSL_KEYSTORE_FILE_NAME)
        keystoreFilePathTmp = os.path.join(tempfile.gettempdir(), \
                                           SSL_KEYSTORE_FILE_NAME)
        passFilePath = os.path.join(security_server_keys_dir, \
                                    SSL_KEY_PASSWORD_FILE_NAME)
        passFilePathTmp = os.path.join(tempfile.gettempdir(), \
                                       SSL_KEY_PASSWORD_FILE_NAME)
        passinFilePath = os.path.join(tempfile.gettempdir(), \
                                      SSL_PASSIN_FILE)
        passwordFilePath = os.path.join(tempfile.gettempdir(), \
                                        SSL_PASSWORD_FILE)

        with open(passFilePathTmp, 'w+') as passFile:
            passFile.write(pem_password)
            passFile.close
            pass

        set_file_permissions(passFilePath, "660", read_ambari_user(), False)

        copy_file(passFilePathTmp, passinFilePath)
        copy_file(passFilePathTmp, passwordFilePath)

        retcode, out, err = run_os_command(EXPRT_KSTR_CMD.format(import_cert_path, \
                                                                 import_key_path, passwordFilePath, passinFilePath, keystoreFilePathTmp))
    if retcode == 0:
        print 'Importing and saving Certificate...done.'
        import_file_to_keystore(keystoreFilePathTmp, keystoreFilePath)
        import_file_to_keystore(passFilePathTmp, passFilePath)

        import_file_to_keystore(import_cert_path, os.path.join( \
            security_server_keys_dir, SSL_CERT_FILE_NAME))
        import_file_to_keystore(import_key_path, os.path.join( \
            security_server_keys_dir, SSL_KEY_FILE_NAME))

        #Validate keystore
        retcode, out, err = run_os_command(VALIDATE_KEYSTORE_CMD.format(keystoreFilePath, \
                                                                        passwordFilePath, passinFilePath))

        remove_file(passinFilePath)
        remove_file(passwordFilePath)

        if not retcode == 0:
            print 'Error during keystore validation occured!:'
            print err
            return False

        return True
    else:
        print_error_msg('Could not import Certificate and Private Key.')
        print 'SSL error on exporting keystore: ' + err.rstrip() + \
            '.\nPlease ensure that provided Private Key password is correct and ' + \
            're-import Certificate.'

        return False
def update_host_names(args, options):
  services_stopped = userInput.get_YN_input("Please, confirm Ambari services are stopped [y/n] (n)? ", False)
  if not services_stopped:
    print 'Exiting...'
    sys.exit(1)

  pending_commands = userInput.get_YN_input("Please, confirm there are no pending commands on cluster [y/n] (n)? ", False)
  if not pending_commands:
    print 'Exiting...'
    sys.exit(1)

  db_backup_done = userInput.get_YN_input("Please, confirm you have made backup of the Ambari db [y/n] (n)? ", False)
  if not db_backup_done:
    print 'Exiting...'
    sys.exit(1)

  status, pid = serverUtils.is_server_runing()
  if status:
    raise FatalException(1, "Ambari Server should be stopped")

  try:
    host_mapping_file_path = args[1]
  except IndexError:
    #host_mapping file is mandatory
    raise FatalException(1, "Invalid number of host update arguments. Probably, you forgot to add json file with "
                            "host changes.")

  if not os.path.isfile(host_mapping_file_path):
    raise FatalException(1, "Invalid file path or file doesn't exist")

  if not os.access(host_mapping_file_path, os.R_OK):
    raise FatalException(1, "File is not readable")

  jdk_path = serverConfiguration.get_java_exe_path()

  if jdk_path is None:
    print_error_msg("No JDK found, please run the \"setup\" "
                    "command to install a JDK automatically or install any "
                    "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
    sys.exit(1)

  properties = serverConfiguration.get_ambari_properties()
  serverConfiguration.parse_properties_file(options)
  options.database_index = LINUX_DBMS_KEYS_LIST.index(properties[JDBC_DATABASE_PROPERTY])

  dbConfiguration.ensure_jdbc_driver_is_installed(options, serverConfiguration.get_ambari_properties())

  serverClassPath = ServerClassPath(serverConfiguration.get_ambari_properties(), options)
  class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()

  command = HOST_UPDATE_HELPER_CMD.format(jdk_path, class_path, host_mapping_file_path)

  ambari_user = serverConfiguration.read_ambari_user()
  current_user = setupSecurity.ensure_can_start_under_current_user(ambari_user)
  environ = setupSecurity.generate_env(options, ambari_user, current_user)

  (retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ)
  print_info_msg("Return code from update host names command, retcode = " + str(retcode))

  if retcode > 0:
    print_error_msg("Error executing update host names, please check the server logs.")
    raise FatalException(1, 'Host names update failed.')
  else:
    print_info_msg('Host names update completed successfully')
Example #33
0
def server_process_main(options, scmStatus=None):
  # debug mode, including stop Java process at startup
  try:
    set_debug_mode_from_options(options)
  except AttributeError:
    pass

  if not check_reverse_lookup():
    print_warning_msg("The hostname was not found in the reverse DNS lookup. "
                      "This may result in incorrect behavior. "
                      "Please check the DNS setup and fix the issue.")

  check_database_name_property()
  parse_properties_file(options)

  ambari_user = read_ambari_user()
  current_user = ensure_can_start_under_current_user(ambari_user)

  print_info_msg("Ambari Server is not running...")

  jdk_path = find_jdk()
  if jdk_path is None:
    err = "No JDK found, please run the \"ambari-server setup\" " \
          "command to install a JDK automatically or install any " \
          "JDK manually to " + configDefaults.JDK_INSTALL_DIR
    raise FatalException(1, err)

  properties = get_ambari_properties()

  # Preparations
  if is_root():
    print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT

  ensure_jdbc_driver_is_installed(options, properties)

  ensure_dbms_is_running(options, properties, scmStatus)

  if scmStatus is not None:
    scmStatus.reportStartPending()

  refresh_stack_hash(properties)

  if scmStatus is not None:
    scmStatus.reportStartPending()

  ensure_server_security_is_configured()

  if scmStatus is not None:
    scmStatus.reportStartPending()

  java_exe = get_java_exe_path()

  serverClassPath = ServerClassPath(properties, options)

  debug_mode = get_debug_mode()
  debug_start = (debug_mode & 1) or SERVER_START_DEBUG
  suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
  suspend_mode = 'y' if suspend_start else 'n'

  param_list = generate_child_process_param_list(ambari_user, java_exe,
                                                 serverClassPath.get_full_ambari_classpath_escaped_for_shell(), debug_start,
                                                 suspend_mode)
  environ = generate_env(options, ambari_user, current_user)

  if not os.path.exists(configDefaults.PID_DIR):
    os.makedirs(configDefaults.PID_DIR, 0755)

  # The launched shell process and sub-processes should have a group id that
  # is different from the parent.
  def make_process_independent():
    processId = os.getpid()
    if processId > 0:
      try:
        os.setpgid(processId, processId)
      except OSError, e:
        print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(pidJava, str(e)))
        pass
Example #34
0
def import_cert_and_key(security_server_keys_dir):
  import_cert_path = get_validated_filepath_input( \
      "Enter path to Certificate: ", \
      "Certificate not found")
  import_key_path  =  get_validated_filepath_input( \
      "Enter path to Private Key: ", "Private Key not found")
  pem_password = get_validated_string_input("Please enter password for Private Key: ", "", None, None, True)

  certInfoDict = get_cert_info(import_cert_path)

  if not certInfoDict:
    print_warning_msg('Unable to get Certificate information')
  else:
    #Validate common name of certificate
    if not is_valid_cert_host(certInfoDict):
      print_warning_msg('Unable to validate Certificate hostname')

    #Validate issue and expirations dates of certificate
    if not is_valid_cert_exp(certInfoDict):
      print_warning_msg('Unable to validate Certificate issue and expiration dates')

  #jetty requires private key files with non-empty key passwords
  retcode = 0
  err = ''
  if not pem_password:
    print 'Generating random password for HTTPS keystore...done.'
    pem_password = generate_random_string()
    retcode, out, err = run_os_command(CHANGE_KEY_PWD_CND.format(
        import_key_path, pem_password))
    import_key_path += '.secured'

  if retcode == 0:
    keystoreFilePath = os.path.join(security_server_keys_dir, \
                                    SSL_KEYSTORE_FILE_NAME)
    keystoreFilePathTmp = os.path.join(tempfile.gettempdir(), \
                                       SSL_KEYSTORE_FILE_NAME)
    passFilePath = os.path.join(security_server_keys_dir, \
                                SSL_KEY_PASSWORD_FILE_NAME)
    passFilePathTmp = os.path.join(tempfile.gettempdir(), \
                                   SSL_KEY_PASSWORD_FILE_NAME)
    passinFilePath = os.path.join(tempfile.gettempdir(), \
                                  SSL_PASSIN_FILE)
    passwordFilePath = os.path.join(tempfile.gettempdir(), \
                                    SSL_PASSWORD_FILE)

    with open(passFilePathTmp, 'w+') as passFile:
      passFile.write(pem_password)
      passFile.close
      pass

    set_file_permissions(passFilePath, "660", read_ambari_user(), False)

    copy_file(passFilePathTmp, passinFilePath)
    copy_file(passFilePathTmp, passwordFilePath)

    retcode, out, err = run_os_command(EXPRT_KSTR_CMD.format(import_cert_path, \
                                                             import_key_path, passwordFilePath, passinFilePath, keystoreFilePathTmp))
  if retcode == 0:
    print 'Importing and saving Certificate...done.'
    import_file_to_keystore(keystoreFilePathTmp, keystoreFilePath)
    import_file_to_keystore(passFilePathTmp, passFilePath)

    import_file_to_keystore(import_cert_path, os.path.join( \
        security_server_keys_dir, SSL_CERT_FILE_NAME))
    import_file_to_keystore(import_key_path, os.path.join( \
        security_server_keys_dir, SSL_KEY_FILE_NAME))

    #Validate keystore
    retcode, out, err = run_os_command(VALIDATE_KEYSTORE_CMD.format(keystoreFilePath, \
                                                                    passwordFilePath, passinFilePath))

    remove_file(passinFilePath)
    remove_file(passwordFilePath)

    if not retcode == 0:
      print 'Error during keystore validation occured!:'
      print err
      return False

    return True
  else:
    print_error_msg('Could not import Certificate and Private Key.')
    print 'SSL error on exporting keystore: ' + err.rstrip() + \
        '.\nPlease ensure that provided Private Key password is correct and ' + \
        're-import Certificate.'

    return False
Example #35
0
def _install_mpack(options, replay_mode=False, is_upgrade=False):
  """
  Install management pack
  :param options: Command line options
  :param replay_mode: Flag to indicate if executing command in replay mode
  """

  mpack_path = options.mpack_path
  if not mpack_path:
    print_error_msg("Management pack not specified!")
    raise FatalException(-1, 'Management pack not specified!')

  print_info_msg("Installing management pack {0}".format(mpack_path))

  # Download management pack to a temp location
  tmp_archive_path = download_mpack(mpack_path)
  if not (tmp_archive_path and os.path.exists(tmp_archive_path)):
    print_error_msg("Management pack could not be downloaded!")
    raise FatalException(-1, 'Management pack could not be downloaded!')

  # Expand management pack in temp directory
  tmp_root_dir = expand_mpack(tmp_archive_path)

  # Read mpack metadata
  mpack_metadata = read_mpack_metadata(tmp_root_dir)
  if not mpack_metadata:
    raise FatalException(-1, 'Malformed management pack {0}. Metadata file missing!'.format(mpack_path))

  # Validate management pack prerequisites
  # Skip validation in replay mode
  if not replay_mode:
    validate_mpack_prerequisites(mpack_metadata)

  if is_upgrade:
    # Execute pre upgrade hook
    _execute_hook(mpack_metadata, BEFORE_UPGRADE_HOOK_NAME, tmp_root_dir)
  else:
    # Execute pre install hook
    _execute_hook(mpack_metadata, BEFORE_INSTALL_HOOK_NAME, tmp_root_dir)

  # Purge previously installed stacks and management packs
  if options.purge and options.purge_list:
    purge_resources = options.purge_list.split(",")
    validate_purge(options, purge_resources, tmp_root_dir, mpack_metadata, replay_mode)
    purge_stacks_and_mpacks(purge_resources, replay_mode)

  adjust_ownership_list = []
  change_ownership_list = []

  # Get ambari mpack properties
  stack_location, extension_location, service_definitions_location, mpacks_staging_location, dashboard_location = get_mpack_properties()
  mpacks_cache_location = os.path.join(mpacks_staging_location, MPACKS_CACHE_DIRNAME)
  # Create directories
  if not os.path.exists(stack_location):
    sudo.makedir(stack_location, 0755)
  adjust_ownership_list.append((stack_location, "0755", "{0}", True))
  change_ownership_list.append((stack_location,"{0}",True))
  if not os.path.exists(extension_location):
    sudo.makedir(extension_location, 0755)
  adjust_ownership_list.append((extension_location, "0755", "{0}", True))
  change_ownership_list.append((extension_location,"{0}",True))
  if not os.path.exists(service_definitions_location):
    sudo.makedir(service_definitions_location, 0755)
  adjust_ownership_list.append((service_definitions_location, "0755", "{0}", True))
  change_ownership_list.append((service_definitions_location,"{0}",True))
  if not os.path.exists(mpacks_staging_location):
    sudo.makedir(mpacks_staging_location, 0755)
  adjust_ownership_list.append((mpacks_staging_location, "0755", "{0}", True))
  change_ownership_list.append((mpacks_staging_location,"{0}",True))
  if not os.path.exists(mpacks_cache_location):
    sudo.makedir(mpacks_cache_location, 0755)
  adjust_ownership_list.append((mpacks_cache_location, "0755", "{0}", True))
  change_ownership_list.append((mpacks_cache_location,"{0}",True))
  if not os.path.exists(dashboard_location):
    sudo.makedir(dashboard_location, 0755)
    sudo.makedir(os.path.join(dashboard_location, GRAFANA_DASHBOARDS_DIRNAME), 0755)
    sudo.makedir(os.path.join(dashboard_location, SERVICE_METRICS_DIRNAME), 0755)
  adjust_ownership_list.append((dashboard_location, "0755", "{0}", True))
  change_ownership_list.append((dashboard_location,"{0}",True))

  # Stage management pack (Stage at /var/lib/ambari-server/resources/mpacks/mpack_name-mpack_version)
  mpack_name = mpack_metadata.name
  mpack_version = mpack_metadata.version
  mpack_dirname = mpack_name + "-" + mpack_version
  mpack_staging_dir = os.path.join(mpacks_staging_location, mpack_dirname)
  mpack_archive_path = os.path.join(mpacks_cache_location, os.path.basename(tmp_archive_path))

  print_info_msg("Stage management pack {0}-{1} to staging location {2}".format(
          mpack_name, mpack_version, mpack_staging_dir))
  if os.path.exists(mpack_staging_dir):
    if options.force:
      print_info_msg("Force removing previously installed management pack from {0}".format(mpack_staging_dir))
      sudo.rmtree(mpack_staging_dir)
    else:
      error_msg = "Management pack {0}-{1} already installed!".format(mpack_name, mpack_version)
      print_error_msg(error_msg)
      raise FatalException(-1, error_msg)
  shutil.move(tmp_root_dir, mpack_staging_dir)
  shutil.move(tmp_archive_path, mpack_archive_path)

  # Process setup steps for all artifacts (stack-definitions, extension-definitions,
  # service-definitions, stack-addon-service-definitions) in the management pack
  for artifact in mpack_metadata.artifacts:
    # Artifact name (Friendly name)
    artifact_name = artifact.name
    # Artifact type (stack-definitions, extension-definitions, service-definitions, etc)
    artifact_type = artifact.type
    # Artifact directory with contents of the artifact
    artifact_source_dir = os.path.join(mpack_staging_dir, artifact.source_dir)

    print_info_msg("Processing artifact {0} of type {1} in {2}".format(
            artifact_name, artifact_type, artifact_source_dir))
    if artifact.type == STACK_DEFINITIONS_ARTIFACT_NAME:
      process_stack_definitions_artifact(artifact, artifact_source_dir, options)
    elif artifact.type == EXTENSION_DEFINITIONS_ARTIFACT_NAME:
      process_extension_definitions_artifact(artifact, artifact_source_dir, options)
    elif artifact.type == SERVICE_DEFINITIONS_ARTIFACT_NAME:
      process_service_definitions_artifact(artifact, artifact_source_dir, options)
    elif artifact.type == STACK_ADDON_SERVICE_DEFINITIONS_ARTIFACT_NAME:
      process_stack_addon_service_definitions_artifact(artifact, artifact_source_dir, options)
    else:
      print_info_msg("Unknown artifact {0} of type {1}".format(artifact_name, artifact_type))

  ambari_user = read_ambari_user()

  if ambari_user:
     # This is required when a non-admin user is configured to setup ambari-server
    print_info_msg("Adjusting file permissions and ownerships")
    for pack in adjust_ownership_list:
      file = pack[0]
      mod = pack[1]
      user = pack[2].format(ambari_user)
      recursive = pack[3]
      logger.info("Setting file permissions: {0} {1} {2} {3}".format(file, mod, user, recursive))
      set_file_permissions(file, mod, user, recursive)

    for pack in change_ownership_list:
      path = pack[0]
      user = pack[1].format(ambari_user)
      recursive = pack[2]
      logger.info("Changing ownership: {0} {1} {2}".format(path, user, recursive))
      change_owner(path, user, recursive)

  print_info_msg("Management pack {0}-{1} successfully installed! Please restart ambari-server.".format(mpack_name, mpack_version))
  return mpack_metadata, mpack_name, mpack_version, mpack_staging_dir, mpack_archive_path
Example #36
0
def upgrade(args):
    print_info_msg("Upgrade Ambari Server", True)
    if not is_root():
        err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
        raise FatalException(4, err)
    print_info_msg(
        'Updating Ambari Server properties in {0} ...'.format(
            AMBARI_PROPERTIES_FILE), True)
    retcode = update_ambari_properties()
    if not retcode == 0:
        err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
        raise FatalException(retcode, err)

    print_info_msg(
        'Updating Ambari Server properties in {0} ...'.format(AMBARI_ENV_FILE),
        True)
    retcode = update_ambari_env()
    if not retcode == 0:
        err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting'
        raise FatalException(retcode, err)

    retcode = update_krb_jaas_login_properties()
    if retcode == -2:
        pass  # no changes done, let's be silent
    elif retcode == 0:
        print_info_msg("File {0} updated.".format(AMBARI_KRB_JAAS_LOGIN_FILE),
                       True)
    elif not retcode == 0:
        err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting'
        raise FatalException(retcode, err)

    restore_custom_services()
    replay_mpack_logs()
    try:
        update_database_name_property(upgrade=True)
    except FatalException:
        return -1

    # Ignore the server version & database options passed via command-line arguments
    parse_properties_file(args)

    #TODO check database version
    change_objects_owner(args)

    retcode = run_schema_upgrade(args)
    if not retcode == 0:
        print_error_msg(
            "Ambari server upgrade failed. Please look at {0}, for more details."
            .format(configDefaults.SERVER_LOG_FILE))
        raise FatalException(11, 'Schema upgrade failed.')

    user = read_ambari_user()
    if user is None:
        warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
        print_warning_msg(warn)
    else:
        adjust_directory_permissions(user)

    # create jdbc symlinks if jdbc drivers are available in resources
    check_jdbc_drivers(args)

    properties = get_ambari_properties()
    if properties == -1:
        err = "Error getting ambari properties"
        print_error_msg(err)
        raise FatalException(-1, err)

    # Move *.py files from custom_actions to custom_actions/scripts
    # This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
    ambari_version = get_ambari_version(properties)
    if ambari_version is None:
        args.warnings.append(
            "*.py files were not moved from custom_actions to custom_actions/scripts."
        )
    elif compare_versions(ambari_version, "2.0.0") == 0:
        move_user_custom_actions()

    # Move files installed by package to default views directory to a custom one
    for views_dir in get_views_dir(properties):
        root_views_dir = views_dir + "/../"

        if os.path.samefile(root_views_dir, get_default_views_dir()):
            continue

        for file in glob.glob(get_default_views_dir() + '/*'):
            shutil.move(file, root_views_dir)

    # Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
    admin_views_dirs = get_admin_views_dir(properties)
    for admin_views_dir in admin_views_dirs:
        shutil.rmtree(admin_views_dir)

    # Modify timestamp of views jars to current time
    views_jars = get_views_jars(properties)
    for views_jar in views_jars:
        os.utime(views_jar, None)

    # check if ambari is configured to use LDAP authentication
    if properties.get_property(CLIENT_SECURITY) == "ldap":
        args.warnings.append(
            "LDAP authentication is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration."
        )

    # adding custom jdbc name and previous custom jdbc properties
    # we need that to support new dynamic jdbc names for upgraded ambari
    add_jdbc_properties(properties)

    json_url = get_json_url_from_repo_file()
    if json_url:
        print "Ambari repo file contains latest json url {0}, updating stacks repoinfos with it...".format(
            json_url)
        properties = get_ambari_properties()
        stack_root = get_stack_location(properties)
        update_latest_in_repoinfos_for_stacks(stack_root, json_url)
    else:
        print "Ambari repo file doesn't contain latest json url, skipping repoinfos modification"
Example #37
0
def setup(options):
  modifyGlobalConstant(options.use_default)
  retcode = verify_setup_allowed()
  if not retcode == 0:
    raise FatalException(1, None)

  if not is_root():
    err = configDefaults.MESSAGE_ERROR_SETUP_NOT_ROOT
    raise FatalException(4, err)

  if _check_repo_options(options):
    configureRepoURL(options.repo_url)
    
  if _check_hostname(options):
    configureHostname(options.hostname)

  # proceed jdbc properties if they were set
  if _check_jdbc_options(options):
    proceedJDBCProperties(options)
    return

  (retcode, err) = disable_security_enhancements()
  if not retcode == 0:
    raise FatalException(retcode, err)

  #Create tbds user, if needed
  retcode = check_ambari_user()
  if not retcode == 0:
    err = 'Failed to create user. Exiting.'
    raise FatalException(retcode, err)

  print configDefaults.MESSAGE_CHECK_FIREWALL
  check_firewall()

  # proceed jdbc properties if they were set
  if _check_jdbc_options(options):
    proceedJDBCProperties(options)

  print 'Checking JDK...'
  try:
    download_and_install_jdk(options)
  except FatalException as e:
    err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e)
    raise FatalException(e.code, err)

  print 'Completing setup...'
  retcode = configure_os_settings()
  if not retcode == 0:
    err = 'Configure of OS settings in tbds.properties failed. Exiting.'
    raise FatalException(retcode, err)

  print 'Configuring database...'
  prompt_db_properties(options)

  #DB setup should be done last after doing any setup.

  _setup_database(options)

  check_jdbc_drivers(options)

  print 'Extracting system views...'
  retcode = extract_views()
  if not retcode == 0:
    err = 'Error while extracting system views. Exiting'
    raise FatalException(retcode, err)

  # we've already done this, but new files were created so run it one time.
  adjust_directory_permissions(read_ambari_user())