Example #1
0
def get_check_command(oozie_url, host_name, configurations, parameters):
  kerberos_env = None

  user = USER_DEFAULT
  if USER_KEY in configurations:
    user = configurations[USER_KEY]

  security_enabled = False
  if SECURITY_ENABLED in configurations:
    security_enabled = str(configurations[SECURITY_ENABLED]).upper() == 'TRUE'

  if security_enabled:
    # defaults
    user_keytab = USER_KEYTAB_DEFAULT
    user_principal = USER_PRINCIPAL_DEFAULT

    # check script params
    if USER_PRINCIPAL_SCRIPT_PARAM_KEY in parameters:
      user_principal = parameters[USER_PRINCIPAL_SCRIPT_PARAM_KEY]
      user_principal = user_principal.replace('_HOST', host_name.lower())
    if USER_KEYTAB_SCRIPT_PARAM_KEY in parameters:
      user_keytab = parameters[USER_KEYTAB_SCRIPT_PARAM_KEY]

    # check configurations last as they should always take precedence
    if USER_PRINCIPAL_KEY in configurations:
      user_principal = configurations[USER_PRINCIPAL_KEY]
      user_principal = user_principal.replace('_HOST', host_name.lower())
    if USER_KEYTAB_KEY in configurations:
      user_keytab = configurations[USER_KEYTAB_KEY]

    # Create the kerberos credentials cache (ccache) file and set it in the environment to use
    # when executing curl
    env = Environment.get_instance()
    ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid())
    kerberos_env = {'KRB5CCNAME': ccache_file}

    # Get the configured Kerberos executable search paths, if any
    kerberos_executable_search_paths = None
    if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
      kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]

    klist_path_local = get_klist_path(kerberos_executable_search_paths)
    kinit_path_local = get_kinit_path(kerberos_executable_search_paths)

    # Determine if we need to kinit by testing to see if the relevant cache exists and has
    # non-expired tickets.  Tickets are marked to expire after 5 minutes to help reduce the number
    # it kinits we do but recover quickly when keytabs are regenerated

    kinit_command = "{0} -s {1} || ".format(klist_path_local, ccache_file) + format("{kinit_path_local} -l 5m20s -c {ccache_file} -kt {user_keytab} {user_principal}; ")
    Execute(kinit_command, environment=kerberos_env, user=user)

  # oozie configuration directory uses a symlink when > HDP 2.2
  oozie_config_directory = OOZIE_CONF_DIR_LEGACY
  if os.path.exists(OOZIE_CONF_DIR):
    oozie_config_directory = OOZIE_CONF_DIR

  command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format(
    oozie_config_directory, oozie_url)

  return (command, kerberos_env, user)
def get_check_command(oozie_url, host_name, configurations):
  if OOZIE_USER in configurations:
    oozie_user = configurations[OOZIE_USER]
  else:
    raise Exception("Oozie user is required")
    
  security_enabled = False
  if SECURITY_ENABLED in configurations:
    security_enabled = str(configurations[SECURITY_ENABLED]).upper() == 'TRUE'
  kerberos_env = None
  if security_enabled:
    if OOZIE_KEYTAB in configurations and OOZIE_PRINCIPAL in configurations:
      oozie_keytab = configurations[OOZIE_KEYTAB]
      oozie_principal = configurations[OOZIE_PRINCIPAL]

      # substitute _HOST in kerberos principal with actual fqdn
      oozie_principal = oozie_principal.replace('_HOST', host_name)
    else:
      raise KerberosPropertiesNotFound('The Oozie keytab and principal are required configurations when security is enabled.')

    # Create the kerberos credentials cache (ccache) file and set it in the environment to use
    # when executing curl
    env = Environment.get_instance()
    ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid())
    kerberos_env = {'KRB5CCNAME': ccache_file}

    # Get the configured Kerberos executable search paths, if any
    if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
      kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
    else:
      kerberos_executable_search_paths = None

    klist_path_local = get_klist_path(kerberos_executable_search_paths)
    klist_command = format("{klist_path_local} -s {ccache_file}")

    # Determine if we need to kinit by testing to see if the relevant cache exists and has
    # non-expired tickets.  Tickets are marked to expire after 5 minutes to help reduce the number
    # it kinits we do but recover quickly when keytabs are regenerated
    return_code, _ = call(klist_command, user=oozie_user)
    if return_code != 0:
      kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
      kinit_command = format("{kinit_path_local} -l 5m -kt {oozie_keytab} {oozie_principal}; ")

      # kinit
      Execute(kinit_command, 
              environment=kerberos_env,
              user=oozie_user,
      )

  # oozie configuration directory uses a symlink when > HDP 2.2
  oozie_config_directory = OOZIE_CONF_DIR_LEGACY
  if os.path.exists(OOZIE_CONF_DIR):
    oozie_config_directory = OOZIE_CONF_DIR

  command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format(
    oozie_config_directory, oozie_url)

  return (command, kerberos_env, oozie_user)
def get_check_command(oozie_url, host_name, configurations):
    security_enabled = False
    if SECURITY_ENABLED in configurations:
        security_enabled = str(
            configurations[SECURITY_ENABLED]).upper() == 'TRUE'
    kerberos_env = None
    if security_enabled:
        if OOZIE_KEYTAB in configurations and OOZIE_PRINCIPAL in configurations:
            oozie_keytab = configurations[OOZIE_KEYTAB]
            oozie_principal = configurations[OOZIE_PRINCIPAL]

            # substitute _HOST in kerberos principal with actual fqdn
            oozie_principal = oozie_principal.replace('_HOST', host_name)
        else:
            raise KerberosPropertiesNotFound(
                'The Oozie keytab and principal are required configurations when security is enabled.'
            )

        # Create the kerberos credentials cache (ccache) file and set it in the environment to use
        # when executing curl
        env = Environment.get_instance()
        ccache_file = "{0}{1}oozie_alert_cc_{2}".format(
            env.tmp_dir, os.sep, os.getpid())
        kerberos_env = {'KRB5CCNAME': ccache_file}

        # Get the configured Kerberos executable search paths, if any
        if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
            kerberos_executable_search_paths = configurations[
                KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
        else:
            kerberos_executable_search_paths = None

        klist_path_local = get_klist_path(kerberos_executable_search_paths)
        klist_command = format("{klist_path_local} -s {ccache_file}")

        # Determine if we need to kinit by testing to see if the relevant cache exists and has
        # non-expired tickets.  Tickets are marked to expire after 5 minutes to help reduce the number
        # it kinits we do but recover quickly when keytabs are regenerated
        return_code, _ = call(klist_command)
        if return_code != 0:
            kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
            kinit_command = format(
                "{kinit_path_local} -l 5m -kt {oozie_keytab} {oozie_principal}; "
            )

            # kinit
            Execute(kinit_command, environment=kerberos_env)

    # oozie configuration directory uses a symlink when > HDP 2.2
    oozie_config_directory = OOZIE_CONF_DIR_LEGACY
    if os.path.exists(OOZIE_CONF_DIR):
        oozie_config_directory = OOZIE_CONF_DIR

    command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format(
        oozie_config_directory, oozie_url)

    return (command, kerberos_env)
Example #4
0
def get_check_command(oozie_url, host_name, parameters):
  security_enabled = False
  if SECURITY_ENABLED in parameters:
    security_enabled = str(parameters[SECURITY_ENABLED]).upper() == 'TRUE'
  kerberos_env = None
  if security_enabled:
    if OOZIE_KEYTAB in parameters and OOZIE_PRINCIPAL in parameters:
      oozie_keytab = parameters[OOZIE_KEYTAB]
      oozie_principal = parameters[OOZIE_PRINCIPAL]

      # substitute _HOST in kerberos principal with actual fqdn
      oozie_principal = oozie_principal.replace('_HOST', host_name)
    else:
      raise KerberosPropertiesNotFound('The Oozie keytab and principal are required parameters when security is enabled.')

    # Create the kerberos credentials cache (ccache) file and set it in the environment to use
    # when executing curl
    env = Environment.get_instance()
    ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid())
    kerberos_env = {'KRB5CCNAME': ccache_file}

    klist_path_local = get_klist_path()
    klist_command = format("{klist_path_local} -s {ccache_file}")

    # Determine if we need to kinit by testing to see if the relevant cache exists and has
    # non-expired tickets.  Tickets are marked to expire after 5 minutes to help reduce the number
    # it kinits we do but recover quickly when keytabs are regenerated
    return_code, _ = call(klist_command)
    if return_code != 0:
      kinit_path_local = get_kinit_path()
      kinit_command = format("{kinit_path_local} -l 5m -kt {oozie_keytab} {oozie_principal}; ")

      # kinit
      Execute(kinit_command, environment=kerberos_env)
  command = format("source /etc/oozie/conf/oozie-env.sh ; oozie admin -oozie {oozie_url} -status")
  return (command, kerberos_env)
Example #5
0
so_src_x86 = format("{so_src_dir_x86}/{snappy_so}")
so_src_x64 = format("{so_src_dir_x64}/{snappy_so}")

#security params
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
falcon_user = config['configurations']['falcon-env']['falcon_user']

#exclude file
hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
update_exclude_file_only = default("/commandParams/update_exclude_file_only",
                                   False)
command_phase = default("/commandParams/phase", "")

klist_path_local = get_klist_path(
    default('/configurations/kerberos-env/executable_search_paths', None))
kinit_path_local = get_kinit_path(
    default('/configurations/kerberos-env/executable_search_paths', None))
#hosts
hostname = config["hostname"]
rm_host = default("/clusterHostInfo/rm_host", [])
slave_hosts = default("/clusterHostInfo/slave_hosts", [])
oozie_servers = default("/clusterHostInfo/oozie_server", [])
hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", [])
hive_server_host = default("/clusterHostInfo/hive_server_host", [])
hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
hs_host = default("/clusterHostInfo/hs_host", [])
jtnode_host = default("/clusterHostInfo/jtnode_host", [])
namenode_host = default("/clusterHostInfo/namenode_host", [])
nm_host = default("/clusterHostInfo/nm_host", [])
ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
def get_check_command(oozie_url, host_name, configurations, parameters, only_kinit):
  kerberos_env = None

  user = USER_DEFAULT
  if USER_KEY in configurations:
    user = configurations[USER_KEY]

  if is_security_enabled(configurations):
    # defaults
    user_keytab = USER_KEYTAB_DEFAULT
    user_principal = USER_PRINCIPAL_DEFAULT

    # check script params
    if USER_PRINCIPAL_SCRIPT_PARAM_KEY in parameters:
      user_principal = parameters[USER_PRINCIPAL_SCRIPT_PARAM_KEY]
      user_principal = user_principal.replace('_HOST', host_name.lower())
    if USER_KEYTAB_SCRIPT_PARAM_KEY in parameters:
      user_keytab = parameters[USER_KEYTAB_SCRIPT_PARAM_KEY]

    # check configurations last as they should always take precedence
    if USER_PRINCIPAL_KEY in configurations:
      user_principal = configurations[USER_PRINCIPAL_KEY]
      user_principal = user_principal.replace('_HOST', host_name.lower())
    if USER_KEYTAB_KEY in configurations:
      user_keytab = configurations[USER_KEYTAB_KEY]

    # Create the kerberos credentials cache (ccache) file and set it in the environment to use
    # when executing curl
    env = Environment.get_instance()
    ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid())
    kerberos_env = {'KRB5CCNAME': ccache_file}

    # Get the configured Kerberos executable search paths, if any
    kerberos_executable_search_paths = None
    if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
      kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]

    klist_path_local = get_klist_path(kerberos_executable_search_paths)
    kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
    kinit_part_command = format("{kinit_path_local} -l 5m20s -c {ccache_file} -kt {user_keytab} {user_principal}; ")

    # Determine if we need to kinit by testing to see if the relevant cache exists and has
    # non-expired tickets.  Tickets are marked to expire after 5 minutes to help reduce the number
    # it kinits we do but recover quickly when keytabs are regenerated

    if only_kinit:
      kinit_command = kinit_part_command
    else:
      kinit_command = "{0} -s {1} || ".format(klist_path_local, ccache_file) + kinit_part_command

    Execute(kinit_command, environment=kerberos_env, user=user)

  # oozie configuration directory uses a symlink when > HDP 2.2
  oozie_config_directory = OOZIE_CONF_DIR_LEGACY
  if os.path.exists(OOZIE_CONF_DIR):
    oozie_config_directory = OOZIE_CONF_DIR

  command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format(
    oozie_config_directory, oozie_url)

  return (command, kerberos_env, user)
Example #7
0
def get_check_command(oozie_url, host_name, configurations, parameters,
                      only_kinit):
    kerberos_env = None

    user = USER_DEFAULT
    if USER_KEY in configurations:
        user = configurations[USER_KEY]

    if is_security_enabled(configurations):
        # defaults
        user_keytab = USER_KEYTAB_DEFAULT
        user_principal = USER_PRINCIPAL_DEFAULT

        # check script params
        if USER_PRINCIPAL_SCRIPT_PARAM_KEY in parameters:
            user_principal = parameters[USER_PRINCIPAL_SCRIPT_PARAM_KEY]
            user_principal = user_principal.replace('_HOST', host_name.lower())
        if USER_KEYTAB_SCRIPT_PARAM_KEY in parameters:
            user_keytab = parameters[USER_KEYTAB_SCRIPT_PARAM_KEY]

        # check configurations last as they should always take precedence
        if USER_PRINCIPAL_KEY in configurations:
            user_principal = configurations[USER_PRINCIPAL_KEY]
            user_principal = user_principal.replace('_HOST', host_name.lower())
        if USER_KEYTAB_KEY in configurations:
            user_keytab = configurations[USER_KEYTAB_KEY]

        # Create the kerberos credentials cache (ccache) file and set it in the environment to use
        # when executing curl
        env = Environment.get_instance()
        ccache_file = "{0}{1}oozie_alert_cc_{2}".format(
            env.tmp_dir, os.sep, os.getpid())
        kerberos_env = {'KRB5CCNAME': ccache_file}

        # Get the configured Kerberos executable search paths, if any
        kerberos_executable_search_paths = None
        if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
            kerberos_executable_search_paths = configurations[
                KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]

        klist_path_local = get_klist_path(kerberos_executable_search_paths)
        kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
        kinit_part_command = format(
            "{kinit_path_local} -l 5m20s -c {ccache_file} -kt {user_keytab} {user_principal}; "
        )

        # Determine if we need to kinit by testing to see if the relevant cache exists and has
        # non-expired tickets.  Tickets are marked to expire after 5 minutes to help reduce the number
        # it kinits we do but recover quickly when keytabs are regenerated

        if only_kinit:
            kinit_command = kinit_part_command
        else:
            kinit_command = "{0} -s {1} || ".format(
                klist_path_local, ccache_file) + kinit_part_command

        # prevent concurrent kinit
        kinit_lock = global_lock.get_lock(global_lock.LOCK_TYPE_KERBEROS)
        kinit_lock.acquire()
        try:
            Execute(kinit_command, environment=kerberos_env, user=user)
        finally:
            kinit_lock.release()

    # Configure stack root
    stack_root = STACK_ROOT_DEFAULT
    if STACK_NAME_KEY in configurations and STACK_ROOT_KEY in configurations:
        stack_root = stack_tools.get_stack_root(
            configurations[STACK_NAME_KEY],
            configurations[STACK_ROOT_KEY]).lower()

    # oozie configuration directory using a symlink
    oozie_config_directory = OOZIE_CONF_DIR.replace(STACK_ROOT_PATTERN,
                                                    stack_root)
    if not os.path.exists(oozie_config_directory):
        oozie_config_directory = OOZIE_CONF_DIR_LEGACY

    command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format(
        oozie_config_directory, oozie_url)

    return (command, kerberos_env, user)
def get_check_command(oozie_url, host_name, configurations, parameters):
  kerberos_env = None

  smokeuser = SMOKEUSER_DEFAULT
  if SMOKEUSER_KEY in configurations:
    smokeuser = configurations[SMOKEUSER_KEY]

  security_enabled = False
  if SECURITY_ENABLED in configurations:
    security_enabled = str(configurations[SECURITY_ENABLED]).upper() == 'TRUE'

  if security_enabled:
    # defaults
    smokeuser_keytab = SMOKEUSER_KEYTAB_DEFAULT
    smokeuser_principal = SMOKEUSER_PRINCIPAL_DEFAULT

    # check script params
    if SMOKEUSER_PRINCIPAL_SCRIPT_PARAM_KEY in parameters:
      smokeuser_principal = parameters[SMOKEUSER_PRINCIPAL_SCRIPT_PARAM_KEY]
    if SMOKEUSER_KEYTAB_SCRIPT_PARAM_KEY in parameters:
      smokeuser_keytab = parameters[SMOKEUSER_KEYTAB_SCRIPT_PARAM_KEY]

    # check configurations last as they should always take precedence
    if SMOKEUSER_PRINCIPAL_KEY in configurations:
      smokeuser_principal = configurations[SMOKEUSER_PRINCIPAL_KEY]
    if SMOKEUSER_KEYTAB_KEY in configurations:
      smokeuser_keytab = configurations[SMOKEUSER_KEYTAB_KEY]

    # Create the kerberos credentials cache (ccache) file and set it in the environment to use
    # when executing curl
    env = Environment.get_instance()
    ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid())
    kerberos_env = {'KRB5CCNAME': ccache_file}

    # Get the configured Kerberos executable search paths, if any
    kerberos_executable_search_paths = None
    if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
      kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]

    klist_path_local = get_klist_path(kerberos_executable_search_paths)
    klist_command = format("{klist_path_local} -s {ccache_file}")

    # Determine if we need to kinit by testing to see if the relevant cache exists and has
    # non-expired tickets.  Tickets are marked to expire after 5 minutes to help reduce the number
    # it kinits we do but recover quickly when keytabs are regenerated
    return_code, _ = call(klist_command, user=smokeuser)
    if return_code != 0:
      kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
      kinit_command = format("{kinit_path_local} -l 5m -kt {smokeuser_keytab} {smokeuser_principal}; ")

      # kinit
      Execute(kinit_command, environment=kerberos_env, user=smokeuser)

  # oozie configuration directory uses a symlink when > HDP 2.2
  oozie_config_directory = OOZIE_CONF_DIR_LEGACY
  if os.path.exists(OOZIE_CONF_DIR):
    oozie_config_directory = OOZIE_CONF_DIR

  command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format(
    oozie_config_directory, oozie_url)

  return (command, kerberos_env, smokeuser)
def execute(configurations={}, parameters={}, host_name=None):
  """
  Returns a tuple containing the result code and a pre-formatted result label

  Keyword arguments:
  configurations (dictionary): a mapping of configuration key to value
  parameters (dictionary): a mapping of script parameter key to value
  host_name (string): the name of this host where the alert is running
  """

  result_code = RESULT_CODE_UNKNOWN

  if configurations is None:
    return (result_code, ['There were no configurations supplied to the script.'])

  webhcat_port = WEBHCAT_PORT_DEFAULT
  if TEMPLETON_PORT_KEY in configurations:
    webhcat_port = int(configurations[TEMPLETON_PORT_KEY])

  security_enabled = False
  if SECURITY_ENABLED_KEY in configurations:
    security_enabled = configurations[SECURITY_ENABLED_KEY].lower() == 'true'

  # parse script arguments
  connection_timeout = CONNECTION_TIMEOUT_DEFAULT
  curl_connection_timeout = CURL_CONNECTION_TIMEOUT_DEFAULT
  if CONNECTION_TIMEOUT_KEY in parameters:
    connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY])
    curl_connection_timeout = str(int(connection_timeout))


  # the alert will always run on the webhcat host
  if host_name is None:
    host_name = socket.getfqdn()

  # webhcat always uses http, never SSL
  query_url = "http://{0}:{1}/templeton/v1/status".format(host_name, webhcat_port)

  # initialize
  total_time = 0
  json_response = {}

  if security_enabled:
    if WEBHCAT_KEYTAB_KEY not in configurations or WEBHCAT_PRINCIPAL_KEY not in configurations:
      return (RESULT_CODE_UNKNOWN, [str(configurations)])

    try:
      webhcat_keytab = configurations[WEBHCAT_KEYTAB_KEY]
      webhcat_principal = configurations[WEBHCAT_PRINCIPAL_KEY]

      # substitute _HOST in kerberos principal with actual fqdn
      webhcat_principal = webhcat_principal.replace('_HOST', host_name)

      # Create the kerberos credentials cache (ccache) file and set it in the environment to use
      # when executing curl
      env = Environment.get_instance()
      ccache_file = "{0}{1}webhcat_alert_cc_{2}".format(env.tmp_dir, sep, getpid())
      kerberos_env = {'KRB5CCNAME': ccache_file}

      # Get the configured Kerberos executable search paths, if any
      if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
        kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
      else:
        kerberos_executable_search_paths = None

      klist_path_local = get_klist_path(kerberos_executable_search_paths)
      klist_command = format("{klist_path_local} -s {ccache_file}")

      # Determine if we need to kinit by testing to see if the relevant cache exists and has
      # non-expired tickets.  Tickets are marked to expire after 5 minutes to help reduce the number
      # it kinits we do but recover quickly when keytabs are regenerated
      return_code, _ = call(klist_command)
      if return_code != 0:
        kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
        kinit_command = format("{kinit_path_local} -l 5m -c {ccache_file} -kt {webhcat_keytab} {webhcat_principal}; ")

        # kinit so that curl will work with --negotiate
        Execute(kinit_command)

      # make a single curl call to get just the http code
      curl = subprocess.Popen(['curl', '--negotiate', '-u', ':', '-sL', '-w',
        '%{http_code}', '--connect-timeout', curl_connection_timeout,
        '-o', '/dev/null', query_url], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=kerberos_env)

      stdout, stderr = curl.communicate()

      if stderr != '':
        raise Exception(stderr)

      # check the response code
      response_code = int(stdout)

      # 0 indicates no connection
      if response_code == 0:
        label = CRITICAL_CONNECTION_MESSAGE.format(query_url)
        return (RESULT_CODE_CRITICAL, [label])

      # any other response aside from 200 is a problem
      if response_code != 200:
        label = CRITICAL_HTTP_MESSAGE.format(response_code, query_url)
        return (RESULT_CODE_CRITICAL, [label])

      # now that we have the http status and it was 200, get the content
      start_time = time.time()
      curl = subprocess.Popen(['curl', '--negotiate', '-u', ':', '-sL',
        '--connect-timeout', curl_connection_timeout, query_url, ],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=kerberos_env)

      stdout, stderr = curl.communicate()
      total_time = time.time() - start_time

      if stderr != '':
        raise Exception(stderr)

      json_response = json.loads(stdout)
    except Exception, exception:
      return (RESULT_CODE_CRITICAL, [str(exception)])
Example #10
0
def execute(configurations={}, parameters={}, host_name=None):
    """
  Returns a tuple containing the result code and a pre-formatted result label

  Keyword arguments:
  configurations (dictionary): a mapping of configuration key to value
  parameters (dictionary): a mapping of script parameter key to value
  host_name (string): the name of this host where the alert is running
  """

    result_code = RESULT_CODE_UNKNOWN

    if configurations is None:
        return (result_code,
                ['There were no configurations supplied to the script.'])

    webhcat_port = WEBHCAT_PORT_DEFAULT
    if TEMPLETON_PORT_KEY in configurations:
        webhcat_port = int(configurations[TEMPLETON_PORT_KEY])

    security_enabled = False
    if SECURITY_ENABLED_KEY in configurations:
        security_enabled = configurations[SECURITY_ENABLED_KEY].lower(
        ) == 'true'

    # parse script arguments
    connection_timeout = CONNECTION_TIMEOUT_DEFAULT
    curl_connection_timeout = CURL_CONNECTION_TIMEOUT_DEFAULT
    if CONNECTION_TIMEOUT_KEY in parameters:
        connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY])
        curl_connection_timeout = str(int(connection_timeout))

    # the alert will always run on the webhcat host
    if host_name is None:
        host_name = socket.getfqdn()

    smokeuser = SMOKEUSER_DEFAULT

    if SMOKEUSER_KEY in configurations:
        smokeuser = configurations[SMOKEUSER_KEY]

    if SMOKEUSER_SCRIPT_PARAM_KEY in parameters:
        smokeuser = parameters[SMOKEUSER_SCRIPT_PARAM_KEY]

    # webhcat always uses http, never SSL
    query_url = "http://{0}:{1}/templeton/v1/status?user.name={2}".format(
        host_name, webhcat_port, smokeuser)

    # initialize
    total_time = 0
    json_response = {}

    if security_enabled:
        if WEBHCAT_KEYTAB_KEY not in configurations or WEBHCAT_PRINCIPAL_KEY not in configurations:
            return (RESULT_CODE_UNKNOWN, [str(configurations)])

        try:
            webhcat_keytab = configurations[WEBHCAT_KEYTAB_KEY]
            webhcat_principal = configurations[WEBHCAT_PRINCIPAL_KEY]

            # substitute _HOST in kerberos principal with actual fqdn
            webhcat_principal = webhcat_principal.replace('_HOST', host_name)

            # Create the kerberos credentials cache (ccache) file and set it in the environment to use
            # when executing curl
            env = Environment.get_instance()
            ccache_file = "{0}{1}webhcat_alert_cc_{2}".format(
                env.tmp_dir, sep, getpid())
            kerberos_env = {'KRB5CCNAME': ccache_file}

            # Get the configured Kerberos executable search paths, if any
            if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
                kerberos_executable_search_paths = configurations[
                    KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
            else:
                kerberos_executable_search_paths = None

            klist_path_local = get_klist_path(kerberos_executable_search_paths)
            klist_command = format("{klist_path_local} -s {ccache_file}")

            # Determine if we need to kinit by testing to see if the relevant cache exists and has
            # non-expired tickets.  Tickets are marked to expire after 5 minutes to help reduce the number
            # it kinits we do but recover quickly when keytabs are regenerated
            return_code, _ = call(klist_command)
            if return_code != 0:
                kinit_path_local = get_kinit_path(
                    kerberos_executable_search_paths)
                kinit_command = format(
                    "{kinit_path_local} -l 5m -c {ccache_file} -kt {webhcat_keytab} {webhcat_principal}; "
                )

                # kinit so that curl will work with --negotiate
                Execute(kinit_command)

            # make a single curl call to get just the http code
            curl = subprocess.Popen([
                'curl', '--negotiate', '-u', ':', '-sL', '-w', '%{http_code}',
                '--connect-timeout', curl_connection_timeout, '-o',
                '/dev/null', query_url
            ],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    env=kerberos_env)

            stdout, stderr = curl.communicate()

            if stderr != '':
                raise Exception(stderr)

            # check the response code
            response_code = int(stdout)

            # 0 indicates no connection
            if response_code == 0:
                label = CRITICAL_CONNECTION_MESSAGE.format(query_url)
                return (RESULT_CODE_CRITICAL, [label])

            # any other response aside from 200 is a problem
            if response_code != 200:
                label = CRITICAL_HTTP_MESSAGE.format(response_code, query_url)
                return (RESULT_CODE_CRITICAL, [label])

            # now that we have the http status and it was 200, get the content
            start_time = time.time()
            curl = subprocess.Popen([
                'curl',
                '--negotiate',
                '-u',
                ':',
                '-sL',
                '--connect-timeout',
                curl_connection_timeout,
                query_url,
            ],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    env=kerberos_env)

            stdout, stderr = curl.communicate()
            total_time = time.time() - start_time

            if stderr != '':
                raise Exception(stderr)

            json_response = json.loads(stdout)
        except Exception, exception:
            return (RESULT_CODE_CRITICAL, [str(exception)])
so_src_dir_x64 = format("{hadoop_home}/lib64")
so_src_x86 = format("{so_src_dir_x86}/{snappy_so}")
so_src_x64 = format("{so_src_dir_x64}/{snappy_so}")

#security params
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
falcon_user = config['configurations']['falcon-env']['falcon_user']

#exclude file
hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
update_exclude_file_only = default("/commandParams/update_exclude_file_only",False)
command_phase = default("/commandParams/phase","")

klist_path_local = get_klist_path(default('/configurations/kerberos-env/executable_search_paths', None))
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
#hosts
hostname = config["hostname"]
rm_host = default("/clusterHostInfo/rm_host", [])
slave_hosts = default("/clusterHostInfo/slave_hosts", [])
oozie_servers = default("/clusterHostInfo/oozie_server", [])
hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", [])
hive_server_host =  default("/clusterHostInfo/hive_server_host", [])
hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
hs_host = default("/clusterHostInfo/hs_host", [])
jtnode_host = default("/clusterHostInfo/jtnode_host", [])
namenode_host = default("/clusterHostInfo/namenode_host", [])
nm_host = default("/clusterHostInfo/nm_host", [])
ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
journalnode_hosts = default("/clusterHostInfo/journalnode_hosts", [])
Example #12
0
def curl_krb_request(tmp_dir,
                     keytab,
                     principal,
                     url,
                     cache_file_prefix,
                     krb_exec_search_paths,
                     return_only_http_code,
                     caller_label,
                     user,
                     connection_timeout=CONNECTION_TIMEOUT_DEFAULT,
                     ca_certs=None,
                     kinit_timer_ms=DEFAULT_KERBEROS_KINIT_TIMER_MS,
                     method='',
                     body='',
                     header=''):
    """
    Makes a curl request using the kerberos credentials stored in a calculated cache file. The
    cache file is created by combining the supplied principal, keytab, user, and request name into
    a unique hash.

    This function will use the klist command to determine if the cache is expired and will perform
    a kinit if necessary. Additionally, it has an internal timer to force a kinit after a
    configurable amount of time. This is to prevent boundary issues where requests hit the edge
    of a ticket's lifetime.

    :param tmp_dir: the directory to use for storing the local kerberos cache for this request.
    :param keytab: the location of the keytab to use when performing a kinit
    :param principal: the principal to use when performing a kinit
    :param url: the URL to request
    :param cache_file_prefix: an identifier used to build the unique cache name for this request.
                              This ensures that multiple requests can use the same cache.
    :param krb_exec_search_paths: the search path to use for invoking kerberos binaries
    :param return_only_http_code: True to return only the HTTP code, False to return GET content
    :param caller_label: an identifier to give context into the caller of this module (used for logging)
    :param user: the user to invoke the curl command as
    :param connection_timeout: if specified, a connection timeout for curl (default 10 seconds)
    :param ca_certs: path to certificates
    :param kinit_timer_ms: if specified, the time (in ms), before forcing a kinit even if the
                           klist cache is still valid.
    :return:
    """

    import uuid
    # backward compatibility with old code and management packs, etc. All new code need pass ca_certs explicitly
    if ca_certs is None:
        try:
            from ambari_agent.AmbariConfig import AmbariConfig
            ca_certs = AmbariConfig.get_resolved_config(
            ).get_ca_cert_file_path()
        except:
            pass
    # start off false
    is_kinit_required = False

    # Create the kerberos credentials cache (ccache) file and set it in the environment to use
    # when executing curl. Use a hash of the combination of the principal and keytab file
    # to generate a (relatively) unique cache filename so that we can use it as needed. Scope
    # this file by user in order to prevent sharing of cache files by multiple users.
    ccache_file_name = HASH_ALGORITHM("{0}|{1}".format(principal,
                                                       keytab)).hexdigest()

    curl_krb_cache_path = os.path.join(tmp_dir, "curl_krb_cache")
    if not os.path.exists(curl_krb_cache_path):
        os.makedirs(curl_krb_cache_path)
    os.chmod(curl_krb_cache_path, 01777)

    ccache_file_path = "{0}{1}{2}_{3}_cc_{4}".format(curl_krb_cache_path,
                                                     os.sep, cache_file_prefix,
                                                     user, ccache_file_name)
    kerberos_env = {'KRB5CCNAME': ccache_file_path}

    # concurrent kinit's can cause the following error:
    # Internal credentials cache error while storing credentials while getting initial credentials
    kinit_lock = global_lock.get_lock(global_lock.LOCK_TYPE_KERBEROS)
    kinit_lock.acquire()
    try:
        # If there are no tickets in the cache or they are expired, perform a kinit, else use what
        # is in the cache
        if krb_exec_search_paths:
            klist_path_local = get_klist_path(krb_exec_search_paths)
        else:
            klist_path_local = get_klist_path()

        # take a look at the last time kinit was run for the specified cache and force a new
        # kinit if it's time; this helps to avoid problems approaching ticket boundary when
        # executing a klist and then a curl
        last_kinit_time = _KINIT_CACHE_TIMES.get(ccache_file_name, 0)
        current_time = long(time.time())
        if current_time - kinit_timer_ms > last_kinit_time:
            is_kinit_required = True

        # if the time has not expired, double-check that the cache still has a valid ticket
        if not is_kinit_required:
            klist_command = "{0} -s {1}".format(klist_path_local,
                                                ccache_file_path)
            is_kinit_required = (shell.call(klist_command, user=user)[0] != 0)

        # if kinit is required, the perform the kinit
        if is_kinit_required:
            if krb_exec_search_paths:
                kinit_path_local = get_kinit_path(krb_exec_search_paths)
            else:
                kinit_path_local = get_kinit_path()

            logger.debug(
                "Enabling Kerberos authentication for %s via GSSAPI using ccache at %s",
                caller_label, ccache_file_path)

            # kinit; there's no need to set a ticket timeout as this will use the default invalidation
            # configured in the krb5.conf - regenerating keytabs will not prevent an existing cache
            # from working correctly
            shell.checked_call("{0} -c {1} -kt {2} {3} > /dev/null".format(
                kinit_path_local, ccache_file_path, keytab, principal),
                               user=user)

            # record kinit time
            _KINIT_CACHE_TIMES[ccache_file_name] = current_time
        else:
            # no kinit needed, use the cache
            logger.debug(
                "Kerberos authentication for %s via GSSAPI already enabled using ccache at %s.",
                caller_label, ccache_file_path)
    finally:
        kinit_lock.release()

    # check if cookies dir exists, if not then create it
    cookies_dir = os.path.join(tmp_dir, "cookies")

    if not os.path.exists(cookies_dir):
        os.makedirs(cookies_dir)

    cookie_file_name = str(uuid.uuid4())
    cookie_file = os.path.join(cookies_dir, cookie_file_name)

    start_time = time.time()
    error_msg = None

    # setup timeouts for the request; ensure we use integers since that is what curl needs
    connection_timeout = int(connection_timeout)
    maximum_timeout = connection_timeout + 2

    ssl_options = ['-k']
    if ca_certs:
        ssl_options = ['--cacert', ca_certs]
    try:
        if return_only_http_code:
            _, curl_stdout, curl_stderr = get_user_call_output(
                ['curl', '--location-trusted'] + ssl_options + [
                    '--negotiate', '-u', admin_username + ':' + admin_password,
                    '-b', cookie_file, '-c', cookie_file, '-w', '%{http_code}',
                    url, '--connect-timeout',
                    str(connection_timeout), '--max-time',
                    str(maximum_timeout), '-o', '/dev/null'
                ],
                user=user,
                env=kerberos_env)
        else:
            curl_command = ['curl', '--location-trusted'] + ssl_options + [
                '--negotiate', '-u', admin_username + ':' + admin_password,
                '-b', cookie_file, '-c', cookie_file, url, '--connect-timeout',
                str(connection_timeout), '--max-time',
                str(maximum_timeout)
            ]
            # returns response body
            if len(method) > 0 and len(body) == 0 and len(header) == 0:
                curl_command.extend(['-X', method])

            elif len(method) > 0 and len(body) == 0 and len(header) > 0:
                curl_command.extend(['-H', header, '-X', method])

            elif len(method) > 0 and len(body) > 0 and len(header) == 0:
                curl_command.extend(['-X', method, '-d', body])

            elif len(method) > 0 and len(body) > 0 and len(header) > 0:
                curl_command.extend(['-H', header, '-X', method, '-d', body])

            _, curl_stdout, curl_stderr = get_user_call_output(
                curl_command, user=user, env=kerberos_env)

    except Fail:
        if logger.isEnabledFor(logging.DEBUG):
            logger.exception(
                "Unable to make a curl request for {0}.".format(caller_label))
        raise
    finally:
        if os.path.isfile(cookie_file):
            os.remove(cookie_file)

    # empty quotes evaluates to false
    if curl_stderr:
        error_msg = curl_stderr

    time_millis = time.time() - start_time

    # empty quotes evaluates to false
    if curl_stdout:
        if return_only_http_code:
            return (int(curl_stdout), error_msg, time_millis)
        else:
            return (curl_stdout, error_msg, time_millis)

    logger.debug("The curl response for %s is empty; standard error = %s",
                 caller_label, str(error_msg))

    return ("", error_msg, time_millis)
Example #13
0
def execute(parameters=None, host_name=None):
    """
  Returns a tuple containing the result code and a pre-formatted result label

  Keyword arguments:
  parameters (dictionary): a mapping of parameter key to value
  host_name (string): the name of this host where the alert is running
  """

    if parameters is None:
        return (RESULT_CODE_UNKNOWN,
                ['There were no parameters supplied to the script.'])

    if not OOZIE_URL_KEY in parameters:
        return (RESULT_CODE_UNKNOWN,
                ['The Oozie URL is a required parameter.'])

    # use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all
    # interfaces, which doesn't work on Windows
    localhost_address = 'localhost' if OSCheck.get_os_family(
    ) == OSConst.WINSRV_FAMILY else '0.0.0.0'

    oozie_url = parameters[OOZIE_URL_KEY]
    oozie_url = oozie_url.replace(
        urlparse(oozie_url).hostname, localhost_address)

    security_enabled = False
    if SECURITY_ENABLED in parameters:
        security_enabled = str(parameters[SECURITY_ENABLED]).upper() == 'TRUE'

    command = format(
        "source /etc/oozie/conf/oozie-env.sh ; oozie admin -oozie {oozie_url} -status"
    )

    try:
        # kinit if security is enabled so that oozie-env.sh can make the web request
        kerberos_env = None

        if security_enabled:
            if OOZIE_KEYTAB in parameters and OOZIE_PRINCIPAL in parameters:
                oozie_keytab = parameters[OOZIE_KEYTAB]
                oozie_principal = parameters[OOZIE_PRINCIPAL]

                # substitute _HOST in kerberos principal with actual fqdn
                oozie_principal = oozie_principal.replace('_HOST', host_name)
            else:
                return (RESULT_CODE_UNKNOWN, [
                    'The Oozie keytab and principal are required parameters when security is enabled.'
                ])

            # Create the kerberos credentials cache (ccache) file and set it in the environment to use
            # when executing curl
            env = Environment.get_instance()
            ccache_file = "{0}{1}oozie_alert_cc_{2}".format(
                env.tmp_dir, sep, getpid())
            kerberos_env = {'KRB5CCNAME': ccache_file}

            klist_path_local = get_klist_path()
            klist_command = format("{klist_path_local} -s {ccache_file}")

            # Determine if we need to kinit by testing to see if the relevant cache exists and has
            # non-expired tickets.  Tickets are marked to expire after 5 minutes to help reduce the number
            # it kinits we do but recover quickly when keytabs are regenerated
            return_code, _ = call(klist_command)
            if return_code != 0:
                kinit_path_local = get_kinit_path()
                kinit_command = format(
                    "{kinit_path_local} -l 5m -kt {oozie_keytab} {oozie_principal}; "
                )

                # kinit
                Execute(kinit_command, environment=kerberos_env)

        # execute the command
        Execute(command, environment=kerberos_env)

        return (RESULT_CODE_OK,
                ["Successful connection to {0}".format(oozie_url)])

    except Exception, ex:
        return (RESULT_CODE_CRITICAL, [str(ex)])
def execute(parameters=None, host_name=None):
  """
  Returns a tuple containing the result code and a pre-formatted result label

  Keyword arguments:
  parameters (dictionary): a mapping of parameter key to value
  host_name (string): the name of this host where the alert is running
  """

  if parameters is None:
    return (RESULT_CODE_UNKNOWN, ['There were no parameters supplied to the script.'])

  if not OOZIE_URL_KEY in parameters:
    return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.'])

  # use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all
  # interfaces, which doesn't work on Windows
  localhost_address = 'localhost' if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY else '0.0.0.0'

  oozie_url = parameters[OOZIE_URL_KEY]
  oozie_url = oozie_url.replace(urlparse(oozie_url).hostname,localhost_address)

  security_enabled = False
  if SECURITY_ENABLED in parameters:
    security_enabled = str(parameters[SECURITY_ENABLED]).upper() == 'TRUE'

  command = format("source /etc/oozie/conf/oozie-env.sh ; oozie admin -oozie {oozie_url} -status")

  try:
    # kinit if security is enabled so that oozie-env.sh can make the web request
    kerberos_env = None

    if security_enabled:
      if OOZIE_KEYTAB in parameters and OOZIE_PRINCIPAL in parameters:
        oozie_keytab = parameters[OOZIE_KEYTAB]
        oozie_principal = parameters[OOZIE_PRINCIPAL]

        # substitute _HOST in kerberos principal with actual fqdn
        oozie_principal = oozie_principal.replace('_HOST', host_name)
      else:
        return (RESULT_CODE_UNKNOWN, ['The Oozie keytab and principal are required parameters when security is enabled.'])

      # Create the kerberos credentials cache (ccache) file and set it in the environment to use
      # when executing curl
      env = Environment.get_instance()
      ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, sep, getpid())
      kerberos_env = {'KRB5CCNAME': ccache_file}

      klist_path_local = get_klist_path()
      klist_command = format("{klist_path_local} -s {ccache_file}")

      # Determine if we need to kinit by testing to see if the relevant cache exists and has
      # non-expired tickets.  Tickets are marked to expire after 5 minutes to help reduce the number
      # it kinits we do but recover quickly when keytabs are regenerated
      return_code, _ = call(klist_command)
      if return_code != 0:
        kinit_path_local = get_kinit_path()
        kinit_command = format("{kinit_path_local} -l 5m -kt {oozie_keytab} {oozie_principal}; ")

        # kinit
        Execute(kinit_command, environment=kerberos_env)

    # execute the command
    Execute(command, environment=kerberos_env)

    return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)])

  except Exception, ex:
    return (RESULT_CODE_CRITICAL, [str(ex)])
Example #15
0
  def _make_web_request(self, url):
    """
    Makes an http(s) request to a web resource and returns the http code. If
    there was an error making the request, return 0 for the status code.
    """
    error_msg = None
    try:
      response_code = 0
      kerberos_keytab = None
      kerberos_principal = None

      if self.uri_property_keys.kerberos_principal is not None:
        kerberos_principal = self._get_configuration_value(
          self.uri_property_keys.kerberos_principal)

        if kerberos_principal is not None:
          # substitute _HOST in kerberos principal with actual fqdn
          kerberos_principal = kerberos_principal.replace('_HOST', self.host_name)

      if self.uri_property_keys.kerberos_keytab is not None:
        kerberos_keytab = self._get_configuration_value(self.uri_property_keys.kerberos_keytab)

      if kerberos_principal is not None and kerberos_keytab is not None:
        # Create the kerberos credentials cache (ccache) file and set it in the environment to use
        # when executing curl. Use the md5 hash of the combination of the principal and keytab file
        # to generate a (relatively) unique cache filename so that we can use it as needed.
        tmp_dir = self.config.get('agent', 'tmp_dir')
        if tmp_dir is None:
          tmp_dir = gettempdir()

        ccache_file_name = _md5("{0}|{1}".format(kerberos_principal, kerberos_keytab)).hexdigest()
        ccache_file_path = "{0}{1}web_alert_cc_{2}".format(tmp_dir, os.sep, ccache_file_name)
        kerberos_env = {'KRB5CCNAME': ccache_file_path}

        # If there are no tickets in the cache or they are expired, perform a kinit, else use what
        # is in the cache
        klist_path_local = get_klist_path()

        if os.system("{0} -s {1}".format(klist_path_local, ccache_file_path)) != 0:
          kinit_path_local = get_kinit_path()
          logger.debug("[Alert][{0}] Enabling Kerberos authentication via GSSAPI using ccache at {1}.".format(
            self.get_name(), ccache_file_path))

          os.system("{0} -l 5m -c {1} -kt {2} {3} > /dev/null".format(
            kinit_path_local, ccache_file_path, kerberos_keytab,
            kerberos_principal))
        else:
          logger.debug("[Alert][{0}] Kerberos authentication via GSSAPI already enabled using ccache at {1}.".format(
            self.get_name(), ccache_file_path))

        # check if cookies dir exists, if not then create it
        tmp_dir = self.config.get('agent', 'tmp_dir')
        cookies_dir = os.path.join(tmp_dir, "cookies")

        if not os.path.exists(cookies_dir):
          os.makedirs(cookies_dir)

        cookie_file_name = str(uuid.uuid4())
        cookie_file = os.path.join(cookies_dir, cookie_file_name)

        start_time = time.time()

        try:
          curl = subprocess.Popen(['curl', '--negotiate', '-u', ':', '-b', cookie_file, '-c', cookie_file, '-sL', '-w',
            '%{http_code}', url, '--connect-timeout', CURL_CONNECTION_TIMEOUT,
            '-o', '/dev/null'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=kerberos_env)

          curl_stdout, curl_stderr = curl.communicate()
        finally:
          if os.path.isfile(cookie_file):
            os.remove(cookie_file)

        # empty quotes evaluates to false
        if curl_stderr:
          error_msg = curl_stderr

        # empty quotes evaluates to false
        if curl_stdout:
          response_code = int(curl_stdout)

        time_millis = time.time() - start_time
      else:
        # kerberos is not involved; use urllib2
        response_code, time_millis, error_msg = self._make_web_request_urllib(url)

      return WebResponse(status_code=response_code, time_millis=time_millis,
        error_msg=error_msg)

    except Exception, exception:
      if logger.isEnabledFor(logging.DEBUG):
        logger.exception("[Alert][{0}] Unable to make a web request.".format(self.get_name()))

      return WebResponse(status_code=0, time_millis=0, error_msg=str(exception))
Example #16
0
    def _make_web_request(self, url):
        """
    Makes an http(s) request to a web resource and returns the http code. If
    there was an error making the request, return 0 for the status code.
    """
        error_msg = None
        try:
            response_code = 0
            kerberos_keytab = None
            kerberos_principal = None

            if self.uri_property_keys.kerberos_principal is not None:
                kerberos_principal = self._get_configuration_value(self.uri_property_keys.kerberos_principal)

                if kerberos_principal is not None:
                    # substitute _HOST in kerberos principal with actual fqdn
                    kerberos_principal = kerberos_principal.replace("_HOST", self.host_name)

            if self.uri_property_keys.kerberos_keytab is not None:
                kerberos_keytab = self._get_configuration_value(self.uri_property_keys.kerberos_keytab)

            if kerberos_principal is not None and kerberos_keytab is not None:
                # Create the kerberos credentials cache (ccache) file and set it in the environment to use
                # when executing curl. Use the md5 hash of the combination of the principal and keytab file
                # to generate a (relatively) unique cache filename so that we can use it as needed.
                tmp_dir = self.config.get("agent", "tmp_dir")
                if tmp_dir is None:
                    tmp_dir = gettempdir()

                ccache_file_name = _md5("{0}|{1}".format(kerberos_principal, kerberos_keytab)).hexdigest()
                ccache_file_path = "{0}{1}web_alert_cc_{2}".format(tmp_dir, os.sep, ccache_file_name)
                kerberos_env = {"KRB5CCNAME": ccache_file_path}

                # If there are no tickets in the cache or they are expired, perform a kinit, else use what
                # is in the cache
                klist_path_local = get_klist_path()

                if os.system("{0} -s {1}".format(klist_path_local, ccache_file_path)) != 0:
                    kinit_path_local = get_kinit_path()
                    logger.debug(
                        "[Alert][{0}] Enabling Kerberos authentication via GSSAPI using ccache at {1}.".format(
                            self.get_name(), ccache_file_path
                        )
                    )

                    os.system(
                        "{0} -l 5m -c {1} -kt {2} {3} > /dev/null".format(
                            kinit_path_local, ccache_file_path, kerberos_keytab, kerberos_principal
                        )
                    )
                else:
                    logger.debug(
                        "[Alert][{0}] Kerberos authentication via GSSAPI already enabled using ccache at {1}.".format(
                            self.get_name(), ccache_file_path
                        )
                    )

                # check if cookies dir exists, if not then create it
                tmp_dir = self.config.get("agent", "tmp_dir")
                cookies_dir = os.path.join(tmp_dir, "cookies")

                if not os.path.exists(cookies_dir):
                    os.makedirs(cookies_dir)

                cookie_file_name = str(uuid.uuid4())
                cookie_file = os.path.join(cookies_dir, cookie_file_name)

                start_time = time.time()

                try:
                    curl = subprocess.Popen(
                        [
                            "curl",
                            "--negotiate",
                            "-u",
                            ":",
                            "-b",
                            cookie_file,
                            "-c",
                            cookie_file,
                            "-sL",
                            "-w",
                            "%{http_code}",
                            url,
                            "--connect-timeout",
                            CURL_CONNECTION_TIMEOUT,
                            "-o",
                            "/dev/null",
                        ],
                        stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE,
                        env=kerberos_env,
                    )

                    curl_stdout, curl_stderr = curl.communicate()
                finally:
                    if os.path.isfile(cookie_file):
                        os.remove(cookie_file)

                # empty quotes evaluates to false
                if curl_stderr:
                    error_msg = curl_stderr

                # empty quotes evaluates to false
                if curl_stdout:
                    response_code = int(curl_stdout)

                time_millis = time.time() - start_time
            else:
                # kerberos is not involved; use urllib2
                response_code, time_millis, error_msg = self._make_web_request_urllib(url)

            return WebResponse(status_code=response_code, time_millis=time_millis, error_msg=error_msg)

        except Exception, exception:
            if logger.isEnabledFor(logging.DEBUG):
                logger.exception("[Alert][{0}] Unable to make a web request.".format(self.get_name()))

            return WebResponse(status_code=0, time_millis=0, error_msg=str(exception))