Ejemplo n.º 1
0
 def check_command(command, keyword):
   cmd = "{0} | grep -E '{1}'".format(command, keyword)
   Logger.info("check command: {0}".format(command))
   output = Toolkit.exe(cmd)
   if (output == ""):
     Logger.error("command {0} not running".format(command))
     raise ComponentIsNotRunning()
Ejemplo n.º 2
0
 def check_service(service, keyword = "running|active|运行"):
   cmd = "service {0} status | grep -E '{1}'".format(service, keyword)
   Logger.info("check service: {0}".format(service))
   output = Toolkit.exe(cmd)
   if (output == ""):
     Logger.error("service {0} not running".format(service))
     raise ComponentIsNotRunning()
Ejemplo n.º 3
0
def get_hdp_version():
  try:
    command = 'hdp-select status hadoop-client'
    return_code, hdp_output = shell.call(command, timeout=20)
  except Exception, e:
    Logger.error(str(e))
    raise Fail('Unable to execute hdp-select command to retrieve the version.')
def pre_rolling_upgrade_shutdown(hdfs_binary):
  """
  Runs the "shutdownDatanode {ipc_address} upgrade" command to shutdown the
  DataNode in preparation for an upgrade. This will then periodically check
  "getDatanodeInfo" to ensure the DataNode has shutdown correctly.
  This function will obtain the Kerberos ticket if security is enabled.
  :param hdfs_binary: name/path of the HDFS binary to use
  :return: Return True if ran ok (even with errors), and False if need to stop the datanode forcefully.
  """
  import params

  Logger.info('DataNode executing "shutdownDatanode" command in preparation for upgrade...')
  if params.security_enabled:
    Execute(params.dn_kinit_cmd, user = params.hdfs_user)

  dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary)
  command = format('{dfsadmin_base_command} -shutdownDatanode {dfs_dn_ipc_address} upgrade')

  code, output = shell.call(command, user=params.hdfs_user)
  if code == 0:
    # verify that the datanode is down
    _check_datanode_shutdown(hdfs_binary)
  else:
    # Due to bug HDFS-7533, DataNode may not always shutdown during stack upgrade, and it is necessary to kill it.
    if output is not None and re.search("Shutdown already in progress", output):
      Logger.error("Due to a known issue in DataNode, the command {0} did not work, so will need to shutdown the datanode forcefully.".format(command))
      return False
  return True
def _copy_files(source_and_dest_pairs, component_user, file_owner, group_owner, kinit_if_needed):
  """
  :param source_and_dest_pairs: List of tuples (x, y), where x is the source file in the local file system,
  and y is the destination file path in HDFS
  :param component_user:  User that will execute the Hadoop commands, usually smokeuser
  :param file_owner: Owner to set for the file copied to HDFS (typically hdfs account)
  :param group_owner: Owning group to set for the file copied to HDFS (typically hadoop group)
  :param kinit_if_needed: kinit command if it is needed, otherwise an empty string
  :return: Returns 0 if at least one file was copied and no exceptions occurred, and 1 otherwise.

  Must kinit before calling this function.
  """
  import params

  return_value = 1
  if source_and_dest_pairs and len(source_and_dest_pairs) > 0:
    return_value = 0
    for (source, destination) in source_and_dest_pairs:
      try:
        destination_dir = os.path.dirname(destination)

        params.HdfsDirectory(destination_dir,
                             action="create",
                             owner=file_owner,
                             hdfs_user=params.hdfs_user,   # this will be the user to run the commands as
                             mode=0555
        )

        # Because CopyFromLocal does not guarantee synchronization, it's possible for two processes to first attempt to
        # copy the file to a temporary location, then process 2 fails because the temporary file was already created by
        # process 1, so process 2 tries to clean up by deleting the temporary file, and then process 1
        # cannot finish the copy to the final destination, and both fail!
        # For this reason, the file name on the destination must be unique, and we then rename it to the intended value.
        # The rename operation is synchronized by the Namenode.
        orig_dest_file_name = os.path.split(destination)[1]
        unique_string = str(uuid.uuid4())[:8]
        new_dest_file_name = orig_dest_file_name + "." + unique_string
        new_destination = os.path.join(destination_dir, new_dest_file_name)
        CopyFromLocal(source,
                      mode=0444,
                      owner=file_owner,
                      group=group_owner,
                      user=params.hdfs_user,               # this will be the user to run the commands as
                      dest_dir=destination_dir,
                      dest_file=new_dest_file_name,
                      kinnit_if_needed=kinit_if_needed,
                      hdfs_user=params.hdfs_user,
                      hadoop_bin_dir=params.hadoop_bin_dir,
                      hadoop_conf_dir=params.hadoop_conf_dir
        )

        mv_command = format("fs -mv {new_destination} {destination}")
        ExecuteHadoop(mv_command,
                      user=params.hdfs_user,
                      bin_dir=params.hadoop_bin_dir,
                      conf_dir=params.hadoop_conf_dir
        )
      except Exception, e:
        Logger.error("Failed to copy file. Source: %s, Destination: %s. Error: %s" % (source, destination, e.message))
        return_value = 1
Ejemplo n.º 6
0
 def check_service_status(self, service, keyword):
   cmd = "service {0} status | grep -E '{1}'".format(service, keyword)
   Logger.info("run service check on {0} : ".format(service))
   (status, output) = commands.getstatusoutput(cmd)
   if (output == ""):
     Logger.error("service {0} not running".format(service))
     raise ComponentIsNotRunning() 
Ejemplo n.º 7
0
  def syncCluster(self, coors, datanodes, coord_port, datanode_port, host, sql_str):
    for coor in coors:
      name = "coordinator_" + coor.replace(".", "_")
      sql = "CREATE NODE {0} WITH (TYPE = '{1}', HOST = '{2}', PORT = {3});".format(name, "coordinator", coor, coord_port)
      cmd = sql_str.format(host, coord_port, sql)
      Logger.info(cmd)
      (ret, out) = commands.getstatusoutput(cmd)
      if (ret == 0 and out.rfind("CREATE NODE") != -1):
        # insert record
        Logger.info("coordinator {0} create success on coordinator {1}".format(name, host))
      elif (ret != 0 and out.rfind(name) != -1):
        # update record
        Logger.info("coordinator {0} exist on coordinator {1}, updating coordinator ...".format(name, host))
        update_cmd = cmd.replace("CREATE", "ALTER")
        Logger.info(update_cmd)
        (ret, out) = commands.getstatusoutput(update_cmd)
        if (ret == 0 and out.rfind("ALTER NODE") != -1):
          Logger.info("coordinator {0} update success on coordinator {1}".format(name, host))
        elif (ret != 0 and out.rfind(name) != -1):
          # update coordinator self
          # update pgxc_node set node_host = 'local', node_port = 5000 WHERE node_name = 'datanode_10_151_0_123';
          self_sql = "update pgxc_node set node_host = '{0}', node_port = {1} WHERE node_name = '{2}';".format(host, datanode_port, name)
          self_cmd = sql_str.format(coor, coord_port, self_sql)
          Logger.info(self_cmd)
          (ret, out) = commands.getstatusoutput(self_cmd)
          if (ret == 0 and out.rfind("UPDATE 1") != -1):
            Logger.info("coordinator {0} update success on coordinator {1} itself".format(name, host))
          else:
            Logger.error("coordinator {0} update failed on coordinator {1} itself".format(name, host))
            raise Fail()
        else:
          Logger.error("coordinator {0} update failed on coordinator {1}".format(name, host))
          raise Fail()
      else:
        Logger.error("coordinator {0} create failed on coordinator {1}".format(name, host))
        raise Fail()

    for datanode in datanodes:
      name = "datanode_" + datanode.replace(".", "_")
      sql = "CREATE NODE {0} WITH (TYPE = '{1}', HOST = '{2}', PORT = {3});".format(name, "datanode", datanode, datanode_port)
      cmd = sql_str.format(host, coord_port, sql)
      Logger.info(cmd)
      (ret, out) = commands.getstatusoutput(cmd)
      if (ret == 0 and out.rfind("CREATE NODE") != -1):
        # insert record
        Logger.info("datanode {0} create success on coordinator {1}".format(name, host))
      elif (ret != 0 and out.rfind(name) != -1):
        # update record
        Logger.info("datanode {0} exist on coordinator {1}, updating datanode ...".format(name, host))
        update_cmd = cmd.replace("CREATE", "ALTER")
        Logger.info(update_cmd)
        (ret, out) = commands.getstatusoutput(update_cmd)
        if (ret == 0 and out.rfind("ALTER NODE") != -1):
          Logger.info("datannode {0} update success on coordinator {1}".format(name, host))
        else:
          Logger.error("datanode {0} update failed on coordinator {1}".format(name, host))
          raise Fail()
      else:
        Logger.error("datanode {0} create failed on coordinator {1}".format(name, host))
        raise Fail()
Ejemplo n.º 8
0
 def check_process(keyword):
   Logger.info("check process by: {0}".format(keyword))
   cmd = "ps aux | grep -E '" + keyword + "' | grep -v grep | cat"
   result = Toolkit.exe(cmd)
   if (result == ""):
     Logger.error("process checked by {0} not exist".format(keyword))
     raise ComponentIsNotRunning()
Ejemplo n.º 9
0
def service_check(cmd, user, label):
    """
    Executes a service check command that adheres to LSB-compliant
    return codes.  The return codes are interpreted as defined
    by the LSB.

    See http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
    for more information.

    :param cmd: The service check command to execute.
    :param label: The name of the service.
    """
    Logger.info("Performing service check; cmd={0}, user={1}, label={2}".format(cmd, user, label))
    rc, out, err = get_user_call_output(cmd, user, is_checked_call=False)

    if len(err) > 0:
      Logger.error(err)

    if rc in [1, 2, 3]:
      # if return code in [1, 2, 3], then 'program is not running' or 'program is dead'
      Logger.info("{0} is not running".format(label))
      raise ComponentIsNotRunning()

    elif rc == 0:
      # if return code = 0, then 'program is running or service is OK'
      Logger.info("{0} is running".format(label))

    else:
      # else service state is unknown
      err_msg = "{0} service check failed; cmd '{1}' returned {2}".format(label, cmd, rc)
      Logger.error(err_msg)
      raise ExecutionFailed(err_msg, rc, out, err)
Ejemplo n.º 10
0
def _get_directory_mappings_during_upgrade():
  """
  Gets a dictionary of directory to archive name that represents the
  directories that need to be backed up and their output tarball archive targets
  :return:  the dictionary of directory to tarball mappings
  """
  import params

  # Must be performing an Upgrade
  if params.upgrade_direction is None or params.upgrade_direction != Direction.UPGRADE or \
          params.upgrade_from_version is None or params.upgrade_from_version == "":
    Logger.error("Function _get_directory_mappings_during_upgrade() can only be called during a Stack Upgrade in direction UPGRADE.")
    return {}

  # By default, use this for all stacks.
  knox_data_dir = '/var/lib/knox/data'

  if params.stack_name and params.stack_name.upper() == "HDP" and \
          compare_versions(format_hdp_stack_version(params.upgrade_from_version), "2.3.0.0") > 0:
    # Use the version that is being upgraded from.
    knox_data_dir = format('/usr/hdp/{upgrade_from_version}/knox/data')

  # the trailing "/" is important here so as to not include the "conf" folder itself
  directories = {knox_data_dir: BACKUP_DATA_ARCHIVE, params.knox_conf_dir + "/": BACKUP_CONF_ARCHIVE}

  Logger.info(format("Knox directories to backup:\n{directories}"))
  return directories
Ejemplo n.º 11
0
 def check_process(self, keyword):
   Logger.info("check process with: {0}".format(keyword))
   cmd = "ps aux | grep -E '" + keyword + "' | grep -v grep | cat"
   result = self.exe(cmd)
   if (result == ""):
     Logger.error("process {0} not exist".format(keyword))
     raise ComponentIsNotRunning()
def write_actual_version_to_history_file(repository_version, actual_version):
  """
  Save the tuple of repository_version,actual_version to the repo version history file if the repository_version
  doesn't already exist
  :param repository_version: normalized repo version (without build number) as received from the server
  :param actual_version: Repo version with the build number, as determined using hdp-select
  :returns Return True if appended the values to the file, otherwise, return False.
  """
  wrote_value = False
  if repository_version is None or actual_version is None:
    return

  if repository_version == "" or actual_version == "":
    return

  value = repository_version + "," + actual_version
  key_exists = False
  try:
    if os.path.isfile(REPO_VERSION_HISTORY_FILE):
      with open(REPO_VERSION_HISTORY_FILE, "r") as f:
        for line in f.readlines():
          line_parts = line.split(",")
          if line_parts and len(line_parts) == 2 and line_parts[0] == repository_version and line_parts[1] == actual_version:
            key_exists = True
            break

    if not key_exists:
      with open(REPO_VERSION_HISTORY_FILE, "a") as f:
        f.write(repository_version + "," + actual_version + "\n")
        wrote_value = True
    if wrote_value:
      Logger.info("Appended value \"{0}\" to file {1} to track this as a new version.".format(value, REPO_VERSION_HISTORY_FILE))
  except Exception, err:
    Logger.error("Failed to write to file {0} the value: {1}. Error: {2}".format(REPO_VERSION_HISTORY_FILE, value, str(err)))
Ejemplo n.º 13
0
def __update_sysctl_file_suse():
  """
  Updates /etc/sysctl.conf file with the HAWQ parameters on SUSE.
  """
  # Backup file
  backup_file_name = hawq_constants.sysctl_backup_file.format(str(int(time.time())))
  try:
    # Generate file with kernel parameters needed by hawq to temp file
    File(hawq_constants.hawq_sysctl_tmp_file, content=__convert_sysctl_dict_to_text(), owner=hawq_constants.hawq_user,
        group=hawq_constants.hawq_group)

    sysctl_file_dict = utils.read_file_to_dict(hawq_constants.sysctl_suse_file)
    sysctl_file_dict_original = sysctl_file_dict.copy()
    hawq_sysctl_dict = utils.read_file_to_dict(hawq_constants.hawq_sysctl_tmp_file)

    # Merge common system file with hawq specific file
    sysctl_file_dict.update(hawq_sysctl_dict)

    if sysctl_file_dict_original != sysctl_file_dict:
      # Backup file
      Execute("cp {0} {1}".format(hawq_constants.sysctl_suse_file, backup_file_name), timeout=hawq_constants.default_exec_timeout)
      # Write merged properties to file
      utils.write_dict_to_file(sysctl_file_dict, hawq_constants.sysctl_suse_file)
      # Reload kernel sysctl parameters from /etc/sysctl.conf
      Execute("sysctl -e -p", timeout=hawq_constants.default_exec_timeout)

  except Exception as e:
    Logger.error("Error occurred while updating sysctl.conf file, reverting the contents" + str(e))
    Execute("cp {0} {1}".format(hawq_constants.sysctl_suse_file, hawq_constants.hawq_sysctl_tmp_file))
    Execute("mv {0} {1}".format(backup_file_name, hawq_constants.sysctl_suse_file), timeout=hawq_constants.default_exec_timeout)
    Logger.error("Please execute `sysctl -e -p` on the command line manually to reload the contents of file {0}".format(
      hawq_constants.hawq_sysctl_tmp_file))
    raise Fail("Failed to update sysctl.conf file ")
Ejemplo n.º 14
0
 def get_hdp_version():
   if not options.hdp_version:
     # Ubuntu returns: "stdin: is not a tty", as subprocess output.
     tmpfile = tempfile.NamedTemporaryFile()
     out = None
     with open(tmpfile.name, 'r+') as file:
       get_hdp_version_cmd = '/usr/bin/hdp-select status %s > %s' % ('hadoop-mapreduce-historyserver', tmpfile.name)
       code, stdoutdata = shell.call(get_hdp_version_cmd)
       out = file.read()
     pass
     if code != 0 or out is None:
       Logger.warning("Could not verify HDP version by calling '%s'. Return Code: %s, Output: %s." %
                      (get_hdp_version_cmd, str(code), str(out)))
       return 1
    
     matches = re.findall(r"([\d\.]+\-\d+)", out)
     hdp_version = matches[0] if matches and len(matches) > 0 else None
    
     if not hdp_version:
       Logger.error("Could not parse HDP version from output of hdp-select: %s" % str(out))
       return 1
   else:
     hdp_version = options.hdp_version
     
   return hdp_version
def _get_current_hiveserver_version():
  """
  Runs "hive --version" and parses the result in order
  to obtain the current version of hive.

  :return:  the hiveserver2 version, returned by "hive --version"
  """
  import params

  try:
    # When downgrading the source version should be the version we are downgrading from
    if "downgrade" == params.upgrade_direction:
      if not params.downgrade_from_version:
        raise Fail('The version from which we are downgrading from should be provided in \'downgrade_from_version\'')
      source_version = params.downgrade_from_version
    else:
      source_version = params.current_version
    hive_execute_path = _get_hive_execute_path(source_version)
    version_hive_bin = params.hive_bin
    formatted_source_version = format_hdp_stack_version(source_version)
    if formatted_source_version and compare_versions(formatted_source_version, "2.2") >= 0:
      version_hive_bin = format('/usr/hdp/{source_version}/hive/bin')
    command = format('{version_hive_bin}/hive --version')
    return_code, hdp_output = shell.call(command, user=params.hive_user, path=hive_execute_path)
  except Exception, e:
    Logger.error(str(e))
    raise Fail('Unable to execute hive --version command to retrieve the hiveserver2 version.')
Ejemplo n.º 16
0
def get_write_lock_files_solr_cloud(hadoop_prefix, collections):
    import params

    write_locks_to_delete = ''

    for collection_path in collections:
        code, output = call(format('{hadoop_prefix} -ls {collection_path}'))
        core_paths = get_core_paths(output, collection_path)

        collection_name = collection_path.replace(format('{solr_hdfs_directory}/'), '')
        zk_code, zk_output = call(format(
            '{zk_client_prefix} -cmd get {solr_cloud_zk_directory}/collections/{collection_name}/state.json'),
            env={'JAVA_HOME': params.java64_home},
            timeout=60
        )
        if zk_code != 0:
            Logger.error(format('Cannot determine cores owned by [{solr_hostname}] in collection [{collection_name}] due to ZK error.'))
            continue

        for core_path in core_paths:
            core_node_name = core_path.replace(format('{collection_path}/'), '')
            pattern = re.compile(format(HOSTNAME_VERIFIER_PATTERN), re.MULTILINE|re.DOTALL)
            core_on_hostname = re.search(pattern, zk_output)
            if core_on_hostname is not None:
                write_locks_to_delete += WRITE_LOCK_PATTERN.format(core_path)

    return write_locks_to_delete
Ejemplo n.º 17
0
def bootstrap_standby_namenode(params, use_path=False):

  bin_path = os.path.join(params.hadoop_bin_dir, '') if use_path else ""

  try:
    iterations = 50
    bootstrap_cmd = format("{bin_path}hdfs namenode -bootstrapStandby -nonInteractive")
    # Blue print based deployments start both NN in parallel and occasionally
    # the first attempt to bootstrap may fail. Depending on how it fails the
    # second attempt may not succeed (e.g. it may find the folder and decide that
    # bootstrap succeeded). The solution is to call with -force option but only
    # during initial start
    if params.command_phase == "INITIAL_START":
      bootstrap_cmd = format("{bin_path}hdfs namenode -bootstrapStandby -nonInteractive -force")
    Logger.info("Boostrapping standby namenode: %s" % (bootstrap_cmd))
    for i in range(iterations):
      Logger.info('Try %d out of %d' % (i+1, iterations))
      code, out = shell.call(bootstrap_cmd, logoutput=False, user=params.hdfs_user)
      if code == 0:
        Logger.info("Standby namenode bootstrapped successfully")
        return True
      elif code == 5:
        Logger.info("Standby namenode already bootstrapped")
        return True
      else:
        Logger.warning('Bootstrap standby namenode failed with %d error code. Will retry' % (code))
  except Exception as ex:
    Logger.error('Bootstrap standby namenode threw an exception. Reason %s' %(str(ex)))
  return False
def prepare_upgrade_check_for_previous_dir():
  """
  During a NonRolling (aka Express Upgrade), preparing the NameNode requires backing up some data.
  Check that there is no "previous" folder inside the NameNode Name Dir.
  """
  import params

  if params.dfs_ha_enabled:
    namenode_ha = NamenodeHAState()
    if namenode_ha.is_active(params.hostname):
      Logger.info("NameNode High Availability is enabled and this is the Active NameNode.")

      problematic_previous_namenode_dirs = set()
      nn_name_dirs = params.dfs_name_dir.split(',')
      for nn_dir in nn_name_dirs:
        if os.path.isdir(nn_dir):
          # Check for a previous folder, which is not allowed.
          previous_dir = os.path.join(nn_dir, "previous")
          if os.path.isdir(previous_dir):
            problematic_previous_namenode_dirs.add(previous_dir)

      if len(problematic_previous_namenode_dirs) > 0:
        message = 'WARNING. The following NameNode Name Dir(s) have a "previous" folder from an older version.\n' \
                  'Please back it up first, and then delete it, OR Finalize (E.g., "hdfs dfsadmin -finalizeUpgrade").\n' \
                  'NameNode Name Dir(s): {0}\n' \
                  '***** Then, retry this step. *****'.format(", ".join(problematic_previous_namenode_dirs))
        Logger.error(message)
        raise Fail(message)
Ejemplo n.º 19
0
 def check_postgre_running(self, statusCmd):
   Logger.info("check service by: {0}".format(statusCmd))
   cmd = "{0} | grep -E 'is running'".format(statusCmd)
   result = self.exe1(cmd)
   if (result != ""):
     Logger.error("service not exist")
     raise ComponentIsNotRunning()
Ejemplo n.º 20
0
def wait_for_safemode_off(hdfs_binary, afterwait_sleep=0, execute_kinit=False):
  """
  During NonRolling (aka Express Upgrade), after starting NameNode, which is still in safemode, and then starting
  all of the DataNodes, we need for NameNode to receive all of the block reports and leave safemode.
  If HA is present, then this command will run individually on each NameNode, which checks for its own address.
  """
  import params

  Logger.info("Wait to leafe safemode since must transition from ON to OFF.")

  if params.security_enabled and execute_kinit:
    kinit_command = format("{params.kinit_path_local} -kt {params.hdfs_user_keytab} {params.hdfs_principal_name}")
    Execute(kinit_command, user=params.hdfs_user, logoutput=True)

  try:
    # Note, this fails if namenode_address isn't prefixed with "params."

    dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary, use_specific_namenode=True)
    is_namenode_safe_mode_off = dfsadmin_base_command + " -safemode get | grep 'Safe mode is OFF'"

    # Wait up to 30 mins
    Execute(is_namenode_safe_mode_off,
            tries=115,
            try_sleep=10,
            user=params.hdfs_user,
            logoutput=True
            )

    # Wait a bit more since YARN still depends on block reports coming in.
    # Also saw intermittent errors with HBASE service check if it was done too soon.
    time.sleep(afterwait_sleep)
  except Fail:
    Logger.error("NameNode is still in safemode, please be careful with commands that need safemode OFF.")
Ejemplo n.º 21
0
    def service_check(self, env):
        import params
        env.set_params(params)

        if not os.path.isfile(params.solr_config_pid_file):
            Logger.error(format("PID file {solr_config_pid_file} does not exist"))
            exit(1)

        if not params.solr_collection_sample_create:
            Logger.info("Create sample collection unchecked, skipping ...")
            return

        if exists_collection(params.solr_collection_name):
            Logger.warning(format("Collection {solr_collection_name} already exists, skipping ..."))
            return

        if not params.solr_cloud_mode:
            Execute(
                    format(
                            '{solr_config_bin_dir}/solr create_core -c {solr_collection_name}' +
                            ' -d {solr_collection_config_dir} -p {solr_config_port} >> {solr_config_service_log_file} 2>&1'
                    ),
                    environment={'JAVA_HOME': params.java64_home},
                    user=params.solr_config_user
            )
        else:
            Execute(format(
                    '{solr_config_bin_dir}/solr create_collection -c {solr_collection_name}' +
                    ' -d {solr_collection_config_dir} -p {solr_config_port}' +
                    ' -s {solr_collection_shards} -rf {solr_collection_replicas}' +
                    ' >> {solr_config_service_log_file} 2>&1'),
                    environment={'JAVA_HOME': params.java64_home},
                    user=params.solr_config_user
            )
Ejemplo n.º 22
0
 def check_url(self, url):
   Logger.info("check url: {0}".format(url))
   cmd = "curl -I \"" + url + "\" 2> /dev/null | awk 'NR==1{print}' | awk '{print $2}'"
   result = self.exe(cmd)
   if (result != "200"):
     Logger.error("service not exist")
     raise ComponentIsNotRunning()
Ejemplo n.º 23
0
  def delete_storm_local_data(self, env):
    """
    Deletes Storm data from local directories. This will create a marker file
    with JSON data representing the upgrade stack and request/stage ID. This
    will prevent multiple Storm components on the same host from removing
    the local directories more than once.
    :return:
    """
    import params

    Logger.info('Clearing Storm data from local directories...')

    storm_local_directory = params.local_dir
    if storm_local_directory is None:
      raise Fail("The storm local directory specified by storm-site/storm.local.dir must be specified")

    request_id = default("/requestId", None)
    stage_id = default("/stageId", None)
    stack_version = params.version
    stack_name = params.stack_name

    json_map = {}
    json_map["requestId"] = request_id
    json_map["stageId"] = stage_id
    json_map["stackVersion"] = stack_version
    json_map["stackName"] = stack_name

    temp_directory = params.tmp_dir
    upgrade_file = os.path.join(temp_directory, "storm-upgrade-{0}.json".format(stack_version))

    if os.path.exists(upgrade_file):
      try:
        with open(upgrade_file) as file_pointer:
          existing_json_map = json.load(file_pointer)

        if cmp(json_map, existing_json_map) == 0:
          Logger.info("The storm upgrade has already removed the local directories for {0}-{1} for request {2} and stage {3}".format(
            stack_name, stack_version, request_id, stage_id))

          # nothing else to do here for this as it appears to have already been
          # removed by another component being upgraded
          return

      except:
        Logger.error("The upgrade file {0} appears to be corrupt; removing...".format(upgrade_file))
        File(upgrade_file, action="delete")
    else:
      # delete the upgrade file since it does not match
      File(upgrade_file, action="delete")

    # delete from local directory
    Directory(storm_local_directory, action="delete", recursive=True)

    # recreate storm local directory
    Directory(storm_local_directory, mode=0755, owner = params.storm_user,
      group = params.user_group, recursive = True)

    # the file doesn't exist, so create it
    with open(upgrade_file, 'w') as file_pointer:
      json.dump(json_map, file_pointer, indent=2)
Ejemplo n.º 24
0
def check_ranger_service():
  import params

  ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url)
  ambari_username_password_for_ranger = format("{ambari_ranger_admin}:{ambari_ranger_password}")
  response_code = ranger_adm_obj.check_ranger_login_urllib2(params.policymgr_mgr_url)

  if response_code is not None and response_code == 200:
    user_resp_code = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password)
    if user_resp_code is not None and user_resp_code == 200:
      get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger)
      if not get_repo_flag:
        create_repo_flag = create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger)
        if create_repo_flag:
          return True
        else:
          return False
      else:
        return True
    else:
      Logger.error('Ambari admin user creation failed')
      return False
  else:
    Logger.error('Ranger service is not reachable host')
    return False
Ejemplo n.º 25
0
def create_repo(url, data, usernamepassword):
  try:
    base_url = url + '/service/public/v2/api/service'
    base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '')
    headers = {
      'Accept': 'application/json',
      "Content-Type": "application/json"
    }
    request = urllib2.Request(base_url, data, headers)
    request.add_header("Authorization", "Basic {0}".format(base64string))
    result = urllib2.urlopen(request, timeout=20)
    response_code = result.getcode()
    response = json.loads(json.JSONEncoder().encode(result.read()))
    if response_code == 200:
      Logger.info('Repository created Successfully')
      return True
    else:
      Logger.info('Repository not created')
      return False
  except urllib2.URLError, e:
    if isinstance(e, urllib2.HTTPError):
      Logger.error("Error creating service. Http status code - {0}. \n {1}".format(e.code, e.read()))
      return False
    else:
      Logger.error("Error creating service. Reason - {0}.".format(e.reason))
      return False
Ejemplo n.º 26
0
 def update_ranger_policy(self, policyId, data, usernamepassword):
   """
   :param policyId: policy id which needs to be updated
   :param data: policy data that needs to be updated
   :param usernamepassword: user credentials using which policy needs to be updated
   :return Returns successful response and response code else None
   """
   try:
     searchRepoURL = self.urlPolicies + "/" + str(policyId)
     base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '')
     headers = {
       'Accept': 'application/json',
       "Content-Type": "application/json"
     }
     request = urllib2.Request(searchRepoURL, data, headers)
     request.add_header("Authorization", "Basic {0}".format(base64string))
     request.get_method = lambda: 'PUT'
     result = openurl(request, timeout=20)
     response_code = result.getcode()
     response = json.loads(json.JSONEncoder().encode(result.read()))
     if response_code == 200:
       Logger.info('Policy updated Successfully')
       return response_code
     else:
       Logger.error('Update Policy failed')
       return None
   except urllib2.URLError, e:
     if isinstance(e, urllib2.HTTPError):
       raise Fail("Error updating policy. Http status code - {0}. \n {1}".format(e.code, e.read()))
     else:
       raise Fail("Error updating policy. Reason - {0}.".format(e.reason))
Ejemplo n.º 27
0
def get_repo(url, name, usernamepassword):
  try:
    base_url = url + '/service/public/v2/api/service?serviceName=' + name + '&serviceType=kms&isEnabled=true'
    request = urllib2.Request(base_url)
    base64string = base64.encodestring(usernamepassword).replace('\n', '')
    request.add_header("Content-Type", "application/json")
    request.add_header("Accept", "application/json")
    request.add_header("Authorization", "Basic {0}".format(base64string))
    result = urllib2.urlopen(request, timeout=20)
    response_code = result.getcode()
    response = json.loads(result.read())
    if response_code == 200 and len(response) > 0:
      for repo in response:
        if repo.get('name').lower() == name.lower() and repo.has_key('name'):
          Logger.info('KMS repository exist')
          return True
        else:
          Logger.info('KMS repository doesnot exist')
          return False
    else:
      Logger.info('KMS repository doesnot exist')
      return False
  except urllib2.URLError, e:
    if isinstance(e, urllib2.HTTPError):
      Logger.error("Error getting {0} service. Http status code - {1}. \n {2}".format(name, e.code, e.read()))
      return False
    else:
      Logger.error("Error getting {0} service. Reason - {1}.".format(name, e.reason))
      return False
Ejemplo n.º 28
0
 def status(self, env):
   import params
   cmd = "curl -I \"" + params.lhotse_web_url + "\" 2> /dev/null | awk 'NR==1{print}' | awk '{print $2}'"
   Logger.error("run cmd = {0}".format(cmd))
   (ret, output) = commands.getstatusoutput(cmd)
   if output != "200" :
     Logger.error("lhotse web not exists")
     raise ComponentIsNotRunning()
Ejemplo n.º 29
0
 def check_data_correctness(self):
   expected_data = "55"
   Logger.info("Validating data inserted, finding sum of all the inserted entries. Expected output: {0}".format(expected_data))
   sql_cmd = "select sum(col1) from {0}".format(hawq_constants.smoke_check_table_name)
   _, stdout, _ = exec_psql_cmd(sql_cmd, self.active_master_host, tuples_only=False)
   if expected_data != stdout.strip():
     Logger.error("Incorrect data returned. Expected Data: {0} Actual Data: {1}".format(expected_data, stdout))
     raise Fail("Incorrect data returned.")
Ejemplo n.º 30
0
 def check_state(self):
   import params
   command = "source {0} && hawq state -d {1}".format(hawq_constants.hawq_greenplum_path_file, params.hawq_master_dir)
   Logger.info("Executing hawq status check..")
   (retcode, out, err) = exec_ssh_cmd(self.active_master_host, command)
   if retcode:
     Logger.error("hawq state command returned non-zero result: {0}. Out: {1} Error: {2}".format(retcode, out, err))
     raise Fail("Unexpected result of hawq state command.")
   Logger.info("Output of command:\n{0}".format(str(out) + "\n"))
Ejemplo n.º 31
0
def enable_kms_plugin():

    import params

    if params.has_ranger_admin:

        ranger_flag = False

        if params.stack_supports_ranger_kerberos and params.security_enabled:
            if not is_empty(params.rangerkms_principal
                            ) and params.rangerkms_principal != '':
                ranger_flag = check_ranger_service_support_kerberos(
                    params.kms_user, params.rangerkms_keytab,
                    params.rangerkms_principal)
            else:
                ranger_flag = check_ranger_service_support_kerberos(
                    params.kms_user, params.spengo_keytab,
                    params.spnego_principal)
        else:
            ranger_flag = check_ranger_service()

        if not ranger_flag:
            Logger.error('Error in Get/Create service for Ranger Kms.')

        current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

        File(format('{kms_conf_dir}/ranger-security.xml'),
             owner=params.kms_user,
             group=params.kms_group,
             mode=0644,
             content=format(
                 '<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))

        Directory([
            os.path.join('/etc', 'ranger', params.repo_name),
            os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
        ],
                  owner=params.kms_user,
                  group=params.kms_group,
                  mode=0775,
                  create_parents=True)

        File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
                          format('kms_{repo_name}.json')),
             owner=params.kms_user,
             group=params.kms_group,
             mode=0644)

        # remove plain-text password from xml configs
        plugin_audit_properties_copy = {}
        plugin_audit_properties_copy.update(
            params.config['configurations']['ranger-kms-audit'])

        if params.plugin_audit_password_property in plugin_audit_properties_copy:
            plugin_audit_properties_copy[
                params.plugin_audit_password_property] = "crypted"

        XmlConfig(
            "ranger-kms-audit.xml",
            conf_dir=params.kms_conf_dir,
            configurations=plugin_audit_properties_copy,
            configuration_attributes=params.config['configuration_attributes']
            ['ranger-kms-audit'],
            owner=params.kms_user,
            group=params.kms_group,
            mode=0744)

        XmlConfig(
            "ranger-kms-security.xml",
            conf_dir=params.kms_conf_dir,
            configurations=params.config['configurations']
            ['ranger-kms-security'],
            configuration_attributes=params.config['configuration_attributes']
            ['ranger-kms-security'],
            owner=params.kms_user,
            group=params.kms_group,
            mode=0744)

        # remove plain-text password from xml configs
        ranger_kms_policymgr_ssl_copy = {}
        ranger_kms_policymgr_ssl_copy.update(
            params.config['configurations']['ranger-kms-policymgr-ssl'])

        for prop in params.kms_plugin_password_properties:
            if prop in ranger_kms_policymgr_ssl_copy:
                ranger_kms_policymgr_ssl_copy[prop] = "crypted"

        XmlConfig(
            "ranger-policymgr-ssl.xml",
            conf_dir=params.kms_conf_dir,
            configurations=ranger_kms_policymgr_ssl_copy,
            configuration_attributes=params.config['configuration_attributes']
            ['ranger-kms-policymgr-ssl'],
            owner=params.kms_user,
            group=params.kms_group,
            mode=0744)

        if params.xa_audit_db_is_enabled:
            cred_setup = params.cred_setup_prefix + (
                '-f', params.credential_file, '-k', 'auditDBCred', '-v',
                PasswordString(params.xa_audit_db_password), '-c', '1')
            Execute(cred_setup,
                    environment={'JAVA_HOME': params.java_home},
                    logoutput=True,
                    sudo=True)

        cred_setup = params.cred_setup_prefix + (
            '-f', params.credential_file, '-k', 'sslKeyStore', '-v',
            PasswordString(params.ssl_keystore_password), '-c', '1')
        Execute(cred_setup,
                environment={'JAVA_HOME': params.java_home},
                logoutput=True,
                sudo=True)

        cred_setup = params.cred_setup_prefix + (
            '-f', params.credential_file, '-k', 'sslTrustStore', '-v',
            PasswordString(params.ssl_truststore_password), '-c', '1')
        Execute(cred_setup,
                environment={'JAVA_HOME': params.java_home},
                logoutput=True,
                sudo=True)

        File(params.credential_file,
             owner=params.kms_user,
             group=params.kms_group,
             mode=0640)

        # create ranger kms audit directory
        if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node:
            params.HdfsResource("/ranger/audit",
                                type="directory",
                                action="create_on_execute",
                                owner=params.hdfs_user,
                                group=params.hdfs_user,
                                mode=0755,
                                recursive_chmod=True)
            params.HdfsResource("/ranger/audit/kms",
                                type="directory",
                                action="create_on_execute",
                                owner=params.kms_user,
                                group=params.kms_group,
                                mode=0750,
                                recursive_chmod=True)
            params.HdfsResource(None, action="execute")

        if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1:
            Logger.info(
                'Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml'
            )
            XmlConfig(
                "hdfs-site.xml",
                conf_dir=params.kms_conf_dir,
                configurations=params.config['configurations']['hdfs-site'],
                configuration_attributes=params.
                config['configuration_attributes']['hdfs-site'],
                owner=params.kms_user,
                group=params.kms_group,
                mode=0644)
        else:
            File(format('{kms_conf_dir}/hdfs-site.xml'), action="delete")
Ejemplo n.º 32
0
def create_ams_dashboards():
    """
  Create dashboards in grafana from the json files
  """
    import params
    server = Server(protocol=params.ams_grafana_protocol.strip(),
                    host=params.ams_grafana_host.strip(),
                    port=params.ams_grafana_port,
                    user=params.ams_grafana_admin_user,
                    password=params.ams_grafana_admin_pwd)

    dashboard_files = params.get_grafana_dashboard_defs()
    version = params.get_ambari_version()
    Logger.info("Checking dashboards to update for Ambari version : %s" %
                version)
    # Friendly representation of dashboard
    Dashboard = namedtuple('Dashboard', ['uri', 'id', 'title', 'tags'])

    existing_dashboards = []
    response = perform_grafana_get_call(GRAFANA_SEARCH_BULTIN_DASHBOARDS,
                                        server)
    if response and response.status == 200:
        data = response.read()
        try:
            datasources = json.loads(data)
        except:
            Logger.error(
                "Unable to parse JSON response from grafana request: %s" %
                GRAFANA_SEARCH_BULTIN_DASHBOARDS)
            Logger.info(data)
            return

        for datasource in datasources:
            existing_dashboards.append(
                Dashboard(uri=datasource['uri'],
                          id=datasource['id'],
                          title=datasource['title'],
                          tags=datasource['tags']))
        pass
    else:
        Logger.error(
            "Failed to execute search query on Grafana dashboards. "
            "query = %s\n statuscode = %s\n reason = %s\n data = %s\n" %
            (GRAFANA_SEARCH_BULTIN_DASHBOARDS, response.status,
             response.reason, response.read()))
        return

    Logger.debug('Dashboard definitions found = %s' % str(dashboard_files))

    if dashboard_files:
        for dashboard_file in dashboard_files:
            try:
                with open(dashboard_file, 'r') as file:
                    dashboard_def = json.load(file)
            except Exception, e:
                Logger.error('Unable to load dashboard json file %s' %
                             dashboard_file)
                Logger.error(str(e))
                continue

            if dashboard_def:
                update_def = True
                # Make sure static json does not have id
                if "id" in dashboard_def:
                    dashboard_def['id'] = None
                # Set correct tags
                if 'tags' in dashboard_def:
                    dashboard_def['tags'].append('builtin')
                    dashboard_def['tags'].append(version)
                else:
                    dashboard_def['tags'] = ['builtin', version]

                dashboard_def['overwrite'] = True

                for dashboard in existing_dashboards:
                    if dashboard.title == dashboard_def['title']:
                        if version not in dashboard.tags:
                            # Found existing dashboard with wrong version - update dashboard
                            update_def = True
                        else:
                            update_def = False  # Skip update
                pass

                if update_def:
                    Logger.info(
                        "Updating dashboard definition for %s with tags: %s" %
                        (dashboard_def['title'], dashboard_def['tags']))

                    # Discrepancy in grafana export vs import format
                    dashboard_def_payload = {"dashboard": dashboard_def}
                    paylaod = json.dumps(dashboard_def_payload).strip()

                    (response,
                     data) = perform_grafana_post_call(GRAFANA_DASHBOARDS_URL,
                                                       paylaod, server)

                    if response and response.status == 200:
                        Logger.info("Dashboard created successfully.\n %s" %
                                    str(data))
                    else:
                        Logger.error("Failed creating dashboard: %s" %
                                     dashboard_def['title'])
                    pass
                else:
                    Logger.info('No update needed for dashboard = %s' %
                                dashboard_def['title'])
            pass
        pass
Ejemplo n.º 33
0
def falcon(type, action=None, upgrade_type=None):
    import params

    if action == 'config':
        Directory(
            params.falcon_pid_dir,
            owner=params.falcon_user,
            create_parents=True,
            mode=0755,
            cd_access="a",
        )

        Directory(
            params.falcon_log_dir,
            owner=params.falcon_user,
            create_parents=True,
            mode=0755,
            cd_access="a",
        )

        Directory(params.falcon_webapp_dir,
                  owner=params.falcon_user,
                  create_parents=True)

        Directory(params.falcon_home,
                  owner=params.falcon_user,
                  create_parents=True)

        Directory(params.etc_prefix_dir, mode=0755, create_parents=True)

        Directory(params.falcon_conf_dir,
                  owner=params.falcon_user,
                  create_parents=True)

        File(
            params.falcon_conf_dir + '/falcon-env.sh',
            content=InlineTemplate(params.falcon_env_sh_template),
            owner=params.falcon_user,
            group=params.user_group,
        )

        PropertiesFile(params.falcon_conf_dir + '/client.properties',
                       properties=params.falcon_client_properties,
                       mode=0644,
                       owner=params.falcon_user)

        PropertiesFile(params.falcon_conf_dir + '/runtime.properties',
                       properties=params.falcon_runtime_properties,
                       mode=0644,
                       owner=params.falcon_user)

        PropertiesFile(params.falcon_conf_dir + '/startup.properties',
                       properties=params.falcon_startup_properties,
                       mode=0644,
                       owner=params.falcon_user)

        if params.falcon_graph_storage_directory:
            Directory(params.falcon_graph_storage_directory,
                      owner=params.falcon_user,
                      group=params.user_group,
                      mode=0775,
                      create_parents=True,
                      cd_access="a")

        if params.falcon_graph_serialize_path:
            Directory(params.falcon_graph_serialize_path,
                      owner=params.falcon_user,
                      group=params.user_group,
                      mode=0775,
                      create_parents=True,
                      cd_access="a")

        # Generate atlas-application.properties.xml file
        if params.falcon_atlas_support:
            # If Atlas is added later than Falcon, this package will be absent.
            install_atlas_hook_packages(
                params.atlas_plugin_package,
                params.atlas_ubuntu_plugin_package, params.host_sys_prepped,
                params.agent_stack_retry_on_unavailability,
                params.agent_stack_retry_count)

            atlas_hook_filepath = os.path.join(params.falcon_conf_dir,
                                               params.atlas_hook_filename)
            setup_atlas_hook(SERVICE.FALCON,
                             params.falcon_atlas_application_properties,
                             atlas_hook_filepath, params.falcon_user,
                             params.user_group)

            # Falcon 0.10 uses FALCON_EXTRA_CLASS_PATH.
            # Setup symlinks for older versions.
            if params.current_version_formatted and check_stack_feature(
                    StackFeature.FALCON_ATLAS_SUPPORT_2_3,
                    params.current_version_formatted):
                setup_atlas_jar_symlinks("falcon", params.falcon_webinf_lib)

    if type == 'server':
        if action == 'config':
            if params.store_uri[0:4] == "hdfs":
                params.HdfsResource(params.store_uri,
                                    type="directory",
                                    action="create_on_execute",
                                    owner=params.falcon_user,
                                    mode=0755)
            elif params.store_uri[0:4] == "file":
                Directory(params.store_uri[7:],
                          owner=params.falcon_user,
                          create_parents=True)

            # TODO change to proper mode
            params.HdfsResource(params.falcon_apps_dir,
                                type="directory",
                                action="create_on_execute",
                                owner=params.falcon_user,
                                mode=0777)

            # In HDP 2.4 and earlier, the data-mirroring directory was copied to HDFS.
            if params.supports_data_mirroring:
                params.HdfsResource(params.dfs_data_mirroring_dir,
                                    type="directory",
                                    action="create_on_execute",
                                    owner=params.falcon_user,
                                    group=params.proxyuser_group,
                                    recursive_chown=True,
                                    recursive_chmod=True,
                                    mode=0770,
                                    source=params.local_data_mirroring_dir)

            # Falcon Extensions were supported in HDP 2.5 and higher.
            effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(
                params.version)
            supports_falcon_extensions = effective_version and check_stack_feature(
                StackFeature.FALCON_EXTENSIONS, effective_version)

            if supports_falcon_extensions:
                params.HdfsResource(params.falcon_extensions_dest_dir,
                                    type="directory",
                                    action="create_on_execute",
                                    owner=params.falcon_user,
                                    group=params.proxyuser_group,
                                    recursive_chown=True,
                                    recursive_chmod=True,
                                    mode=0755,
                                    source=params.falcon_extensions_source_dir)
                # Create the extensons HiveDR store
                params.HdfsResource(os.path.join(
                    params.falcon_extensions_dest_dir, "mirroring"),
                                    type="directory",
                                    action="create_on_execute",
                                    owner=params.falcon_user,
                                    group=params.proxyuser_group,
                                    mode=0770)

            # At least one HDFS Dir should be created, so execute the change now.
            params.HdfsResource(None, action="execute")

            Directory(params.falcon_local_dir,
                      owner=params.falcon_user,
                      create_parents=True,
                      cd_access="a")

            if params.falcon_embeddedmq_enabled == True:
                Directory(os.path.abspath(
                    os.path.join(params.falcon_embeddedmq_data, "..")),
                          owner=params.falcon_user,
                          create_parents=True)

                Directory(params.falcon_embeddedmq_data,
                          owner=params.falcon_user,
                          create_parents=True)

        # although Falcon's falcon-config.sh will use 'which hadoop' to figure
        # this out, in an upgraded cluster, it's possible that 'which hadoop'
        # still points to older binaries; it's safer to just pass in the
        # hadoop home directory to use
        environment_dictionary = {"HADOOP_HOME": params.hadoop_home_dir}

        pid = get_user_call_output.get_user_call_output(
            format("cat {server_pid_file}"),
            user=params.falcon_user,
            is_checked_call=False)[1]
        process_exists = format("ls {server_pid_file} && ps -p {pid}")

        if action == 'start':
            try:
                Execute(
                    format('{falcon_home}/bin/falcon-config.sh server falcon'),
                    user=params.falcon_user,
                    path=params.hadoop_bin_dir,
                    environment=environment_dictionary,
                    not_if=process_exists,
                )
            except:
                show_logs(params.falcon_log_dir, params.falcon_user)
                raise

            if not os.path.exists(params.target_jar_file):
                try:
                    File(params.target_jar_file,
                         content=DownloadSource(params.bdb_resource_name),
                         mode=0755)
                except:
                    exc_msg = traceback.format_exc()
                    exception_message = format(
                        "Caught Exception while downloading {bdb_resource_name}:\n{exc_msg}"
                    )
                    Logger.error(exception_message)

                if not os.path.isfile(params.target_jar_file):
                    error_message = """
If you are using bdb as the Falcon graph db store, please run
ambari-server setup --jdbc-db=bdb --jdbc-driver=<path to je5.0.73.jar>
on the ambari server host.  Otherwise falcon startup will fail.
Otherwise please configure Falcon to use HBase as the backend as described
in the Falcon documentation.
"""
                    Logger.error(error_message)
            try:
                Execute(
                    format(
                        '{falcon_home}/bin/falcon-start -port {falcon_port}'),
                    user=params.falcon_user,
                    path=params.hadoop_bin_dir,
                    environment=environment_dictionary,
                    not_if=process_exists,
                )
            except:
                show_logs(params.falcon_log_dir, params.falcon_user)
                raise

        if action == 'stop':
            try:
                Execute(format('{falcon_home}/bin/falcon-stop'),
                        user=params.falcon_user,
                        path=params.hadoop_bin_dir,
                        environment=environment_dictionary)
            except:
                show_logs(params.falcon_log_dir, params.falcon_user)
                raise

            File(params.server_pid_file, action='delete')
Ejemplo n.º 34
0
def hive_interactive(name=None):
    import params

    # list of properties that should be excluded from the config
    # this approach is a compromise against adding a dedicated config
    # type for hive_server_interactive or needed config groups on a
    # per component basis
    exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting']

    # List of configs to be excluded from hive2 client, but present in Hive2 server.
    exclude_list_for_hive2_client = ['javax.jdo.option.ConnectionPassword']

    # Copy Tarballs in HDFS.
    if params.stack_version_formatted_major and check_stack_feature(
            StackFeature.ROLLING_UPGRADE,
            params.stack_version_formatted_major):
        resource_created = copy_to_hdfs(
            "tez_hive2",
            params.user_group,
            params.hdfs_user,
            file_mode=params.tarballs_mode,
            host_sys_prepped=params.host_sys_prepped)

        if resource_created:
            params.HdfsResource(None, action="execute")

    Directory(params.hive_interactive_etc_dir_prefix, mode=0755)

    Logger.info("Directories to fill with configs: %s" %
                str(params.hive_conf_dirs_list))
    for conf_dir in params.hive_conf_dirs_list:
        fill_conf_dir(conf_dir)
    '''
  As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml,
  we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml.
  '''
    merged_hive_interactive_site = {}
    merged_hive_interactive_site.update(
        params.config['configurations']['hive-site'])
    merged_hive_interactive_site.update(
        params.config['configurations']['hive-interactive-site'])
    for item in exclude_list:
        if item in merged_hive_interactive_site.keys():
            del merged_hive_interactive_site[item]
    '''
  Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
  which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed.
  '''
    remove_atlas_hook_if_exists(merged_hive_interactive_site)
    '''
  As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml,
  we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml.
  '''
    merged_tez_interactive_site = {}
    if 'tez-site' in params.config['configurations']:
        merged_tez_interactive_site.update(
            params.config['configurations']['tez-site'])
        Logger.info(
            "Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'."
        )
    else:
        Logger.error(
            "Tez's 'tez-site' couldn't be retrieved from passed-in configurations."
        )

    merged_tez_interactive_site.update(
        params.config['configurations']['tez-interactive-site'])
    XmlConfig(
        "tez-site.xml",
        conf_dir=params.tez_interactive_config_dir,
        configurations=merged_tez_interactive_site,
        configuration_attributes=params.config['configuration_attributes']
        ['tez-interactive-site'],
        owner=params.tez_interactive_user,
        group=params.user_group,
        mode=0664)
    '''
  Merge properties from hiveserver2-interactive-site into hiveserver2-site
  '''
    merged_hiveserver2_interactive_site = {}
    if 'hiveserver2-site' in params.config['configurations']:
        merged_hiveserver2_interactive_site.update(
            params.config['configurations']['hiveserver2-site'])
        Logger.info(
            "Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'."
        )
    else:
        Logger.error(
            "'hiveserver2-site' couldn't be retrieved from passed-in configurations."
        )
    merged_hiveserver2_interactive_site.update(
        params.config['configurations']['hiveserver2-interactive-site'])

    # Create config files under /etc/hive2/conf and /etc/hive2/conf/conf.server:
    #   hive-site.xml
    #   hive-env.sh
    #   llap-daemon-log4j2.properties
    #   llap-cli-log4j2.properties
    #   hive-log4j2.properties
    #   hive-exec-log4j2.properties
    #   beeline-log4j2.properties

    hive2_conf_dirs_list = params.hive_conf_dirs_list
    hive2_client_conf_path = format(
        "{stack_root}/current/{component_directory}/conf")

    # Making copy of 'merged_hive_interactive_site' in 'merged_hive_interactive_site_copy', and deleting 'javax.jdo.option.ConnectionPassword'
    # config from there, as Hive2 client shouldn't have that config.
    merged_hive_interactive_site_copy = merged_hive_interactive_site.copy()
    for item in exclude_list_for_hive2_client:
        if item in merged_hive_interactive_site.keys():
            del merged_hive_interactive_site_copy[item]

    for conf_dir in hive2_conf_dirs_list:
        if conf_dir == hive2_client_conf_path:
            XmlConfig(
                "hive-site.xml",
                conf_dir=conf_dir,
                configurations=merged_hive_interactive_site_copy,
                configuration_attributes=params.
                config['configuration_attributes']['hive-interactive-site'],
                owner=params.hive_user,
                group=params.user_group,
                mode=0644)
        else:
            XmlConfig(
                "hive-site.xml",
                conf_dir=conf_dir,
                configurations=merged_hive_interactive_site,
                configuration_attributes=params.
                config['configuration_attributes']['hive-interactive-site'],
                owner=params.hive_user,
                group=params.user_group,
                mode=0644)

        XmlConfig(
            "hiveserver2-site.xml",
            conf_dir=conf_dir,
            configurations=merged_hiveserver2_interactive_site,
            configuration_attributes=params.config['configuration_attributes']
            ['hiveserver2-interactive-site'],
            owner=params.hive_user,
            group=params.user_group,
            mode=0644)

        hive_server_interactive_conf_dir = conf_dir

        File(format("{hive_server_interactive_conf_dir}/hive-env.sh"),
             owner=params.hive_user,
             group=params.user_group,
             content=InlineTemplate(params.hive_interactive_env_sh_template))

        llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties'
        File(format(
            "{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"),
             mode=0644,
             group=params.user_group,
             owner=params.hive_user,
             content=params.llap_daemon_log4j)

        llap_cli_log4j2_filename = 'llap-cli-log4j2.properties'
        File(format(
            "{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"),
             mode=0644,
             group=params.user_group,
             owner=params.hive_user,
             content=params.llap_cli_log4j2)

        hive_log4j2_filename = 'hive-log4j2.properties'
        File(format(
            "{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"),
             mode=0644,
             group=params.user_group,
             owner=params.hive_user,
             content=params.hive_log4j2)

        hive_exec_log4j2_filename = 'hive-exec-log4j2.properties'
        File(format(
            "{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"),
             mode=0644,
             group=params.user_group,
             owner=params.hive_user,
             content=params.hive_exec_log4j2)

        beeline_log4j2_filename = 'beeline-log4j2.properties'
        File(format(
            "{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"),
             mode=0644,
             group=params.user_group,
             owner=params.hive_user,
             content=params.beeline_log4j2)

        File(format(
            "{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"
        ),
             owner=params.hive_user,
             group=params.user_group,
             content=Template("hadoop-metrics2-llapdaemon.j2"))

        File(format(
            "{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"
        ),
             owner=params.hive_user,
             group=params.user_group,
             content=Template("hadoop-metrics2-llaptaskscheduler.j2"))

    # On some OS this folder could be not exists, so we will create it before pushing there files
    Directory(params.limits_conf_dir,
              create_parents=True,
              owner='root',
              group='root')

    File(os.path.join(params.limits_conf_dir, 'hive.conf'),
         owner='root',
         group='root',
         mode=0644,
         content=Template("hive.conf.j2"))

    if not os.path.exists(params.target_hive_interactive):
        jdbc_connector(params.target_hive_interactive,
                       params.hive_intaractive_previous_jdbc_jar)

    File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
         content=DownloadSource(
             format("{jdk_location}{check_db_connection_jar_name}")),
         mode=0644)
    File(params.start_hiveserver2_interactive_path,
         mode=0755,
         content=Template(format('{start_hiveserver2_interactive_script}')))

    Directory(params.hive_pid_dir,
              create_parents=True,
              cd_access='a',
              owner=params.hive_user,
              group=params.user_group,
              mode=0755)
    Directory(params.hive_log_dir,
              create_parents=True,
              cd_access='a',
              owner=params.hive_user,
              group=params.user_group,
              mode=0755)
    Directory(params.hive_interactive_var_lib,
              create_parents=True,
              cd_access='a',
              owner=params.hive_user,
              group=params.user_group,
              mode=0755)
Ejemplo n.º 35
0
    def create_ambari_admin_user(self, ambari_admin_username,
                                 ambari_admin_password, usernamepassword):
        """
    :param ambari_admin_username: username of user to be created
    :param ambari_admin_password: user password of user to be created
    :param usernamepassword: user credentials using which repository needs to be searched.
    :return: Returns user credentials if user exist otherwise rerutns credentials of  created user.
    """
        flag_ambari_admin_present = False
        match = re.match('[a-zA-Z0-9_\S]+$', ambari_admin_password)
        if match is None:
            raise Fail(
                'Invalid password given for Ranger Admin user for Ambari')
        try:
            url = self.url_users + '?name=' + str(ambari_admin_username)
            request = urllib2.Request(url)
            base_64_string = base64.encodestring(usernamepassword).replace(
                '\n', '')
            request.add_header("Content-Type", "application/json")
            request.add_header("Accept", "application/json")
            request.add_header("Authorization",
                               "Basic {0}".format(base_64_string))
            result = openurl(request, timeout=20)
            response_code = result.getcode()
            response = json.loads(result.read())
            if response_code == 200 and len(response['vXUsers']) >= 0:
                for vxuser in response['vXUsers']:
                    if vxuser['name'] == ambari_admin_username:
                        flag_ambari_admin_present = True
                        break
                    else:
                        flag_ambari_admin_present = False

                if flag_ambari_admin_present:
                    Logger.info(ambari_admin_username +
                                ' user already exists.')
                    return response_code
                else:
                    Logger.info(
                        ambari_admin_username +
                        ' user is not present, creating user using given configurations'
                    )
                    url = self.url_sec_users
                    admin_user = dict()
                    admin_user['status'] = 1
                    admin_user['userRoleList'] = ['ROLE_SYS_ADMIN']
                    admin_user['name'] = ambari_admin_username
                    admin_user['password'] = ambari_admin_password
                    admin_user['description'] = ambari_admin_username
                    admin_user['firstName'] = ambari_admin_username
                    data = json.dumps(admin_user)
                    base_64_string = base64.encodestring(
                        '{0}'.format(usernamepassword)).replace('\n', '')
                    headers = {
                        'Accept': 'application/json',
                        "Content-Type": "application/json"
                    }
                    request = urllib2.Request(url, data, headers)
                    request.add_header("Authorization",
                                       "Basic {0}".format(base_64_string))
                    result = openurl(request, timeout=20)
                    response_code = result.getcode()
                    response = json.loads(json.JSONEncoder().encode(
                        result.read()))
                    if response_code == 200 and response is not None:
                        Logger.info('Ambari admin user creation successful.')
                        return response_code
                    else:
                        Logger.error('Ambari admin user creation failed.')
                        return None
            else:
                return None
        except urllib2.URLError, e:
            if isinstance(e, urllib2.HTTPError):
                raise Fail(
                    "Error creating ambari admin user. Http status code - {0}. \n {1}"
                    .format(e.code, e.read()))
            else:
                raise Fail(
                    "Error creating ambari admin user. Reason - {0}.".format(
                        e.reason))
Ejemplo n.º 36
0
def download_database_connector_if_needed():
    """
  Downloads the database connector to use when connecting to the metadata storage
  """
    import params
    if params.registry_storage_type != 'mysql' and params.registry_storage_type != 'oracle':
        # In any other case than oracle and mysql, e.g. postgres, just return.
        return

    if params.jdbc_driver_jar == None:
        if "mysql" in params.registry_storage_type:
            Logger.error(
                "Failed to find mysql-java-connector jar. Make sure you followed the steps to register mysql driver"
            )
            Logger.info("Users should register the mysql java driver jar.")
            Logger.info("yum install mysql-connector-java*")
            Logger.info(
                "sudo ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar"
            )
            raise Fail('Unable to establish jdbc connection to your ' +
                       params.registry_storage_type + ' instance.')
        if "oracle" in params.registry_storage_type:
            Logger.error(
                "Failed to find ojdbc jar. Please download and make sure you followed the steps to register oracle jdbc driver"
            )
            Logger.info("Users should register the oracle ojdbc driver jar.")
            Logger.info(
                "Create a symlink e.g. ln -s /usr/share/java/ojdbc6.jar /usr/share/java/ojdbc.jar"
            )
            Logger.info(
                "sudo ambari-server setup --jdbc-db=oracle --jdbc-driver=/usr/share/java/ojdbc.jar"
            )
            raise Fail('Unable to establish jdbc connection to your ' +
                       params.registry_storage_type + ' instance.')

    File(params.check_db_connection_jar,
         content=DownloadSource(
             format("{jdk_location}/{check_db_connection_jar_name}")))

    target_jar_with_directory = params.connector_download_dir + os.path.sep + params.jdbc_driver_jar
    target_jar_bootstrap_dir = params.connector_bootstrap_download_dir + os.path.sep + params.jdbc_driver_jar

    if not os.path.exists(target_jar_with_directory):
        File(params.downloaded_custom_connector,
             content=DownloadSource(params.connector_curl_source))

        Execute(
            ('cp', '--remove-destination', params.downloaded_custom_connector,
             target_jar_with_directory),
            path=["/bin", "/usr/bin/"],
            sudo=True)

        File(target_jar_with_directory, owner="root", group=params.user_group)

    if not os.path.exists(target_jar_bootstrap_dir):
        File(params.downloaded_custom_connector,
             content=DownloadSource(params.connector_curl_source))

        Execute(('cp', '--remove-destination',
                 params.downloaded_custom_connector, target_jar_bootstrap_dir),
                path=["/bin", "/usr/bin/"],
                sudo=True)

        File(target_jar_with_directory, owner="root", group=params.user_group)
Ejemplo n.º 37
0
def copy_jdbc_connector(stack_version=None):
    import params

    if params.jdbc_jar_name is None and params.driver_curl_source.endswith(
            "/None"):
        error_message = "Error! Sorry, but we can't find jdbc driver related to {0} database to download from {1}. \
    Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'".format(
            params.db_flavor, params.jdk_location)
        Logger.error(error_message)

    if params.driver_curl_source and not params.driver_curl_source.endswith(
            "/None"):
        if params.previous_jdbc_jar and os.path.isfile(
                params.previous_jdbc_jar):
            File(params.previous_jdbc_jar, action='delete')

    kms_home = params.kms_home
    if stack_version is not None:
        kms_home = format("{stack_root}/{stack_version}/ranger-kms")

    driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")

    File(params.downloaded_custom_connector,
         content=DownloadSource(params.driver_curl_source),
         mode=0644)

    Directory(os.path.join(kms_home, 'ews', 'lib'), mode=0755)

    if params.db_flavor.lower() == 'sqla':
        Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C',
                 params.tmp_dir),
                sudo=True)

        Execute(('cp', '--remove-destination', params.jar_path_in_archive,
                 os.path.join(kms_home, 'ews', 'webapp', 'lib')),
                path=["/bin", "/usr/bin/"],
                sudo=True)

        Directory(params.jdbc_libs_dir, cd_access="a", create_parents=True)

        Execute(as_sudo([
            'yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir
        ],
                        auto_escape=False),
                path=["/bin", "/usr/bin/"])

        File(os.path.join(kms_home, 'ews', 'webapp', 'lib', 'sajdbc4.jar'),
             mode=0644)
    else:
        Execute(
            ('cp', '--remove-destination', params.downloaded_custom_connector,
             os.path.join(kms_home, 'ews', 'webapp', 'lib')),
            path=["/bin", "/usr/bin/"],
            sudo=True)

        File(os.path.join(kms_home, 'ews', 'webapp', 'lib',
                          params.jdbc_jar_name),
             mode=0644)

    ModifyPropertiesFile(
        format("{kms_home}/install.properties"),
        properties=params.config['configurations']['kms-properties'],
        owner=params.kms_user)

    if params.db_flavor.lower() == 'sqla':
        ModifyPropertiesFile(
            format("{kms_home}/install.properties"),
            properties={
                'SQL_CONNECTOR_JAR':
                format('{kms_home}/ews/webapp/lib/sajdbc4.jar')
            },
            owner=params.kms_user,
        )
    else:
        ModifyPropertiesFile(
            format("{kms_home}/install.properties"),
            properties={'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
            owner=params.kms_user,
        )
Ejemplo n.º 38
0
def get_tarball_paths(name,
                      use_upgrading_version_during_upgrade=True,
                      custom_source_file=None,
                      custom_dest_file=None):
    """
  For a given tarball name, get the source and destination paths to use.
  :param name: Tarball name
  :param use_upgrading_version_during_upgrade:
  :param custom_source_file: If specified, use this source path instead of the default one from the map.
  :param custom_dest_file: If specified, use this destination path instead of the default one from the map.
  :return: A tuple of (success status, source path, destination path, optional preparation function which is invoked to setup the tarball)
  """
    stack_name = Script.get_stack_name()

    if not stack_name:
        Logger.error(
            "Cannot copy {0} tarball to HDFS because stack name could not be determined."
            .format(str(name)))
        return False, None, None

    if name is None or name.lower() not in TARBALL_MAP:
        Logger.error(
            "Cannot copy tarball to HDFS because {0} is not supported in stack {1} for this operation."
            .format(str(name), str(stack_name)))
        return False, None, None

    service = TARBALL_MAP[name.lower()]['service']

    stack_version = get_current_version(service=service,
                                        use_upgrading_version_during_upgrade=
                                        use_upgrading_version_during_upgrade)
    if not stack_version:
        Logger.error(
            "Cannot copy {0} tarball to HDFS because stack version could be be determined."
            .format(str(name)))
        return False, None, None

    stack_root = Script.get_stack_root()
    if not stack_root:
        Logger.error(
            "Cannot copy {0} tarball to HDFS because stack root could be be determined."
            .format(str(name)))
        return False, None, None

    (source_file, dest_file) = TARBALL_MAP[name.lower()]['dirs']

    if custom_source_file is not None:
        source_file = custom_source_file

    if custom_dest_file is not None:
        dest_file = custom_dest_file

    source_file = source_file.replace(STACK_NAME_PATTERN, stack_name.lower())
    dest_file = dest_file.replace(STACK_NAME_PATTERN, stack_name.lower())

    source_file = source_file.replace(STACK_ROOT_PATTERN, stack_root.lower())
    dest_file = dest_file.replace(STACK_ROOT_PATTERN, stack_root.lower())

    source_file = source_file.replace(STACK_VERSION_PATTERN, stack_version)
    dest_file = dest_file.replace(STACK_VERSION_PATTERN, stack_version)

    prepare_function = None
    if "prepare_function" in TARBALL_MAP[name.lower()]:
        prepare_function = TARBALL_MAP[name.lower()]['prepare_function']

    return True, source_file, dest_file, prepare_function
Ejemplo n.º 39
0
    except Exception, e:
        Logger.logger.exception(
            "Could not parse output of {0}. Error: {1}".format(
                str(tmp_file), str(e)))
    finally:
        try:
            if os.path.exists(tmp_file):
                os.remove(tmp_file)
        except Exception, e:
            Logger.logger.exception(
                "Could not remove file {0}. Error: {1}".format(
                    str(tmp_file), str(e)))

    if code != 0 or out is None or out == "":
        Logger.error(
            "Could not verify stack version by calling '{0}'. Return Code: {1}, Output: {2}."
            .format(get_stack_versions_cmd, str(code), str(out)))
        return None

    matches = re.findall(r"([\d\.]+(?:-\d+)?)", out)

    if matches and len(matches) == 1:
        stack_version = matches[0]
    elif matches and len(matches) > 1:
        Logger.error(
            "Found multiple matches for stack version, cannot identify the correct one from: {0}"
            .format(", ".join(matches)))

    return stack_version

Ejemplo n.º 40
0
if 'httpProxyPassword' in config['configurations']['streamline-common']:
    http_proxy_password = config['configurations']['streamline-common'][
        'httpProxyPassword']
else:
    http_proxy_password = None

streamline_catalog_root_url = 'http://{0}:{1}/api/v1/catalog'.format(
    hostname, streamline_port)

# mysql jar
jdk_location = config['hostLevelParams']['jdk_location']
if 'mysql' == streamline_storage_type:
    jdbc_driver_jar = default("/hostLevelParams/custom_mysql_jdbc_name", None)
    if jdbc_driver_jar == None:
        Logger.error(
            "Failed to find mysql-java-connector jar. Make sure you followed the steps to register mysql driver"
        )
        Logger.info("Users should register the mysql java driver jar.")
        Logger.info("yum install mysql-connector-java*")
        Logger.info(
            "sudo ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar"
        )

    connector_curl_source = format("{jdk_location}/{jdbc_driver_jar}")
    connector_download_dir = format("{streamline_home}/libs")
    connector_bootstrap_download_dir = format(
        "{streamline_home}/bootstrap/lib")
    downloaded_custom_connector = format("{tmp_dir}/{jdbc_driver_jar}")

check_db_connection_jar_name = "DBConnectionVerification.jar"
check_db_connection_jar = format(
Ejemplo n.º 41
0
def enable_kms_plugin():

    import params

    if params.has_ranger_admin:
        count = 0
        while count < 5:
            ranger_flag = check_ranger_service()
            if ranger_flag:
                break
            else:
                time.sleep(5)  # delay for 5 seconds
                count = count + 1
        else:
            Logger.error(
                "Ranger service is not reachable after {0} tries".format(
                    count))

        current_datetime = datetime.now()

        File(
            format('{kms_conf_dir}/ranger-security.xml'),
            owner=params.kms_user,
            group=params.kms_group,
            mode=0644,
            content=InlineTemplate(
                format(
                    '<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
                )))

        Directory([
            os.path.join('/etc', 'ranger', params.repo_name),
            os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
        ],
                  owner=params.kms_user,
                  group=params.kms_group,
                  mode=0775,
                  recursive=True)

        File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
                          format('kms_{repo_name}.json')),
             owner=params.kms_user,
             group=params.kms_group,
             mode=0644)

        XmlConfig(
            "ranger-kms-audit.xml",
            conf_dir=params.kms_conf_dir,
            configurations=params.config['configurations']['ranger-kms-audit'],
            configuration_attributes=params.config['configuration_attributes']
            ['ranger-kms-audit'],
            owner=params.kms_user,
            group=params.kms_group,
            mode=0744)

        XmlConfig(
            "ranger-kms-security.xml",
            conf_dir=params.kms_conf_dir,
            configurations=params.config['configurations']
            ['ranger-kms-security'],
            configuration_attributes=params.config['configuration_attributes']
            ['ranger-kms-security'],
            owner=params.kms_user,
            group=params.kms_group,
            mode=0744)

        XmlConfig(
            "ranger-policymgr-ssl.xml",
            conf_dir=params.kms_conf_dir,
            configurations=params.config['configurations']
            ['ranger-kms-policymgr-ssl'],
            configuration_attributes=params.config['configuration_attributes']
            ['ranger-kms-policymgr-ssl'],
            owner=params.kms_user,
            group=params.kms_group,
            mode=0744)

        if params.xa_audit_db_is_enabled:
            cred_setup = params.cred_setup_prefix + (
                '-f', params.credential_file, '-k', 'auditDBCred', '-v',
                PasswordString(params.xa_audit_db_password), '-c', '1')
            Execute(cred_setup,
                    environment={'JAVA_HOME': params.java_home},
                    logoutput=True,
                    sudo=True)

        cred_setup = params.cred_setup_prefix + (
            '-f', params.credential_file, '-k', 'sslKeyStore', '-v',
            PasswordString(params.ssl_keystore_password), '-c', '1')
        Execute(cred_setup,
                environment={'JAVA_HOME': params.java_home},
                logoutput=True,
                sudo=True)

        cred_setup = params.cred_setup_prefix + (
            '-f', params.credential_file, '-k', 'sslTrustStore', '-v',
            PasswordString(params.ssl_truststore_password), '-c', '1')
        Execute(cred_setup,
                environment={'JAVA_HOME': params.java_home},
                logoutput=True,
                sudo=True)

        File(params.credential_file,
             owner=params.kms_user,
             group=params.kms_group,
             mode=0640)
Ejemplo n.º 42
0
    def service_check(self, env):
        import params
        env.set_params(params)

        if params.streamline_ssl_enabled:
            streamline_api = format(
                "https://{params.hostname}:{params.streamline_ssl_port}/api/v1/catalog/streams/componentbundles"
            )
        else:
            streamline_api = format(
                "http://{params.hostname}:{params.streamline_port}/api/v1/catalog/streams/componentbundles"
            )

        Logger.info(streamline_api)
        max_retries = 3
        success = False

        if (params.security_enabled) and (not params.streamline_sso_enabled):
            kinit_cmd = format(
                "{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};"
            )
            return_code, out = shell.checked_call(
                kinit_cmd,
                path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
                user=params.smokeuser,
            )

        for num in range(0, max_retries):
            try:
                Logger.info(format("Making http requests to {streamline_api}"))

                if params.security_enabled:
                    get_app_info_cmd = "curl --negotiate -u : -ks --location-trusted --connect-timeout " + CURL_CONNECTION_TIMEOUT + " " + streamline_api
                    return_code, stdout, _ = get_user_call_output(
                        get_app_info_cmd,
                        user=params.smokeuser,
                        path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
                    )
                    try:
                        json_response = json.loads(stdout)
                        success = True
                        Logger.info(
                            format(
                                "Successfully made a API request to SAM. {stdout}"
                            ))
                        break
                    except Exception as e:
                        Logger.error(
                            format(
                                "Response from SAM API was not a valid JSON. Response: {stdout}"
                            ))
                else:
                    response = urllib2.urlopen(streamline_api)
                    api_response = response.read()
                    response_code = response.getcode()
                    Logger.info(format("SAM response http status {response}"))
                    if response.getcode() != 200:
                        Logger.error(
                            format(
                                "Failed to fetch response for {streamline_api}"
                            ))
                        show_logs(params.streamline_log_dir,
                                  params.streamline_user)
                        raise
                    else:
                        success = True
                        Logger.info(
                            format(
                                "Successfully made a API request to SAM. {api_response}"
                            ))
                        break
            except (urllib2.URLError, ExecutionFailed) as e:
                Logger.error(
                    format(
                        "Failed to make API request to SAM server at {streamline_api},retrying.. {num} out of {max_retries}"
                    ))
                time.sleep(num * 10)  #exponential back-off
                continue

        if success != True:
            Logger.error(
                format(
                    "Failed to make API request to  SAM server at {streamline_api} after {max_retries}"
                ))
            raise
                else:
                    return None

        except urllib2.HTTPError, e:
            Logger.error(
                "Error during Ranger service authentication. Http status code - {0}. {1}"
                .format(e.code, e.read()))
            return None

        except urllib2.URLError, e:
            Logger.error(
                "Error during Ranger service authentication. {0}".format(
                    e.reason))
            return None
        except Exception, e:
            Logger.error(
                "Error occured when connecting Ranger admin. {0} ".format(e))
            return None

    def check_if_not_policy_exist(self, service_name):
        try:
            url = self.rangerurl + "/service/public/v2/api/service/" + \
                  service_name + "/policy?policyName=dpprofiler-audit-read"
            Logger.info("Checking ranger policy. Url : {0}".format(url))
            usernamepassword = '******'.format(self.username, self.password)
            base_64_string = base64.encodestring(usernamepassword).replace(
                '\n', '')
            request = urllib2.Request(url)
            request.add_header("Content-Type", "application/json")
            request.add_header("Accept", "application/json")
            request.add_header("Authorization",
                               "Basic {0}".format(base_64_string))
Ejemplo n.º 44
0
    def _llap_start(self, env, cleanup=False):
      import params
      env.set_params(params)
      Logger.info("Starting LLAP")
      LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()
      LLAP_APP_NAME = 'llap0'

      unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')

      cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}"
                   " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m "
                   " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}"
                   " --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
      if params.security_enabled:
        llap_keytab_splits = params.hive_llap_keytab_file.split("/")
        Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
        cmd += format(" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
                      "{llap_keytab_splits[4]} --slider-principal {params.hive_llap_principal}")

      # Append args.
      llap_java_args = InlineTemplate(params.llap_app_java_opts).get_content()
      cmd += format(" --args \" {llap_java_args}\"")

      run_file_path = None
      try:
        Logger.info(format("Command: {cmd}"))
        code, output, error = shell.checked_call(cmd, user=params.hive_user, stderr=subprocess.PIPE, logoutput=True)

        if code != 0 or output is None:
          raise Fail("Command failed with either non-zero return code or no output.")

        # E.g., output:
        # Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider
        exp = r"Prepared (.*?run.sh) for running LLAP"
        run_file_path = None
        out_splits = output.split("\n")
        for line in out_splits:
          line = line.strip()
          m = re.match(exp, line, re.I)
          if m and len(m.groups()) == 1:
            run_file_name = m.group(1)
            run_file_path = os.path.join(params.hive_user_home_dir, run_file_name)
            break
        if not run_file_path:
          raise Fail("Did not find run.sh file in output: " + str(output))

        Logger.info(format("Run file path: {run_file_path}"))
        Execute(run_file_path, user=params.hive_user, logoutput=True)
        Logger.info("Submitted LLAP app name : {0}".format(LLAP_APP_NAME))

        # We need to check the status of LLAP app to figure out it got
        # launched properly and is in running state. Then go ahead with Hive Interactive Server start.
        status = self.check_llap_app_status(LLAP_APP_NAME, params.num_retries_for_checking_llap_status)
        if status:
          Logger.info("LLAP app '{0}' deployed successfully.".format(LLAP_APP_NAME))
          return True
        else:
          Logger.error("LLAP app '{0}' deployment unsuccessful.".format(LLAP_APP_NAME))
          return False
      except:
        # Attempt to clean up the packaged application, or potentially rename it with a .bak
        if run_file_path is not None and cleanup:
          try:
            parent_dir = os.path.dirname(run_file_path)
            if os.path.isdir(parent_dir):
              shutil.rmtree(parent_dir)
          except Exception, e:
            Logger.error("Could not cleanup LLAP app package. Error: " + str(e))

        # throw the original exception
        raise
Ejemplo n.º 45
0
    def _llap_start(self, env, cleanup=False):
      import params
      env.set_params(params)

      LLAP_APP_NAME = 'llap0'

      if params.hive_server_interactive_ha:
        """
        Check llap app state
        """
        Logger.info("HSI HA is enabled. Checking if LLAP is already running ...")
        status = self.check_llap_app_status(LLAP_APP_NAME, 2, params.hive_server_interactive_ha)
        if status:
          Logger.info("LLAP app '{0}' is already running.".format(LLAP_APP_NAME))
          return True
        else:
          Logger.info("LLAP app '{0}' is not running. llap will be started.".format(LLAP_APP_NAME))
        pass

      # Call for cleaning up the earlier run(s) LLAP package folders.
      self._cleanup_past_llap_package_dirs()

      Logger.info("Starting LLAP")
      LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()

      unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')

      # Figure out the Slider Anti-affinity to be used.
      # YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it
      # starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP
      # container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided.
      slider_placement = 4
      if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)):
        slider_placement = 0
        Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * "
                    "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))
      else:
        Logger.info("Setting slider_placement: 4, as llap_daemon_container_size : {0} <= 0.5 * "
                    "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))


      cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}"
                   " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m"
                   " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}"
                   " --slider-placement {slider_placement} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}"
                   " {params.llap_extra_slider_opts} --skiphadoopversion --skiphbasecp --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
      if params.security_enabled:
        llap_keytab_splits = params.hive_llap_keytab_file.split("/")
        Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
        cmd += format(" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
                      "{llap_keytab_splits[4]} --slider-principal {params.hive_llap_principal}")

      # Add the aux jars if they are specified. If empty, dont need to add this param.
      if params.hive_aux_jars:
        cmd+= format(" --auxjars {params.hive_aux_jars}")

      # Append args.
      llap_java_args = InlineTemplate(params.llap_app_java_opts).get_content()
      cmd += format(" --args \" {llap_java_args}\"")

      run_file_path = None
      try:
        Logger.info(format("Command: {cmd}"))
        code, output, error = shell.checked_call(cmd, user=params.hive_user, stderr=subprocess.PIPE, logoutput=True)

        if code != 0 or output is None:
          raise Fail("Command failed with either non-zero return code or no output.")

        # E.g., output:
        # Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider
        exp = r"Prepared (.*?run.sh) for running LLAP"
        run_file_path = None
        out_splits = output.split("\n")
        for line in out_splits:
          line = line.strip()
          m = re.match(exp, line, re.I)
          if m and len(m.groups()) == 1:
            run_file_name = m.group(1)
            run_file_path = os.path.join(params.hive_user_home_dir, run_file_name)
            break
        if not run_file_path:
          raise Fail("Did not find run.sh file in output: " + str(output))

        Logger.info(format("Run file path: {run_file_path}"))
        Execute(run_file_path, user=params.hive_user, logoutput=True)
        Logger.info("Submitted LLAP app name : {0}".format(LLAP_APP_NAME))

        # We need to check the status of LLAP app to figure out it got
        # launched properly and is in running state. Then go ahead with Hive Interactive Server start.
        status = self.check_llap_app_status(LLAP_APP_NAME, params.num_retries_for_checking_llap_status)
        if status:
          Logger.info("LLAP app '{0}' deployed successfully.".format(LLAP_APP_NAME))
          return True
        else:
          Logger.error("LLAP app '{0}' deployment unsuccessful.".format(LLAP_APP_NAME))
          return False
      except:
        # Attempt to clean up the packaged application, or potentially rename it with a .bak
        if run_file_path is not None and cleanup:
          parent_dir = os.path.dirname(run_file_path)
          Directory(parent_dir,
                    action = "delete",
                    ignore_failures = True,
          )

        # throw the original exception
        raise
Ejemplo n.º 46
0
                    raise Exception("Code is nonzero or output is empty")

                Logger.debug("Command: %s\nOutput: %s" %
                             (get_stack_comp_version_cmd, str(out)))
                matches = re.findall(r"( [\d\.]+(\-\d+)?)", out)
                version = matches[0][0].strip() if matches and len(
                    matches) > 0 and len(matches[0]) > 0 else None
                Logger.debug("Version for component %s: %s" %
                             (component_name, str(version)))
            except Exception, e:
                Logger.error(
                    "Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s."
                    % (component_name, get_stack_comp_version_cmd, str(code),
                       str(out)))
        else:
            Logger.error("Could not find stack selector for stack: %s" %
                         str(stack_name))

    return version


def get_component_version_with_stack_selector(stack_selector_path,
                                              component_name):
    """
   For specific cases where we deal with HDP add on services from a management pack, the version
   needs to be determined by using the specific stack selector itself.
   :param stack_selector_path: /usr/bin/hdf-select
   Comes from the service which calls for this function.
   :param component_name: Component name as a string necessary to get the version
   :return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
   This function can be called by custom services, hence should not be removed
  """
Ejemplo n.º 47
0
def kms(upgrade_type=None):
    import params

    if params.has_ranger_admin:

        Directory(params.kms_conf_dir,
                  owner=params.kms_user,
                  group=params.kms_group,
                  create_parents=True)

        Directory("/etc/security/serverKeys",
                  create_parents=True,
                  cd_access="a")

        Directory("/etc/ranger/kms", create_parents=True, cd_access="a")

        copy_jdbc_connector()

        File(
            format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
            content=DownloadSource(
                format("{jdk_location}{check_db_connection_jar_name}")),
            mode=0644,
        )

        cp = format("{check_db_connection_jar}")
        if params.db_flavor.lower() == 'sqla':
            cp = cp + os.pathsep + format(
                "{kms_home}/ews/webapp/lib/sajdbc4.jar")
        else:
            path_to_jdbc = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
            if not os.path.isfile(path_to_jdbc):
                path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + \
                               params.default_connectors_map[params.db_flavor.lower()] if params.db_flavor.lower() in params.default_connectors_map else None
                if not os.path.isfile(path_to_jdbc):
                    path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + "*"
                    error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.db_flavor] + \
                          " in ranger kms lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
                    Logger.error(error_message)

            cp = cp + os.pathsep + path_to_jdbc

        db_connection_check_command = format(
            "{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_kms_jdbc_connection_url}' {db_user} {db_password!p} {ranger_kms_jdbc_driver}"
        )

        env_dict = {}
        if params.db_flavor.lower() == 'sqla':
            env_dict = {'LD_LIBRARY_PATH': params.ld_library_path}

        Execute(db_connection_check_command,
                path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
                tries=5,
                try_sleep=10,
                environment=env_dict)

        if params.xa_audit_db_is_enabled and params.driver_source is not None and not params.driver_source.endswith(
                "/None"):
            if params.xa_previous_jdbc_jar and os.path.isfile(
                    params.xa_previous_jdbc_jar):
                File(params.xa_previous_jdbc_jar, action='delete')

            File(params.downloaded_connector_path,
                 content=DownloadSource(params.driver_source),
                 mode=0644)

            Execute(('cp', '--remove-destination',
                     params.downloaded_connector_path, params.driver_target),
                    path=["/bin", "/usr/bin/"],
                    sudo=True)

            File(params.driver_target, mode=0644)

        Directory(os.path.join(params.kms_home, 'ews', 'webapp', 'WEB-INF',
                               'classes', 'lib'),
                  mode=0755,
                  owner=params.kms_user,
                  group=params.kms_group)

        Execute(('cp', format('{kms_home}/ranger-kms-initd'),
                 '/etc/init.d/ranger-kms'),
                not_if=format('ls /etc/init.d/ranger-kms'),
                only_if=format('ls {kms_home}/ranger-kms-initd'),
                sudo=True)

        File('/etc/init.d/ranger-kms', mode=0755)

        Directory(
            format('{kms_home}/'),
            owner=params.kms_user,
            group=params.kms_group,
            recursive_ownership=True,
        )

        Directory(params.ranger_kms_pid_dir,
                  mode=0755,
                  owner=params.kms_user,
                  group=params.user_group,
                  cd_access="a",
                  create_parents=True)

        if params.stack_supports_pid:
            File(
                format('{kms_conf_dir}/ranger-kms-env-piddir.sh'),
                content=format(
                    "export RANGER_KMS_PID_DIR_PATH={ranger_kms_pid_dir}\nexport KMS_USER={kms_user}"
                ),
                owner=params.kms_user,
                group=params.kms_group,
                mode=0755)

        Directory(params.kms_log_dir,
                  owner=params.kms_user,
                  group=params.kms_group,
                  cd_access='a',
                  create_parents=True,
                  mode=0755)

        File(format('{kms_conf_dir}/ranger-kms-env-logdir.sh'),
             content=format("export RANGER_KMS_LOG_DIR={kms_log_dir}"),
             owner=params.kms_user,
             group=params.kms_group,
             mode=0755)

        Execute(('ln', '-sf', format('{kms_home}/ranger-kms'),
                 '/usr/bin/ranger-kms'),
                not_if=format('ls /usr/bin/ranger-kms'),
                only_if=format('ls {kms_home}/ranger-kms'),
                sudo=True)

        File('/usr/bin/ranger-kms', mode=0755)

        Execute(('ln', '-sf', format('{kms_home}/ranger-kms'),
                 '/usr/bin/ranger-kms-services.sh'),
                not_if=format('ls /usr/bin/ranger-kms-services.sh'),
                only_if=format('ls {kms_home}/ranger-kms'),
                sudo=True)

        File('/usr/bin/ranger-kms-services.sh', mode=0755)

        Execute(('ln', '-sf', format('{kms_home}/ranger-kms-initd'),
                 format('{kms_home}/ranger-kms-services.sh')),
                not_if=format('ls {kms_home}/ranger-kms-services.sh'),
                only_if=format('ls {kms_home}/ranger-kms-initd'),
                sudo=True)

        File(format('{kms_home}/ranger-kms-services.sh'), mode=0755)

        Directory(params.kms_log_dir,
                  owner=params.kms_user,
                  group=params.kms_group,
                  mode=0775)

        do_keystore_setup(params.credential_provider_path, params.jdbc_alias,
                          params.db_password)
        do_keystore_setup(params.credential_provider_path,
                          params.masterkey_alias,
                          params.kms_master_key_password)
        if params.stack_support_kms_hsm and params.enable_kms_hsm:
            do_keystore_setup(params.credential_provider_path,
                              params.hms_partition_alias,
                              unicode(params.hms_partition_passwd))
        if params.stack_supports_ranger_kms_ssl and params.ranger_kms_ssl_enabled:
            do_keystore_setup(params.ranger_kms_cred_ssl_path,
                              params.ranger_kms_ssl_keystore_alias,
                              params.ranger_kms_ssl_passwd)

        # remove plain-text password from xml configs
        dbks_site_copy = {}
        dbks_site_copy.update(params.config['configurations']['dbks-site'])

        for prop in params.dbks_site_password_properties:
            if prop in dbks_site_copy:
                dbks_site_copy[prop] = "_"

        XmlConfig(
            "dbks-site.xml",
            conf_dir=params.kms_conf_dir,
            configurations=dbks_site_copy,
            configuration_attributes=params.config['configuration_attributes']
            ['dbks-site'],
            owner=params.kms_user,
            group=params.kms_group,
            mode=0644)

        ranger_kms_site_copy = {}
        ranger_kms_site_copy.update(
            params.config['configurations']['ranger-kms-site'])
        if params.stack_supports_ranger_kms_ssl:
            # remove plain-text password from xml configs
            for prop in params.ranger_kms_site_password_properties:
                if prop in ranger_kms_site_copy:
                    ranger_kms_site_copy[prop] = "_"

        XmlConfig(
            "ranger-kms-site.xml",
            conf_dir=params.kms_conf_dir,
            configurations=ranger_kms_site_copy,
            configuration_attributes=params.config['configuration_attributes']
            ['ranger-kms-site'],
            owner=params.kms_user,
            group=params.kms_group,
            mode=0644)

        XmlConfig(
            "kms-site.xml",
            conf_dir=params.kms_conf_dir,
            configurations=params.config['configurations']['kms-site'],
            configuration_attributes=params.config['configuration_attributes']
            ['kms-site'],
            owner=params.kms_user,
            group=params.kms_group,
            mode=0644)

        File(os.path.join(params.kms_conf_dir, "kms-log4j.properties"),
             owner=params.kms_user,
             group=params.kms_group,
             content=InlineTemplate(params.kms_log4j),
             mode=0644)
        if params.security_enabled:
            # core-site.xml linking required by setup for HDFS encryption
            XmlConfig(
                "core-site.xml",
                conf_dir=params.kms_conf_dir,
                configurations=params.config['configurations']['core-site'],
                configuration_attributes=params.
                config['configuration_attributes']['core-site'],
                owner=params.kms_user,
                group=params.kms_group,
                mode=0644)
        else:
            File(format('{kms_conf_dir}/core-site.xml'), action="delete")
Ejemplo n.º 48
0
def copy_tarballs_to_hdfs(source, dest, hdp_select_component_name,
                          component_user, file_owner, group_owner):
    """
  :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
  :param hdp_select_component_name: Component name to get the status to determine the version
  :param component_user: User that will execute the Hadoop commands
  :param file_owner: Owner of the files copied to HDFS (typically hdfs account)
  :param group_owner: Group owner of the files copied to HDFS (typically hadoop group)
  :return: Returns 0 on success, 1 if no files were copied, and in some cases may raise an exception.
 
  In order to call this function, params.py must have all of the following,
  hdp_stack_version, kinit_path_local, security_enabled, hdfs_user, hdfs_principal_name, hdfs_user_keytab,
  hadoop_bin_dir, hadoop_conf_dir, and HdfsDirectory as a partial function.
  """

    component_tar_source_file, component_tar_destination_folder = source, dest

    if not os.path.exists(component_tar_source_file):
        Logger.warning("Could not find file: %s" %
                       str(component_tar_source_file))
        return 1

    # Ubuntu returns: "stdin: is not a tty", as subprocess output.
    tmpfile = tempfile.NamedTemporaryFile()
    out = None
    with open(tmpfile.name, 'r+') as file:
        get_hdp_version_cmd = '/usr/bin/hdp-select status %s > %s' % (
            hdp_select_component_name, tmpfile.name)
        code, stdoutdata = shell.call(get_hdp_version_cmd)
        out = file.read()
    pass
    if code != 0 or out is None:
        Logger.warning(
            "Could not verify HDP version by calling '%s'. Return Code: %s, Output: %s."
            % (get_hdp_version_cmd, str(code), str(out)))
        return 1

    matches = re.findall(r"([\d\.]+\-\d+)", out)
    hdp_version = matches[0] if matches and len(matches) > 0 else None

    if not hdp_version:
        Logger.error(
            "Could not parse HDP version from output of hdp-select: %s" %
            str(out))
        return 1

    file_name = os.path.basename(component_tar_source_file)
    destination_file = os.path.join(component_tar_destination_folder,
                                    file_name)
    destination_file = destination_file.replace("{{ hdp_stack_version }}",
                                                hdp_version)

    kinit_if_needed = ""
    if params.security_enabled:
        kinit_if_needed = format(
            "{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};")

    if kinit_if_needed:
        Execute(kinit_if_needed, user=component_user, path='/bin')

    does_hdfs_file_exist_cmd = "fs -ls %s" % destination_file
    does_hdfs_file_exist = False
    try:
        ExecuteHadoop(does_hdfs_file_exist_cmd,
                      user=component_user,
                      logoutput=True,
                      conf_dir=params.hadoop_conf_dir,
                      bin_dir=params.hadoop_bin_dir)
        does_hdfs_file_exist = True
    except Fail:
        pass

    if not does_hdfs_file_exist:
        source_and_dest_pairs = [
            (component_tar_source_file, destination_file),
        ]
        return _copy_files(source_and_dest_pairs, file_owner, group_owner,
                           kinit_if_needed)
    return 1
Ejemplo n.º 49
0
  get_stack_versions_cmd = "/usr/bin/hdp-select versions > {0}".format(tmp_file)
  try:
    code, stdoutdata = shell.call(get_stack_versions_cmd, logoutput=True)
    with open(tmp_file, 'r+') as file:
      out = file.read()
  except Exception, e:
    Logger.logger.exception("Could not parse output of {0}. Error: {1}".format(str(tmp_file), str(e)))
  finally:
    try:
      if os.path.exists(tmp_file):
        os.remove(tmp_file)
    except Exception, e:
      Logger.logger.exception("Could not remove file {0}. Error: {1}".format(str(tmp_file), str(e)))

  if code != 0 or out is None or out == "":
    Logger.error("Could not verify HDP version by calling '{0}'. Return Code: {1}, Output: {2}.".format(get_stack_versions_cmd, str(code), str(out)))
    return None

  matches = re.findall(r"([\d\.]+\-\d+)", out)

  if matches and len(matches) == 1:
    stack_version = matches[0]
  elif matches and len(matches) > 1:
    Logger.error("Found multiple matches for HDP version, cannot identify the correct one from: {0}".format(", ".join(matches)))

  return stack_version

def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=None, custom_dest_file=None, force_execute=False,
                 use_upgrading_version_during_uprade=True, replace_existing_files=False, host_sys_prepped=False):
  """
  :param name: Tarball name, e.g., tez, hive, pig, sqoop.
Ejemplo n.º 50
0
    def create_ranger_repository(self,
                                 component,
                                 repo_name,
                                 repo_properties,
                                 ambari_ranger_admin,
                                 ambari_ranger_password,
                                 admin_uname,
                                 admin_password,
                                 policy_user,
                                 is_security_enabled=False,
                                 is_stack_supports_ranger_kerberos=False,
                                 component_user=None,
                                 component_user_principal=None,
                                 component_user_keytab=None):
        if not is_stack_supports_ranger_kerberos or not is_security_enabled:
            response_code = self.check_ranger_login_urllib2(self.base_url)
            repo_data = json.dumps(repo_properties)
            ambari_ranger_password = unicode(ambari_ranger_password)
            admin_password = unicode(admin_password)
            ambari_username_password_for_ranger = format(
                '{ambari_ranger_admin}:{ambari_ranger_password}')

            if response_code is not None and response_code == 200:
                user_resp_code = self.create_ambari_admin_user(
                    ambari_ranger_admin, ambari_ranger_password,
                    format("{admin_uname}:{admin_password}"))
                if user_resp_code is not None and user_resp_code == 200:
                    retryCount = 0
                    while retryCount <= 5:
                        repo = self.get_repository_by_name_urllib2(
                            repo_name, component, 'true',
                            ambari_username_password_for_ranger)
                        if repo is not None:
                            Logger.info('{0} Repository {1} exist'.format(
                                component.title(), repo['name']))
                            break
                        else:
                            response = self.create_repository_urllib2(
                                repo_data, ambari_username_password_for_ranger)
                            if response is not None:
                                Logger.info(
                                    '{0} Repository created in Ranger admin'.
                                    format(component.title()))
                                break
                            else:
                                if retryCount < 5:
                                    Logger.info(
                                        "Retry Repository Creation is being called"
                                    )
                                    time.sleep(30)  # delay for 30 seconds
                                    retryCount += 1
                                else:
                                    Logger.error(
                                        '{0} Repository creation failed in Ranger admin'
                                        .format(component.title()))
                                    break
                else:
                    Logger.error('Ambari admin user creation failed')
            elif not self.skip_if_rangeradmin_down:
                Logger.error("Connection failed to Ranger Admin !")
        elif is_stack_supports_ranger_kerberos and is_security_enabled:
            response = self.check_ranger_login_curl(component_user,
                                                    component_user_keytab,
                                                    component_user_principal,
                                                    self.url_login, True)

            if response and response[0] == 200:
                retryCount = 0
                repo_data = json.dumps(repo_properties)
                while retryCount <= 5:
                    response = self.get_repository_by_name_curl(
                        component_user, component_user_keytab,
                        component_user_principal, repo_name, component, 'true')
                    if response is not None:
                        Logger.info('{0} Repository {1} exist'.format(
                            component.title(), (response['name'])))
                        break
                    else:
                        response = self.create_repository_curl(
                            component_user, component_user_keytab,
                            component_user_principal, repo_name, repo_data,
                            policy_user)
                        if response and len(response) > 0:
                            Logger.info(
                                '{0} Repository created in Ranger admin'.
                                format(component.title()))
                            break
                        else:
                            if retryCount < 5:
                                time.sleep(30)  # delay for 30 seconds
                                retryCount += 1
                            else:
                                Logger.error(
                                    '{0} Repository creation failed in Ranger admin'
                                    .format(component.title()))
                                break
            else:
                Logger.error("Connection failed to Ranger Admin !")
Ejemplo n.º 51
0
 def __check_if_client_exists(self, serviceName):
     Logger.info("Checking if " + serviceName + " client libraries exist")
     if not self.__package_exists(serviceName):
         error_msg = serviceName + " client libraries do not exist on the PXF node"
         Logger.error(error_msg)
         raise Fail(error_msg)
Ejemplo n.º 52
0
            return True
        else:
            Logger.info('Repository not created')
            return False
    except urllib2.URLError, e:
        if isinstance(e, urllib2.HTTPError):
            Logger.error(
                "Error creating service. Http status code - {0}. \n {1}".
                format(e.code, e.read()))
            return False
        else:
            Logger.error("Error creating service. Reason - {0}.".format(
                e.reason))
            return False
    except socket.timeout as e:
        Logger.error("Error creating service. Reason - {0}".format(e))
        return False


def get_repo(url, name, usernamepassword):
    try:
        base_url = url + '/service/public/v2/api/service?serviceName=' + name + '&serviceType=kms&isEnabled=true'
        request = urllib2.Request(base_url)
        base64string = base64.encodestring(usernamepassword).replace('\n', '')
        request.add_header("Content-Type", "application/json")
        request.add_header("Accept", "application/json")
        request.add_header("Authorization", "Basic {0}".format(base64string))
        result = urllib2.urlopen(request, timeout=20)
        response_code = result.getcode()
        response = json.loads(result.read())
        if response_code == 200 and len(response) > 0:
Ejemplo n.º 53
0
def hive_service(name, action='start', upgrade_type=None):

    import params
    import status_params

    if name == 'metastore':
        pid_file = status_params.hive_metastore_pid
        cmd = format(
            "{start_metastore_path} {hive_log_dir}/hive.out {hive_log_dir}/hive.err {pid_file} {hive_server_conf_dir} {hive_log_dir}"
        )
    elif name == 'hiveserver2':
        pid_file = status_params.hive_pid
        cmd = format(
            "{start_hiveserver2_path} {hive_log_dir}/hive-server2.out {hive_log_dir}/hive-server2.err {pid_file} {hive_server_conf_dir} {hive_log_dir}"
        )

        if params.security_enabled and check_stack_feature(
                StackFeature.HIVE_SERVER2_KERBERIZED_ENV,
                params.version_for_stack_feature_checks):
            hive_kinit_cmd = format(
                "{kinit_path_local} -kt {hive_server2_keytab} {hive_principal}; "
            )
            Execute(hive_kinit_cmd, user=params.hive_user)

    pid = get_user_call_output.get_user_call_output(format("cat {pid_file}"),
                                                    user=params.hive_user,
                                                    is_checked_call=False)[1]
    process_id_exists_command = format(
        "ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1")

    if action == 'start':
        if name == 'hiveserver2':
            check_fs_root(params.hive_server_conf_dir, params.execute_path)

        daemon_cmd = cmd
        hadoop_home = params.hadoop_home
        hive_bin = "hive"

        # upgrading hiveserver2 (rolling_restart) means that there is an existing,
        # de-registering hiveserver2; the pid will still exist, but the new
        # hiveserver is spinning up on a new port, so the pid will be re-written
        if upgrade_type == UPGRADE_TYPE_ROLLING:
            process_id_exists_command = None

            if params.version and params.stack_root:
                hadoop_home = format("{stack_root}/{version}/hadoop")
                hive_bin = os.path.join(params.hive_bin, hive_bin)

        Execute(daemon_cmd,
                user=params.hive_user,
                environment={
                    'HADOOP_HOME': hadoop_home,
                    'JAVA_HOME': params.java64_home,
                    'HIVE_BIN': hive_bin
                },
                path=params.execute_path,
                not_if=process_id_exists_command)

        if params.hive_jdbc_driver == "com.mysql.jdbc.Driver" or \
           params.hive_jdbc_driver == "org.postgresql.Driver" or \
           params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":

            validation_called = False

            if params.hive_jdbc_target is not None:
                validation_called = True
                validate_connection(params.hive_jdbc_target, params.hive_lib)
            if params.hive2_jdbc_target is not None:
                validation_called = True
                validate_connection(params.hive2_jdbc_target,
                                    params.hive_server2_hive2_lib)

            if not validation_called:
                emessage = "ERROR! DB connection check should be executed at least one time!"
                Logger.error(emessage)

    elif action == 'stop':

        daemon_kill_cmd = format("{sudo} kill {pid}")
        daemon_hard_kill_cmd = format("{sudo} kill -9 {pid}")

        Execute(daemon_kill_cmd,
                not_if=format("! ({process_id_exists_command})"))

        wait_time = 5
        Execute(
            daemon_hard_kill_cmd,
            not_if=format(
                "! ({process_id_exists_command}) || ( sleep {wait_time} && ! ({process_id_exists_command}) )"
            ),
            ignore_failures=True)

        try:
            # check if stopped the process, else fail the task
            Execute(
                format("! ({process_id_exists_command})"),
                tries=20,
                try_sleep=3,
            )
        except:
            show_logs(params.hive_log_dir, params.hive_user)
            raise

        File(pid_file, action="delete")
Ejemplo n.º 54
0
      run_prepare_war = True
      Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} has contents which differ.\n" \
      "Expected: {command_to_file}.\nActual: {cmd}."))
  else:
    run_prepare_war = True
    Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} is missing."))

  if run_prepare_war:
    # Time-consuming to run
    return_code, output = shell.call(command, user=params.oozie_user)
    if output is None:
      output = ""

    if return_code != 0 or "New Oozie WAR file with added".lower() not in output.lower():
      message = "Unexpected Oozie WAR preparation output {0}".format(output)
      Logger.error(message)
      raise Fail(message)

    # Generate marker file
    File(prepare_war_cmd_file,
         content=command_to_file,
         mode=0644,
    )
  else:
    Logger.info(format("No need to run prepare-war since marker file {prepare_war_cmd_file} already exists."))
  ###############################################
  # PREPARE-WAR END [BEGIN]
  ###############################################
  oozie_shared_lib = format("/usr/hdp/{hdp_version}/oozie/share")
  oozie_user = '******'
  oozie_hdfs_user_dir = format("{hdfs_path_prefix}/user/{oozie_user}")
Ejemplo n.º 55
0
    error_message = ""
    for check_name in FALLIBLE_CHECKS:
      if check_name in structured_output and "exit_code" in structured_output[check_name] \
          and structured_output[check_name]["exit_code"] != 0:
        error_message += "Check {0} was unsuccessful. Exit code: {1}.".format(check_name, \
                                                                             structured_output[check_name]["exit_code"])
        if "message" in structured_output[check_name]:
          error_message += " Message: {0}".format(structured_output[check_name]["message"])
        error_message += "\n"

    Logger.info("Host checks completed.")
    Logger.debug("Structured output: " + str(structured_output))

    if error_message:
      Logger.error(error_message)
      raise Fail(error_message)


  def execute_transparent_huge_page_check(self, config):
    Logger.info("Transparent huge page check started.")

    thp_regex = "\[(.+)\]"
    file_name = None
    if OSCheck.is_ubuntu_family():
      file_name = THP_FILE_UBUNTU
    elif OSCheck.is_redhat_family():
      file_name = THP_FILE_REDHAT
    if file_name and os.path.isfile(file_name):
      with open(file_name) as f:
        file_content = f.read()
Ejemplo n.º 56
0
def hive_interactive(name=None):
    import params

    MB_TO_BYTES = 1048576

    # if warehouse directory is in DFS
    if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(
            params.default_fs).scheme:
        # Create Hive Metastore Warehouse Dir
        params.HdfsResource(params.hive_metastore_warehouse_dir,
                            type="directory",
                            action="create_on_execute",
                            owner=params.hive_user,
                            group=params.user_group,
                            mode=0700)
        # create directories for llap package
        pkg_dir = '/user/' + params.hive_user + '/.yarn'
        for dir in [pkg_dir, pkg_dir + '/package', pkg_dir + '/package/LLAP']:
            # hdfsresouces handles parent creation badly
            params.HdfsResource(dir,
                                type="directory",
                                action="create_on_execute",
                                owner=params.hive_user,
                                group=params.user_group,
                                mode=0755)

        if not is_empty(params.tez_hook_proto_base_directory):
            params.HdfsResource(params.tez_hook_proto_base_directory,
                                type="directory",
                                action="create_on_execute",
                                owner=params.hive_user,
                                mode=01755)

        if not is_empty(params.hive_hook_proto_base_directory):
            params.HdfsResource(params.hive_hook_proto_base_directory,
                                type="directory",
                                action="create_on_execute",
                                owner=params.hive_user,
                                mode=01777)

            dag_meta = params.tez_hook_proto_base_directory + "dag_meta"
            params.HdfsResource(dag_meta,
                                type="directory",
                                action="create_on_execute",
                                owner=params.hive_user,
                                mode=01777)

            dag_data = params.tez_hook_proto_base_directory + "dag_data"
            params.HdfsResource(dag_data,
                                type="directory",
                                action="create_on_execute",
                                owner=params.hive_user,
                                mode=01777)

            app_data = params.tez_hook_proto_base_directory + "app_data"
            params.HdfsResource(app_data,
                                type="directory",
                                action="create_on_execute",
                                owner=params.hive_user,
                                mode=01777)

    else:
        Logger.info(
            format(
                "Not creating warehouse directory '{hive_metastore_warehouse_dir}', as the location is not in DFS."
            ))

    # Create Hive User Dir
    params.HdfsResource(params.hive_hdfs_user_dir,
                        type="directory",
                        action="create_on_execute",
                        owner=params.hive_user,
                        mode=params.hive_hdfs_user_mode)

    # list of properties that should be excluded from the config
    # this approach is a compromise against adding a dedicated config
    # type for hive_server_interactive or needed config groups on a
    # per component basis
    exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting']

    # List of configs to be excluded from hive2 client, but present in Hive2 server.
    exclude_list_for_hive2_client = [
        'javax.jdo.option.ConnectionPassword',
        'hadoop.security.credential.provider.path'
    ]

    Logger.info("Directories to fill with configs: %s" %
                str(params.hive_conf_dirs_list))
    for conf_dir in params.hive_conf_dirs_list:
        fill_conf_dir(conf_dir)
    '''
  As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml,
  we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml.
  '''
    merged_hive_interactive_site = {}
    merged_hive_interactive_site.update(params.hive_site_config)
    merged_hive_interactive_site.update(
        params.config['configurations']['hive-interactive-site'])
    for item in exclude_list:
        if item in merged_hive_interactive_site.keys():
            del merged_hive_interactive_site[item]

    merged_hive_interactive_site[
        'hive.llap.daemon.vcpus.per.instance'] = format(
            merged_hive_interactive_site['hive.llap.daemon.vcpus.per.instance']
        )
    merged_hive_interactive_site[
        'hive.server2.active.passive.ha.enable'] = str(
            params.hive_server_interactive_ha).lower()
    '''
  Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to
  convert it to bytes before we write it down to config file.
  '''
    if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys():
        hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get(
            "hive.llap.io.memory.size")
        hive_llap_io_mem_size_in_bytes = long(
            hive_llap_io_mem_size_in_mb) * MB_TO_BYTES
        merged_hive_interactive_site[
            'hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes
        Logger.info(
            "Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing "
            "it to config file.".format(hive_llap_io_mem_size_in_mb,
                                        hive_llap_io_mem_size_in_bytes))
    '''
  Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
  which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed.
  '''
    # Generate atlas-application.properties.xml file
    if params.enable_atlas_hook and params.stack_supports_atlas_hook_for_hive_interactive:
        Logger.info("Setup for Atlas Hive2 Hook started.")

        atlas_hook_filepath = os.path.join(
            params.hive_server_interactive_conf_dir,
            params.atlas_hook_filename)
        setup_atlas_hook(SERVICE.HIVE,
                         params.hive_atlas_application_properties,
                         atlas_hook_filepath, params.hive_user,
                         params.user_group)

        Logger.info("Setup for Atlas Hive2 Hook done.")
    else:
        # Required for HDP 2.5 stacks
        Logger.info(
            "Skipping setup for Atlas Hook, as it is disabled/ not supported.")
        remove_atlas_hook_if_exists(merged_hive_interactive_site)
    '''
  As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml,
  we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml.
  '''
    merged_tez_interactive_site = {}
    if 'tez-site' in params.config['configurations']:
        merged_tez_interactive_site.update(
            params.config['configurations']['tez-site'])
        Logger.info(
            "Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'."
        )
    else:
        Logger.error(
            "Tez's 'tez-site' couldn't be retrieved from passed-in configurations."
        )

    merged_tez_interactive_site.update(
        params.config['configurations']['tez-interactive-site'])

    XmlConfig("tez-site.xml",
              conf_dir=params.tez_interactive_conf_dir,
              configurations=merged_tez_interactive_site,
              configuration_attributes=params.config['configurationAttributes']
              ['tez-interactive-site'],
              owner=params.tez_interactive_user,
              group=params.user_group,
              mode=0664)
    '''
  Merge properties from hiveserver2-interactive-site into hiveserver2-site
  '''
    merged_hiveserver2_interactive_site = {}
    if 'hiveserver2-site' in params.config['configurations']:
        merged_hiveserver2_interactive_site.update(
            params.config['configurations']['hiveserver2-site'])
        Logger.info(
            "Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'."
        )
    else:
        Logger.error(
            "'hiveserver2-site' couldn't be retrieved from passed-in configurations."
        )
    merged_hiveserver2_interactive_site.update(
        params.config['configurations']['hiveserver2-interactive-site'])

    # Create config files under hive_server_interactive_conf_dir:
    #   hive-site.xml
    #   hive-env.sh
    #   llap-daemon-log4j2.properties
    #   llap-cli-log4j2.properties
    #   hive-log4j2.properties
    #   hive-exec-log4j2.properties
    #   beeline-log4j2.properties

    hive_server_interactive_conf_dir = params.hive_server_interactive_conf_dir

    mode_identified = 0600
    merged_hive_interactive_site = update_credential_provider_path(
        merged_hive_interactive_site, 'hive-site',
        os.path.join(conf_dir, 'hive-site.jceks'), params.hive_user,
        params.user_group)
    XmlConfig("hive-site.xml",
              conf_dir=hive_server_interactive_conf_dir,
              configurations=merged_hive_interactive_site,
              configuration_attributes=params.config['configurationAttributes']
              ['hive-interactive-site'],
              owner=params.hive_user,
              group=params.user_group,
              mode=0644)
    XmlConfig("hiveserver2-site.xml",
              conf_dir=hive_server_interactive_conf_dir,
              configurations=merged_hiveserver2_interactive_site,
              configuration_attributes=params.config['configurationAttributes']
              ['hiveserver2-interactive-site'],
              owner=params.hive_user,
              group=params.user_group,
              mode=mode_identified)

    File(format("{hive_server_interactive_conf_dir}/hive-env.sh"),
         owner=params.hive_user,
         group=params.user_group,
         mode=0755,
         content=InlineTemplate(params.hive_interactive_env_sh_template))

    llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties'
    File(format(
        "{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"),
         mode=mode_identified,
         group=params.user_group,
         owner=params.hive_user,
         content=InlineTemplate(params.llap_daemon_log4j))

    llap_cli_log4j2_filename = 'llap-cli-log4j2.properties'
    File(format(
        "{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"),
         mode=mode_identified,
         group=params.user_group,
         owner=params.hive_user,
         content=InlineTemplate(params.llap_cli_log4j2))

    hive_log4j2_filename = 'hive-log4j2.properties'
    File(format("{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"),
         mode=mode_identified,
         group=params.user_group,
         owner=params.hive_user,
         content=InlineTemplate(params.hive_log4j2))

    hive_exec_log4j2_filename = 'hive-exec-log4j2.properties'
    File(format(
        "{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"),
         mode=mode_identified,
         group=params.user_group,
         owner=params.hive_user,
         content=InlineTemplate(params.hive_exec_log4j2))

    beeline_log4j2_filename = 'beeline-log4j2.properties'
    File(
        format("{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"),
        mode=mode_identified,
        group=params.user_group,
        owner=params.hive_user,
        content=InlineTemplate(params.beeline_log4j2))

    XmlConfig("beeline-site.xml",
              conf_dir=conf_dir,
              configurations=params.beeline_site_config,
              owner=params.hive_user,
              group=params.user_group,
              mode=mode_identified)

    File(os.path.join(hive_server_interactive_conf_dir,
                      "hadoop-metrics2-hiveserver2.properties"),
         owner=params.hive_user,
         group=params.user_group,
         mode=mode_identified,
         content=Template("hadoop-metrics2-hiveserver2.properties.j2"))

    File(format(
        "{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"
    ),
         owner=params.hive_user,
         group=params.user_group,
         mode=mode_identified,
         content=Template("hadoop-metrics2-llapdaemon.j2"))

    File(format(
        "{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"
    ),
         owner=params.hive_user,
         group=params.user_group,
         mode=mode_identified,
         content=Template("hadoop-metrics2-llaptaskscheduler.j2"))

    # On some OS this folder could be not exists, so we will create it before pushing there files
    Directory(params.limits_conf_dir,
              create_parents=True,
              owner='root',
              group='root')

    File(os.path.join(params.limits_conf_dir, 'hive.conf'),
         owner='root',
         group='root',
         mode=0644,
         content=Template("hive.conf.j2"))

    if not os.path.exists(params.target_hive_interactive):
        jdbc_connector(params.target_hive_interactive,
                       params.hive_intaractive_previous_jdbc_jar)

    File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
         content=DownloadSource(
             format("{jdk_location}/{check_db_connection_jar_name}")),
         mode=0644)
    File(params.start_hiveserver2_interactive_path,
         mode=0755,
         content=Template(format('{start_hiveserver2_interactive_script}')))

    Directory(params.hive_pid_dir,
              create_parents=True,
              cd_access='a',
              owner=params.hive_user,
              group=params.user_group,
              mode=0755)
    Directory(params.hive_log_dir,
              create_parents=True,
              cd_access='a',
              owner=params.hive_user,
              group=params.user_group,
              mode=0755)
    Directory(params.hive_interactive_var_lib,
              create_parents=True,
              cd_access='a',
              owner=params.hive_user,
              group=params.user_group,
              mode=0755)
    generate_logfeeder_input_config(
        'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
Ejemplo n.º 57
0
    def _llap_start(self, env, cleanup=False):
      import params
      env.set_params(params)
      Logger.info("Starting LLAP")

      # TODO, start only if not already running.
      # TODO : Currently hardcoded the params. Need to read the suggested values from hive2/hive-site.xml.
      # TODO, ensure that script works as hive from cmd when not cd'ed in /homve/hive
      # Needs permission to write to hive home dir.

      cmd = ''
      if params.security_enabled:
        cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb "
                     "{slider_am_container_mb} --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
                     "{hive_llap_keytab_file} --slider-principal {hive_headless_keytab} --loglevel INFO")
      else:
        cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb {slider_am_container_mb} --loglevel INFO")

      run_file_path = None
      try:
        Logger.info(format("Command: {cmd}"))
        cmd = cmd.split()
        code, output, error = shell.checked_call(cmd, user=params.hive_user, stderr=subprocess.PIPE, logoutput=True)

        if code != 0 or output is None:
          raise Fail("Command failed with either non-zero return code or no output.")

        # E.g., output:
        # Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider
        exp = r"Prepared (.*?run.sh) for running LLAP"
        m = re.match(exp, output, re.I)
        if m and len(m.groups()) == 1:
          run_file_name = m.group(1)
          run_file_path = os.path.join(params.hive_user_home_dir, run_file_name)
        else:
          raise Fail("Did not find run.sh file in output: " + str(output))

        Logger.info(format("Run file path: {run_file_path}"))
        if os.path.isfile(run_file_path):
          Execute(run_file_path, user=params.hive_user)

          # TODO : Sleep below is not a good idea. We need to check the status of LLAP app to figure out it got
          # launched properly and is in running state. Then go ahead with Hive Interactive Server start.
          Logger.info("Sleeping for 30 secs")
          time.sleep(30)
          Logger.info("LLAP app deployed successfully.")
          return True
        else:
          raise Fail(format("Did not find run file {run_file_path}"))
      except:
        # Attempt to clean up the packaged application, or potentially rename it with a .bak
        if run_file_path is not None and cleanup:
          try:
            parent_dir = os.path.dirname(run_file_path)
            if os.path.isdir(parent_dir):
              shutil.rmtree(parent_dir)
          except Exception, e:
            Logger.error("Could not cleanup LLAP app package. Error: " + str(e))

        # throw the original exception
        raise
Ejemplo n.º 58
0
def setup_ranger_plugin(component_select_name,
                        service_name,
                        previous_jdbc_jar,
                        component_downloaded_custom_connector,
                        component_driver_curl_source,
                        component_driver_curl_target,
                        java_home,
                        repo_name,
                        plugin_repo_dict,
                        ranger_env_properties,
                        plugin_properties,
                        policy_user,
                        policymgr_mgr_url,
                        plugin_enabled,
                        conf_dict,
                        component_user,
                        component_group,
                        cache_service_list,
                        plugin_audit_properties,
                        plugin_audit_attributes,
                        plugin_security_properties,
                        plugin_security_attributes,
                        plugin_policymgr_ssl_properties,
                        plugin_policymgr_ssl_attributes,
                        component_list,
                        audit_db_is_enabled,
                        credential_file,
                        xa_audit_db_password,
                        ssl_truststore_password,
                        ssl_keystore_password,
                        api_version=None,
                        stack_version_override=None,
                        skip_if_rangeradmin_down=True,
                        is_security_enabled=False,
                        is_stack_supports_ranger_kerberos=False,
                        component_user_principal=None,
                        component_user_keytab=None):

    if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith(
            "/None"):
        if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar):
            File(previous_jdbc_jar, action='delete')

        File(component_downloaded_custom_connector,
             content=DownloadSource(component_driver_curl_source),
             mode=0644)

        Execute(('cp', '--remove-destination',
                 component_downloaded_custom_connector,
                 component_driver_curl_target),
                path=["/bin", "/usr/bin/"],
                sudo=True)

        File(component_driver_curl_target, mode=0644)

    if policymgr_mgr_url.endswith('/'):
        policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
    stack_version = get_stack_version(component_select_name)
    if stack_version_override is not None:
        stack_version = stack_version_override

    component_conf_dir = conf_dict

    if plugin_enabled:

        service_name_exist = False
        policycache_path = os.path.join('/etc', 'ranger', repo_name,
                                        'policycache')
        try:
            for cache_service in cache_service_list:
                policycache_json_file = format(
                    '{policycache_path}/{cache_service}_{repo_name}.json')
                if os.path.isfile(policycache_json_file) and os.path.getsize(
                        policycache_json_file) > 0:
                    with open(policycache_json_file) as json_file:
                        json_data = json.load(json_file)
                        if 'serviceName' in json_data and json_data[
                                'serviceName'] == repo_name:
                            service_name_exist = True
                            Logger.info(
                                "Skipping Ranger API calls, as policy cache file exists for {0}"
                                .format(service_name))
                            break
        except Exception, err:
            Logger.error(
                "Error occurred while fetching service name from policy cache file.\nError: {0}"
                .format(err))

        if not service_name_exist:
            if api_version is not None and api_version == 'v2':
                ranger_adm_obj = RangeradminV2(
                    url=policymgr_mgr_url,
                    skip_if_rangeradmin_down=skip_if_rangeradmin_down)
                ranger_adm_obj.create_ranger_repository(
                    service_name, repo_name, plugin_repo_dict,
                    ranger_env_properties['ranger_admin_username'],
                    ranger_env_properties['ranger_admin_password'],
                    ranger_env_properties['admin_username'],
                    ranger_env_properties['admin_password'], policy_user,
                    is_security_enabled, is_stack_supports_ranger_kerberos,
                    component_user, component_user_principal,
                    component_user_keytab)
            else:
                ranger_adm_obj = Rangeradmin(
                    url=policymgr_mgr_url,
                    skip_if_rangeradmin_down=skip_if_rangeradmin_down)
                ranger_adm_obj.create_ranger_repository(
                    service_name, repo_name, plugin_repo_dict,
                    ranger_env_properties['ranger_admin_username'],
                    ranger_env_properties['ranger_admin_password'],
                    ranger_env_properties['admin_username'],
                    ranger_env_properties['admin_password'], policy_user)

        current_datetime = datetime.now()

        File(
            format('{component_conf_dir}/ranger-security.xml'),
            owner=component_user,
            group=component_group,
            mode=0644,
            content=InlineTemplate(
                format(
                    '<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
                )))

        Directory([
            os.path.join('/etc', 'ranger', repo_name),
            os.path.join('/etc', 'ranger', repo_name, 'policycache')
        ],
                  owner=component_user,
                  group=component_group,
                  mode=0775,
                  create_parents=True,
                  cd_access='a')

        for cache_service in cache_service_list:
            File(os.path.join('/etc', 'ranger', repo_name, 'policycache',
                              format('{cache_service}_{repo_name}.json')),
                 owner=component_user,
                 group=component_group,
                 mode=0644)

        # remove plain-text password from xml configs
        plugin_audit_password_property = 'xasecure.audit.destination.db.password'
        plugin_audit_properties_copy = {}
        plugin_audit_properties_copy.update(plugin_audit_properties)

        if plugin_audit_password_property in plugin_audit_properties_copy:
            plugin_audit_properties_copy[
                plugin_audit_password_property] = "crypted"

        XmlConfig(format('ranger-{service_name}-audit.xml'),
                  conf_dir=component_conf_dir,
                  configurations=plugin_audit_properties_copy,
                  configuration_attributes=plugin_audit_attributes,
                  owner=component_user,
                  group=component_group,
                  mode=0744)

        XmlConfig(format('ranger-{service_name}-security.xml'),
                  conf_dir=component_conf_dir,
                  configurations=plugin_security_properties,
                  configuration_attributes=plugin_security_attributes,
                  owner=component_user,
                  group=component_group,
                  mode=0744)

        # remove plain-text password from xml configs
        plugin_password_properties = [
            'xasecure.policymgr.clientssl.keystore.password',
            'xasecure.policymgr.clientssl.truststore.password'
        ]
        plugin_policymgr_ssl_properties_copy = {}
        plugin_policymgr_ssl_properties_copy.update(
            plugin_policymgr_ssl_properties)

        for prop in plugin_password_properties:
            if prop in plugin_policymgr_ssl_properties_copy:
                plugin_policymgr_ssl_properties_copy[prop] = "crypted"

        if str(service_name).lower() == 'yarn':
            XmlConfig("ranger-policymgr-ssl-yarn.xml",
                      conf_dir=component_conf_dir,
                      configurations=plugin_policymgr_ssl_properties_copy,
                      configuration_attributes=plugin_policymgr_ssl_attributes,
                      owner=component_user,
                      group=component_group,
                      mode=0744)
        else:
            XmlConfig("ranger-policymgr-ssl.xml",
                      conf_dir=component_conf_dir,
                      configurations=plugin_policymgr_ssl_properties_copy,
                      configuration_attributes=plugin_policymgr_ssl_attributes,
                      owner=component_user,
                      group=component_group,
                      mode=0744)

        # creating symblink should be done by rpm package
        # setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list)

        setup_ranger_plugin_keystore(service_name, audit_db_is_enabled,
                                     stack_version, credential_file,
                                     xa_audit_db_password,
                                     ssl_truststore_password,
                                     ssl_keystore_password, component_user,
                                     component_group, java_home)
Ejemplo n.º 59
0
def setup_ranger_hbase():
  import params
  
  if params.has_ranger_admin:
    File(params.downloaded_custom_connector,
         content = DownloadSource(params.driver_curl_source)
    )

    if not os.path.isfile(params.driver_curl_target):
      Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),
              path=["/bin", "/usr/bin/"],
              sudo=True)

    try:
      command = 'hdp-select status hbase-client'
      return_code, hdp_output = shell.call(command, timeout=20)
    except Exception, e:
      Logger.error(str(e))
      raise Fail('Unable to execute hdp-select command to retrieve the version.')

    if return_code != 0:
      raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))

    hdp_version = re.sub('hbase-client - ', '', hdp_output).strip()
    match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)

    if match is None:
      raise Fail('Failed to get extracted version')

    file_path = '/usr/hdp/'+ hdp_version +'/ranger-hbase-plugin/install.properties'
    if not os.path.isfile(file_path):
      raise Fail('Ranger HBase plugin install.properties file does not exist at {0}'.format(file_path))
    
    ranger_hbase_dict = ranger_hbase_properties()
    hbase_repo_data = hbase_repo_properties()

    write_properties_to_file(file_path, ranger_hbase_dict)

    if params.enable_ranger_hbase:
      cmd = format('cd /usr/hdp/{hdp_version}/ranger-hbase-plugin/ && sh enable-hbase-plugin.sh')
      ranger_adm_obj = Rangeradmin(url=ranger_hbase_dict['POLICY_MGR_URL'])
      response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2(ranger_hbase_dict['POLICY_MGR_URL'] + '/login.jsp', 'test:test')

      if response_code is not None and response_code == 200:
        ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password)
        ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password
        if ambari_ranger_admin != '' and ambari_ranger_password != '':
          repo = ranger_adm_obj.get_repository_by_name_urllib2(ranger_hbase_dict['REPOSITORY_NAME'], 'hbase', 'true', ambari_username_password_for_ranger)
          if repo and repo['name'] == ranger_hbase_dict['REPOSITORY_NAME']:
            Logger.info('Hbase Repository exist')
          else:
            response = ranger_adm_obj.create_repository_urllib2(hbase_repo_data, ambari_username_password_for_ranger, params.policy_user)
            if response is not None:
              Logger.info('Hbase Repository created in Ranger admin')
            else:
              Logger.info('Hbase Repository creation failed in Ranger admin')
        else:
          Logger.info('Ambari admin username and password are blank ')
      else:
          Logger.info('Ranger service is not started on given host')
    else:
      cmd = format('cd /usr/hdp/{hdp_version}/ranger-hbase-plugin/ && sh disable-hbase-plugin.sh')

    Execute(cmd, environment={'JAVA_HOME': params.java64_home}, logoutput=True)                    
Ejemplo n.º 60
0
def hive_service_interactive(name, action='start'):
    import params
    import status_params

    pid_file = status_params.hive_interactive_pid
    cmd = format(
        "{start_hiveserver2_interactive_path} {hive_pid_dir}/hive-server2-interactive.out {hive_log_dir}/hive-server2-interactive.err {pid_file} {hive_server_interactive_conf_dir} {tez_interactive_conf_dir}"
    )

    pid = get_user_call_output.get_user_call_output(format("cat {pid_file}"),
                                                    user=params.hive_user,
                                                    is_checked_call=False)[1]
    process_id_exists_command = format(
        "ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1")

    if action == 'start':
        check_fs_root(params.hive_server_interactive_conf_dir,
                      params.execute_path_hive_interactive)
        daemon_cmd = cmd
        hadoop_home = params.hadoop_home
        hive_interactive_bin = "hive2"

        Execute(daemon_cmd,
                user=params.hive_user,
                environment={
                    'HADOOP_HOME': hadoop_home,
                    'JAVA_HOME': params.java64_home,
                    'HIVE_BIN': hive_interactive_bin
                },
                path=params.execute_path,
                not_if=process_id_exists_command)

        if params.hive_jdbc_driver == "com.mysql.jdbc.Driver" or \
            params.hive_jdbc_driver == "org.postgresql.Driver" or \
            params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":

            path_to_jdbc = params.target_hive_interactive
            if not params.jdbc_jar_name:
                path_to_jdbc = format("{hive_interactive_lib}/") + \
                               params.default_connectors_map[params.hive_jdbc_driver] if params.hive_jdbc_driver in params.default_connectors_map else None
                if not os.path.isfile(path_to_jdbc):
                    path_to_jdbc = format("{hive_interactive_lib}/") + "*"
                    error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \
                          " in hive lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
                    Logger.error(error_message)

            db_connection_check_command = format(
                "{java64_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}"
            )
            Execute(db_connection_check_command,
                    path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
                    tries=5,
                    try_sleep=10)
    elif action == 'stop':

        daemon_kill_cmd = format("{sudo} kill {pid}")
        daemon_hard_kill_cmd = format("{sudo} kill -9 {pid}")

        Execute(daemon_kill_cmd,
                not_if=format("! ({process_id_exists_command})"))

        # check if stopped the process, otherwise send hard kill command.
        try:
            Execute(
                format("! ({process_id_exists_command})"),
                tries=10,
                try_sleep=3,
            )
        except:
            Execute(daemon_hard_kill_cmd,
                    not_if=format("! ({process_id_exists_command}) "))

        # check if stopped the process, else fail the task
        Execute(
            format("! ({process_id_exists_command})"),
            tries=20,
            try_sleep=3,
        )

        File(pid_file, action="delete")