Example #1
0
  def action_create(self):
    path = self.resource.path

    if not sudo.path_exists(path):
      Logger.info("Creating directory %s since it doesn't exist." % self.resource)
      
      # dead links should be followed, else we gonna have failures on trying to create directories on top of them.
      if self.resource.follow:
        # Follow symlink until the tail.
        followed_links = set()
        while sudo.path_lexists(path):
          if path in followed_links:
            raise Fail("Applying %s failed, looped symbolic links found while resolving %s" % (self.resource, path))
          followed_links.add(path)
          path = sudo.readlink(path)
          
        if path != self.resource.path:
          Logger.info("Following the link {0} to {1} to create the directory".format(self.resource.path, path))

      if self.resource.create_parents:
        sudo.makedirs(path, self.resource.mode or 0755)
      else:
        dirname = os.path.dirname(path)
        if not sudo.path_isdir(dirname):
          raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
        
        sudo.makedir(path, self.resource.mode or 0755)
      
    if not sudo.path_isdir(path):
      raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
    
    _ensure_metadata(path, self.resource.owner, self.resource.group,
                        mode=self.resource.mode, cd_access=self.resource.cd_access,
                        recursive_ownership=self.resource.recursive_ownership, recursive_mode_flags=self.resource.recursive_mode_flags, 
                        recursion_follow_links=self.resource.recursion_follow_links, safemode_folders=self.resource.safemode_folders)
Example #2
0
    def action_create(self):
        path = self.resource.path

        if not sudo.path_exists(path):
            Logger.info("Creating directory %s" % self.resource)
            if self.resource.recursive:
                if self.resource.recursive_permission:
                    DirectoryProvider.makedirs_and_set_permission_recursively(
                        path, self.resource.owner, self.resource.group,
                        self.resource.mode)
                else:
                    sudo.makedirs(path, self.resource.mode or 0755)
            else:
                dirname = os.path.dirname(path)
                if not sudo.path_isdir(dirname):
                    raise Fail(
                        "Applying %s failed, parent directory %s doesn't exist"
                        % (self.resource, dirname))

                sudo.makedir(path, self.resource.mode or 0755)

        if not sudo.path_isdir(path):
            raise Fail("Applying %s failed, file %s already exists" %
                       (self.resource, path))

        _ensure_metadata(path,
                         self.resource.owner,
                         self.resource.group,
                         mode=self.resource.mode,
                         cd_access=self.resource.cd_access)
Example #3
0
  def action_create(self):
    path = self.resource.path
    
    if sudo.path_isdir(path):
      raise Fail("Applying %s failed, directory with name %s exists" % (self.resource, path))
    
    dirname = os.path.dirname(path)
    if not sudo.path_isdir(dirname):
      raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
    
    write = False
    content = self._get_content()
    if not sudo.path_exists(path):
      write = True
      reason = "it doesn't exist"
    elif self.resource.replace:
      if content is not None:
        old_content = sudo.read_file(path, encoding=self.resource.encoding)
        if content != old_content:
          write = True
          reason = "contents don't match"
          if self.resource.backup:
            self.resource.env.backup_file(path)

    if write:
      Logger.info("Writing %s because %s" % (self.resource, reason))
      sudo.create_file(path, content, encoding=self.resource.encoding)

    _ensure_metadata(self.resource.path, self.resource.owner,
                        self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access)
Example #4
0
  def action_create(self):
    path = self.resource.path
    
    if sudo.path_isdir(path):
      raise Fail("Applying %s failed, directory with name %s exists" % (self.resource, path))
    
    dirname = os.path.dirname(path)
    if not sudo.path_isdir(dirname):
      raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
    
    write = False
    content = self._get_content()
    if not sudo.path_exists(path):
      write = True
      reason = "it doesn't exist"
    elif self.resource.replace:
      if content is not None:
        old_content = sudo.read_file(path, encoding=self.resource.encoding)
        if content != old_content:
          write = True
          reason = "contents don't match"
          if self.resource.backup:
            self.resource.env.backup_file(path)

    if write:
      Logger.info("Writing %s because %s" % (self.resource, reason))
      sudo.create_file(path, content, encoding=self.resource.encoding)

    _ensure_metadata(self.resource.path, self.resource.owner,
                        self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access)
Example #5
0
def copytree(src, dst, exclude_sub_dirs=set(), force_replace=False):
  """
  Copy content of directory from source path to the target path with possibility to exclude some directories

  :type src str
  :type dst str
  :type exclude_sub_dirs list|set
  :type force_replace bool
  """

  def copy(_src, _dst):
    from resource_management.core import shell
    # full path is required to avoid usage of system command aliases like 'cp -i', which block overwrite
    if force_replace:
      shell.checked_call(["/bin/cp", "-rfp", _src, _dst], sudo=True)
    else:
      shell.checked_call(["/bin/cp", "-rp", _src, _dst], sudo=True)

  if not sudo.path_isdir(src) or not sudo.path_isdir(dst):
    raise Fail("The source or the destination is not a folder")

  sub_dirs_to_copy = sudo.listdir(src)
  for d in sub_dirs_to_copy:
    if d not in exclude_sub_dirs:
      src_path = os.path.join(src, d)
      copy(src_path, dst)
Example #6
0
    def action_create(self):
        path = self.resource.path

        if not sudo.path_exists(path):
            Logger.info("Creating directory %s since it doesn't exist." %
                        self.resource)

            # dead links should be followed, else we gonna have failures on trying to create directories on top of them.
            if self.resource.follow:
                # Follow symlink until the tail.
                followed_links = set()
                while sudo.path_islink(path):
                    if path in followed_links:
                        raise Fail(
                            "Applying %s failed, looped symbolic links found while resolving %s"
                            % (self.resource, path))
                    followed_links.add(path)
                    prev_path = path
                    path = sudo.readlink(path)

                    if not os.path.isabs(path):
                        path = os.path.join(os.path.dirname(prev_path), path)

                if path != self.resource.path:
                    Logger.info(
                        "Following the link {0} to {1} to create the directory"
                        .format(self.resource.path, path))

            if self.resource.create_parents:
                sudo.makedirs(path, self.resource.mode or 0755)
            else:
                dirname = os.path.dirname(path)
                if not sudo.path_isdir(dirname):
                    raise Fail(
                        "Applying %s failed, parent directory %s doesn't exist"
                        % (self.resource, dirname))

                try:
                    sudo.makedir(path, self.resource.mode or 0755)
                except Exception as ex:
                    # race condition (somebody created the file before us)
                    if "File exists" in str(ex):
                        sudo.makedirs(path, self.resource.mode or 0755)
                    else:
                        raise

        if not sudo.path_isdir(path):
            raise Fail("Applying %s failed, file %s already exists" %
                       (self.resource, path))

        _ensure_metadata(
            path,
            self.resource.owner,
            self.resource.group,
            mode=self.resource.mode,
            cd_access=self.resource.cd_access,
            recursive_ownership=self.resource.recursive_ownership,
            recursive_mode_flags=self.resource.recursive_mode_flags,
            recursion_follow_links=self.resource.recursion_follow_links,
            safemode_folders=self.resource.safemode_folders)
Example #7
0
def get_hadoop_dir(target):
  """
  Return the hadoop shared directory which should be used for the command's component. The
  directory including the component's version is tried first, but if that doesn't exist,
  this will fallback to using "current".

  :target: the target directory
  """
  stack_root = Script.get_stack_root()
  stack_version = Script.get_stack_version()

  if not target in HADOOP_DIR_DEFAULTS:
    raise Fail("Target {0} not defined".format(target))

  hadoop_dir = HADOOP_DIR_DEFAULTS[target]

  formatted_stack_version = format_stack_version(stack_version)

  if stack_features.check_stack_feature(StackFeature.ROLLING_UPGRADE, formatted_stack_version):
    # read the desired version from the component map and use that for building the hadoop home
    version = component_version.get_component_repository_version()
    if version is None:
      version = default("/commandParams/version", None)

    # home uses a different template
    if target == "home":
      hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, version, "hadoop")
      if version is None or sudo.path_isdir(hadoop_dir) is False:
        hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, "current", "hadoop-client")
    else:
      hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_root, version, "hadoop", target)
      if version is None or sudo.path_isdir(hadoop_dir) is False:
        hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_root, "current", "hadoop-client", target)

  return hadoop_dir
Example #8
0
  def action_create(self):
    path = self.resource.path

    if not sudo.path_exists(path):
      Logger.info("Creating directory %s since it doesn't exist." % self.resource)
      
      # dead links should be followed, else we gonna have failures on trying to create directories on top of them.
      if self.resource.follow:
        # Follow symlink until the tail.
        followed_links = set()
        while sudo.path_lexists(path):
          if path in followed_links:
            raise Fail("Applying %s failed, looped symbolic links found while resolving %s" % (self.resource, path))
          followed_links.add(path)
          path = sudo.readlink(path)
          
        if path != self.resource.path:
          Logger.info("Following the link {0} to {1} to create the directory".format(self.resource.path, path))

      if self.resource.recursive:
        sudo.makedirs(path, self.resource.mode or 0755)
      else:
        dirname = os.path.dirname(path)
        if not sudo.path_isdir(dirname):
          raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
        
        sudo.makedir(path, self.resource.mode or 0755)
      
    if not sudo.path_isdir(path):
      raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
    
    _ensure_metadata(path, self.resource.owner, self.resource.group,
                        mode=self.resource.mode, cd_access=self.resource.cd_access)
Example #9
0
def _ensure_metadata(path, user, group, mode=None, cd_access=None):
    stat = sudo.stat(path)

    if user:
        uid = _coerce_uid(user)
        if stat.st_uid != uid:
            Logger.info("Changing owner for %s from %d to %s" %
                        (path, stat.st_uid, user))

            sudo.chown(path, user, None)

    if group:
        gid = _coerce_gid(group)
        if stat.st_gid != gid:
            Logger.info("Changing group for %s from %d to %s" %
                        (path, stat.st_gid, group))
            sudo.chown(path, None, group)

    if mode:
        if stat.st_mode != mode:
            Logger.info("Changing permission for %s from %o to %o" %
                        (path, stat.st_mode, mode))
            sudo.chmod(path, mode)

    if cd_access:
        if not re.match("^[ugoa]+$", cd_access):
            raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))

        dir_path = path
        while dir_path != os.sep:
            if sudo.path_isdir(dir_path):
                sudo.chmod_extended(dir_path, cd_access + "+x")

            dir_path = os.path.split(dir_path)[0]
Example #10
0
    def action_create(self):
        path = self.resource.path

        if sudo.path_lexists(path):
            oldpath = os.path.realpath(path)
            if oldpath == self.resource.to:
                return
            if not sudo.path_lexists(path):
                raise Fail(
                    "%s trying to create a symlink with the same name as an existing file or directory"
                    % self.resource)
            Logger.info("%s replacing old symlink to %s" %
                        (self.resource, oldpath))
            sudo.unlink(path)

        if self.resource.hard:
            if not sudo.path_exists(self.resource.to):
                raise Fail(
                    "Failed to apply %s, linking to nonexistent location %s" %
                    (self.resource, self.resource.to))
            if sudo.path_isdir(self.resource.to):
                raise Fail(
                    "Failed to apply %s, cannot create hard link to a directory (%s)"
                    % (self.resource, self.resource.to))

            Logger.info("Creating hard %s" % self.resource)
            sudo.link(self.resource.to, path)
        else:
            if not sudo.path_exists(self.resource.to):
                Logger.info("Warning: linking to nonexistent location %s" %
                            self.resource.to)

            Logger.info("Creating symbolic %s to %s" %
                        (self.resource, self.resource.to))
            sudo.symlink(self.resource.to, path)
Example #11
0
def copy_toolkit_scripts(toolkit_files_dir, toolkit_tmp_dir, user, group,
                         upgrade_type):
    nifiToolkitDirFilesPath = None
    nifiToolkitDirTmpPath = None

    Logger.info("Toolkit files dir is " + toolkit_files_dir)
    Logger.info("Toolkit tmp dir is " + toolkit_tmp_dir)

    for dir in os.listdir(toolkit_files_dir):
        if dir.startswith('nifi-toolkit-'):
            nifiToolkitDirFilesPath = os.path.join(toolkit_files_dir, dir)
            nifiToolkitDirTmpPath = os.path.join(toolkit_tmp_dir, dir)

    if not sudo.path_isdir(nifiToolkitDirTmpPath) or not (upgrade_type is
                                                          None):
        os.system("\cp -r " + nifiToolkitDirFilesPath + " " + toolkit_tmp_dir)
        Directory(nifiToolkitDirTmpPath,
                  owner=user,
                  group=group,
                  create_parents=False,
                  recursive_ownership=True,
                  cd_access="a",
                  mode=0755)
        os.system("\/var/lib/ambari-agent/ambari-sudo.sh chmod -R 755 " +
                  nifiToolkitDirTmpPath)
Example #12
0
def get_hadoop_conf_dir():
    """
  Return the hadoop shared conf directory which should be used for the command's component. The
  directory including the component's version is tried first, but if that doesn't exist,
  this will fallback to using "current".
  """
    stack_root = Script.get_stack_root()
    stack_version = Script.get_stack_version()

    hadoop_conf_dir = os.path.join(os.path.sep, "etc", "hadoop", "conf")
    if check_stack_feature(StackFeature.CONFIG_VERSIONING, stack_version):
        # read the desired version from the component map and use that for building the hadoop home
        version = component_version.get_component_repository_version()
        if version is None:
            version = default("/commandParams/version", None)

        hadoop_conf_dir = os.path.join(stack_root, str(version), "hadoop",
                                       "conf")
        if version is None or sudo.path_isdir(hadoop_conf_dir) is False:
            hadoop_conf_dir = os.path.join(stack_root, "current",
                                           "hadoop-client", "conf")

        Logger.info("Using hadoop conf dir: {0}".format(hadoop_conf_dir))

    return hadoop_conf_dir
Example #13
0
def _ensure_metadata(path, user, group, mode=None, cd_access=None):
  stat = sudo.stat(path)

  if user:
    uid = _coerce_uid(user)
    if stat.st_uid != uid:
      Logger.info(
        "Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
      
      sudo.chown(path, user, None)
      
  if group:
    gid = _coerce_gid(group)
    if stat.st_gid != gid:
      Logger.info(
        "Changing group for %s from %d to %s" % (path, stat.st_gid, group))
      sudo.chown(path, None, group)
      
  if mode:
    if stat.st_mode != mode:
      Logger.info("Changing permission for %s from %o to %o" % (
      path, stat.st_mode, mode))
      sudo.chmod(path, mode)
      
  if cd_access:
    if not re.match("^[ugoa]+$", cd_access):
      raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
    
    dir_path = path
    while dir_path != os.sep:
      if sudo.path_isdir(dir_path):
        sudo.chmod_extended(dir_path, cd_access+"+x")
        
      dir_path = os.path.split(dir_path)[0]
Example #14
0
  def action_create(self):
    path = self.resource.path

    if sudo.path_lexists(path):
      oldpath = os.path.realpath(path)
      if oldpath == self.resource.to:
        return
      if not sudo.path_lexists(path):
        raise Fail(
          "%s trying to create a symlink with the same name as an existing file or directory" % self.resource)
      Logger.info("%s replacing old symlink to %s" % (self.resource, oldpath))
      sudo.unlink(path)
      
    if self.resource.hard:
      if not sudo.path_exists(self.resource.to):
        raise Fail("Failed to apply %s, linking to nonexistent location %s" % (self.resource, self.resource.to))
      if sudo.path_isdir(self.resource.to):
        raise Fail("Failed to apply %s, cannot create hard link to a directory (%s)" % (self.resource, self.resource.to))
      
      Logger.info("Creating hard %s" % self.resource)
      sudo.link(self.resource.to, path)
    else:
      if not sudo.path_exists(self.resource.to):
        Logger.info("Warning: linking to nonexistent location %s" % self.resource.to)
        
      Logger.info("Creating symbolic %s to %s" % (self.resource, self.resource.to))
      sudo.symlink(self.resource.to, path)
Example #15
0
 def action_delete(self):
   path = self.resource.path
   if sudo.path_exists(path):
     if not sudo.path_isdir(path):
       raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path))
     
     Logger.info("Removing directory %s and all its content" % self.resource)
     sudo.rmtree(path)
Example #16
0
 def action_delete(self):
   path = self.resource.path
   if sudo.path_exists(path):
     if not sudo.path_isdir(path):
       raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path))
     
     Logger.info("Removing directory %s and all its content" % self.resource)
     sudo.rmtree(path)
Example #17
0
 def action_delete(self):
   path = self.resource.path
   
   if sudo.path_isdir(path):
     raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path))
   
   if sudo.path_exists(path):
     Logger.info("Deleting %s" % self.resource)
     sudo.unlink(path)
Example #18
0
 def action_delete(self):
   path = self.resource.path
   
   if sudo.path_isdir(path):
     raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path))
   
   if sudo.path_exists(path):
     Logger.info("Deleting %s" % self.resource)
     sudo.unlink(path)
Example #19
0
 def _copy_from_local_directory(self, target, source):
     for next_path_part in sudo.listdir(source):
         new_source = os.path.join(source, next_path_part)
         new_target = format("{target}/{next_path_part}")
         if sudo.path_isdir(new_source):
             Logger.info(format("Creating DFS directory {new_target}"))
             self._create_directory(new_target)
             self._copy_from_local_directory(new_target, new_source)
         else:
             self._create_file(new_target, new_source)
Example #20
0
 def _copy_from_local_directory(self, target, source):
   for next_path_part in sudo.listdir(source):
     new_source = os.path.join(source, next_path_part)
     new_target = format("{target}/{next_path_part}")
     if sudo.path_isdir(new_source):
       Logger.info(format("Creating DFS directory {new_target}"))
       self._create_directory(new_target)
       self._copy_from_local_directory(new_target, new_source)
     else:
       self._create_file(new_target, new_source)
def copy_toolkit_scripts(toolkit_files_dir, toolkit_tmp_dir, user, group,
                         upgrade_type, service):
    import params

    if service == NIFI:
        run_ca_tmp_script = os.path.join(toolkit_tmp_dir, 'run_ca.sh')
        new_run_ca_tmp_script = StaticFile("run_ca.sh")

        if not sudo.path_isfile(run_ca_tmp_script) or sudo.read_file(
                run_ca_tmp_script) != new_run_ca_tmp_script:
            File(format(run_ca_tmp_script),
                 content=new_run_ca_tmp_script,
                 mode=0755,
                 owner=user,
                 group=group)

    if not params.stack_support_nifi_toolkit_package:
        nifiToolkitDirFilesPath = None
        nifiToolkitDirTmpPath = None

        Logger.info("Toolkit files dir is " + toolkit_files_dir)
        Logger.info("Toolkit tmp dir is " + toolkit_tmp_dir)

        for dir in os.listdir(toolkit_files_dir):
            if dir.startswith('nifi-toolkit-'):
                nifiToolkitDirFilesPath = os.path.join(toolkit_files_dir, dir)
                nifiToolkitDirTmpPath = os.path.join(toolkit_tmp_dir, dir)

        if not sudo.path_isdir(nifiToolkitDirTmpPath) or not (upgrade_type is
                                                              None):
            os.system("\cp -r " + nifiToolkitDirFilesPath + " " +
                      toolkit_tmp_dir)
            Directory(nifiToolkitDirTmpPath,
                      owner=user,
                      group=group,
                      create_parents=False,
                      recursive_ownership=True,
                      cd_access="a",
                      mode=0755)
            os.system("\/var/lib/ambari-agent/ambari-sudo.sh chmod -R 755 " +
                      nifiToolkitDirTmpPath)
    else:
        Logger.info("Changing owner of package files")
        package_toolkit_dir = os.path.join(params.stack_root, 'current',
                                           'nifi-toolkit')
        Directory(package_toolkit_dir,
                  owner=user,
                  group=group,
                  create_parents=False,
                  recursive_ownership=True,
                  cd_access="a",
                  mode=0755,
                  recursion_follow_links=True)
Example #22
0
  def action_create(self):
    path = self.resource.path

    if not sudo.path_exists(path):
      Logger.info("Creating directory %s" % self.resource)
      if self.resource.recursive:
        if self.resource.recursive_permission:
          DirectoryProvider.makedirs_and_set_permission_recursively(path, self.resource.owner,
                                                                    self.resource.group, self.resource.mode)
        else:
          sudo.makedirs(path, self.resource.mode or 0755)
      else:
        dirname = os.path.dirname(path)
        if not sudo.path_isdir(dirname):
          raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
        
        sudo.makedir(path, self.resource.mode or 0755)
      
    if not sudo.path_isdir(path):
      raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
    
    _ensure_metadata(path, self.resource.owner, self.resource.group,
                        mode=self.resource.mode, cd_access=self.resource.cd_access)
Example #23
0
def _ensure_metadata(path, user, group, mode=None, cd_access=None):
  user_entity = group_entity = None

  if user or group:
    stat = sudo.stat(path)

  if user:
    _user_entity = pwd.getpwnam(user)
    if stat.st_uid != _user_entity.pw_uid:
      user_entity = _user_entity
      Logger.info(
        "Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
      
  if group:
    _group_entity = grp.getgrnam(group)
    if stat.st_gid != _group_entity.gr_gid:
      group_entity = _group_entity
      Logger.info(
        "Changing group for %s from %d to %s" % (path, stat.st_gid, group))
  
  sudo.chown(path, user_entity, group_entity)

  if mode:
    stat = sudo.stat(path)
    if stat.st_mode != mode:
      Logger.info("Changing permission for %s from %o to %o" % (
      path, stat.st_mode, mode))
      sudo.chmod(path, mode)
      
  if cd_access:
    if not re.match("^[ugoa]+$", cd_access):
      raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
    
    dir_path = path
    while dir_path != os.sep:
      if sudo.path_isdir(dir_path):
        sudo.chmod_extended(dir_path, cd_access+"+rx")
        
      dir_path = os.path.split(dir_path)[0]
Example #24
0
def _ensure_metadata(path, user, group, mode=None, cd_access=None):
    user_entity = group_entity = None

    if user or group:
        stat = sudo.stat(path)

    if user:
        _user_entity = pwd.getpwnam(user)
        if stat.st_uid != _user_entity.pw_uid:
            user_entity = _user_entity
            Logger.info("Changing owner for %s from %d to %s" %
                        (path, stat.st_uid, user))

    if group:
        _group_entity = grp.getgrnam(group)
        if stat.st_gid != _group_entity.gr_gid:
            group_entity = _group_entity
            Logger.info("Changing group for %s from %d to %s" %
                        (path, stat.st_gid, group))

    sudo.chown(path, user_entity, group_entity)

    if mode:
        stat = sudo.stat(path)
        if stat.st_mode != mode:
            Logger.info("Changing permission for %s from %o to %o" %
                        (path, stat.st_mode, mode))
            sudo.chmod(path, mode)

    if cd_access:
        if not re.match("^[ugoa]+$", cd_access):
            raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))

        dir_path = path
        while dir_path != os.sep:
            if sudo.path_isdir(dir_path):
                sudo.chmod_extended(dir_path, cd_access + "+x")

            dir_path = os.path.split(dir_path)[0]
Example #25
0
def _prepare_tez_tarball():
    """
  Prepares the Tez tarball by adding the Hadoop native libraries found in the mapreduce tarball.
  It's very important to use the version of mapreduce which matches tez here.
  Additionally, this will also copy native LZO to the tez tarball if LZO is enabled and the
  GPL license has been accepted.
  :return:  the full path of the newly created tez tarball to use
  """
    import tempfile

    Logger.info("Preparing the Tez tarball...")

    # get the mapreduce tarball which matches the version of tez
    # tez installs the mapreduce tar, so it should always be present
    _, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
    _, tez_source_file, _, _ = get_tarball_paths("tez")

    temp_dir = Script.get_tmp_dir()

    # create the temp staging directories ensuring that non-root agents using tarfile can work with them
    mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-",
                                          dir=temp_dir)
    tez_temp_dir = tempfile.mkdtemp(prefix="tez-tarball-", dir=temp_dir)
    sudo.chmod(mapreduce_temp_dir, 0777)
    sudo.chmod(tez_temp_dir, 0777)

    Logger.info("Extracting {0} to {1}".format(mapreduce_source_file,
                                               mapreduce_temp_dir))
    tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir)

    Logger.info("Extracting {0} to {1}".format(tez_source_file, tez_temp_dir))
    tar_archive.untar_archive(tez_source_file, tez_temp_dir)

    hadoop_lib_native_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib",
                                         "native")
    tez_lib_dir = os.path.join(tez_temp_dir, "lib")

    if not os.path.exists(hadoop_lib_native_dir):
        raise Fail(
            "Unable to seed the Tez tarball with native libraries since the source Hadoop native lib directory {0} does not exist"
            .format(hadoop_lib_native_dir))

    if not os.path.exists(tez_lib_dir):
        raise Fail(
            "Unable to seed the Tez tarball with native libraries since the target Tez lib directory {0} does not exist"
            .format(tez_lib_dir))

    # copy native libraries from hadoop to tez
    Execute(("cp", "-a", hadoop_lib_native_dir, tez_lib_dir), sudo=True)

    # if enabled, LZO GPL libraries must be copied as well
    if lzo_utils.should_install_lzo():
        stack_root = Script.get_stack_root()
        service_version = component_version.get_component_repository_version(
            service_name="TEZ")

        # some installations might not have Tez, but MapReduce2 should be a fallback to get the LZO libraries from
        if service_version is None:
            Logger.warning(
                "Tez does not appear to be installed, using the MapReduce version to get the LZO libraries"
            )
            service_version = component_version.get_component_repository_version(
                service_name="MAPREDUCE2")

        hadoop_lib_native_lzo_dir = os.path.join(stack_root, service_version,
                                                 "hadoop", "lib", "native")

        if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
            Logger.warning(
                "Unable to located native LZO libraries at {0}, falling back to hadoop home"
                .format(hadoop_lib_native_lzo_dir))
            hadoop_lib_native_lzo_dir = os.path.join(stack_root, "current",
                                                     "hadoop-client", "lib",
                                                     "native")

        if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
            raise Fail(
                "Unable to seed the Tez tarball with native libraries since LZO is enabled but the native LZO libraries could not be found at {0}"
                .format(hadoop_lib_native_lzo_dir))

        Execute(("cp", "-a", hadoop_lib_native_lzo_dir, tez_lib_dir),
                sudo=True)

    # ensure that the tez/lib directory is readable by non-root (which it typically is not)
    Directory(tez_lib_dir, mode=0755, cd_access='a', recursive_ownership=True)

    # create the staging directory so that non-root agents can write to it
    tez_native_tarball_staging_dir = os.path.join(
        temp_dir, "tez-native-tarball-staging")
    if not os.path.exists(tez_native_tarball_staging_dir):
        Directory(tez_native_tarball_staging_dir,
                  mode=0777,
                  cd_access='a',
                  create_parents=True,
                  recursive_ownership=True)

    tez_tarball_with_native_lib = os.path.join(tez_native_tarball_staging_dir,
                                               "tez-native.tar.gz")
    Logger.info("Creating a new Tez tarball at {0}".format(
        tez_tarball_with_native_lib))

    # tar up Tez, making sure to specify nothing for the arcname so that it does not include an absolute path
    with closing(tarfile.open(tez_tarball_with_native_lib,
                              "w:gz")) as new_tez_tarball:
        new_tez_tarball.add(tez_temp_dir, arcname=os.path.sep)

    # ensure that the tarball can be read and uploaded
    sudo.chmod(tez_tarball_with_native_lib, 0744)

    # cleanup
    sudo.rmtree(mapreduce_temp_dir)
    sudo.rmtree(tez_temp_dir)

    return tez_tarball_with_native_lib
Example #26
0
    # defaults set to current based on role
    hadoop_mapr_home = format("{stack_root}/current/{mapred_role_root}")
    hadoop_yarn_home = format("{stack_root}/current/{yarn_role_root}")

    # try to render the specific version
    version = component_version.get_component_repository_version()
    if version is None:
        version = default("/commandParams/version", None)

    if version is not None:
        hadoop_mapr_versioned_home = format(
            "{stack_root}/{version}/hadoop-mapreduce")
        hadoop_yarn_versioned_home = format(
            "{stack_root}/{version}/hadoop-yarn")

        if sudo.path_isdir(hadoop_mapr_versioned_home):
            hadoop_mapr_home = hadoop_mapr_versioned_home

        if sudo.path_isdir(hadoop_yarn_versioned_home):
            hadoop_yarn_home = hadoop_yarn_versioned_home

    hadoop_mapred2_jar_location = hadoop_mapr_home
    mapred_bin = format("{hadoop_mapr_home}/sbin")

    yarn_bin = format("{hadoop_yarn_home}/sbin")
    yarn_container_bin = format("{hadoop_yarn_home}/bin")

if stack_supports_timeline_state_store:
    # Timeline Service property that was added timeline_state_store stack feature
    ats_leveldb_state_store_dir = default(
        '/configurations/yarn-site/yarn.timeline-service.leveldb-state-store.path',
Example #27
0
def _ensure_metadata(path,
                     user,
                     group,
                     mode=None,
                     cd_access=None,
                     recursive_ownership=False,
                     recursive_mode_flags=None,
                     recursion_follow_links=False,
                     safemode_folders=[]):
    user_entity = group_entity = None
    _user_entity = _group_entity = None

    if user or group:
        stat = sudo.stat(path)

    if user:
        try:
            _user_entity = pwd.getpwnam(user)
        except KeyError:
            raise Fail("User '{0}' doesn't exist".format(user))

        if stat.st_uid != _user_entity.pw_uid:
            user_entity = _user_entity
            Logger.info("Changing owner for %s from %d to %s" %
                        (path, stat.st_uid, user))

    if group:
        try:
            _group_entity = grp.getgrnam(group)
        except KeyError:
            raise Fail("Group '{0}' doesn't exist".format(group))

        if stat.st_gid != _group_entity.gr_gid:
            group_entity = _group_entity
            Logger.info("Changing group for %s from %d to %s" %
                        (path, stat.st_gid, group))

    if recursive_ownership:
        assert_not_safemode_folder(path, safemode_folders)
        sudo.chown_recursive(path, _user_entity, _group_entity,
                             recursion_follow_links)

    sudo.chown(path, user_entity, group_entity)

    if recursive_mode_flags:
        if not isinstance(recursive_mode_flags, dict):
            raise Fail(
                "'recursion_follow_links' value should be a dictionary with 'f' and(or) 'd' key (for file and directory permission flags)"
            )

        regexp_to_match = "^({0},)*({0})$".format("[ugoa]+[+=-][rwx]+")
        for key, flags in recursive_mode_flags.iteritems():
            if key != 'd' and key != 'f':
                raise Fail(
                    "'recursive_mode_flags' with value '%s' has unknown key '%s', only keys 'f' and 'd' are valid"
                    % (str(recursive_mode_flags), str(key)))

            if not re.match(regexp_to_match, flags):
                raise Fail(
                    "'recursive_mode_flags' found '%s', but should value format have the following format: [ugoa...][[+-=][perms...]...]."
                    % (str(flags)))

        assert_not_safemode_folder(path, safemode_folders)
        sudo.chmod_recursive(path, recursive_mode_flags,
                             recursion_follow_links)

    if mode:
        stat = sudo.stat(path)
        if stat.st_mode != mode:
            Logger.info("Changing permission for %s from %o to %o" %
                        (path, stat.st_mode, mode))
            sudo.chmod(path, mode)

    if cd_access:
        if not re.match("^[ugoa]+$", cd_access):
            raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))

        dir_path = re.sub('/+', '/', path)
        while dir_path != os.sep:
            if sudo.path_isdir(dir_path):
                sudo.chmod_extended(dir_path, cd_access + "+rx")

            dir_path = os.path.split(dir_path)[0]
Example #28
0
def _ensure_metadata(path, user, group, mode=None, cd_access=None, recursive_ownership=False, recursive_mode_flags=None, recursion_follow_links=False, safemode_folders=[]):
  user_entity = group_entity = None
  _user_entity = _group_entity = None

  if user or group:
    stat = sudo.stat(path)

  if user:
    try:
      _user_entity = pwd.getpwnam(user)
    except KeyError:
      raise Fail("User '{0}' doesn't exist".format(user))
    
    if stat.st_uid != _user_entity.pw_uid:
      user_entity = _user_entity
      Logger.info(
        "Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
      
  if group:
    try:
      _group_entity = grp.getgrnam(group)
    except KeyError:
      raise Fail("Group '{0}' doesn't exist".format(group))
    
    if stat.st_gid != _group_entity.gr_gid:
      group_entity = _group_entity
      Logger.info(
        "Changing group for %s from %d to %s" % (path, stat.st_gid, group))
  
  if recursive_ownership:
    assert_not_safemode_folder(path, safemode_folders)
    sudo.chown_recursive(path, _user_entity, _group_entity, recursion_follow_links)
  
  sudo.chown(path, user_entity, group_entity)
  
  if recursive_mode_flags:
    if not isinstance(recursive_mode_flags, dict):
      raise Fail("'recursion_follow_links' value should be a dictionary with 'f' and(or) 'd' key (for file and directory permission flags)")
    
    regexp_to_match = "^({0},)*({0})$".format("[ugoa]+[+=-][rwx]+" )
    for key, flags in recursive_mode_flags.iteritems():
      if key != 'd' and key != 'f':
        raise Fail("'recursive_mode_flags' with value '%s' has unknown key '%s', only keys 'f' and 'd' are valid" % (str(recursive_mode_flags), str(key)))
          
      if not re.match(regexp_to_match, flags):
        raise Fail("'recursive_mode_flags' found '%s', but should value format have the following format: [ugoa...][[+-=][perms...]...]." % (str(flags)))
    
    assert_not_safemode_folder(path, safemode_folders)
    sudo.chmod_recursive(path, recursive_mode_flags, recursion_follow_links)

  if mode:
    stat = sudo.stat(path)
    if stat.st_mode != mode:
      Logger.info("Changing permission for %s from %o to %o" % (
      path, stat.st_mode, mode))
      sudo.chmod(path, mode)
      
  if cd_access:
    if not re.match("^[ugoa]+$", cd_access):
      raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
    
    dir_path = re.sub('/+', '/', path)
    while dir_path != os.sep:
      if sudo.path_isdir(dir_path):
        sudo.chmod_extended(dir_path, cd_access+"+rx")
        
      dir_path = os.path.split(dir_path)[0]