Пример #1
0
  def _create_deployment_dir(self):
    """
    Return the job deployment directory in HDFS, creating it if necessary.
    The actual deployment dir should be 0711 owned by the user
    """
    # Automatic setup of the required directories if needed
    create_directories(self.fs)

    # Check if job owner owns the deployment directory.
    has_deployment_dir_access = False
    if self.job.deployment_dir and self.fs.exists(self.job.deployment_dir):
      statbuf = self.fs.stats(self.job.deployment_dir)
      has_deployment_dir_access = statbuf and (statbuf.user == self.user.username)

    # Case of a shared job
    if self.job.document and self.user != self.job.document.owner or not has_deployment_dir_access:
      path = REMOTE_DEPLOYMENT_DIR.get().replace('$USER', self.user.username).replace('$TIME', str(time.time())).replace('$JOBID', str(self.job.id))
      # Shared coords or bundles might not have any existing workspaces
      if self.fs.exists(self.job.deployment_dir):
        self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user)
      else:
        self._create_dir(path)
    else:
      path = self.job.deployment_dir
      self._create_dir(path)
    return path
Пример #2
0
    def _create_deployment_dir(self):
        """
    Return the job deployment directory in HDFS, creating it if necessary.
    The actual deployment dir should be 0711 owned by the user
    """
        # Automatic setup of the required directories if needed
        create_directories(self.fs)

        # Case of a shared job
        if self.user != self.job.owner:
            path = Hdfs.join(
                REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' %
                (self.user.username, self.job.id, time.time()))
            # Shared coords or bundles might not have any existing workspaces
            if self.fs.exists(self.job.deployment_dir):
                self.fs.copy_remote_dir(self.job.deployment_dir,
                                        path,
                                        owner=self.user,
                                        dir_mode=0711)
            else:
                self._create_dir(path)
        else:
            path = self.job.deployment_dir
            self._create_dir(path)
        return path
Пример #3
0
    def _create_deployment_dir(self):
        """
    Return the job deployment directory in HDFS, creating it if necessary.
    The actual deployment dir should be 0711 owned by the user
    """
        # Automatic setup of the required directories if needed
        create_directories(self.fs)

        # Case of a shared job
        if self.user != self.job.document.owner:
            path = (
                REMOTE_DEPLOYMENT_DIR.get()
                .replace("$USER", self.user.username)
                .replace("$TIME", str(time.time()))
                .replace("$JOBID", str(self.job.id))
            )
            # Shared coords or bundles might not have any existing workspaces
            if self.fs.exists(self.job.deployment_dir):
                self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user)
            else:
                self._create_dir(path)
        else:
            path = self.job.deployment_dir
            self._create_dir(path)
        return path
Пример #4
0
  def _create_deployment_dir(self):
    """
    Return the job deployment directory in HDFS, creating it if necessary.
    The actual deployment dir should be 0711 owned by the user
    """
    # Automatic setup of the required directories if needed
    create_directories(self.fs)

    # Check if job owner owns the deployment directory.
    has_deployment_dir_access = False
    if self.job.deployment_dir and self.fs.exists(self.job.deployment_dir):
      statbuf = self.fs.stats(self.job.deployment_dir)
      has_deployment_dir_access = statbuf and (statbuf.user == self.user.username)

    # Case of a shared job
    if self.job.document and self.user != self.job.document.owner or not has_deployment_dir_access:
      path = REMOTE_DEPLOYMENT_DIR.get().replace('$USER', self.user.username).replace('$TIME', str(time.time())).replace('$JOBID', str(self.job.id))
      # Shared coords or bundles might not have any existing workspaces
      if self.fs.exists(self.job.deployment_dir):
        self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user)
      else:
        self._create_dir(path)
    else:
      path = self.job.deployment_dir
      self._create_dir(path)
    return path
Пример #5
0
 def _create_deployment_dir(self):
   """
   Return the job deployment directory in HDFS, creating it if necessary.
   The actual deployment dir should be 0711 owned by the user
   """
   if self.user != self.job.owner:
     path = Hdfs.join(REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' % (self.user.username, self.job.id, time.time()))
     self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user, dir_mode=0711)
   else:
     path = self.job.deployment_dir
     self._create_dir(path)
   return path
Пример #6
0
 def _create_deployment_dir(self):
   """
   Return the job deployment directory in HDFS, creating it if necessary.
   The actual deployment dir should be 0711 owned by the user
   """
   if self.user != self.job.owner:
     path = Hdfs.join(REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' % (self.user.username, self.job.id, time.time()))
     self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user)
   else:
     path = self.job.deployment_dir
     self._create_dir(path)
   return path
Пример #7
0
def create_directories(fs, directory_list=[]):
  # If needed, create the remote home, deployment and data directories
  directories = [REMOTE_DEPLOYMENT_DIR.get()] + directory_list

  for directory in directories:
    if not fs.do_as_user(fs.DEFAULT_USER, fs.exists, directory):
      remote_home_dir = Hdfs.join('/user', fs.DEFAULT_USER)
      if directory.startswith(remote_home_dir):
        # Home is 755
        fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir)
      # Shared by all the users
      fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777)
      fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
Пример #8
0
def create_directories(fs, directory_list=[]):
  # If needed, create the remote home, deployment and data directories
  directories = [REMOTE_DEPLOYMENT_DIR.get()] + directory_list

  for directory in directories:
    if not fs.do_as_user(fs.DEFAULT_USER, fs.exists, directory):
      remote_home_dir = Hdfs.join('/user', fs.DEFAULT_USER)
      if directory.startswith(remote_home_dir):
        # Home is 755
        fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir)
      # Shared by all the users
      fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777)
      fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
def create_directories(fs):
  # If needed, create the remote home, deployment and data directories
  directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get())

  for directory in directories:
    if not fs.do_as_user("hdfs", fs.exists, directory):
      remote_home_dir = Hdfs.join('/user', "hdfs")
      if directory.startswith(remote_home_dir):
        # Home is 755
        fs.do_as_user("hdfs", fs.create_home_dir, remote_home_dir)
      # Shared by all the users
      fs.do_as_user("hdfs", fs.mkdir, directory, 511)
      fs.do_as_user("hdfs", fs.chmod, directory, 511) # To remove after https://issues.apache.org/jira/browse/HDFS-3491

  return REMOTE_SAMPLE_DIR.get()
Пример #10
0
def create_directories(fs):
    # If needed, create the remote home, deployment and data directories
    directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get())

    for directory in directories:
        if not fs.do_as_user("hdfs", fs.exists, directory):
            remote_home_dir = Hdfs.join('/user', "hdfs")
            if directory.startswith(remote_home_dir):
                # Home is 755
                fs.do_as_user("hdfs", fs.create_home_dir, remote_home_dir)
            # Shared by all the users
            fs.do_as_user("hdfs", fs.mkdir, directory, 511)
            fs.do_as_user(
                "hdfs", fs.chmod, directory, 511
            )  # To remove after https://issues.apache.org/jira/browse/HDFS-3491

    return REMOTE_SAMPLE_DIR.get()
Пример #11
0
  def _create_deployment_dir(self):
    """
    Return the job deployment directory in HDFS, creating it if necessary.
    The actual deployment dir should be 0711 owned by the user
    """
    # Automatic setup of the required directories if needed
    create_directories(self.fs)

    # Case of a shared job
    if self.user != self.job.owner:
      path = Hdfs.join(REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' % (self.user.username, self.job.id, time.time()))
      # Shared coords or bundles might not have any existing workspaces
      if self.fs.exists(self.job.deployment_dir):
        self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user, dir_mode=0711)
      else:
        self._create_dir(path)
    else:
      path = self.job.deployment_dir
      self._create_dir(path)
    return path