def _create_deployment_dir(self):
"""
Return the job deployment directory in HDFS, creating it if necessary.
The actual deployment dir should be 0711 owned by the user
"""
# Automatic setup of the required directories if needed
create_directories(self.fs)
# Check if job owner owns the deployment directory.
has_deployment_dir_access = False
if self.job.deployment_dir and self.fs.exists(self.job.deployment_dir):
statbuf = self.fs.stats(self.job.deployment_dir)
has_deployment_dir_access = statbuf and (statbuf.user == self.user.username)
# Case of a shared job
if self.job.document and self.user != self.job.document.owner or not has_deployment_dir_access:
path = REMOTE_DEPLOYMENT_DIR.get().replace('$USER', self.user.username).replace('$TIME', str(time.time())).replace('$JOBID', str(self.job.id))
# Shared coords or bundles might not have any existing workspaces
if self.fs.exists(self.job.deployment_dir):
self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user)
else:
self._create_dir(path)
else:
path = self.job.deployment_dir
self._create_dir(path)
return path
def _create_deployment_dir(self):
"""
Return the job deployment directory in HDFS, creating it if necessary.
The actual deployment dir should be 0711 owned by the user
"""
# Automatic setup of the required directories if needed
create_directories(self.fs)
# Case of a shared job
if self.user != self.job.owner:
path = Hdfs.join(
REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' %
(self.user.username, self.job.id, time.time()))
# Shared coords or bundles might not have any existing workspaces
if self.fs.exists(self.job.deployment_dir):
self.fs.copy_remote_dir(self.job.deployment_dir,
path,
owner=self.user,
dir_mode=0711)
else:
self._create_dir(path)
else:
path = self.job.deployment_dir
self._create_dir(path)
return path
def _create_deployment_dir(self):
"""
Return the job deployment directory in HDFS, creating it if necessary.
The actual deployment dir should be 0711 owned by the user
"""
# Automatic setup of the required directories if needed
create_directories(self.fs)
# Case of a shared job
if self.user != self.job.document.owner:
path = (
REMOTE_DEPLOYMENT_DIR.get()
.replace("$USER", self.user.username)
.replace("$TIME", str(time.time()))
.replace("$JOBID", str(self.job.id))
)
# Shared coords or bundles might not have any existing workspaces
if self.fs.exists(self.job.deployment_dir):
self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user)
else:
self._create_dir(path)
else:
path = self.job.deployment_dir
self._create_dir(path)
return path
def _create_deployment_dir(self):
"""
Return the job deployment directory in HDFS, creating it if necessary.
The actual deployment dir should be 0711 owned by the user
"""
# Automatic setup of the required directories if needed
create_directories(self.fs)
# Check if job owner owns the deployment directory.
has_deployment_dir_access = False
if self.job.deployment_dir and self.fs.exists(self.job.deployment_dir):
statbuf = self.fs.stats(self.job.deployment_dir)
has_deployment_dir_access = statbuf and (statbuf.user == self.user.username)
# Case of a shared job
if self.job.document and self.user != self.job.document.owner or not has_deployment_dir_access:
path = REMOTE_DEPLOYMENT_DIR.get().replace('$USER', self.user.username).replace('$TIME', str(time.time())).replace('$JOBID', str(self.job.id))
# Shared coords or bundles might not have any existing workspaces
if self.fs.exists(self.job.deployment_dir):
self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user)
else:
self._create_dir(path)
else:
path = self.job.deployment_dir
self._create_dir(path)
return path
def _create_deployment_dir(self):
"""
Return the job deployment directory in HDFS, creating it if necessary.
The actual deployment dir should be 0711 owned by the user
"""
if self.user != self.job.owner:
path = Hdfs.join(REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' % (self.user.username, self.job.id, time.time()))
self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user, dir_mode=0711)
else:
path = self.job.deployment_dir
self._create_dir(path)
return path
def _create_deployment_dir(self):
"""
Return the job deployment directory in HDFS, creating it if necessary.
The actual deployment dir should be 0711 owned by the user
"""
if self.user != self.job.owner:
path = Hdfs.join(REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' % (self.user.username, self.job.id, time.time()))
self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user)
else:
path = self.job.deployment_dir
self._create_dir(path)
return path
def create_directories(fs, directory_list=[]):
# If needed, create the remote home, deployment and data directories
directories = [REMOTE_DEPLOYMENT_DIR.get()] + directory_list
for directory in directories:
if not fs.do_as_user(fs.DEFAULT_USER, fs.exists, directory):
remote_home_dir = Hdfs.join('/user', fs.DEFAULT_USER)
if directory.startswith(remote_home_dir):
# Home is 755
fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir)
# Shared by all the users
fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777)
fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
def create_directories(fs, directory_list=[]):
# If needed, create the remote home, deployment and data directories
directories = [REMOTE_DEPLOYMENT_DIR.get()] + directory_list
for directory in directories:
if not fs.do_as_user(fs.DEFAULT_USER, fs.exists, directory):
remote_home_dir = Hdfs.join('/user', fs.DEFAULT_USER)
if directory.startswith(remote_home_dir):
# Home is 755
fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir)
# Shared by all the users
fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777)
fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
def create_directories(fs):
# If needed, create the remote home, deployment and data directories
directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get())
for directory in directories:
if not fs.do_as_user("hdfs", fs.exists, directory):
remote_home_dir = Hdfs.join('/user', "hdfs")
if directory.startswith(remote_home_dir):
# Home is 755
fs.do_as_user("hdfs", fs.create_home_dir, remote_home_dir)
# Shared by all the users
fs.do_as_user("hdfs", fs.mkdir, directory, 511)
fs.do_as_user("hdfs", fs.chmod, directory, 511) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
return REMOTE_SAMPLE_DIR.get()
def create_directories(fs):
# If needed, create the remote home, deployment and data directories
directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get())
for directory in directories:
if not fs.do_as_user("hdfs", fs.exists, directory):
remote_home_dir = Hdfs.join('/user', "hdfs")
if directory.startswith(remote_home_dir):
# Home is 755
fs.do_as_user("hdfs", fs.create_home_dir, remote_home_dir)
# Shared by all the users
fs.do_as_user("hdfs", fs.mkdir, directory, 511)
fs.do_as_user(
"hdfs", fs.chmod, directory, 511
) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
return REMOTE_SAMPLE_DIR.get()
def _create_deployment_dir(self):
"""
Return the job deployment directory in HDFS, creating it if necessary.
The actual deployment dir should be 0711 owned by the user
"""
# Automatic setup of the required directories if needed
create_directories(self.fs)
# Case of a shared job
if self.user != self.job.owner:
path = Hdfs.join(REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' % (self.user.username, self.job.id, time.time()))
# Shared coords or bundles might not have any existing workspaces
if self.fs.exists(self.job.deployment_dir):
self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user, dir_mode=0711)
else:
self._create_dir(path)
else:
path = self.job.deployment_dir
self._create_dir(path)
return path