def test_get_raw_binary(self, ctx, db_get_raw_data, i_s_get_raw_data,
i_s_get_raw_data_with_context, m_s_get_file_info):
ctx.return_value = mock.Mock()
job_binary = mock.Mock()
job_binary.url = 'internal-db://somebinary'
dispatch.get_raw_binary(job_binary)
self.assertEqual(1, db_get_raw_data.call_count)
job_binary.url = 'swift://container/object'
proxy_configs = dict(proxy_username='******',
proxy_password='******',
proxy_trust_id='proxytrust')
dispatch.get_raw_binary(job_binary, proxy_configs)
dispatch.get_raw_binary(job_binary, proxy_configs, with_context=True)
dispatch.get_raw_binary(job_binary, with_context=True)
self.assertEqual(1, i_s_get_raw_data.call_count)
self.assertEqual(2, i_s_get_raw_data_with_context.call_count)
job_binary.url = 'manila://the_share_id/the_path'
remote = mock.Mock()
remote.instance.node_group.cluster.shares = []
remote.instance.node_group.shares = []
dispatch.get_raw_binary(job_binary, remote=remote)
self.assertEqual(1, m_s_get_file_info.call_count)
def _upload_job_files_to_hdfs(self,
where,
job_dir,
job,
configs,
proxy_configs=None):
mains = job.mains or []
libs = job.libs or []
builtin_libs = edp.get_builtin_binaries(job, configs)
uploaded_paths = []
hdfs_user = self.get_hdfs_user()
lib_dir = job_dir + '/lib'
with where.remote() as r:
for m in mains:
raw_data = d.get_raw_binary(m, proxy_configs)
mfs.put_file_to_maprfs(r, raw_data, m.name, job_dir, hdfs_user)
uploaded_paths.append(os.path.join(job_dir, m.name))
if len(libs) > 0:
self.create_hdfs_dir(r, lib_dir)
for l in libs:
raw_data = d.get_raw_binary(l, proxy_configs)
mfs.put_file_to_maprfs(r, raw_data, l.name, lib_dir, hdfs_user)
uploaded_paths.append(os.path.join(lib_dir, l.name))
for lib in builtin_libs:
mfs.put_file_to_maprfs(r, lib['raw'], lib['name'], lib_dir,
hdfs_user)
uploaded_paths.append(lib_dir + '/' + lib['name'])
return uploaded_paths
def _upload_job_files_to_hdfs(self,
where,
job_dir,
job,
configs,
proxy_configs=None):
mains = job.mains or []
libs = job.libs or []
builtin_libs = edp.get_builtin_binaries(job, configs)
uploaded_paths = []
hdfs_user = self.get_hdfs_user()
lib_dir = job_dir + '/lib'
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(main, proxy_configs)
h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user)
uploaded_paths.append(job_dir + '/' + main.name)
if len(libs) > 0:
# HDFS 2.2.0 fails to put file if the lib dir does not exist
self.create_hdfs_dir(r, lib_dir)
for lib in libs:
raw_data = dispatch.get_raw_binary(lib, proxy_configs)
h.put_file_to_hdfs(r, raw_data, lib.name, lib_dir, hdfs_user)
uploaded_paths.append(lib_dir + '/' + lib.name)
for lib in builtin_libs:
h.put_file_to_hdfs(r, lib['raw'], lib['name'], lib_dir,
hdfs_user)
uploaded_paths.append(lib_dir + '/' + lib['name'])
return uploaded_paths
def _upload_job_files_to_hdfs(self, where, job_dir, job, configs, proxy_configs=None):
mains = job.mains or []
libs = job.libs or []
builtin_libs = edp.get_builtin_binaries(job, configs)
uploaded_paths = []
hdfs_user = self.get_hdfs_user()
job_dir_suffix = "lib" if job.type != edp.JOB_TYPE_SHELL else ""
lib_dir = os.path.join(job_dir, job_dir_suffix)
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(main, proxy_configs)
h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user)
uploaded_paths.append(job_dir + "/" + main.name)
if len(libs) and job_dir_suffix:
# HDFS 2.2.0 fails to put file if the lib dir does not exist
self.create_hdfs_dir(r, lib_dir)
for lib in libs:
raw_data = dispatch.get_raw_binary(lib, proxy_configs)
h.put_file_to_hdfs(r, raw_data, lib.name, lib_dir, hdfs_user)
uploaded_paths.append(lib_dir + "/" + lib.name)
for lib in builtin_libs:
h.put_file_to_hdfs(r, lib["raw"], lib["name"], lib_dir, hdfs_user)
uploaded_paths.append(lib_dir + "/" + lib["name"])
return uploaded_paths
def _upload_job_files_to_hdfs(self, where, job_dir, job, configs,
proxy_configs=None):
mains = job.mains or []
libs = job.libs or []
builtin_libs = edp.get_builtin_binaries(job, configs)
uploaded_paths = []
hdfs_user = self.get_hdfs_user()
lib_dir = job_dir + '/lib'
with where.remote() as r:
for m in mains:
raw_data = d.get_raw_binary(m, proxy_configs)
mfs.put_file_to_maprfs(r, raw_data, m.name, job_dir, hdfs_user)
uploaded_paths.append(os.path.join(job_dir, m.name))
if len(libs) > 0:
self.create_hdfs_dir(r, lib_dir)
for l in libs:
raw_data = d.get_raw_binary(l, proxy_configs)
mfs.put_file_to_maprfs(r, raw_data, l.name, lib_dir,
hdfs_user)
uploaded_paths.append(os.path.join(lib_dir, l.name))
for lib in builtin_libs:
mfs.put_file_to_maprfs(r, lib['raw'], lib['name'], lib_dir,
hdfs_user)
uploaded_paths.append(lib_dir + '/' + lib['name'])
return uploaded_paths
def test_get_raw_binary(self, ctx, db_get_raw_data, i_s_get_raw_data,
i_s_get_raw_data_with_context, m_s_get_file_info):
ctx.return_value = mock.Mock()
job_binary = mock.Mock()
job_binary.url = 'internal-db://somebinary'
dispatch.get_raw_binary(job_binary)
self.assertEqual(1, db_get_raw_data.call_count)
job_binary.url = 'swift://container/object'
proxy_configs = dict(proxy_username='******',
proxy_password='******',
proxy_trust_id='proxytrust')
dispatch.get_raw_binary(job_binary, proxy_configs)
dispatch.get_raw_binary(job_binary, proxy_configs, with_context=True)
dispatch.get_raw_binary(job_binary, with_context=True)
self.assertEqual(1, i_s_get_raw_data.call_count)
self.assertEqual(2, i_s_get_raw_data_with_context.call_count)
job_binary.url = 'manila://the_share_id/the_path'
remote = mock.Mock()
remote.instance.node_group.cluster.shares = []
remote.instance.node_group.shares = []
dispatch.get_raw_binary(job_binary, remote=remote)
self.assertEqual(1, m_s_get_file_info.call_count)
def _upload_job_files_to_hdfs(self, where, job_dir, job):
mains = job.mains or []
libs = job.libs or []
uploaded_paths = []
hdfs_user = self.get_hdfs_user()
with where.remote() as r:
for m in mains:
raw_data = d.get_raw_binary(m)
mfs.put_file_to_maprfs(r, raw_data, m.name, job_dir, hdfs_user)
uploaded_paths.append(job_dir + '/' + m.name)
for l in libs:
raw_data = d.get_raw_binary(l)
lib_dir = job_dir + '/lib/'
self.create_hdfs_dir(r, lib_dir)
mfs.put_file_to_maprfs(r, raw_data, l.name, lib_dir, hdfs_user)
uploaded_paths.append(lib_dir + l.name)
return uploaded_paths
def _upload_job_files_to_hdfs(self, where, job_dir, job):
mains = job.mains or []
libs = job.libs or []
uploaded_paths = []
hdfs_user = self.get_hdfs_user()
with where.remote() as r:
for m in mains:
raw_data = d.get_raw_binary(m)
mfs.put_file_to_maprfs(r, raw_data, m.name, job_dir, hdfs_user)
uploaded_paths.append(job_dir + '/' + m.name)
for l in libs:
raw_data = d.get_raw_binary(l)
lib_dir = job_dir + '/lib/'
self.create_hdfs_dir(r, lib_dir)
mfs.put_file_to_maprfs(r, raw_data, l.name, lib_dir,
hdfs_user)
uploaded_paths.append(lib_dir + l.name)
return uploaded_paths
def upload(r, dir, job_file, proxy_configs):
dst = os.path.join(dir, job_file.name)
raw_data = dispatch.get_raw_binary(
job_file, proxy_configs=proxy_configs, remote=r)
if isinstance(raw_data, dict) and raw_data["type"] == "path":
dst = raw_data['path']
else:
r.write_file_to(dst, raw_data)
return dst
def upload_job_files(where, job_dir, job, hdfs_user):
mains = job.mains or []
libs = job.libs or []
uploaded_paths = []
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(main)
h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user)
uploaded_paths.append(job_dir + '/' + main.name)
for lib in libs:
raw_data = dispatch.get_raw_binary(lib)
# HDFS 2.2.0 fails to put file if the lib dir does not exist
h.create_dir(r, job_dir + "/lib", hdfs_user)
h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib",
hdfs_user)
uploaded_paths.append(job_dir + '/lib/' + lib.name)
return uploaded_paths
def upload(r, dir, job_file, proxy_configs):
dst = os.path.join(dir, job_file.name)
raw_data = dispatch.get_raw_binary(job_file,
proxy_configs=proxy_configs,
remote=remote)
if isinstance(raw_data, dict) and raw_data["type"] == "path":
dst = raw_data['path']
else:
r.write_file_to(dst, raw_data)
return dst
def _upload_job_files_to_hdfs(self,
where,
job_dir,
job,
configs,
proxy_configs=None):
mains = job.mains or []
libs = job.libs or []
builtin_libs = edp.get_builtin_binaries(job, configs)
uploaded_paths = []
hdfs_user = self.get_hdfs_user()
job_dir_suffix = 'lib' if job.type != edp.JOB_TYPE_SHELL else ''
lib_dir = os.path.join(job_dir, job_dir_suffix)
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(main,
proxy_configs=proxy_configs,
remote=r)
if isinstance(raw_data, dict) and raw_data["type"] == "path":
h.copy_from_local(r, raw_data['path'], job_dir, hdfs_user)
else:
h.put_file_to_hdfs(r, raw_data, main.name, job_dir,
hdfs_user)
uploaded_paths.append(job_dir + '/' + main.name)
if len(libs) and job_dir_suffix:
# HDFS 2.2.0 fails to put file if the lib dir does not exist
self.create_hdfs_dir(r, lib_dir)
for lib in libs:
raw_data = dispatch.get_raw_binary(lib,
proxy_configs=proxy_configs,
remote=remote)
if isinstance(raw_data, dict) and raw_data["type"] == "path":
h.copy_from_local(r, raw_data['path'], lib_dir, hdfs_user)
else:
h.put_file_to_hdfs(r, raw_data, lib.name, lib_dir,
hdfs_user)
uploaded_paths.append(lib_dir + '/' + lib.name)
for lib in builtin_libs:
h.put_file_to_hdfs(r, lib['raw'], lib['name'], lib_dir,
hdfs_user)
uploaded_paths.append(lib_dir + '/' + lib['name'])
return uploaded_paths
def upload_job_files(where, job_dir, job, hdfs_user):
mains = job.mains or []
libs = job.libs or []
uploaded_paths = []
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(main)
h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user)
uploaded_paths.append(job_dir + '/' + main.name)
for lib in libs:
raw_data = dispatch.get_raw_binary(lib)
# HDFS 2.2.0 fails to put file if the lib dir does not exist
h.create_dir(r, job_dir + "/lib", hdfs_user)
h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib",
hdfs_user)
uploaded_paths.append(job_dir + '/lib/' + lib.name)
return uploaded_paths
def _upload_job_files_to_hdfs(self, where, job_dir, job,
proxy_configs=None):
mains = job.mains or []
libs = job.libs or []
uploaded_paths = []
hdfs_user = self.get_hdfs_user()
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(main, proxy_configs)
h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user)
uploaded_paths.append(job_dir + '/' + main.name)
for lib in libs:
raw_data = dispatch.get_raw_binary(lib, proxy_configs)
# HDFS 2.2.0 fails to put file if the lib dir does not exist
self.create_hdfs_dir(r, job_dir + "/lib")
h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib",
hdfs_user)
uploaded_paths.append(job_dir + '/lib/' + lib.name)
return uploaded_paths
def _upload_job_files_to_hdfs(self, where, job_dir, job,
proxy_configs=None):
mains = job.mains or []
libs = job.libs or []
uploaded_paths = []
hdfs_user = self.get_hdfs_user()
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(main, proxy_configs)
h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user)
uploaded_paths.append(job_dir + '/' + main.name)
for lib in libs:
raw_data = dispatch.get_raw_binary(lib, proxy_configs)
# HDFS 2.2.0 fails to put file if the lib dir does not exist
self.create_hdfs_dir(r, job_dir + "/lib")
h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib",
hdfs_user)
uploaded_paths.append(job_dir + '/lib/' + lib.name)
return uploaded_paths
def _upload_job_files_to_hdfs(self, where, job_dir, job, configs,
proxy_configs=None):
mains = job.mains or []
libs = job.libs or []
builtin_libs = edp.get_builtin_binaries(job, configs)
uploaded_paths = []
hdfs_user = self.get_hdfs_user()
job_dir_suffix = 'lib' if job.type != edp.JOB_TYPE_SHELL else ''
lib_dir = os.path.join(job_dir, job_dir_suffix)
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(
main, proxy_configs=proxy_configs, remote=r)
if isinstance(raw_data, dict) and raw_data["type"] == "path":
h.copy_from_local(r, raw_data['path'],
job_dir, hdfs_user)
else:
h.put_file_to_hdfs(r, raw_data, main.name,
job_dir, hdfs_user)
uploaded_paths.append(job_dir + '/' + main.name)
if len(libs) and job_dir_suffix:
# HDFS 2.2.0 fails to put file if the lib dir does not exist
self.create_hdfs_dir(r, lib_dir)
for lib in libs:
raw_data = dispatch.get_raw_binary(
lib, proxy_configs=proxy_configs, remote=remote)
if isinstance(raw_data, dict) and raw_data["type"] == "path":
h.copy_from_local(r, raw_data['path'],
lib_dir, hdfs_user)
else:
h.put_file_to_hdfs(r, raw_data, lib.name,
lib_dir, hdfs_user)
uploaded_paths.append(lib_dir + '/' + lib.name)
for lib in builtin_libs:
h.put_file_to_hdfs(r, lib['raw'], lib['name'], lib_dir,
hdfs_user)
uploaded_paths.append(lib_dir + '/' + lib['name'])
return uploaded_paths
def test_get_raw_binary(self, ctx, db_get_raw_data, i_s_get_raw_data,
i_s_get_raw_data_with_context):
ctx.return_value = mock.Mock()
job_binary = mock.Mock()
job_binary.url = 'internal-db://somebinary'
dispatch.get_raw_binary(job_binary)
self.assertEqual(1, db_get_raw_data.call_count)
job_binary.url = 'swift://container/object'
proxy_configs = dict(proxy_username='******',
proxy_password='******',
proxy_trust_id='proxytrust')
dispatch.get_raw_binary(job_binary, proxy_configs)
dispatch.get_raw_binary(job_binary, proxy_configs, with_context=True)
dispatch.get_raw_binary(job_binary, with_context=True)
self.assertEqual(1, i_s_get_raw_data.call_count)
self.assertEqual(2, i_s_get_raw_data_with_context.call_count)
def upload(r, dir, job_file):
dst = os.path.join(dir, job_file.name)
raw_data = dispatch.get_raw_binary(job_file, proxy_configs)
r.write_file_to(dst, raw_data)
uploaded_paths.append(dst)
def get_job_binary_data(id):
job_binary = conductor.job_binary_get(context.ctx(), id)
return dispatch.get_raw_binary(job_binary, with_context=True)
def upload(r, dir, job_file):
dst = os.path.join(dir, job_file.name)
raw_data = dispatch.get_raw_binary(job_file, proxy_configs)
r.write_file_to(dst, raw_data)
uploaded_paths.append(dst)
def upload(r, dir, job_file, proxy_configs):
dst = os.path.join(dir, job_file.name)
raw_data = dispatch.get_raw_binary(job_file, proxy_configs)
r.write_file_to(dst, raw_data)
return dst
def upload(r, dir, job_file, proxy_configs):
dst = os.path.join(dir, job_file.name)
raw_data = dispatch.get_raw_binary(job_file, proxy_configs)
r.write_file_to(dst, raw_data)
return dst
def get_job_binary_data(id):
job_binary = conductor.job_binary_get(context.ctx(), id)
return dispatch.get_raw_binary(job_binary, with_context=True)
