def test_get_raw_binary(self, ctx, db_get_raw_data, i_s_get_raw_data, i_s_get_raw_data_with_context, m_s_get_file_info): ctx.return_value = mock.Mock() job_binary = mock.Mock() job_binary.url = 'internal-db://somebinary' dispatch.get_raw_binary(job_binary) self.assertEqual(1, db_get_raw_data.call_count) job_binary.url = 'swift://container/object' proxy_configs = dict(proxy_username='******', proxy_password='******', proxy_trust_id='proxytrust') dispatch.get_raw_binary(job_binary, proxy_configs) dispatch.get_raw_binary(job_binary, proxy_configs, with_context=True) dispatch.get_raw_binary(job_binary, with_context=True) self.assertEqual(1, i_s_get_raw_data.call_count) self.assertEqual(2, i_s_get_raw_data_with_context.call_count) job_binary.url = 'manila://the_share_id/the_path' remote = mock.Mock() remote.instance.node_group.cluster.shares = [] remote.instance.node_group.shares = [] dispatch.get_raw_binary(job_binary, remote=remote) self.assertEqual(1, m_s_get_file_info.call_count)
def _upload_job_files_to_hdfs(self, where, job_dir, job, configs, proxy_configs=None): mains = job.mains or [] libs = job.libs or [] builtin_libs = edp.get_builtin_binaries(job, configs) uploaded_paths = [] hdfs_user = self.get_hdfs_user() lib_dir = job_dir + '/lib' with where.remote() as r: for m in mains: raw_data = d.get_raw_binary(m, proxy_configs) mfs.put_file_to_maprfs(r, raw_data, m.name, job_dir, hdfs_user) uploaded_paths.append(os.path.join(job_dir, m.name)) if len(libs) > 0: self.create_hdfs_dir(r, lib_dir) for l in libs: raw_data = d.get_raw_binary(l, proxy_configs) mfs.put_file_to_maprfs(r, raw_data, l.name, lib_dir, hdfs_user) uploaded_paths.append(os.path.join(lib_dir, l.name)) for lib in builtin_libs: mfs.put_file_to_maprfs(r, lib['raw'], lib['name'], lib_dir, hdfs_user) uploaded_paths.append(lib_dir + '/' + lib['name']) return uploaded_paths
def _upload_job_files_to_hdfs(self, where, job_dir, job, configs, proxy_configs=None): mains = job.mains or [] libs = job.libs or [] builtin_libs = edp.get_builtin_binaries(job, configs) uploaded_paths = [] hdfs_user = self.get_hdfs_user() lib_dir = job_dir + '/lib' with remote.get_remote(where) as r: for main in mains: raw_data = dispatch.get_raw_binary(main, proxy_configs) h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user) uploaded_paths.append(job_dir + '/' + main.name) if len(libs) > 0: # HDFS 2.2.0 fails to put file if the lib dir does not exist self.create_hdfs_dir(r, lib_dir) for lib in libs: raw_data = dispatch.get_raw_binary(lib, proxy_configs) h.put_file_to_hdfs(r, raw_data, lib.name, lib_dir, hdfs_user) uploaded_paths.append(lib_dir + '/' + lib.name) for lib in builtin_libs: h.put_file_to_hdfs(r, lib['raw'], lib['name'], lib_dir, hdfs_user) uploaded_paths.append(lib_dir + '/' + lib['name']) return uploaded_paths
def _upload_job_files_to_hdfs(self, where, job_dir, job, configs, proxy_configs=None): mains = job.mains or [] libs = job.libs or [] builtin_libs = edp.get_builtin_binaries(job, configs) uploaded_paths = [] hdfs_user = self.get_hdfs_user() job_dir_suffix = "lib" if job.type != edp.JOB_TYPE_SHELL else "" lib_dir = os.path.join(job_dir, job_dir_suffix) with remote.get_remote(where) as r: for main in mains: raw_data = dispatch.get_raw_binary(main, proxy_configs) h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user) uploaded_paths.append(job_dir + "/" + main.name) if len(libs) and job_dir_suffix: # HDFS 2.2.0 fails to put file if the lib dir does not exist self.create_hdfs_dir(r, lib_dir) for lib in libs: raw_data = dispatch.get_raw_binary(lib, proxy_configs) h.put_file_to_hdfs(r, raw_data, lib.name, lib_dir, hdfs_user) uploaded_paths.append(lib_dir + "/" + lib.name) for lib in builtin_libs: h.put_file_to_hdfs(r, lib["raw"], lib["name"], lib_dir, hdfs_user) uploaded_paths.append(lib_dir + "/" + lib["name"]) return uploaded_paths
def _upload_job_files_to_hdfs(self, where, job_dir, job): mains = job.mains or [] libs = job.libs or [] uploaded_paths = [] hdfs_user = self.get_hdfs_user() with where.remote() as r: for m in mains: raw_data = d.get_raw_binary(m) mfs.put_file_to_maprfs(r, raw_data, m.name, job_dir, hdfs_user) uploaded_paths.append(job_dir + '/' + m.name) for l in libs: raw_data = d.get_raw_binary(l) lib_dir = job_dir + '/lib/' self.create_hdfs_dir(r, lib_dir) mfs.put_file_to_maprfs(r, raw_data, l.name, lib_dir, hdfs_user) uploaded_paths.append(lib_dir + l.name) return uploaded_paths
def upload(r, dir, job_file, proxy_configs): dst = os.path.join(dir, job_file.name) raw_data = dispatch.get_raw_binary( job_file, proxy_configs=proxy_configs, remote=r) if isinstance(raw_data, dict) and raw_data["type"] == "path": dst = raw_data['path'] else: r.write_file_to(dst, raw_data) return dst
def upload_job_files(where, job_dir, job, hdfs_user): mains = job.mains or [] libs = job.libs or [] uploaded_paths = [] with remote.get_remote(where) as r: for main in mains: raw_data = dispatch.get_raw_binary(main) h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user) uploaded_paths.append(job_dir + '/' + main.name) for lib in libs: raw_data = dispatch.get_raw_binary(lib) # HDFS 2.2.0 fails to put file if the lib dir does not exist h.create_dir(r, job_dir + "/lib", hdfs_user) h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib", hdfs_user) uploaded_paths.append(job_dir + '/lib/' + lib.name) return uploaded_paths
def upload(r, dir, job_file, proxy_configs): dst = os.path.join(dir, job_file.name) raw_data = dispatch.get_raw_binary(job_file, proxy_configs=proxy_configs, remote=remote) if isinstance(raw_data, dict) and raw_data["type"] == "path": dst = raw_data['path'] else: r.write_file_to(dst, raw_data) return dst
def _upload_job_files_to_hdfs(self, where, job_dir, job, configs, proxy_configs=None): mains = job.mains or [] libs = job.libs or [] builtin_libs = edp.get_builtin_binaries(job, configs) uploaded_paths = [] hdfs_user = self.get_hdfs_user() job_dir_suffix = 'lib' if job.type != edp.JOB_TYPE_SHELL else '' lib_dir = os.path.join(job_dir, job_dir_suffix) with remote.get_remote(where) as r: for main in mains: raw_data = dispatch.get_raw_binary(main, proxy_configs=proxy_configs, remote=r) if isinstance(raw_data, dict) and raw_data["type"] == "path": h.copy_from_local(r, raw_data['path'], job_dir, hdfs_user) else: h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user) uploaded_paths.append(job_dir + '/' + main.name) if len(libs) and job_dir_suffix: # HDFS 2.2.0 fails to put file if the lib dir does not exist self.create_hdfs_dir(r, lib_dir) for lib in libs: raw_data = dispatch.get_raw_binary(lib, proxy_configs=proxy_configs, remote=remote) if isinstance(raw_data, dict) and raw_data["type"] == "path": h.copy_from_local(r, raw_data['path'], lib_dir, hdfs_user) else: h.put_file_to_hdfs(r, raw_data, lib.name, lib_dir, hdfs_user) uploaded_paths.append(lib_dir + '/' + lib.name) for lib in builtin_libs: h.put_file_to_hdfs(r, lib['raw'], lib['name'], lib_dir, hdfs_user) uploaded_paths.append(lib_dir + '/' + lib['name']) return uploaded_paths
def _upload_job_files_to_hdfs(self, where, job_dir, job, proxy_configs=None): mains = job.mains or [] libs = job.libs or [] uploaded_paths = [] hdfs_user = self.get_hdfs_user() with remote.get_remote(where) as r: for main in mains: raw_data = dispatch.get_raw_binary(main, proxy_configs) h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user) uploaded_paths.append(job_dir + '/' + main.name) for lib in libs: raw_data = dispatch.get_raw_binary(lib, proxy_configs) # HDFS 2.2.0 fails to put file if the lib dir does not exist self.create_hdfs_dir(r, job_dir + "/lib") h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib", hdfs_user) uploaded_paths.append(job_dir + '/lib/' + lib.name) return uploaded_paths
def _upload_job_files_to_hdfs(self, where, job_dir, job, configs, proxy_configs=None): mains = job.mains or [] libs = job.libs or [] builtin_libs = edp.get_builtin_binaries(job, configs) uploaded_paths = [] hdfs_user = self.get_hdfs_user() job_dir_suffix = 'lib' if job.type != edp.JOB_TYPE_SHELL else '' lib_dir = os.path.join(job_dir, job_dir_suffix) with remote.get_remote(where) as r: for main in mains: raw_data = dispatch.get_raw_binary( main, proxy_configs=proxy_configs, remote=r) if isinstance(raw_data, dict) and raw_data["type"] == "path": h.copy_from_local(r, raw_data['path'], job_dir, hdfs_user) else: h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user) uploaded_paths.append(job_dir + '/' + main.name) if len(libs) and job_dir_suffix: # HDFS 2.2.0 fails to put file if the lib dir does not exist self.create_hdfs_dir(r, lib_dir) for lib in libs: raw_data = dispatch.get_raw_binary( lib, proxy_configs=proxy_configs, remote=remote) if isinstance(raw_data, dict) and raw_data["type"] == "path": h.copy_from_local(r, raw_data['path'], lib_dir, hdfs_user) else: h.put_file_to_hdfs(r, raw_data, lib.name, lib_dir, hdfs_user) uploaded_paths.append(lib_dir + '/' + lib.name) for lib in builtin_libs: h.put_file_to_hdfs(r, lib['raw'], lib['name'], lib_dir, hdfs_user) uploaded_paths.append(lib_dir + '/' + lib['name']) return uploaded_paths
def test_get_raw_binary(self, ctx, db_get_raw_data, i_s_get_raw_data, i_s_get_raw_data_with_context): ctx.return_value = mock.Mock() job_binary = mock.Mock() job_binary.url = 'internal-db://somebinary' dispatch.get_raw_binary(job_binary) self.assertEqual(1, db_get_raw_data.call_count) job_binary.url = 'swift://container/object' proxy_configs = dict(proxy_username='******', proxy_password='******', proxy_trust_id='proxytrust') dispatch.get_raw_binary(job_binary, proxy_configs) dispatch.get_raw_binary(job_binary, proxy_configs, with_context=True) dispatch.get_raw_binary(job_binary, with_context=True) self.assertEqual(1, i_s_get_raw_data.call_count) self.assertEqual(2, i_s_get_raw_data_with_context.call_count)
def upload(r, dir, job_file): dst = os.path.join(dir, job_file.name) raw_data = dispatch.get_raw_binary(job_file, proxy_configs) r.write_file_to(dst, raw_data) uploaded_paths.append(dst)
def get_job_binary_data(id): job_binary = conductor.job_binary_get(context.ctx(), id) return dispatch.get_raw_binary(job_binary, with_context=True)
def upload(r, dir, job_file, proxy_configs): dst = os.path.join(dir, job_file.name) raw_data = dispatch.get_raw_binary(job_file, proxy_configs) r.write_file_to(dst, raw_data) return dst