def upload_job_files(where, job_dir, job_origin): mains = job_origin.mains or [] libs = job_origin.libs or [] uploaded_paths = [] with remote.get_remote(where) as r: for main in mains: raw_data = dispatch.get_raw_binary(main) h.put_file_to_hdfs(r, raw_data, main.name, job_dir) uploaded_paths.append(job_dir + '/' + main.name) for lib in libs: raw_data = dispatch.get_raw_binary(lib) h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib") uploaded_paths.append(job_dir + '/lib/' + lib.name) return uploaded_paths
def upload_job_files(where, job_dir, job, hdfs_user): mains = job.mains or [] libs = job.libs or [] uploaded_paths = [] with remote.get_remote(where) as r: for main in mains: raw_data = dispatch.get_raw_binary(main) h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user) uploaded_paths.append(job_dir + '/' + main.name) for lib in libs: raw_data = dispatch.get_raw_binary(lib) h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib", hdfs_user) uploaded_paths.append(job_dir + '/lib/' + lib.name) return uploaded_paths
def get_job_binary_data(id): job_binary = conductor.job_binary_get(context.ctx(), id) return dispatch.get_raw_binary(job_binary)
def get_job_binary_data(id): job_binary = conductor.job_binary_get(context.ctx(), id) return dispatch.get_raw_binary(job_binary)