def upload_job_file(where, job_dir, job_origin, job): main_binary = conductor.job_binary_get_raw_data(context.ctx(), job_origin.url) if job.type == 'Jar': job_dir += '/lib' with remote.get_remote(where) as r: h.put_file_to_hdfs(r, main_binary, main_res_names[job.type], job_dir) return "%s/%s" % (job_dir, main_res_names[job.type])
def upload_job_files(where, job_dir, job_origin): mains = job_origin.mains or [] libs = job_origin.libs or [] uploaded_paths = [] with remote.get_remote(where) as r: for main in mains: raw_data = dispatch.get_raw_binary(main) h.put_file_to_hdfs(r, raw_data, main.name, job_dir) uploaded_paths.append(job_dir + '/' + main.name) for lib in libs: raw_data = dispatch.get_raw_binary(lib) h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib") uploaded_paths.append(job_dir + '/lib/' + lib.name) return uploaded_paths
def upload_job_files(where, job_dir, job, hdfs_user): mains = job.mains or [] libs = job.libs or [] uploaded_paths = [] with remote.get_remote(where) as r: for main in mains: raw_data = dispatch.get_raw_binary(main) h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user) uploaded_paths.append(job_dir + '/' + main.name) for lib in libs: raw_data = dispatch.get_raw_binary(lib) h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib", hdfs_user) uploaded_paths.append(job_dir + '/lib/' + lib.name) return uploaded_paths
def upload_workflow_file(where, job_dir, wf_xml): with remote.get_remote(where) as r: h.put_file_to_hdfs(r, wf_xml, "workflow.xml", job_dir) return "%s/workflow.xml" % job_dir
def upload_workflow_file(where, job_dir, wf_xml, hdfs_user): with remote.get_remote(where) as r: h.put_file_to_hdfs(r, wf_xml, "workflow.xml", job_dir, hdfs_user) return "%s/workflow.xml" % job_dir