def upload_job_file(where, job_dir, job_origin, job):
main_binary = conductor.job_binary_get_raw_data(context.ctx(),
job_origin.url)
if job.type == 'Jar':
job_dir += '/lib'
with remote.get_remote(where) as r:
h.put_file_to_hdfs(r, main_binary, main_res_names[job.type], job_dir)
return "%s/%s" % (job_dir, main_res_names[job.type])
def upload_job_file(where, job_dir, job_origin, job):
main_binary = conductor.job_binary_get_raw_data(context.ctx(),
job_origin.url)
if job.type == 'Jar':
job_dir += '/lib'
with remote.get_remote(where) as r:
h.put_file_to_hdfs(r, main_binary, main_res_names[job.type], job_dir)
return "%s/%s" % (job_dir, main_res_names[job.type])
def upload_job_files(where, job_dir, job_origin):
mains = job_origin.mains or []
libs = job_origin.libs or []
uploaded_paths = []
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(main)
h.put_file_to_hdfs(r, raw_data, main.name, job_dir)
uploaded_paths.append(job_dir + '/' + main.name)
for lib in libs:
raw_data = dispatch.get_raw_binary(lib)
h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib")
uploaded_paths.append(job_dir + '/lib/' + lib.name)
return uploaded_paths
def upload_job_files(where, job_dir, job, hdfs_user):
mains = job.mains or []
libs = job.libs or []
uploaded_paths = []
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(main)
h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user)
uploaded_paths.append(job_dir + '/' + main.name)
for lib in libs:
raw_data = dispatch.get_raw_binary(lib)
h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib",
hdfs_user)
uploaded_paths.append(job_dir + '/lib/' + lib.name)
return uploaded_paths
def upload_workflow_file(where, job_dir, wf_xml):
with remote.get_remote(where) as r:
h.put_file_to_hdfs(r, wf_xml, "workflow.xml", job_dir)
return "%s/workflow.xml" % job_dir
def upload_workflow_file(where, job_dir, wf_xml, hdfs_user):
with remote.get_remote(where) as r:
h.put_file_to_hdfs(r, wf_xml, "workflow.xml", job_dir, hdfs_user)
return "%s/workflow.xml" % job_dir
