def _upload_job_files(self, where, job_dir, job, job_configs):
def upload(r, dir, job_file, proxy_configs):
dst = os.path.join(dir, job_file.name)
raw_data = dispatch.get_raw_binary(job_file, proxy_configs)
r.write_file_to(dst, raw_data)
return dst
def upload_builtin(r, dir, builtin):
dst = os.path.join(dir, builtin['name'])
r.write_file_to(dst, builtin['raw'])
return dst
builtin_libs = []
if edp.is_adapt_spark_for_swift_enabled(job_configs.get('configs',
{})):
path = 'service/edp/resources/edp-spark-wrapper.jar'
name = 'builtin-%s.jar' % six.text_type(uuid.uuid4())
builtin_libs = [{'raw': files.get_file_text(path), 'name': name}]
uploaded_paths = []
builtin_paths = []
with remote.get_remote(where) as r:
mains = list(job.mains) if job.mains else []
libs = list(job.libs) if job.libs else []
for job_file in mains + libs:
uploaded_paths.append(
upload(r, job_dir, job_file,
job_configs.get('proxy_configs')))
for builtin in builtin_libs:
builtin_paths.append(upload_builtin(r, job_dir, builtin))
return uploaded_paths, builtin_paths
def _upload_job_files(self, where, job_dir, job, job_configs):
def upload(r, dir, job_file, proxy_configs):
path = jb_manager.JOB_BINARIES. \
get_job_binary_by_url(job_file.url). \
copy_binary_to_cluster(job_file,
proxy_configs=proxy_configs,
remote=r, context=context.ctx())
return path
def upload_builtin(r, dir, builtin):
dst = os.path.join(dir, builtin['name'])
r.write_file_to(dst, builtin['raw'])
return dst
builtin_libs = []
if edp.is_adapt_spark_for_swift_enabled(
job_configs.get('configs', {})):
path = 'service/edp/resources/edp-spark-wrapper.jar'
name = 'builtin-%s.jar' % uuidutils.generate_uuid()
builtin_libs = [{'raw': files.get_file_text(path),
'name': name}]
uploaded_paths = []
builtin_paths = []
with remote.get_remote(where) as r:
mains = list(job.mains) if job.mains else []
libs = list(job.libs) if job.libs else []
job_binaries = mains + libs
self._prepare_job_binaries(job_binaries, r)
for job_file in job_binaries:
uploaded_paths.append(
upload(r, job_dir, job_file,
job_configs.get('proxy_configs')))
for builtin in builtin_libs:
builtin_paths.append(
upload_builtin(r, job_dir, builtin))
return uploaded_paths, builtin_paths
def _upload_job_files(self, where, job_dir, job, job_configs):
def upload(r, dir, job_file, proxy_configs):
path = jb_manager.JOB_BINARIES. \
get_job_binary_by_url(job_file.url). \
copy_binary_to_cluster(job_file,
proxy_configs=proxy_configs,
remote=r, context=context.ctx())
return path
def upload_builtin(r, dir, builtin):
dst = os.path.join(dir, builtin['name'])
r.write_file_to(dst, builtin['raw'])
return dst
builtin_libs = []
if edp.is_adapt_spark_for_swift_enabled(
job_configs.get('configs', {})):
path = 'service/edp/resources/edp-spark-wrapper.jar'
name = 'builtin-%s.jar' % uuidutils.generate_uuid()
builtin_libs = [{'raw': files.get_file_text(path),
'name': name}]
uploaded_paths = []
builtin_paths = []
with remote.get_remote(where) as r:
mains = list(job.mains) if job.mains else []
libs = list(job.libs) if job.libs else []
job_binaries = mains + libs
self._prepare_job_binaries(job_binaries, r)
for job_file in job_binaries:
uploaded_paths.append(
upload(r, job_dir, job_file,
job_configs.get('proxy_configs')))
for builtin in builtin_libs:
builtin_paths.append(
upload_builtin(r, job_dir, builtin))
return uploaded_paths, builtin_paths
def _upload_job_files(self, where, job_dir, job, job_configs):
def upload(r, dir, job_file, proxy_configs):
dst = os.path.join(dir, job_file.name)
raw_data = dispatch.get_raw_binary(
job_file, proxy_configs=proxy_configs, remote=r)
if isinstance(raw_data, dict) and raw_data["type"] == "path":
dst = raw_data['path']
else:
r.write_file_to(dst, raw_data)
return dst
def upload_builtin(r, dir, builtin):
dst = os.path.join(dir, builtin['name'])
r.write_file_to(dst, builtin['raw'])
return dst
builtin_libs = []
if edp.is_adapt_spark_for_swift_enabled(
job_configs.get('configs', {})):
path = 'service/edp/resources/edp-spark-wrapper.jar'
name = 'builtin-%s.jar' % six.text_type(uuid.uuid4())
builtin_libs = [{'raw': files.get_file_text(path),
'name': name}]
uploaded_paths = []
builtin_paths = []
with remote.get_remote(where) as r:
mains = list(job.mains) if job.mains else []
libs = list(job.libs) if job.libs else []
for job_file in mains+libs:
uploaded_paths.append(
upload(r, job_dir, job_file,
job_configs.get('proxy_configs')))
for builtin in builtin_libs:
builtin_paths.append(
upload_builtin(r, job_dir, builtin))
return uploaded_paths, builtin_paths
