def _upload_job_files(self, where, job_dir, job, job_configs):
def upload(r, dir, job_file, proxy_configs):
path = jb_manager.JOB_BINARIES. \
get_job_binary_by_url(job_file.url). \
copy_binary_to_cluster(job_file,
proxy_configs=proxy_configs,
remote=r, context=context.ctx())
return path
def upload_builtin(r, dir, builtin):
dst = os.path.join(dir, builtin['name'])
r.write_file_to(dst, builtin['raw'])
return dst
builtin_libs = []
if edp.is_adapt_spark_for_swift_enabled(job_configs.get('configs',
{})):
path = 'service/edp/resources/edp-spark-wrapper.jar'
name = 'builtin-%s.jar' % uuidutils.generate_uuid()
builtin_libs = [{
'raw': files.try_get_file_text(path),
'name': name
}]
uploaded_paths = []
builtin_paths = []
with remote.get_remote(where) as r:
mains = list(job.mains) if job.mains else []
libs = list(job.libs) if job.libs else []
job_binaries = mains + libs
self._prepare_job_binaries(job_binaries, r)
for job_file in job_binaries:
uploaded_paths.append(
upload(r, job_dir, job_file,
job_configs.get('proxy_configs')))
for builtin in builtin_libs:
builtin_paths.append(upload_builtin(r, job_dir, builtin))
return uploaded_paths, builtin_paths
def from_spec(cls, spec, validator_map, resource_roots):
"""Builds a copy script validator from a specification.
:param spec: May be a string or a single-length dictionary of name to
configuration values. Configuration values include:
env_vars: A list of environment variable names to send to the
script.
output: A key into which to put the stdout of the script in the
image_arguments of the validation run.
:param validator_map: A map of validator name to class.
:param resource_roots: The roots from which relative paths to
resources (scripts and such) will be referenced. Any resource will
be pulled from the first path in the list at which a file exists.
:return: A validator that will copy a script to the image.
"""
jsonschema.validate(spec, cls.SPEC_SCHEMA)
script_contents = None
if isinstance(spec, six.string_types):
script_path = spec
output_var = None
else:
script_path, properties = list(six.iteritems(spec))[0]
output_var = properties.get('output', None)
script_contents = properties.get('inline')
if not script_contents:
for root in resource_roots:
file_path = path.join(root, script_path)
script_contents = files.try_get_file_text(file_path)
if script_contents:
break
script_name = script_path.split('/')[2]
if not script_contents:
raise p_ex.ImageValidationSpecificationError(
_("Script %s not found in any resource roots.") % script_path)
return SaharaCopyScriptValidator(script_contents, script_name,
output_var)
def from_spec(cls, spec, validator_map, resource_roots):
"""Builds a script validator from a specification.
:param spec: May be a string or a single-length dictionary of name to
configuration values. Configuration values include:
env_vars: A list of environment variable names to send to the
script.
output: A key into which to put the stdout of the script in the
env_map of the validation run.
:param validator_map: A map of validator name to class.
:param resource_roots: The roots from which relative paths to
resources (scripts and such) will be referenced. Any resource will
be pulled from the first path in the list at which a file exists.
:return: A validator that will run a script on the image.
"""
jsonschema.validate(spec, cls.SPEC_SCHEMA)
script_contents = None
if isinstance(spec, six.string_types):
script_path = spec
env_vars, output_var = cls._DEFAULT_ENV_VARS, None
else:
script_path, properties = list(six.iteritems(spec))[0]
env_vars = cls._DEFAULT_ENV_VARS + properties.get('env_vars', [])
output_var = properties.get('output', None)
script_contents = properties.get('inline')
if not script_contents:
for root in resource_roots:
file_path = path.join(root, script_path)
script_contents = files.try_get_file_text(file_path)
if script_contents:
break
if not script_contents:
raise p_ex.ImageValidationSpecificationError(
_("Script %s not found in any resource roots.") % script_path)
return SaharaScriptValidator(script_contents, env_vars, output_var)
def try_get_file_text(file_name, package='sahara', **kwargs):
return files.try_get_file_text(file_name, package)
