def _upload_job_files(self, where, job_dir, job, job_configs): def upload(r, dir, job_file, proxy_configs): path = jb_manager.JOB_BINARIES. \ get_job_binary_by_url(job_file.url). \ copy_binary_to_cluster(job_file, proxy_configs=proxy_configs, remote=r, context=context.ctx()) return path def upload_builtin(r, dir, builtin): dst = os.path.join(dir, builtin['name']) r.write_file_to(dst, builtin['raw']) return dst builtin_libs = [] if edp.is_adapt_spark_for_swift_enabled(job_configs.get('configs', {})): path = 'service/edp/resources/edp-spark-wrapper.jar' name = 'builtin-%s.jar' % uuidutils.generate_uuid() builtin_libs = [{ 'raw': files.try_get_file_text(path), 'name': name }] uploaded_paths = [] builtin_paths = [] with remote.get_remote(where) as r: mains = list(job.mains) if job.mains else [] libs = list(job.libs) if job.libs else [] job_binaries = mains + libs self._prepare_job_binaries(job_binaries, r) for job_file in job_binaries: uploaded_paths.append( upload(r, job_dir, job_file, job_configs.get('proxy_configs'))) for builtin in builtin_libs: builtin_paths.append(upload_builtin(r, job_dir, builtin)) return uploaded_paths, builtin_paths
def from_spec(cls, spec, validator_map, resource_roots): """Builds a copy script validator from a specification. :param spec: May be a string or a single-length dictionary of name to configuration values. Configuration values include: env_vars: A list of environment variable names to send to the script. output: A key into which to put the stdout of the script in the image_arguments of the validation run. :param validator_map: A map of validator name to class. :param resource_roots: The roots from which relative paths to resources (scripts and such) will be referenced. Any resource will be pulled from the first path in the list at which a file exists. :return: A validator that will copy a script to the image. """ jsonschema.validate(spec, cls.SPEC_SCHEMA) script_contents = None if isinstance(spec, six.string_types): script_path = spec output_var = None else: script_path, properties = list(six.iteritems(spec))[0] output_var = properties.get('output', None) script_contents = properties.get('inline') if not script_contents: for root in resource_roots: file_path = path.join(root, script_path) script_contents = files.try_get_file_text(file_path) if script_contents: break script_name = script_path.split('/')[2] if not script_contents: raise p_ex.ImageValidationSpecificationError( _("Script %s not found in any resource roots.") % script_path) return SaharaCopyScriptValidator(script_contents, script_name, output_var)
def from_spec(cls, spec, validator_map, resource_roots): """Builds a script validator from a specification. :param spec: May be a string or a single-length dictionary of name to configuration values. Configuration values include: env_vars: A list of environment variable names to send to the script. output: A key into which to put the stdout of the script in the env_map of the validation run. :param validator_map: A map of validator name to class. :param resource_roots: The roots from which relative paths to resources (scripts and such) will be referenced. Any resource will be pulled from the first path in the list at which a file exists. :return: A validator that will run a script on the image. """ jsonschema.validate(spec, cls.SPEC_SCHEMA) script_contents = None if isinstance(spec, six.string_types): script_path = spec env_vars, output_var = cls._DEFAULT_ENV_VARS, None else: script_path, properties = list(six.iteritems(spec))[0] env_vars = cls._DEFAULT_ENV_VARS + properties.get('env_vars', []) output_var = properties.get('output', None) script_contents = properties.get('inline') if not script_contents: for root in resource_roots: file_path = path.join(root, script_path) script_contents = files.try_get_file_text(file_path) if script_contents: break if not script_contents: raise p_ex.ImageValidationSpecificationError( _("Script %s not found in any resource roots.") % script_path) return SaharaScriptValidator(script_contents, env_vars, output_var)
def try_get_file_text(file_name, package='sahara', **kwargs): return files.try_get_file_text(file_name, package)