def _get_settings_values(self, settings, exe_configuration, job_type,
censor):
"""
:param settings: The settings
:type settings: dict
:param exe_configuration: The execution configuration
:type exe_configuration: :class:`job.execution.configuration.json.exe_config.ExecutionConfiguration`
:param job_type: The job type definition
:type job_type: :class:`job.models.JobType`
:param censor: Whether to censor secrets
:type censor: bool
:return: settings name and the value to replace it with
:rtype: dict
"""
config_settings = exe_configuration.get_dict()
param_replacements = {}
secret_settings = {}
# Isolate the job_type settings and convert to list
for task_dict in config_settings['tasks']:
if task_dict['type'] == 'main':
config_settings_dict = task_dict['settings']
for setting in settings:
setting_name = setting['name']
setting_is_secret = setting['secret']
if setting_is_secret:
job_index = job_type.get_secrets_key()
if not secret_settings:
secret_settings = secrets_mgr.retrieve_job_type_secrets(
job_index)
if setting_name in secret_settings.keys():
if censor:
settings_value = '*****'
else:
settings_value = secret_settings[setting_name]
param_replacements[setting_name] = settings_value
else:
param_replacements[setting_name] = ''
else:
if setting_name in config_settings_dict:
param_replacements[setting_name] = config_settings_dict[
setting_name]
else:
param_replacements[setting_name] = ''
return param_replacements
def _get_settings_values(self, settings, exe_configuration, job_type):
"""
:param settings: The settings
:type settings: JSON
:param exe_configuration: The execution configuration
:type exe_configuration: :class:`job.configuration.json.execution.exe_config.ExecutionConfiguration`
:return: settings name and the value to replace it with
:rtype: dict
"""
config_settings = exe_configuration.get_dict()
param_replacements = {}
secret_settings = {}
# Isolate the job_type settings and convert to list
config_settings = config_settings['job_task']['settings']
config_settings_dict = {
setting['name']: setting['value']
for setting in config_settings
}
for setting in settings:
setting_name = setting['name']
setting_is_secret = setting['secret']
if setting_is_secret:
job_type_name = job_type.get_job_type_name()
job_type_ver = job_type.get_job_type_version()
job_index = '-'.join([job_type_name, job_type_ver])
if not secret_settings:
secret_settings = secrets_mgr.retrieve_job_type_secrets(
job_index)
if setting_name in secret_settings.keys():
settings_value = secret_settings[setting_name]
exe_configuration.add_job_task_setting(
setting_name, '*****')
param_replacements[setting_name] = settings_value
else:
param_replacements[setting_name] = ''
else:
if setting_name in config_settings_dict:
param_replacements[setting_name] = config_settings_dict[
setting_name]
else:
param_replacements[setting_name] = ''
return param_replacements
def _configure_secrets(self, config, job_exe, job_type, interface):
"""Creates a copy of the configuration, configures secrets (masked in one of the copies), and applies any final
configuration
:param config: The execution configuration, where the secrets will be masked out
:type config: :class:`job.configuration.json.execution.exe_config.ExecutionConfiguration`
:param job_exe: The job execution model being scheduled
:type job_exe: :class:`job.models.JobExecution`
:param job_type: The job type model
:type job_type: :class:`job.models.JobType`
:param interface: The job interface
:type interface: :class:`job.configuration.interface.job_interface.JobInterface`
:returns: The copy of the execution configuration that contains the secrets
:rtype: :class:`job.configuration.json.execution.exe_config.ExecutionConfiguration`
"""
# Copy the configuration
config_with_secrets = config.create_copy()
# Configure settings values, some are secret
if job_type.is_system:
config.add_to_task('main', settings=self._system_settings_hidden)
config_with_secrets.add_to_task('main', settings=self._system_settings)
else:
config.add_to_task('pre', settings=self._system_settings_hidden)
config_with_secrets.add_to_task('pre', settings=self._system_settings)
config.add_to_task('post', settings=self._system_settings_hidden)
config_with_secrets.add_to_task('post', settings=self._system_settings)
job_config = job_type.get_job_configuration()
secret_settings = secrets_mgr.retrieve_job_type_secrets(job_type.get_secrets_key())
for _config, secrets_hidden in [(config, True), (config_with_secrets, False)]:
task_settings = {}
# TODO: use better interface method once we switch to Seed
for setting in interface.get_dict()['settings']:
name = setting['name']
if setting['secret']:
value = None
if name in secret_settings:
value = secret_settings[name]
if value is not None and secrets_hidden:
value = '*****'
else:
value = job_config.get_setting_value(name)
if setting['required'] or value is not None:
task_settings[name] = value
# TODO: command args and env var replacement from the interface should be removed once Scale drops
# support for old-style job types
args = config._get_task_dict('main')['args']
args = JobInterface._replace_command_parameters(args, task_settings)
env_vars = interface.populate_env_vars_arguments(task_settings)
_config.add_to_task('main', args=args, env_vars=env_vars, settings=task_settings)
# Configure env vars for settings
for _config in [config, config_with_secrets]:
for task_type in _config.get_task_types():
env_vars = {}
for name, value in _config.get_settings(task_type).items():
if value is not None:
env_name = normalize_env_var_name(name)
env_vars[env_name] = value
_config.add_to_task(task_type, env_vars=env_vars)
# Configure Docker parameters for env vars and Docker volumes
for _config in [config, config_with_secrets]:
existing_volumes = set()
for task_type in _config.get_task_types():
docker_params = []
for name, value in _config.get_env_vars(task_type).items():
docker_params.append(DockerParameter('env', '%s=%s' % (name, value)))
for name, volume in _config.get_volumes(task_type).items():
docker_params.append(volume.to_docker_param(is_created=(name in existing_volumes)))
existing_volumes.add(name)
_config.add_to_task(task_type, docker_params=docker_params)
# TODO: this feature should be removed once Scale drops support for job type docker params
# Configure docker parameters listed in job type
if job_type.docker_params:
docker_params = []
for key, value in job_type.docker_params.items():
docker_params.append(DockerParameter(key, value))
if docker_params:
config.add_to_task('main', docker_params=docker_params)
config_with_secrets.add_to_task('main', docker_params=docker_params)
return config_with_secrets
