def _get_config_with_production_and_tier(self, cluster, config):
tier_configurations = self._get_tier_configurations(cluster)
def to_bool(value):
return Boolean.coerce(value)
task = config.job().taskConfig
if task.tier is None:
backfill_args = {
'tier':
String(
next((t.name for t in tier_configurations.tiers
if not to_bool(t.settings['preemptible']) == task.
production and not to_bool(t.settings['revocable'])),
tier_configurations.defaultTierName))
}
else:
backfill_args = {
'production':
Boolean(
next((not to_bool(t.settings['preemptible'])
for t in tier_configurations.tiers
if t.name == task.tier), task.production))
}
config.update_job(config.raw()(**backfill_args))
return config
def test_choice_type():
IntStr = Choice("IntStrFloat", (Integer, String))
one = IntStr(123)
two = IntStr("123")
three = IntStr("abc")
assert one.unwrap() == Integer(123)
assert two.unwrap() == Integer(123)
assert three.unwrap() == String("abc")
def substitute_thermos(cls, command, task, cluster, **kw):
prefix_command = 'cd %s;' % cls.thermos_sandbox(cluster, **kw)
thermos_namespace = ThermosContext(
task_id=task.assignedTask.taskId,
ports=task.assignedTask.assignedPorts)
mesos_namespace = MesosContext(instance=task.assignedTask.instanceId)
command = String(prefix_command + command) % Environment(
thermos=thermos_namespace, mesos=mesos_namespace)
return command.get()
def interpolate_cmd(task, cmd):
"""
:param task: Assigned task passed from Mesos Agent
:param cmd: Command defined inside shell_command inside config.
:return: Interpolated cmd with filled in values, for example ports.
"""
thermos_namespace = ThermosContext(task_id=task.taskId,
ports=task.assignedPorts)
mesos_namespace = MesosContext(instance=task.instanceId)
command = String(cmd) % Environment(thermos=thermos_namespace,
mesos=mesos_namespace)
return command.get()
def _build_path(context, target):
(task_instance,
path) = ScpCommand._extract_task_instance_and_path(context, target)
# No jobkey is specified therefore we are using a local path.
if (task_instance is None):
return path
# Jobkey specified, we want to convert to the user@host:file scp format
(cluster, role, env, name) = task_instance.jobkey
instance = set([task_instance.instance])
api = context.get_api(cluster)
resp = api.query(
api.build_query(role, name, env=env, instances=instance))
context.log_response_and_raise(
resp,
err_msg=('Unable to get information about instance: %s' %
combine_messages(resp)))
if (resp.result.scheduleStatusResult.tasks is None
or len(resp.result.scheduleStatusResult.tasks) == 0):
raise context.CommandError(
EXIT_INVALID_PARAMETER, ScpCommand.JOB_NOT_FOUND_ERROR_MSG %
(task_instance.jobkey, task_instance.instance))
first_task = resp.result.scheduleStatusResult.tasks[0]
assigned = first_task.assignedTask
role = assigned.task.job.role
slave_host = assigned.slaveHost
# If path is absolute, use that. Else if it is a tilde expansion, throw an error.
# Otherwise, use sandbox as relative root.
normalized_input_path = os.path.normpath(path)
if (os.path.isabs(normalized_input_path)):
final_path = normalized_input_path
elif (normalized_input_path.startswith('~/')
or normalized_input_path == '~'):
raise context.CommandError(EXIT_INVALID_PARAMETER,
ScpCommand.TILDE_USAGE_ERROR_MSG % path)
else:
sandbox_path_pre_format = DistributedCommandRunner.thermos_sandbox(
api.cluster, executor_sandbox=context.options.executor_sandbox)
thermos_namespace = ThermosContext(task_id=assigned.taskId,
ports=assigned.assignedPorts)
sandbox_path = String(sandbox_path_pre_format) % Environment(
thermos=thermos_namespace)
# Join the individual folders to the sandbox path to build safely
final_path = os.path.join(str(sandbox_path),
*normalized_input_path.split(os.sep))
return '%s@%s:%s' % (role, slave_host, final_path)