def task_instance_from_job(job, instance):
instance_context = MesosContext(instance=instance)
# TODO(Sathya): Remove health_check_interval_secs references after deprecation cycle is complete.
health_check_config = HealthCheckConfig()
if job.has_health_check_interval_secs():
health_check_config = HealthCheckConfig(interval_secs=job.health_check_interval_secs().get())
elif job.has_health_check_config():
health_check_config = job.health_check_config()
ti = MesosTaskInstance(task=job.task(),
layout=job.layout(),
role=job.role(),
health_check_interval_secs=health_check_config.interval_secs().get(),
health_check_config=health_check_config,
instance=instance)
if job.has_announce():
ti = ti(announce=job.announce())
if job.has_environment():
ti = ti(environment=job.environment())
return ti.bind(mesos=instance_context).interpolate()
def task_instance_from_job(job, instance):
instance_context = MesosContext(instance=instance)
# TODO(Sathya): Remove health_check_interval_secs references after deprecation cycle is complete.
health_check_config = HealthCheckConfig()
if job.has_health_check_interval_secs():
health_check_config = HealthCheckConfig(
interval_secs=job.health_check_interval_secs().get())
elif job.has_health_check_config():
health_check_config = job.health_check_config()
ti = MesosTaskInstance(
task=job.task(),
layout=job.layout(),
role=job.role(),
health_check_interval_secs=health_check_config.interval_secs().get(),
health_check_config=health_check_config,
instance=instance)
if job.has_announce():
ti = ti(announce=job.announce())
if job.has_environment():
ti = ti(environment=job.environment())
return ti.bind(mesos=instance_context).interpolate()
def mesos_task_instance_from_assigned_task(assigned_task):
"""Deserialize MesosTaskInstance from an AssignedTask thrift."""
thermos_task = assigned_task.task.executorConfig.data
if not thermos_task:
raise ValueError('Task did not have a thermos config!')
try:
json_blob = json.loads(thermos_task)
except (TypeError, ValueError) as e:
raise ValueError('Could not deserialize thermos config: %s' % e)
# As part of the transition for MESOS-2133, we can send either a MesosTaskInstance
# or we can be sending a MesosJob. So handle both possible cases. Once everyone
# is using MesosJob, then we can begin to leverage additional information that
# becomes available such as cluster.
if 'instance' in json_blob:
return MesosTaskInstance.json_loads(thermos_task)
# This is a MesosJob
mti, refs = task_instance_from_job(MesosJob.json_loads(thermos_task),
assigned_task.instanceId)
for ref in refs:
# If the ref is {{thermos.task_id}} or a subscope of
# {{thermos.ports}}, it currently gets bound by the Thermos Runner,
# so we must leave them unbound.
#
# {{thermos.user}} is a legacy binding which we can safely ignore.
#
# TODO(wickman) These should be rewritten by the mesos client to use
# %%style%% replacements in order to allow us to better type-check configs
# client-side.
if ref == Ref.from_address('thermos.task_id'):
continue
if Ref.subscope(Ref.from_address('thermos.ports'), ref):
continue
if ref == Ref.from_address('thermos.user'):
continue
raise ValueError('Unexpected unbound refs: %s' %
' '.join(map(str, refs)))
return mti
def mesos_task_instance_from_assigned_task(assigned_task):
"""Deserialize MesosTaskInstance from an AssignedTask thrift."""
thermos_task = assigned_task.task.executorConfig.data
if not thermos_task:
raise ValueError('Task did not have a thermos config!')
try:
json_blob = json.loads(thermos_task)
except (TypeError, ValueError) as e:
raise ValueError('Could not deserialize thermos config: %s' % e)
# As part of the transition for MESOS-2133, we can send either a MesosTaskInstance
# or we can be sending a MesosJob. So handle both possible cases. Once everyone
# is using MesosJob, then we can begin to leverage additional information that
# becomes available such as cluster.
if 'instance' in json_blob:
return MesosTaskInstance.json_loads(thermos_task)
# This is a MesosJob
mti, refs = task_instance_from_job(MesosJob.json_loads(thermos_task), assigned_task.instanceId)
for ref in refs:
# If the ref is {{thermos.task_id}} or a subscope of
# {{thermos.ports}}, it currently gets bound by the Thermos Runner,
# so we must leave them unbound.
#
# {{thermos.user}} is a legacy binding which we can safely ignore.
#
# TODO(wickman) These should be rewritten by the mesos client to use
# %%style%% replacements in order to allow us to better type-check configs
# client-side.
if ref == Ref.from_address('thermos.task_id'):
continue
if Ref.subscope(Ref.from_address('thermos.ports'), ref):
continue
if ref == Ref.from_address('thermos.user'):
continue
raise ValueError('Unexpected unbound refs: %s' % ' '.join(map(str, refs)))
return mti
Resources,
Task,
)
from twitter.aurora.executor.common.sandbox import DirectorySandbox
from twitter.aurora.executor.common.status_checker import ExitState
from twitter.aurora.executor.thermos_task_runner import ThermosTaskRunner
from twitter.common import log
from twitter.common.log.options import LogOptions
from twitter.common.dirutil import safe_rmtree
from twitter.common.quantity import Amount, Time
from twitter.common.contextutil import temporary_dir
TASK = MesosTaskInstance(
instance=0,
role=getpass.getuser(),
task=Task(
resources=Resources(cpu=1.0, ram=16 * MB, disk=32 * MB),
name='hello_world',
processes=[Process(name='hello_world', cmdline='{{command}}')],
))
class TestThermosTaskRunnerIntegration(object):
PANTS_BUILT = False
LOG_DIR = None
@classmethod
def setup_class(cls):
cls.LOG_DIR = tempfile.mkdtemp()
LogOptions.set_log_dir(cls.LOG_DIR)
LogOptions.set_disk_log_level('DEBUG')
log.init('executor_logger')
task_id = thermos_config.task().name().get() + '-001'
at = AssignedTask(taskId=task_id,
task=TaskConfig(executorConfig=ExecutorConfig(
name=AURORA_EXECUTOR_NAME,
data=thermos_config.json_dumps()),
owner=Identity(role=role, user=role)),
assignedPorts=assigned_ports,
**kw)
td = mesos_pb.TaskInfo()
td.task_id.value = task_id
td.name = thermos_config.task().name().get()
td.data = serialize(at)
return td
BASE_MTI = MesosTaskInstance(instance=0, role=getpass.getuser())
BASE_TASK = Task(resources=Resources(cpu=1.0, ram=16 * MB, disk=32 * MB))
HELLO_WORLD_TASK_ID = 'hello_world-001'
HELLO_WORLD = BASE_TASK(name='hello_world',
processes=[
Process(name='hello_world_{{thermos.task_id}}',
cmdline='echo hello world')
])
HELLO_WORLD_MTI = BASE_MTI(task=HELLO_WORLD)
SLEEP60 = BASE_TASK(processes=[Process(name='sleep60', cmdline='sleep 60')])
SLEEP2 = BASE_TASK(processes=[Process(name='sleep2', cmdline='sleep 2')])
SLEEP60_MTI = BASE_MTI(task=SLEEP60)
MESOS_JOB = MesosJob(
