def test_deserialize_thermos_task_unbound_refs():
# test unbound {{standard}} refs
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=MESOS_JOB(task=HELLO_WORLD_UNBOUND).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
with pytest.raises(TaskInfoError) as execinfo:
mesos_task_instance_from_assigned_task(assigned_task)
assert "Unexpected unbound refs: {{unbound_cmd}} {{unbound}}" in execinfo.value.message
# test bound unscoped refs, valid case.
task = BASE_TASK(name='task_name',
processes=[
Process(name='process_name',
cmdline='echo {{thermos.ports[health]}}')
])
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=MESOS_JOB(task=task).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
assert mesos_task_instance_from_assigned_task(assigned_task) is not None
# test unbound unscoped refs
for cmdline in ('echo {{hello_{{thermos.ports[health]}}}}',
'echo {{hello_{{thermos.user_id}}}}'):
task = BASE_TASK(
name='task_name',
processes=[Process(name='process_name', cmdline=cmdline)])
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=MESOS_JOB(task=task).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
with pytest.raises(UnexpectedUnboundRefsError):
mesos_task_instance_from_assigned_task(assigned_task)
def test_deserialize_thermos_task():
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=MESOS_JOB(task=HELLO_WORLD).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
assert mesos_task_instance_from_assigned_task(assigned_task) == BASE_MTI(
task=HELLO_WORLD)
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=HELLO_WORLD_MTI.json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
assert mesos_task_instance_from_assigned_task(assigned_task) == BASE_MTI(
task=HELLO_WORLD)
def assigned_task_from_mesos_task(task):
"""Deserialize AssignedTask from a launchTask task protocol buffer."""
try:
assigned_task = thrift_deserialize(AssignedTask(), task.data)
except (EOFError, TException) as e:
raise TaskInfoError('Could not deserialize task! %s' % e)
return assigned_task
def test_from_assigned_task_shell(self):
interval_secs = 17
initial_interval_secs = 3
max_consecutive_failures = 2
timeout_secs = 5
shell_config = ShellHealthChecker(shell_command='failed command')
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos-generic',
data=MESOS_JOB(
task=HELLO_WORLD,
health_check_config=HealthCheckConfig(
health_checker=HealthCheckerConfig(shell=shell_config),
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=max_consecutive_failures,
timeout_secs=timeout_secs,
)).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=1)
execconfig_data = json.loads(assigned_task.task.executorConfig.data)
assert execconfig_data['health_check_config']['health_checker'][
'shell']['shell_command'] == 'failed command'
health_checker = HealthCheckerProvider().from_assigned_task(
assigned_task, None)
assert health_checker.threaded_health_checker.interval == interval_secs
assert health_checker.threaded_health_checker.initial_interval == initial_interval_secs
hct_max_fail = health_checker.threaded_health_checker.max_consecutive_failures
assert hct_max_fail == max_consecutive_failures
def create_scheduled_task(instance, start_time):
task = ScheduledTask()
task.assignedTask = AssignedTask()
task.assignedTask.taskId = "task_%s" % instance
task.assignedTask.slaveId = "random_machine_id"
task.assignedTask.slaveHost = "junk.nothing"
task.assignedTask.task = TaskConfig()
task.assignedTask.task.owner = Identity(role="nobody")
task.assignedTask.task.environment = "prod"
task.assignedTask.task.jobName = "flibber"
task.assignedTask.task.isService = False
task.assignedTask.task.numCpus = 2
task.assignedTask.task.ramMb = 2048
task.assignedTask.task.diskMb = 4096
task.assignedTask.task.priority = 7
task.assignedTask.task.maxTaskFailures = 3
task.assignedTask.task.production = False
task.assignedTask.task.requestedPorts = ["http"]
task.assignedTask.assignedPorts = {"http": 1001}
task.assignedTask.instanceId = 0
task.status = 2
task.failureCount = instance + 4
task.taskEvents = create_task_events(start_time)
task.ancestorId = "random_task_ancestor%s" % instance
return task
def create_scheduled_tasks(cls):
tasks = []
for name in ['foo', 'bar', 'baz']:
task = ScheduledTask(
failureCount=0,
assignedTask=AssignedTask(
taskId=1287391823,
slaveHost='slavehost',
task=TaskConfig(
maxTaskFailures=1,
executorConfig=ExecutorConfig(data='fake data'),
metadata=[],
job=JobKey(role=cls.TEST_ROLE,
environment=cls.TEST_ENV,
name=name),
owner=Identity(role=cls.TEST_ROLE),
environment=cls.TEST_ENV,
jobName=name,
numCpus=2,
ramMb=2,
diskMb=2),
instanceId=4237894,
assignedPorts={}),
status=ScheduleStatus.RUNNING,
taskEvents=[
TaskEvent(timestamp=28234726395,
status=ScheduleStatus.RUNNING,
message="Hi there")
])
tasks.append(task)
return tasks
def create_scheduled_task(instance, start_time):
task = ScheduledTask(assignedTask=AssignedTask(
taskId="task_%s" % instance,
slaveId="random_machine_id",
slaveHost="junk.nothing",
task=TaskConfig(job=JobKey(role="nobody",
environment="prod",
name='flibber'),
isService=False,
resources=frozenset([
Resource(numCpus=2),
Resource(ramMb=2048),
Resource(diskMb=4096)
]),
priority=7,
maxTaskFailures=3,
production=False),
assignedPorts={"http": 1001},
instanceId=instance),
status=2,
failureCount=instance + 4,
taskEvents=create_task_events(start_time),
ancestorId="random_task_ancestor%s" %
instance)
return task
def test_interpolate_cmd(self):
"""Making sure thermos.ports[foo] gets correctly substituted with assignedPorts info."""
interval_secs = 17
initial_interval_secs = 3
max_consecutive_failures = 2
timeout_secs = 5
shell_cmd = 'FOO_PORT={{thermos.ports[foo]}} failed command'
shell_config = ShellHealthChecker(shell_command=shell_cmd)
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos-generic',
data=MESOS_JOB(
task=HELLO_WORLD,
health_check_config=HealthCheckConfig(
health_checker=HealthCheckerConfig(shell=shell_config),
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=max_consecutive_failures,
timeout_secs=timeout_secs,
)).json_dumps()))
assigned_task = AssignedTask(task=task_config,
instanceId=1,
assignedPorts={'foo': 9001})
interpolated_cmd = HealthCheckerProvider.interpolate_cmd(assigned_task,
cmd=shell_cmd)
assert interpolated_cmd == 'FOO_PORT=9001 failed command'
def test_from_assigned_task_http_endpoint_style_config(self):
interval_secs = 17
initial_interval_secs = 3
max_consecutive_failures = 2
http_config = HttpHealthChecker(endpoint='/foo',
expected_response='bar',
expected_response_code=201)
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos',
data=MESOS_JOB(
task=HELLO_WORLD,
health_check_config=HealthCheckConfig(
health_checker=HealthCheckerConfig(http=http_config),
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=max_consecutive_failures,
timeout_secs=7)).json_dumps()))
assigned_task = AssignedTask(task=task_config,
instanceId=1,
assignedPorts={'health': 9001})
execconfig_data = json.loads(assigned_task.task.executorConfig.data)
http_exec_config = execconfig_data['health_check_config'][
'health_checker']['http']
assert http_exec_config['endpoint'] == '/foo'
assert http_exec_config['expected_response'] == 'bar'
assert http_exec_config['expected_response_code'] == 201
health_checker = HealthCheckerProvider().from_assigned_task(
assigned_task, None)
assert health_checker.threaded_health_checker.interval == interval_secs
assert health_checker.threaded_health_checker.initial_interval == initial_interval_secs
def test_from_assigned_task_shell_no_demotion(self, mock_getpwnam):
interval_secs = 17
initial_interval_secs = 3
max_consecutive_failures = 2
timeout_secs = 5
shell_config = ShellHealthChecker(shell_command='failed command')
task_config = TaskConfig(
job=JobKey(role='role', environment='env', name='name'),
executorConfig=ExecutorConfig(
name='thermos-generic',
data=MESOS_JOB(
task=HELLO_WORLD,
health_check_config=HealthCheckConfig(
health_checker=HealthCheckerConfig(shell=shell_config),
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=max_consecutive_failures,
timeout_secs=timeout_secs,
)).json_dumps()))
assigned_task = AssignedTask(task=task_config,
instanceId=1,
assignedPorts={'foo': 9001})
execconfig_data = json.loads(assigned_task.task.executorConfig.data)
assert execconfig_data['health_check_config']['health_checker'][
'shell']['shell_command'] == 'failed command'
health_checker = HealthCheckerProvider(
nosetuid_health_checks=True).from_assigned_task(
assigned_task, None)
assert health_checker.threaded_health_checker.interval == interval_secs
assert health_checker.threaded_health_checker.initial_interval == initial_interval_secs
hct_max_fail = health_checker.threaded_health_checker.max_consecutive_failures
assert hct_max_fail == max_consecutive_failures
# Should not be trying to access role's user info.
assert not mock_getpwnam.called
def make_assigned_task(thermos_config, assigned_ports=None):
from gen.apache.aurora.api.constants import AURORA_EXECUTOR_NAME
from gen.apache.aurora.api.ttypes import (
AssignedTask,
ExecutorConfig,
Identity,
JobKey,
TaskConfig
)
assigned_ports = assigned_ports or {}
executor_config = ExecutorConfig(name=AURORA_EXECUTOR_NAME, data=thermos_config.json_dumps())
task_config = TaskConfig(
job=JobKey(
role=thermos_config.role().get(),
environment="prod",
name=thermos_config.name().get()),
owner=Identity(role=thermos_config.role().get(), user=thermos_config.role().get()),
environment=thermos_config.environment().get(),
jobName=thermos_config.name().get(),
executorConfig=executor_config)
return AssignedTask(
instanceId=12345,
task=task_config,
assignedPorts=assigned_ports,
slaveHost='test-host')
def create_task(cls):
return [
ScheduledTask(assignedTask=AssignedTask(
instanceId=0,
task=TaskConfig(
job=JobKey(role='role', environment='test', name='job'))),
status=ScheduleStatus.RUNNING)
]
def test_deserialize_thermos_task_unbound_refs():
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=MESOS_JOB(task=HELLO_WORLD_UNBOUND).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
with pytest.raises(ValueError) as execinfo:
mesos_task_instance_from_assigned_task(assigned_task)
assert execinfo.value.message == "Unexpected unbound refs: {{unbound_cmd}} {{unbound}}"
def create_task(cls):
return [
ScheduledTask(assignedTask=AssignedTask(
instanceId=0,
task=TaskConfig(owner=Identity(role='test_role'),
jobName='test_job')),
status=ScheduleStatus.RUNNING)
]
def mock_status_active_tasks(self, instance_ids):
tasks = []
for i in instance_ids:
tasks.append(ScheduledTask(
status=ScheduleStatus.RUNNING,
assignedTask=AssignedTask(task=TaskConfig(), instanceId=i)
))
response = make_response(result=Result(scheduleStatusResult=ScheduleStatusResult(tasks=tasks)))
self.mock_scheduler.getTasksWithoutConfigs(IgnoreArg()).AndReturn(response)
def expect_get_tasks(self, tasks, ignore_ids=None, response_code=ResponseCode.OK):
scheduled = []
for index, task in enumerate(tasks):
if not ignore_ids or index not in ignore_ids:
scheduled.append(ScheduledTask(assignedTask=AssignedTask(task=task, instanceId=index)))
response = make_response(response_code)
response.result = Result(scheduleStatusResult=ScheduleStatusResult(tasks=scheduled))
query = TaskQuery(jobKeys=[self._job_key], statuses=ACTIVE_STATES)
self._scheduler.getTasksStatus(query).AndReturn(response)
def create_scheduled_task(cls,
instance_id,
status=ScheduleStatus.RUNNING,
task_id=None,
initial_time=None):
task = ScheduledTask(
status=status,
assignedTask=AssignedTask(instanceId=instance_id,
taskId=task_id or "Task%s" % instance_id,
slaveId="Slave%s" % instance_id,
slaveHost="Slave%s" % instance_id,
task=TaskConfig()),
taskEvents=[TaskEvent(timestamp=initial_time or 1000)])
return task
def create_task(self, duration, id, host=None, name=None, prod=None):
return ScheduledTask(assignedTask=AssignedTask(
instanceId=id,
slaveHost=host,
task=TaskConfig(production=prod if prod is not None else True,
job=JobKey(role=self._role,
environment=self._env,
name=name or self._name))),
status=ScheduleStatus.RUNNING,
taskEvents=[
TaskEvent(status=ScheduleStatus.RUNNING,
timestamp=(time.time() - duration) *
1000)
])
def test_from_assigned_task_shell_filesystem_image(self, mock_getpwnam):
interval_secs = 17
initial_interval_secs = 3
max_consecutive_failures = 2
timeout_secs = 5
shell_config = ShellHealthChecker(shell_command='failed command')
task_config = TaskConfig(
job=JobKey(role='role', environment='env', name='name'),
executorConfig=ExecutorConfig(
name='thermos-generic',
data=MESOS_JOB(
task=HELLO_WORLD,
health_check_config=HealthCheckConfig(
health_checker=HealthCheckerConfig(shell=shell_config),
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=max_consecutive_failures,
timeout_secs=timeout_secs,
)).json_dumps()))
assigned_task = AssignedTask(task=task_config,
instanceId=1,
assignedPorts={'foo': 9001})
execconfig_data = json.loads(assigned_task.task.executorConfig.data)
assert execconfig_data['health_check_config']['health_checker'][
'shell']['shell_command'] == 'failed command'
mock_sandbox = mock.Mock(spec_set=SandboxInterface)
type(mock_sandbox).root = mock.PropertyMock(return_value='/some/path')
type(mock_sandbox).is_filesystem_image = mock.PropertyMock(
return_value=True)
with mock.patch(
'apache.aurora.executor.common.health_checker.ShellHealthCheck'
) as mock_shell:
HealthCheckerProvider(
nosetuid_health_checks=False,
mesos_containerizer_path='/some/path/mesos-containerizer'
).from_assigned_task(assigned_task, mock_sandbox)
class NotNone(object):
def __eq__(self, other):
return other is not None
assert mock_shell.mock_calls == [
mock.call(cmd='failed command',
wrapper_fn=NotNone(),
preexec_fn=None,
timeout_secs=5.0)
]
def make_task(thermos_config, assigned_ports={}, **kw):
role = getpass.getuser()
task_id = thermos_config.task().name().get() + '-001'
at = AssignedTask(taskId=task_id,
task=TaskConfig(executorConfig=ExecutorConfig(
name=AURORA_EXECUTOR_NAME,
data=thermos_config.json_dumps()),
owner=Identity(role=role, user=role)),
assignedPorts=assigned_ports,
**kw)
td = mesos_pb2.TaskInfo()
td.task_id.value = task_id
td.name = thermos_config.task().name().get()
td.data = serialize(at)
return td
def mock_status_active_tasks(self, instance_ids):
tasks = []
for i in instance_ids:
tasks.append(
ScheduledTask(status=ScheduleStatus.RUNNING,
assignedTask=AssignedTask(task=TaskConfig(),
instanceId=i)))
response = Response(responseCode=ResponseCode.OK,
messageDEPRECATED='test')
response.result = Result()
response.result.scheduleStatusResult = ScheduleStatusResult(
tasks=tasks)
self.mock_scheduler.getTasksWithoutConfigs(
IgnoreArg()).AndReturn(response)
def make_assigned_task(thermos_config, assigned_ports=None):
from gen.apache.aurora.api.constants import AURORA_EXECUTOR_NAME
from gen.apache.aurora.api.ttypes import AssignedTask, ExecutorConfig, JobKey, TaskConfig
assigned_ports = assigned_ports or {}
executor_config = ExecutorConfig(name=AURORA_EXECUTOR_NAME,
data=thermos_config.json_dumps())
task_config = TaskConfig(job=JobKey(role=thermos_config.role().get(),
environment='test',
name=thermos_config.name().get()),
executorConfig=executor_config)
return AssignedTask(instanceId=12345,
task=task_config,
assignedPorts=assigned_ports,
taskId="taskId-12345")
def expect_get_tasks(self, tasks, ignore_ids=None, response_code=None):
response_code = ResponseCode.OK if response_code is None else response_code
response = Response(responseCode=response_code,
messageDEPRECATED='test')
scheduled = []
for index, task in enumerate(tasks):
if not ignore_ids or index not in ignore_ids:
scheduled.append(
ScheduledTask(assignedTask=AssignedTask(task=task,
instanceId=index)))
response.result = Result(scheduleStatusResult=ScheduleStatusResult(
tasks=scheduled))
query = TaskQuery(owner=Identity(role=self._job_key.role),
environment=self._job_key.environment,
jobName=self._job_key.name,
statuses=ACTIVE_STATES)
self._scheduler.getTasksStatus(query).AndReturn(response)
def create_scheduled_tasks(cls):
tasks = []
for name in ['foo', 'bar', 'baz']:
task = ScheduledTask(failureCount=0,
assignedTask=AssignedTask(
taskId=1287391823,
slaveHost='slavehost',
task=cls.create_task_config(name),
instanceId=4237894,
assignedPorts={}),
status=ScheduleStatus.RUNNING,
taskEvents=[
TaskEvent(timestamp=28234726395,
status=ScheduleStatus.RUNNING,
message="Hi there")
])
tasks.append(task)
return tasks
def test_launchTask_deserialization_fail(self): # noqa
proxy_driver = ProxyDriver()
role = getpass.getuser()
task_info = mesos_pb2.TaskInfo()
task_info.name = task_info.task_id.value = 'broken'
task_info.data = serialize(
AssignedTask(task=TaskConfig(
job=JobKey(role=role, environment='env', name='name'),
executorConfig=ExecutorConfig(name=AURORA_EXECUTOR_NAME,
data='garbage'))))
te = FastThermosExecutor(runner_provider=make_provider(safe_mkdtemp()),
sandbox_provider=DefaultTestSandboxProvider())
te.launchTask(proxy_driver, task_info)
proxy_driver.wait_stopped()
updates = proxy_driver.method_calls['sendStatusUpdate']
assert len(updates) == 2
assert updates[0][0][0].state == mesos_pb2.TASK_STARTING
assert updates[1][0][0].state == mesos_pb2.TASK_FAILED
def test_from_assigned_task_no_health_port(self):
interval_secs = 17
initial_interval_secs = 3
max_consecutive_failures = 2
timeout_secs = 5
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos-generic',
data=MESOS_JOB(
task=HELLO_WORLD,
health_check_config=HealthCheckConfig(
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=max_consecutive_failures,
timeout_secs=timeout_secs,
)).json_dumps()))
# No health port and we don't have a shell_command.
assigned_task = AssignedTask(task=task_config,
instanceId=1,
assignedPorts={'http': 9001})
health_checker = HealthCheckerProvider().from_assigned_task(
assigned_task, None)
self.assertIsNone(health_checker)
def test_from_assigned_task(self):
interval_secs = 17
initial_interval_secs = 3
max_consecutive_failures = 2
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos',
data=MESOS_JOB(
task=HELLO_WORLD,
health_check_config=HealthCheckConfig(
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=max_consecutive_failures,
timeout_secs=7)).json_dumps()))
assigned_task = AssignedTask(task=task_config,
instanceId=1,
assignedPorts={'health': 9001})
health_checker = HealthCheckerProvider().from_assigned_task(
assigned_task, None)
assert health_checker.threaded_health_checker.interval == interval_secs
assert health_checker.threaded_health_checker.initial_interval == initial_interval_secs
hct_max_fail = health_checker.threaded_health_checker.max_consecutive_failures
assert hct_max_fail == max_consecutive_failures
def test_from_assigned_task_shell(self, mock_getpwnam):
interval_secs = 17
initial_interval_secs = 3
max_consecutive_failures = 2
timeout_secs = 5
shell_config = ShellHealthChecker(shell_command='failed command')
task_config = TaskConfig(
job=JobKey(role='role', environment='env', name='name'),
executorConfig=ExecutorConfig(
name='thermos-generic',
data=MESOS_JOB(
task=HELLO_WORLD,
health_check_config=HealthCheckConfig(
health_checker=HealthCheckerConfig(shell=shell_config),
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=max_consecutive_failures,
timeout_secs=timeout_secs,
)).json_dumps()))
assigned_task = AssignedTask(task=task_config,
instanceId=1,
assignedPorts={'foo': 9001})
execconfig_data = json.loads(assigned_task.task.executorConfig.data)
assert execconfig_data['health_check_config']['health_checker'][
'shell']['shell_command'] == 'failed command'
mock_sandbox = mock.Mock(spec_set=SandboxInterface)
type(mock_sandbox).root = mock.PropertyMock(return_value='/some/path')
type(mock_sandbox).is_filesystem_image = mock.PropertyMock(
return_value=False)
health_checker = HealthCheckerProvider().from_assigned_task(
assigned_task, mock_sandbox)
assert health_checker.threaded_health_checker.interval == interval_secs
assert health_checker.threaded_health_checker.initial_interval == initial_interval_secs
hct_max_fail = health_checker.threaded_health_checker.max_consecutive_failures
assert hct_max_fail == max_consecutive_failures
mock_getpwnam.assert_called_once_with(task_config.job.role)
def create_tasks(cls): return [ScheduledTask(assignedTask=AssignedTask(instanceId=index)) for index in cls.INSTANCES]
def create_task(self, status, id):
return ScheduledTask(
assignedTask=AssignedTask(instanceId=id, taskId=id),
status=status,
taskEvents=[TaskEvent(status=status, timestamp=10)])
