def test_deserialize_thermos_task_unbound_refs():
# test unbound {{standard}} refs
task_config = TaskConfig(
executorConfig=ExecutorConfig(name="thermos", data=MESOS_JOB(task=HELLO_WORLD_UNBOUND).json_dumps())
)
assigned_task = AssignedTask(task=task_config, instanceId=0)
with pytest.raises(TaskInfoError) as execinfo:
mesos_task_instance_from_assigned_task(assigned_task)
assert "Unexpected unbound refs: {{unbound_cmd}} {{unbound}}" in execinfo.value.message
# test bound unscoped refs, valid case.
task = BASE_TASK(
name="task_name", processes=[Process(name="process_name", cmdline="echo {{thermos.ports[health]}}")]
)
task_config = TaskConfig(executorConfig=ExecutorConfig(name="thermos", data=MESOS_JOB(task=task).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
assert mesos_task_instance_from_assigned_task(assigned_task) is not None
# test unbound unscoped refs
for cmdline in ("echo {{hello_{{thermos.ports[health]}}}}", "echo {{hello_{{thermos.user_id}}}}"):
task = BASE_TASK(name="task_name", processes=[Process(name="process_name", cmdline=cmdline)])
task_config = TaskConfig(executorConfig=ExecutorConfig(name="thermos", data=MESOS_JOB(task=task).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
with pytest.raises(UnexpectedUnboundRefsError):
mesos_task_instance_from_assigned_task(assigned_task)
def test_deserialize_thermos_task_unbound_refs():
# test unbound {{standard}} refs
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=MESOS_JOB(task=HELLO_WORLD_UNBOUND).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
with pytest.raises(TaskInfoError) as execinfo:
mesos_task_instance_from_assigned_task(assigned_task)
assert "Unexpected unbound refs: {{unbound_cmd}} {{unbound}}" in execinfo.value.message
# test bound unscoped refs, valid case.
task = BASE_TASK(name='task_name',
processes=[
Process(name='process_name',
cmdline='echo {{thermos.ports[health]}}')
])
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=MESOS_JOB(task=task).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
assert mesos_task_instance_from_assigned_task(assigned_task) is not None
# test unbound unscoped refs
for cmdline in ('echo {{hello_{{thermos.ports[health]}}}}',
'echo {{hello_{{thermos.user_id}}}}'):
task = BASE_TASK(
name='task_name',
processes=[Process(name='process_name', cmdline=cmdline)])
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=MESOS_JOB(task=task).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
with pytest.raises(UnexpectedUnboundRefsError):
mesos_task_instance_from_assigned_task(assigned_task)
def test_deserialize_thermos_task_unbound_refs():
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=MESOS_JOB(task=HELLO_WORLD_UNBOUND).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
with pytest.raises(ValueError) as execinfo:
mesos_task_instance_from_assigned_task(assigned_task)
assert execinfo.value.message == "Unexpected unbound refs: {{unbound_cmd}} {{unbound}}"
def test_deserialize_thermos_task_unbound_refs():
task_config = TaskConfig(
executorConfig=ExecutorConfig(
name='thermos', data=MESOS_JOB(task=HELLO_WORLD_UNBOUND).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
with pytest.raises(ValueError) as execinfo:
mesos_task_instance_from_assigned_task(assigned_task)
assert execinfo.value.message == "Unexpected unbound refs: {{unbound_cmd}} {{unbound}}"
def test_deserialize_thermos_task():
task_config = TaskConfig(
executorConfig=ExecutorConfig(name='thermos', data=MESOS_JOB(task=HELLO_WORLD).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
assert mesos_task_instance_from_assigned_task(assigned_task) == BASE_MTI(task=HELLO_WORLD)
task_config = TaskConfig(
executorConfig=ExecutorConfig(name='thermos', data=HELLO_WORLD_MTI.json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
assert mesos_task_instance_from_assigned_task(assigned_task) == BASE_MTI(task=HELLO_WORLD)
def test_deserialize_thermos_task():
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=MESOS_JOB(task=HELLO_WORLD).json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
assert mesos_task_instance_from_assigned_task(assigned_task) == BASE_MTI(
task=HELLO_WORLD)
task_config = TaskConfig(executorConfig=ExecutorConfig(
name='thermos', data=HELLO_WORLD_MTI.json_dumps()))
assigned_task = AssignedTask(task=task_config, instanceId=0)
assert mesos_task_instance_from_assigned_task(assigned_task) == BASE_MTI(
task=HELLO_WORLD)
def from_assigned_task(self, assigned_task, _):
mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
if not mesos_task.has_announce():
return None
portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
# assigned_task.slaveHost is the --hostname argument passed into the mesos slave.
# Using this allows overriding the hostname published into ZK when announcing.
# If no argument was passed to the mesos-slave, the slave falls back to gethostname().
endpoint, additional = make_endpoints(
assigned_task.slaveHost,
portmap,
mesos_task.announce().primary_port().get())
client = self.make_zk_client()
if mesos_task.announce().has_zk_path():
if self.__allow_custom_serverset_path:
path = mesos_task.announce().zk_path().get()
else:
app.error('Executor must be started with --announcer-allow-custom-serverset-path in order '
'to use zk_path in the Announcer config')
else:
path = self.make_zk_path(assigned_task)
initial_interval = mesos_task.health_check_config().initial_interval_secs().get()
interval = mesos_task.health_check_config().interval_secs().get()
consecutive_failures = mesos_task.health_check_config().max_consecutive_failures().get()
timeout_secs = initial_interval + (consecutive_failures * interval)
return AnnouncerChecker(
client, path, timeout_secs, endpoint, additional=additional, shard=assigned_task.instanceId,
name=self.name)
def from_assigned_task(self, assigned_task, _):
mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
if not mesos_task.has_announce():
return None
portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
# assigned_task.slaveHost is the --hostname argument passed into the mesos slave.
# Using this allows overriding the hostname published into ZK when announcing.
# If no argument was passed to the mesos-slave, the slave falls back to gethostname().
endpoint, additional = make_endpoints(
assigned_task.slaveHost,
portmap,
mesos_task.announce().primary_port().get())
client = self.make_zk_client()
if mesos_task.announce().has_zk_path():
if self.__allow_custom_serverset_path:
path = mesos_task.announce().zk_path().get()
else:
app.error('Executor must be started with --announcer-allow-custom-serverset-path in order '
'to use zk_path in the Announcer config')
else:
path = self.make_zk_path(assigned_task)
initial_interval = mesos_task.health_check_config().initial_interval_secs().get()
interval = mesos_task.health_check_config().interval_secs().get()
consecutive_failures = mesos_task.health_check_config().max_consecutive_failures().get()
timeout_secs = initial_interval + (consecutive_failures * interval)
return AnnouncerChecker(
client, path, timeout_secs, endpoint, additional=additional, shard=assigned_task.instanceId,
name=self.name)
def from_assigned_task(self, assigned_task, sandbox):
task_id = assigned_task.taskId
resources = mesos_task_instance_from_assigned_task(
assigned_task).task().resources()
task_monitor = TaskMonitor(self._checkpoint_root, task_id)
resource_monitor = TaskResourceMonitor(
task_id, task_monitor, **self._resource_monitor_options)
return ResourceManager(resources, resource_monitor)
def from_assigned_task(self, assigned_task, sandbox):
task_id = assigned_task.taskId
resources = mesos_task_instance_from_assigned_task(assigned_task).task().resources()
task_monitor = TaskMonitor(self._checkpoint_root, task_id)
resource_monitor = TaskResourceMonitor(
task_id,
task_monitor,
**self._resource_monitor_options)
return ResourceManager(resources, resource_monitor)
def from_assigned_task(self, assigned_task, sandbox):
task_id = assigned_task.taskId
resources = mesos_task_instance_from_assigned_task(assigned_task).task().resources()
task_path = TaskPath(root=self._checkpoint_root, task_id=task_id)
task_monitor = TaskMonitor(task_path, task_id)
resource_monitor = TaskResourceMonitor(
task_monitor,
sandbox.root,
disk_collector=self._disk_collector,
disk_collection_interval=self._disk_collection_interval)
return ResourceManager(resources, resource_monitor)
def from_assigned_task(self, assigned_task, sandbox):
task_id = assigned_task.taskId
resources = mesos_task_instance_from_assigned_task(
assigned_task).task().resources()
task_path = TaskPath(root=self._checkpoint_root, task_id=task_id)
task_monitor = TaskMonitor(task_path, task_id)
resource_monitor = TaskResourceMonitor(
task_monitor,
sandbox.root,
disk_collector=self._disk_collector,
disk_collection_interval=self._disk_collection_interval)
return ResourceManager(resources, resource_monitor)
def from_assigned_task(self, assigned_task, _):
mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
if not mesos_task.has_announce():
return None
portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
endpoint, additional = make_endpoints(
socket.gethostname(),
portmap,
mesos_task.announce().primary_port().get())
serverset = self.make_serverset(assigned_task)
return AnnouncerChecker(
serverset, endpoint, additional=additional, shard=assigned_task.instanceId, name=self.name)
def from_assigned_task(self, assigned_task, _):
mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
if not mesos_task.has_announce():
return None
portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
endpoint, additional = make_endpoints(
socket.gethostname(), portmap,
mesos_task.announce().primary_port().get())
serverset = self.make_serverset(assigned_task)
return AnnouncerChecker(serverset,
endpoint,
additional=additional,
shard=assigned_task.instanceId,
name=self.name)
def from_assigned_task(self, assigned_task, _):
mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
if not mesos_task.has_announce():
return None
portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
endpoint, additional = make_endpoints(socket.gethostname(), portmap, mesos_task.announce().primary_port().get())
client = self.make_zk_client()
path = self.make_zk_path(assigned_task)
initial_interval = mesos_task.health_check_config().initial_interval_secs().get()
interval = mesos_task.health_check_config().interval_secs().get()
consecutive_failures = mesos_task.health_check_config().max_consecutive_failures().get()
timeout_secs = initial_interval + (consecutive_failures * interval)
return AnnouncerChecker(
client, path, timeout_secs, endpoint, additional=additional, shard=assigned_task.instanceId, name=self.name
)
