def allocate_task_resources(
task_config: MesosTaskConfig,
offer_resources: ResourceSet,
) -> Tuple[MesosTaskConfig, ResourceSet]:
""" Allocate a task's resources to a Mesos offer
:param task: the specification for the task to allocate
:param offer_resources: a mapping of resource name -> available resources
(should come from :func:`get_offer_resources`)
:returns: a pair of (`prepared_task_config`, `remaining_resources`), where
`prepared_task_config` is the task_config object modified with the
actual resources consumed
"""
for res, val in offer_resources.items():
if res not in _NUMERIC_RESOURCES:
continue
offer_resources = offer_resources.set(res, val - task_config[res])
port = offer_resources.ports[0]['begin']
if offer_resources.ports[0]['begin'] == offer_resources.ports[0]['end']:
avail_ports = offer_resources.ports[1:]
else:
new_port_range = offer_resources.ports[0].set('begin', port + 1)
avail_ports = offer_resources.ports.set(0, new_port_range)
offer_resources = offer_resources.set('ports', avail_ports)
task_config = task_config.set('ports', v(m(begin=port, end=port)))
return task_config, offer_resources
def mock_task_config():
return MesosTaskConfig(
uuid='mock_uuid',
name='mock_name',
image='mock_image',
cmd='mock_cmd',
timeout=1000,
)
def mock_task_config():
return MesosTaskConfig(
uuid='mock_uuid',
name='mock_name',
image='mock_image',
cmd='mock_cmd',
retries=5,
)
def fake_task():
return MesosTaskConfig(
name='fake_name',
cpus=10.0,
mem=1024.0,
disk=1000.0,
gpus=1,
ports=v(m(begin=31200, end=31200)),
image='fake_image',
cmd='echo "fake"'
)
def test_offer_matches_constraints_no_match(ef, fake_offer):
attributes = {
attribute.name: attribute.text.value for attribute in fake_offer.attributes}
fake_task = MesosTaskConfig(
image='fake_image',
cmd='echo "fake"',
constraints=[
['region', '==', 'another_fake_region_text'],
],
)
match = attributes_match_constraints(attributes, fake_task.constraints)
assert not match
def test_mesos_task_config_factories():
m = MesosTaskConfig(
cmd='/bin/true', cpus=1, mem=64, disk=15, gpus=6.0, image='fake_image')
assert type(m.cpus) is float
assert m.cpus == 1.0
assert type(m.mem) is float
assert m.mem == 64.0
assert type(m.disk) is float
assert m.disk == 15.0
assert type(m.gpus) is int
assert m.gpus == 6
try:
m = m.set(name='a' * 256)
assert False, 'Task id longer than 255 characters was accepted'
except InvariantException as e:
print(e)
assert True
def status_update_test_prep(state, reason=''):
task = MesosTaskConfig(
cmd='/bin/true', name='fake_name', image='fake_image')
task_id = task.task_id
update = Dict(
task_id=Dict(value=task_id),
state=state,
reason=reason
)
task_metadata = TaskMetadata(
task_config=task,
task_state='TASK_INITED',
task_state_history=m(TASK_INITED=time.time()),
)
return update, task_id, task_metadata
def test_run(mock_timeout_executor, mock_downstream):
mock_config = MesosTaskConfig(image='fake', cmd='cat', timeout=60)
mock_timeout_executor.run(mock_config)
assert mock_downstream.run.call_count == 1
assert len(mock_timeout_executor.running_tasks) == 1
def test_mesos_task_config_set_task_id():
m = MesosTaskConfig(cmd='/bin/true', image='fake')
new_task_id = 'new' + m.task_id
result = m.set_task_id(new_task_id)
assert result.task_id == new_task_id
def test_run(mock_logging_executor, mock_downstream):
mock_config = MesosTaskConfig(image='fake', cmd='cat')
mock_logging_executor.run(mock_config)
assert mock_downstream.run.call_count == 1
def test_run_default_retries(mock_retrying_executor, mock_downstream):
mock_config = MesosTaskConfig(image='fake_image', cmd='some command')
mock_retrying_executor.run(mock_config)
assert mock_downstream.run.call_count == 1
assert mock_retrying_executor.task_retries[mock_config.task_id] == 2