def test_create_task_defaults(self, mock_task):
cluster = MesosCluster('mesos-cluster-a.me')
mock_serializer = mock.MagicMock()
task = cluster.create_task(
action_run_id='action_c',
command='echo hi',
cpus=1,
mem=10,
constraints=[],
docker_image='container:latest',
docker_parameters=[],
env={'TESTING': 'true'},
extra_volumes=[],
serializer=mock_serializer,
)
cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with(
name='action_c',
cmd='echo hi',
cpus=1,
mem=10,
constraints=[],
image='container:latest',
docker_parameters=[],
environment={'TESTING': 'true'},
volumes=[],
uris=[],
offer_timeout=None,
)
assert_equal(task, mock_task.return_value)
mock_task.assert_called_once_with(
'action_c',
cluster.runner.TASK_CONFIG_INTERFACE.return_value,
mock_serializer,
)
def test_create_task_with_task_id(self, mock_task):
cluster = MesosCluster('mesos-cluster-a.me')
mock_serializer = mock.MagicMock()
task_id = 'task.0123-fabc'
task = cluster.create_task(
action_run_id='action_c',
command='echo hi',
cpus=1,
mem=10,
disk=20,
constraints=[],
docker_image='container:latest',
docker_parameters=[],
env={'TESTING': 'true'},
extra_volumes=[],
serializer=mock_serializer,
task_id=task_id,
)
assert cluster.runner.TASK_CONFIG_INTERFACE.call_count == 1
assert task == mock_task.return_value
task_config = cluster.runner.TASK_CONFIG_INTERFACE.return_value
task_config.set_task_id.assert_called_once_with(task_id)
mock_task.assert_called_once_with(
'action_c',
task_config.set_task_id.return_value,
mock_serializer,
)
def test_create_task_with_task_id(self, mock_task):
cluster = MesosCluster('mesos-cluster-a.me')
mock_serializer = mock.MagicMock()
task_id = 'task.0123-fabc'
task = cluster.create_task(
action_run_id='action_c',
command='echo hi',
cpus=1,
mem=10,
disk=20,
constraints=[],
docker_image='container:latest',
docker_parameters=[],
env={'TESTING': 'true'},
extra_volumes=[],
serializer=mock_serializer,
task_id=task_id,
)
assert cluster.runner.TASK_CONFIG_INTERFACE.call_count == 1
assert task == mock_task.return_value
task_config = cluster.runner.TASK_CONFIG_INTERFACE.return_value
task_config.set_task_id.assert_called_once_with(task_id)
mock_task.assert_called_once_with(
'action_c',
task_config.set_task_id.return_value,
mock_serializer,
)
def test_submit_with_clusterman(self):
mock_clusterman_metrics = mock.MagicMock()
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock(
get_config=mock.Mock(return_value={
'environment': {
'CLUSTERMAN_RESOURCES': '{"required_cpus|blah=x": 4}',
'EXECUTOR_CLUSTER': 'fake-cluster',
'EXECUTOR_POOL': 'fake-pool',
},
}),
)
mock_task.get_mesos_id.return_value = 'this_task'
with mock.patch(
'tron.mesos.get_clusterman_metrics',
return_value=mock_clusterman_metrics,
autospec=True,
), staticconf.testing.MockConfiguration(
{'clusters': {'fake-cluster': {'aws_region': 'fake-region'}}},
namespace='clusterman',
):
cluster.submit(mock_task)
assert 'this_task' in cluster.tasks
assert cluster.tasks['this_task'] == mock_task
cluster.runner.run.assert_called_once_with(
mock_task.get_config.return_value,
)
assert mock_clusterman_metrics.ClustermanMetricsBotoClient.call_count == 1
def test_recover_disabled(self):
cluster = MesosCluster('mesos-cluster-a.me', enabled=False)
mock_task = mock.MagicMock()
mock_task.get_mesos_id.return_value = 'this_task'
cluster.recover(mock_task)
assert 'this_task' not in cluster.tasks
mock_task.exited.assert_called_once_with(None)
def test_recover_disabled(self):
cluster = MesosCluster('mesos-cluster-a.me', enabled=False)
mock_task = mock.MagicMock()
mock_task.get_mesos_id.return_value = 'this_task'
cluster.recover(mock_task)
assert 'this_task' not in cluster.tasks
mock_task.exited.assert_called_once_with(None)
def test_process_event_task_id_invalid(self):
event = mock_task_event('other_task', 'some_platform_type')
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock(spec_set=MesosTask)
mock_task.get_mesos_id.return_value = 'this_task'
cluster.tasks['this_task'] = mock_task
cluster._process_event(event)
assert_equal(mock_task.handle_event.call_count, 0)
def test_process_event_control_stop(self):
event = mock.MagicMock(
kind='control',
message='stop',
)
cluster = MesosCluster('mesos-cluster-a.me')
cluster._process_event(event)
assert cluster.runner.stop.call_count == 1
assert cluster.deferred is None
def test_process_event_control_stop(self):
event = mock.MagicMock(
kind='control',
message='stop',
)
cluster = MesosCluster('mesos-cluster-a.me')
cluster._process_event(event)
assert cluster.runner.stop.call_count == 1
assert cluster.deferred is None
def test_set_enabled_off(self):
cluster = MesosCluster('mesos-cluster-a.me', enabled=True)
mock_task = mock.Mock()
cluster.tasks = {'task': mock_task}
cluster.set_enabled(False)
assert not cluster.enabled
assert cluster.runner.stop.call_count == 1
assert cluster.tasks == {}
assert mock_task.exited.call_count == 1
def test_set_enabled_off(self):
cluster = MesosCluster('mesos-cluster-a.me', enabled=True)
mock_task = mock.Mock()
cluster.tasks = {'task': mock_task}
cluster.set_enabled(False)
assert not cluster.enabled
assert cluster.runner.stop.call_count == 1
assert cluster.tasks == {}
assert mock_task.exited.call_count == 1
def test_stop(self):
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock()
cluster.tasks = {'task_id': mock_task}
cluster.stop()
assert_equal(cluster.runner.stop.call_count, 1)
assert_equal(cluster.deferred.cancel.call_count, 1)
mock_task.exited.assert_called_once_with(None)
assert_equal(len(cluster.tasks), 0)
def test_process_event_task(self):
event = mock_task_event('this_task', 'some_platform_type')
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock(spec_set=MesosTask)
mock_task.get_mesos_id.return_value = 'this_task'
cluster.tasks['this_task'] = mock_task
cluster._process_event(event)
mock_task.handle_event.assert_called_once_with(event)
def test_process_event_task(self):
event = mock_task_event('this_task', 'some_platform_type')
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock(spec_set=MesosTask)
mock_task.get_mesos_id.return_value = 'this_task'
cluster.tasks['this_task'] = mock_task
cluster._process_event(event)
mock_task.handle_event.assert_called_once_with(event)
def test_process_event_task_id_invalid(self):
event = mock_task_event('other_task', 'some_platform_type')
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock(spec_set=MesosTask)
mock_task.get_mesos_id.return_value = 'this_task'
cluster.tasks['this_task'] = mock_task
cluster._process_event(event)
assert_equal(mock_task.handle_event.call_count, 0)
def test_stop_default(self):
# When stopping, tasks should not exit. They will be recovered
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock()
cluster.tasks = {'task_id': mock_task}
cluster.stop()
assert cluster.runner.stop.call_count == 1
assert cluster.deferred is None
assert mock_task.exited.call_count == 0
assert len(cluster.tasks) == 1
def test_submit(self):
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock()
mock_task.get_mesos_id.return_value = 'this_task'
cluster.submit(mock_task)
assert 'this_task' in cluster.tasks
assert cluster.tasks['this_task'] == mock_task
cluster.runner.run.assert_called_once_with(
mock_task.get_config.return_value, )
def test_stop_default(self):
# When stopping, tasks should not exit. They will be recovered
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock()
cluster.tasks = {'task_id': mock_task}
cluster.stop()
assert cluster.runner.stop.call_count == 1
assert cluster.deferred is None
assert mock_task.exited.call_count == 0
assert len(cluster.tasks) == 1
def test_recover(self):
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock()
mock_task.get_mesos_id.return_value = 'this_task'
cluster.recover(mock_task)
assert 'this_task' in cluster.tasks
assert cluster.tasks['this_task'] == mock_task
cluster.runner.reconcile.assert_called_once_with(
mock_task.get_config.return_value, )
assert mock_task.started.call_count == 1
def test_submit(self):
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock()
mock_task.get_mesos_id.return_value = 'this_task'
cluster.submit(mock_task)
assert 'this_task' in cluster.tasks
assert cluster.tasks['this_task'] == mock_task
cluster.runner.run.assert_called_once_with(
mock_task.get_config.return_value,
)
def test_recover(self):
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock()
mock_task.get_mesos_id.return_value = 'this_task'
cluster.recover(mock_task)
assert 'this_task' in cluster.tasks
assert cluster.tasks['this_task'] == mock_task
cluster.runner.reconcile.assert_called_once_with(
mock_task.get_config.return_value,
)
assert mock_task.started.call_count == 1
def test_submit_disabled(self):
cluster = MesosCluster('mesos-cluster-a.me', enabled=False)
mock_task = mock.MagicMock()
mock_task.get_mesos_id.return_value = 'this_task'
with mock.patch(
'tron.mesos.get_clusterman_metrics',
return_value=(None, None),
autospec=True,
):
cluster.submit(mock_task)
assert 'this_task' not in cluster.tasks
mock_task.exited.assert_called_once_with(1)
def test_init_disabled(self):
cluster = MesosCluster('mesos-cluster-a.me', enabled=False)
assert_equal(cluster.queue, self.mock_queue)
assert_equal(cluster.processor, self.mock_processor)
assert_equal(self.mock_processor.executor_from_config.call_count, 0)
assert cluster.runner is None
def test_create_task_disabled(self, mock_task):
# If Mesos is disabled, should return None
cluster = MesosCluster('mesos-cluster-a.me', enabled=False)
mock_serializer = mock.MagicMock()
task = cluster.create_task(
action_run_id='action_c',
command='echo hi',
cpus=1,
mem=10,
constraints=[],
docker_image='container:latest',
docker_parameters=[],
env={'TESTING': 'true'},
extra_volumes=[],
serializer=mock_serializer,
)
assert task is None
def test_set_enabled_on(self):
cluster = MesosCluster('mesos-cluster-a.me', enabled=False)
cluster.set_enabled(True)
assert_equal(cluster.enabled, True)
# Basically the same as regular initialization
assert_equal(self.mock_processor.executor_from_config.call_count, 2)
self.mock_runner_cls.assert_called_once_with(
self.mock_processor.executor_from_config.return_value,
self.mock_queue,
)
assert_equal(cluster.runner, self.mock_runner_cls.return_value)
get_event_deferred = cluster.deferred
assert_equal(get_event_deferred, self.mock_queue.get.return_value)
get_event_deferred.addCallback.assert_has_calls([
mock.call(cluster._process_event),
mock.call(cluster.handle_next_event),
])
def test_create_task_disabled(self, mock_task):
# If Mesos is disabled, should return None
cluster = MesosCluster('mesos-cluster-a.me', enabled=False)
mock_serializer = mock.MagicMock()
task = cluster.create_task(
action_run_id='action_c',
command='echo hi',
cpus=1,
mem=10,
disk=20,
constraints=[],
docker_image='container:latest',
docker_parameters=[],
env={'TESTING': 'true'},
extra_volumes=[],
serializer=mock_serializer,
)
assert task is None
def test_submit(self):
mock_clusterman_metrics = mock.MagicMock()
cluster = MesosCluster('mesos-cluster-a.me')
mock_task = mock.MagicMock(get_config=mock.Mock(return_value={'environment': {}}))
mock_task.get_mesos_id.return_value = 'this_task'
with mock.patch(
'tron.mesos.get_clusterman_metrics',
return_value=(mock_clusterman_metrics),
autospec=True,
):
cluster.submit(mock_task)
assert 'this_task' in cluster.tasks
assert cluster.tasks['this_task'] == mock_task
cluster.runner.run.assert_called_once_with(
mock_task.get_config.return_value,
)
assert mock_clusterman_metrics.ClustermanMetricsBotoClient.call_count == 0
def test_set_enabled_on(self):
cluster = MesosCluster('mesos-cluster-a.me', enabled=False)
cluster.set_enabled(True)
assert_equal(cluster.enabled, True)
# Basically the same as regular initialization
assert_equal(self.mock_processor.executor_from_config.call_count, 2)
self.mock_runner_cls.assert_called_once_with(
self.mock_processor.executor_from_config.return_value,
self.mock_queue,
)
assert_equal(cluster.runner, self.mock_runner_cls.return_value)
get_event_deferred = cluster.deferred
assert_equal(get_event_deferred, self.mock_queue.get.return_value)
get_event_deferred.addCallback.assert_has_calls([
mock.call(cluster._process_event),
mock.call(cluster.handle_next_event),
])
def test_configure_tasks(self):
cluster = MesosCluster(
'mesos-cluster-a.me',
default_volumes=[],
dockercfg_location='first',
offer_timeout=60,
)
assert_equal(cluster.default_volumes, [])
assert_equal(cluster.dockercfg_location, 'first')
assert_equal(cluster.offer_timeout, 60)
expected_volumes = [{
'container_path': '/tmp',
'host_path': '/host',
'mode': 'RO',
}]
cluster.configure_tasks(
default_volumes=expected_volumes,
dockercfg_location='second',
offer_timeout=300,
)
assert_equal(cluster.default_volumes, expected_volumes)
assert_equal(cluster.dockercfg_location, 'second')
assert_equal(cluster.offer_timeout, 300)
def test_create_task_defaults(self, mock_task):
cluster = MesosCluster('mesos-cluster-a.me')
mock_serializer = mock.MagicMock()
task = cluster.create_task(
action_run_id='action_c',
command='echo hi',
cpus=1,
mem=10,
disk=20,
constraints=[],
docker_image='container:latest',
docker_parameters=[],
env={'TESTING': 'true'},
extra_volumes=[],
serializer=mock_serializer,
)
cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with(
name='action_c',
cmd='echo hi',
cpus=1,
mem=10,
disk=20,
constraints=[],
image='container:latest',
docker_parameters=[],
environment={'TESTING': 'true'},
volumes=[],
uris=[],
offer_timeout=None,
)
assert_equal(task, mock_task.return_value)
mock_task.assert_called_once_with(
'action_c',
cluster.runner.TASK_CONFIG_INTERFACE.return_value,
mock_serializer,
)
def test_configure_tasks(self):
cluster = MesosCluster(
'mesos-cluster-a.me',
default_volumes=[],
dockercfg_location='first',
offer_timeout=60,
)
assert_equal(cluster.default_volumes, [])
assert_equal(cluster.dockercfg_location, 'first')
assert_equal(cluster.offer_timeout, 60)
expected_volumes = [{
'container_path': '/tmp',
'host_path': '/host',
'mode': 'RO',
}]
cluster.configure_tasks(
default_volumes=expected_volumes,
dockercfg_location='second',
offer_timeout=300,
)
assert_equal(cluster.default_volumes, expected_volumes)
assert_equal(cluster.dockercfg_location, 'second')
assert_equal(cluster.offer_timeout, 300)
def test_init(self, mock_socket):
mock_socket.gethostname.return_value = 'hostname'
cluster = MesosCluster(
mesos_address='mesos-cluster-a.me',
mesos_master_port=5000,
secret='my_secret',
mesos_role='tron',
framework_id='fake_framework_id',
principal="fake-principal",
)
assert_equal(cluster.queue, self.mock_queue)
assert_equal(cluster.processor, self.mock_processor)
self.mock_get_leader.assert_called_once_with(
'mesos-cluster-a.me', 5000
)
self.mock_processor.executor_from_config.assert_has_calls([
mock.call(
provider='mesos_task',
provider_config={
'secret': 'my_secret',
'principal': 'fake-principal',
'mesos_address': self.mock_get_leader.return_value,
'role': 'tron',
'framework_name': 'tron-hostname',
'framework_id': 'fake_framework_id',
'failover': True,
},
),
mock.call(
provider='logging',
provider_config=mock.ANY,
),
])
self.mock_runner_cls.assert_called_once_with(
self.mock_processor.executor_from_config.return_value,
self.mock_queue,
)
assert_equal(cluster.runner, self.mock_runner_cls.return_value)
get_event_deferred = cluster.deferred
assert_equal(get_event_deferred, self.mock_queue.get.return_value)
get_event_deferred.addCallback.assert_has_calls([
mock.call(cluster._process_event),
mock.call(cluster.handle_next_event),
])
def test_create_task_with_configuration(self, mock_task):
cluster = MesosCluster(
'mesos-cluster-a.me',
default_volumes=[
{
'container_path': '/tmp',
'host_path': '/host',
'mode': 'RO',
},
{
'container_path': '/other',
'host_path': '/other',
'mode': 'RW',
},
],
dockercfg_location='some_place',
offer_timeout=202,
)
mock_serializer = mock.MagicMock()
task = cluster.create_task(
action_run_id='action_c',
command='echo hi',
cpus=1,
mem=10,
disk=20,
constraints=[],
docker_image='container:latest',
docker_parameters=[],
env={'TESTING': 'true'},
# This should override the default volume for /tmp
extra_volumes=[
{
'container_path': '/tmp',
'host_path': '/custom',
'mode': 'RW',
},
],
serializer=mock_serializer,
)
cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with(
name='action_c',
cmd='echo hi',
cpus=1,
mem=10,
disk=20,
constraints=[],
image='container:latest',
docker_parameters=[],
environment={'TESTING': 'true'},
volumes=[
{
'container_path': '/tmp',
'host_path': '/custom',
'mode': 'RW',
},
{
'container_path': '/other',
'host_path': '/other',
'mode': 'RW',
},
],
uris=['some_place'],
offer_timeout=202,
)
assert_equal(task, mock_task.return_value)
mock_task.assert_called_once_with(
'action_c',
cluster.runner.TASK_CONFIG_INTERFACE.return_value,
mock_serializer,
)
def test_stop_disabled(self):
# Shouldn't raise an error
cluster = MesosCluster('mesos-cluster-a.me', enabled=False)
cluster.stop()
def test_create_task_with_configuration(self, mock_task):
cluster = MesosCluster(
'mesos-cluster-a.me',
default_volumes=[
{
'container_path': '/tmp',
'host_path': '/host',
'mode': 'RO',
},
{
'container_path': '/other',
'host_path': '/other',
'mode': 'RW',
},
],
dockercfg_location='some_place',
offer_timeout=202,
)
mock_serializer = mock.MagicMock()
task = cluster.create_task(
action_run_id='action_c',
command='echo hi',
cpus=1,
mem=10,
disk=20,
constraints=[],
docker_image='container:latest',
docker_parameters=[],
env={'TESTING': 'true'},
# This should override the default volume for /tmp
extra_volumes=[
{
'container_path': '/tmp',
'host_path': '/custom',
'mode': 'RW',
},
],
serializer=mock_serializer,
)
cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with(
name='action_c',
cmd='echo hi',
cpus=1,
mem=10,
disk=20,
constraints=[],
image='container:latest',
docker_parameters=[],
environment={'TESTING': 'true'},
volumes=[
{
'container_path': '/tmp',
'host_path': '/custom',
'mode': 'RW',
},
{
'container_path': '/other',
'host_path': '/other',
'mode': 'RW',
},
],
uris=['some_place'],
offer_timeout=202,
)
assert_equal(task, mock_task.return_value)
mock_task.assert_called_once_with(
'action_c',
cluster.runner.TASK_CONFIG_INTERFACE.return_value,
mock_serializer,
)
def test_set_enabled_on_already(self):
cluster = MesosCluster('mesos-cluster-a.me', enabled=True)
cluster.set_enabled(True)
assert_equal(cluster.enabled, True)
# Runner should have only be created once
assert_equal(self.mock_runner_cls.call_count, 1)
def test_set_enabled_on_already(self):
cluster = MesosCluster('mesos-cluster-a.me', enabled=True)
cluster.set_enabled(True)
assert_equal(cluster.enabled, True)
# Runner should have only be created once
assert_equal(self.mock_runner_cls.call_count, 1)
def test_stop_disabled(self):
# Shouldn't raise an error
cluster = MesosCluster('mesos-cluster-a.me', enabled=False)
cluster.stop()
def test_kill(self):
cluster = MesosCluster('mesos-cluster-a.me')
cluster.kill('fake_task_id')
cluster.runner.kill.assert_called_once_with('fake_task_id')
def test_kill(self):
cluster = MesosCluster('mesos-cluster-a.me')
cluster.kill('fake_task_id')
cluster.runner.kill.assert_called_once_with('fake_task_id')
def test_set_enabled_off(self):
cluster = MesosCluster('mesos-cluster-a.me', enabled=True)
cluster.set_enabled(False)
assert_equal(cluster.enabled, False)
assert_equal(cluster.runner.stop.call_count, 1)