def test_create_task_defaults(self, mock_task): cluster = MesosCluster('mesos-cluster-a.me') mock_serializer = mock.MagicMock() task = cluster.create_task( action_run_id='action_c', command='echo hi', cpus=1, mem=10, constraints=[], docker_image='container:latest', docker_parameters=[], env={'TESTING': 'true'}, extra_volumes=[], serializer=mock_serializer, ) cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with( name='action_c', cmd='echo hi', cpus=1, mem=10, constraints=[], image='container:latest', docker_parameters=[], environment={'TESTING': 'true'}, volumes=[], uris=[], offer_timeout=None, ) assert_equal(task, mock_task.return_value) mock_task.assert_called_once_with( 'action_c', cluster.runner.TASK_CONFIG_INTERFACE.return_value, mock_serializer, )
def test_create_task_with_task_id(self, mock_task): cluster = MesosCluster('mesos-cluster-a.me') mock_serializer = mock.MagicMock() task_id = 'task.0123-fabc' task = cluster.create_task( action_run_id='action_c', command='echo hi', cpus=1, mem=10, disk=20, constraints=[], docker_image='container:latest', docker_parameters=[], env={'TESTING': 'true'}, extra_volumes=[], serializer=mock_serializer, task_id=task_id, ) assert cluster.runner.TASK_CONFIG_INTERFACE.call_count == 1 assert task == mock_task.return_value task_config = cluster.runner.TASK_CONFIG_INTERFACE.return_value task_config.set_task_id.assert_called_once_with(task_id) mock_task.assert_called_once_with( 'action_c', task_config.set_task_id.return_value, mock_serializer, )
def test_submit_with_clusterman(self): mock_clusterman_metrics = mock.MagicMock() cluster = MesosCluster('mesos-cluster-a.me') mock_task = mock.MagicMock( get_config=mock.Mock(return_value={ 'environment': { 'CLUSTERMAN_RESOURCES': '{"required_cpus|blah=x": 4}', 'EXECUTOR_CLUSTER': 'fake-cluster', 'EXECUTOR_POOL': 'fake-pool', }, }), ) mock_task.get_mesos_id.return_value = 'this_task' with mock.patch( 'tron.mesos.get_clusterman_metrics', return_value=mock_clusterman_metrics, autospec=True, ), staticconf.testing.MockConfiguration( {'clusters': {'fake-cluster': {'aws_region': 'fake-region'}}}, namespace='clusterman', ): cluster.submit(mock_task) assert 'this_task' in cluster.tasks assert cluster.tasks['this_task'] == mock_task cluster.runner.run.assert_called_once_with( mock_task.get_config.return_value, ) assert mock_clusterman_metrics.ClustermanMetricsBotoClient.call_count == 1
def test_recover_disabled(self): cluster = MesosCluster('mesos-cluster-a.me', enabled=False) mock_task = mock.MagicMock() mock_task.get_mesos_id.return_value = 'this_task' cluster.recover(mock_task) assert 'this_task' not in cluster.tasks mock_task.exited.assert_called_once_with(None)
def test_process_event_task_id_invalid(self): event = mock_task_event('other_task', 'some_platform_type') cluster = MesosCluster('mesos-cluster-a.me') mock_task = mock.MagicMock(spec_set=MesosTask) mock_task.get_mesos_id.return_value = 'this_task' cluster.tasks['this_task'] = mock_task cluster._process_event(event) assert_equal(mock_task.handle_event.call_count, 0)
def test_process_event_control_stop(self): event = mock.MagicMock( kind='control', message='stop', ) cluster = MesosCluster('mesos-cluster-a.me') cluster._process_event(event) assert cluster.runner.stop.call_count == 1 assert cluster.deferred is None
def test_set_enabled_off(self): cluster = MesosCluster('mesos-cluster-a.me', enabled=True) mock_task = mock.Mock() cluster.tasks = {'task': mock_task} cluster.set_enabled(False) assert not cluster.enabled assert cluster.runner.stop.call_count == 1 assert cluster.tasks == {} assert mock_task.exited.call_count == 1
def test_stop(self): cluster = MesosCluster('mesos-cluster-a.me') mock_task = mock.MagicMock() cluster.tasks = {'task_id': mock_task} cluster.stop() assert_equal(cluster.runner.stop.call_count, 1) assert_equal(cluster.deferred.cancel.call_count, 1) mock_task.exited.assert_called_once_with(None) assert_equal(len(cluster.tasks), 0)
def test_process_event_task(self): event = mock_task_event('this_task', 'some_platform_type') cluster = MesosCluster('mesos-cluster-a.me') mock_task = mock.MagicMock(spec_set=MesosTask) mock_task.get_mesos_id.return_value = 'this_task' cluster.tasks['this_task'] = mock_task cluster._process_event(event) mock_task.handle_event.assert_called_once_with(event)
def test_stop_default(self): # When stopping, tasks should not exit. They will be recovered cluster = MesosCluster('mesos-cluster-a.me') mock_task = mock.MagicMock() cluster.tasks = {'task_id': mock_task} cluster.stop() assert cluster.runner.stop.call_count == 1 assert cluster.deferred is None assert mock_task.exited.call_count == 0 assert len(cluster.tasks) == 1
def test_submit(self): cluster = MesosCluster('mesos-cluster-a.me') mock_task = mock.MagicMock() mock_task.get_mesos_id.return_value = 'this_task' cluster.submit(mock_task) assert 'this_task' in cluster.tasks assert cluster.tasks['this_task'] == mock_task cluster.runner.run.assert_called_once_with( mock_task.get_config.return_value, )
def test_recover(self): cluster = MesosCluster('mesos-cluster-a.me') mock_task = mock.MagicMock() mock_task.get_mesos_id.return_value = 'this_task' cluster.recover(mock_task) assert 'this_task' in cluster.tasks assert cluster.tasks['this_task'] == mock_task cluster.runner.reconcile.assert_called_once_with( mock_task.get_config.return_value, ) assert mock_task.started.call_count == 1
def test_submit_disabled(self): cluster = MesosCluster('mesos-cluster-a.me', enabled=False) mock_task = mock.MagicMock() mock_task.get_mesos_id.return_value = 'this_task' with mock.patch( 'tron.mesos.get_clusterman_metrics', return_value=(None, None), autospec=True, ): cluster.submit(mock_task) assert 'this_task' not in cluster.tasks mock_task.exited.assert_called_once_with(1)
def test_init_disabled(self): cluster = MesosCluster('mesos-cluster-a.me', enabled=False) assert_equal(cluster.queue, self.mock_queue) assert_equal(cluster.processor, self.mock_processor) assert_equal(self.mock_processor.executor_from_config.call_count, 0) assert cluster.runner is None
def test_create_task_disabled(self, mock_task): # If Mesos is disabled, should return None cluster = MesosCluster('mesos-cluster-a.me', enabled=False) mock_serializer = mock.MagicMock() task = cluster.create_task( action_run_id='action_c', command='echo hi', cpus=1, mem=10, constraints=[], docker_image='container:latest', docker_parameters=[], env={'TESTING': 'true'}, extra_volumes=[], serializer=mock_serializer, ) assert task is None
def test_set_enabled_on(self): cluster = MesosCluster('mesos-cluster-a.me', enabled=False) cluster.set_enabled(True) assert_equal(cluster.enabled, True) # Basically the same as regular initialization assert_equal(self.mock_processor.executor_from_config.call_count, 2) self.mock_runner_cls.assert_called_once_with( self.mock_processor.executor_from_config.return_value, self.mock_queue, ) assert_equal(cluster.runner, self.mock_runner_cls.return_value) get_event_deferred = cluster.deferred assert_equal(get_event_deferred, self.mock_queue.get.return_value) get_event_deferred.addCallback.assert_has_calls([ mock.call(cluster._process_event), mock.call(cluster.handle_next_event), ])
def test_create_task_disabled(self, mock_task): # If Mesos is disabled, should return None cluster = MesosCluster('mesos-cluster-a.me', enabled=False) mock_serializer = mock.MagicMock() task = cluster.create_task( action_run_id='action_c', command='echo hi', cpus=1, mem=10, disk=20, constraints=[], docker_image='container:latest', docker_parameters=[], env={'TESTING': 'true'}, extra_volumes=[], serializer=mock_serializer, ) assert task is None
def test_submit(self): mock_clusterman_metrics = mock.MagicMock() cluster = MesosCluster('mesos-cluster-a.me') mock_task = mock.MagicMock(get_config=mock.Mock(return_value={'environment': {}})) mock_task.get_mesos_id.return_value = 'this_task' with mock.patch( 'tron.mesos.get_clusterman_metrics', return_value=(mock_clusterman_metrics), autospec=True, ): cluster.submit(mock_task) assert 'this_task' in cluster.tasks assert cluster.tasks['this_task'] == mock_task cluster.runner.run.assert_called_once_with( mock_task.get_config.return_value, ) assert mock_clusterman_metrics.ClustermanMetricsBotoClient.call_count == 0
def test_configure_tasks(self): cluster = MesosCluster( 'mesos-cluster-a.me', default_volumes=[], dockercfg_location='first', offer_timeout=60, ) assert_equal(cluster.default_volumes, []) assert_equal(cluster.dockercfg_location, 'first') assert_equal(cluster.offer_timeout, 60) expected_volumes = [{ 'container_path': '/tmp', 'host_path': '/host', 'mode': 'RO', }] cluster.configure_tasks( default_volumes=expected_volumes, dockercfg_location='second', offer_timeout=300, ) assert_equal(cluster.default_volumes, expected_volumes) assert_equal(cluster.dockercfg_location, 'second') assert_equal(cluster.offer_timeout, 300)
def test_create_task_defaults(self, mock_task): cluster = MesosCluster('mesos-cluster-a.me') mock_serializer = mock.MagicMock() task = cluster.create_task( action_run_id='action_c', command='echo hi', cpus=1, mem=10, disk=20, constraints=[], docker_image='container:latest', docker_parameters=[], env={'TESTING': 'true'}, extra_volumes=[], serializer=mock_serializer, ) cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with( name='action_c', cmd='echo hi', cpus=1, mem=10, disk=20, constraints=[], image='container:latest', docker_parameters=[], environment={'TESTING': 'true'}, volumes=[], uris=[], offer_timeout=None, ) assert_equal(task, mock_task.return_value) mock_task.assert_called_once_with( 'action_c', cluster.runner.TASK_CONFIG_INTERFACE.return_value, mock_serializer, )
def test_init(self, mock_socket): mock_socket.gethostname.return_value = 'hostname' cluster = MesosCluster( mesos_address='mesos-cluster-a.me', mesos_master_port=5000, secret='my_secret', mesos_role='tron', framework_id='fake_framework_id', principal="fake-principal", ) assert_equal(cluster.queue, self.mock_queue) assert_equal(cluster.processor, self.mock_processor) self.mock_get_leader.assert_called_once_with( 'mesos-cluster-a.me', 5000 ) self.mock_processor.executor_from_config.assert_has_calls([ mock.call( provider='mesos_task', provider_config={ 'secret': 'my_secret', 'principal': 'fake-principal', 'mesos_address': self.mock_get_leader.return_value, 'role': 'tron', 'framework_name': 'tron-hostname', 'framework_id': 'fake_framework_id', 'failover': True, }, ), mock.call( provider='logging', provider_config=mock.ANY, ), ]) self.mock_runner_cls.assert_called_once_with( self.mock_processor.executor_from_config.return_value, self.mock_queue, ) assert_equal(cluster.runner, self.mock_runner_cls.return_value) get_event_deferred = cluster.deferred assert_equal(get_event_deferred, self.mock_queue.get.return_value) get_event_deferred.addCallback.assert_has_calls([ mock.call(cluster._process_event), mock.call(cluster.handle_next_event), ])
def test_create_task_with_configuration(self, mock_task): cluster = MesosCluster( 'mesos-cluster-a.me', default_volumes=[ { 'container_path': '/tmp', 'host_path': '/host', 'mode': 'RO', }, { 'container_path': '/other', 'host_path': '/other', 'mode': 'RW', }, ], dockercfg_location='some_place', offer_timeout=202, ) mock_serializer = mock.MagicMock() task = cluster.create_task( action_run_id='action_c', command='echo hi', cpus=1, mem=10, disk=20, constraints=[], docker_image='container:latest', docker_parameters=[], env={'TESTING': 'true'}, # This should override the default volume for /tmp extra_volumes=[ { 'container_path': '/tmp', 'host_path': '/custom', 'mode': 'RW', }, ], serializer=mock_serializer, ) cluster.runner.TASK_CONFIG_INTERFACE.assert_called_once_with( name='action_c', cmd='echo hi', cpus=1, mem=10, disk=20, constraints=[], image='container:latest', docker_parameters=[], environment={'TESTING': 'true'}, volumes=[ { 'container_path': '/tmp', 'host_path': '/custom', 'mode': 'RW', }, { 'container_path': '/other', 'host_path': '/other', 'mode': 'RW', }, ], uris=['some_place'], offer_timeout=202, ) assert_equal(task, mock_task.return_value) mock_task.assert_called_once_with( 'action_c', cluster.runner.TASK_CONFIG_INTERFACE.return_value, mock_serializer, )
def test_stop_disabled(self): # Shouldn't raise an error cluster = MesosCluster('mesos-cluster-a.me', enabled=False) cluster.stop()
def test_set_enabled_on_already(self): cluster = MesosCluster('mesos-cluster-a.me', enabled=True) cluster.set_enabled(True) assert_equal(cluster.enabled, True) # Runner should have only be created once assert_equal(self.mock_runner_cls.call_count, 1)
def test_kill(self): cluster = MesosCluster('mesos-cluster-a.me') cluster.kill('fake_task_id') cluster.runner.kill.assert_called_once_with('fake_task_id')
def test_set_enabled_off(self): cluster = MesosCluster('mesos-cluster-a.me', enabled=True) cluster.set_enabled(False) assert_equal(cluster.enabled, False) assert_equal(cluster.runner.stop.call_count, 1)