def invocation_flow(self, sreq, extra_sources=None): """ Invoke the backend protocl with gateway :param sreq: StorletRequest instance :param extra_sources (WIP): A list of StorletRequest instance to gather as extra resoureces to feed to storlet container as data source :return: StorletResponse instance """ run_time_sbox = RunTimeSandbox(self.scope, self.conf, self.logger) docker_updated = self.update_docker_container_from_cache(sreq) run_time_sbox.activate_storlet_daemon(sreq, docker_updated) self._add_system_params(sreq) slog_path = self.paths.get_host_slog_path(sreq.storlet_main) storlet_pipe_path = \ self.paths.get_host_storlet_pipe(sreq.storlet_main) sprotocol = StorletInvocationProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout, self.logger, extra_sources=extra_sources) sresp = sprotocol.communicate() self._upload_storlet_logs(slog_path, sreq) return sresp
def invocation_flow(self, sreq, extra_sources=None): """ Invoke the backend protocl with gateway :param sreq: StorletRequest instance :param extra_sources (WIP): A list of StorletRequest instance to gather as extra resoureces to feed to storlet container as data source :return: StorletResponse instance """ run_time_sbox = RunTimeSandbox(self.scope, self.conf, self.logger) docker_updated = self.update_docker_container_from_cache(sreq) run_time_sbox.activate_storlet_daemon(sreq, docker_updated) self._add_system_params(sreq) slog_path = self.paths.slog_path(sreq.storlet_main) storlet_pipe_path = self.paths.host_storlet_pipe(sreq.storlet_main) sprotocol = StorletInvocationProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout, self.logger, extra_sources=extra_sources) sresp = sprotocol.communicate() self._upload_storlet_logs(slog_path, sreq) return sresp
def setUp(self): self.logger = FakeLogger() # TODO(takashi): take these values from config file self.conf = {'docker_repo': 'localhost:5001'} self.scope = '0123456789abc' self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
class TestRunTimeSandbox(unittest.TestCase): def setUp(self): self.logger = FakeLogger() # TODO(takashi): take these values from config file self.conf = {'docker_repo': 'localhost:5001'} self.scope = '0123456789abc' self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger) def test_ping(self): with mock.patch('storlets.gateway.gateways.docker.runtime.' 'SBusClient.ping') as ping: ping.return_value = SBusResponse(True, 'OK') self.assertEqual(self.sbox.ping(), 1) with mock.patch('storlets.gateway.gateways.docker.runtime.' 'SBusClient.ping') as ping: ping.return_value = SBusResponse(False, 'Error') self.assertEqual(self.sbox.ping(), 0) with mock.patch('storlets.gateway.gateways.docker.runtime.' 'SBusClient.ping') as ping: ping.side_effect = SBusClientSendError() self.assertEqual(self.sbox.ping(), -1) with mock.patch('storlets.gateway.gateways.docker.runtime.' 'SBusClient.ping') as ping: ping.side_effect = SBusClientMalformedResponse() self.assertEqual(self.sbox.ping(), -1) with mock.patch('storlets.gateway.gateways.docker.runtime.' 'SBusClient.ping') as ping: ping.side_effect = SBusClientIOError() self.assertEqual(self.sbox.ping(), -1) def test_wait(self): with mock.patch('storlets.gateway.gateways.docker.runtime.' 'SBusClient.ping') as ping, \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'time.sleep') as sleep: ping.return_value = SBusResponse(True, 'OK') self.sbox.wait() self.assertEqual(sleep.call_count, 0) with mock.patch('storlets.gateway.gateways.docker.runtime.' 'SBusClient.ping') as ping, \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'time.sleep') as sleep: ping.side_effect = [SBusResponse(False, 'Error'), SBusResponse(True, 'OK')] self.sbox.wait() self.assertEqual(sleep.call_count, 1) # TODO(takashi): should test timeout case def test_restart(self): class FakeProc(object): def __init__(self, stdout, stderr, code): self.stdout = stdout self.stderr = stderr self.returncode = code def communicate(self): return (self.stdout, self.stderr) with mock.patch('storlets.gateway.gateways.docker.runtime.' 'RunTimePaths.create_host_pipe_dir'), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'subprocess.Popen') as popen: _wait = self.sbox.wait def dummy_wait_success(*args, **kwargs): return 1 self.sbox.wait = dummy_wait_success # Test that popen is called successfully popen.return_value = FakeProc('Try to restart\nOK', '', 0) self.sbox.restart() self.assertEqual(1, popen.call_count) self.sbox.wait = _wait with mock.patch('storlets.gateway.gateways.docker.runtime.' 'RunTimePaths.create_host_pipe_dir'), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'subprocess.Popen') as popen: _wait = self.sbox.wait def dummy_wait_success(*args, **kwargs): return 1 self.sbox.wait = dummy_wait_success # Test double failure to restart the container # for both the tenant image and generic image popen.return_value = FakeProc('Try to restart', 'Some error', 1) with self.assertRaises(StorletRuntimeException): self.sbox.restart() self.assertEqual(2, popen.call_count) self.sbox.wait = _wait with mock.patch('storlets.gateway.gateways.docker.runtime.' 'RunTimePaths.create_host_pipe_dir'), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'subprocess.Popen') as popen: _wait = self.sbox.wait def dummy_wait_success(*args, **kwargs): return 1 self.sbox.wait = dummy_wait_success # Test failure to restart the container for the tenant image # success for the generic image popen.side_effect = [FakeProc('Try to restart', 'Some error', 1), FakeProc('Try to restart\nOK', '', 0)] self.sbox.restart() self.assertEqual(2, popen.call_count) self.sbox.wait = _wait with mock.patch('storlets.gateway.gateways.docker.runtime.' 'RunTimePaths.create_host_pipe_dir'), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'subprocess.Popen') as popen: _wait = self.sbox.wait def dummy_wait_failure(*args, **kwargs): raise StorletTimeout() self.sbox.wait = dummy_wait_failure popen.return_value = FakeProc('OK', '', 0) with self.assertRaises(StorletRuntimeException): self.sbox.restart() self.sbox.wait = _wait def test_get_storlet_classpath(self): storlet_id = 'Storlet.jar' storlet_main = 'org.openstack.storlet.Storlet' dependencies = ['dep1', 'dep2'] self.assertEqual( '/home/swift/org.openstack.storlet.Storlet/Storlet.jar:' '/home/swift/org.openstack.storlet.Storlet/dep1:' '/home/swift/org.openstack.storlet.Storlet/dep2', self.sbox._get_storlet_classpath(storlet_main, storlet_id, dependencies),)
class TestRunTimeSandbox(unittest.TestCase): def setUp(self): self.logger = FakeLogger() # TODO(takashi): take these values from config file self.conf = {'docker_repo': 'localhost:5001'} self.scope = '0123456789abc' self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger) def tearDown(self): pass def test_parse_sandbox_factory_answer(self): status, msg = self.sbox._parse_sandbox_factory_answer('True:message') self.assertTrue(status) self.assertEqual('message', msg) status, msg = self.sbox._parse_sandbox_factory_answer('False:message') self.assertFalse(status) self.assertEqual('message', msg) with self.assertRaises(StorletRuntimeException): self.sbox._parse_sandbox_factory_answer('Foo') def _check_all_pipese_closed(self, pipes): for _pipe in pipes: self.assertTrue(_pipe[0].closed) self.assertTrue(_pipe[1].closed) def test_ping(self): with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0): self.assertEqual(1, self.sbox.ping()) self._check_all_pipese_closed(pipes) with _mock_os_pipe(['False:ERROR']) as pipes, _mock_sbus(-1): self.assertEqual(-1, self.sbox.ping()) self._check_all_pipese_closed(pipes) with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0): with self.assertRaises(StorletRuntimeException): self.sbox.ping() self._check_all_pipese_closed(pipes) def test_wait(self): with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'time.sleep') as _s: self.sbox.wait() self.assertEqual(0, _s.call_count) self._check_all_pipese_closed(pipes) with _mock_os_pipe(['False:ERROR', 'True:OK']) as pipes, \ _mock_sbus(0), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'time.sleep') as _s: self.sbox.wait() self.assertEqual(1, _s.call_count) self._check_all_pipese_closed(pipes) with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'time.sleep') as _s: with self.assertRaises(StorletRuntimeException): self.sbox.wait() self._check_all_pipese_closed(pipes) # TODO(takashi): should test timeout case def test_restart(self): with mock.patch('storlets.gateway.gateways.docker.runtime.' 'RunTimePaths.create_host_pipe_prefix'), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'subprocess.call'): _wait = self.sbox.wait def dummy_wait_success(*args, **kwargs): return 1 self.sbox.wait = dummy_wait_success self.sbox.restart() self.sbox.wait = _wait with mock.patch('storlets.gateway.gateways.docker.runtime.' 'RunTimePaths.create_host_pipe_prefix'), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'subprocess.call'): _wait = self.sbox.wait def dummy_wait_failure(*args, **kwargs): raise StorletRuntimeException() self.sbox.wait = dummy_wait_failure with self.assertRaises(StorletRuntimeException): self.sbox.restart() self.sbox.wait = _wait def test_get_storlet_classpath(self): storlet_id = 'Storlet.jar' storlet_main = 'org.openstack.storlet.Storlet' dependencies = ['dep1', 'dep2'] self.assertEqual( '/home/swift/org.openstack.storlet.Storlet/Storlet.jar:' '/home/swift/org.openstack.storlet.Storlet/dep1:' '/home/swift/org.openstack.storlet.Storlet/dep2', self.sbox._get_storlet_classpath(storlet_main, storlet_id, dependencies), )
class TestRunTimeSandbox(unittest.TestCase): def setUp(self): self.logger = FakeLogger() # TODO(takashi): take these values from config file self.conf = {'docker_repo': 'localhost:5001'} self.scope = '0123456789abc' self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger) def tearDown(self): pass def test_parse_sandbox_factory_answer(self): status, msg = self.sbox._parse_sandbox_factory_answer('True:message') self.assertTrue(status) self.assertEqual(msg, 'message') status, msg = self.sbox._parse_sandbox_factory_answer('False:message') self.assertFalse(status) self.assertEqual(msg, 'message') with self.assertRaises(StorletRuntimeException): self.sbox._parse_sandbox_factory_answer('Foo') def _check_all_pipese_closed(self, pipes): for _pipe in pipes: self.assertTrue(_pipe[0].closed) self.assertTrue(_pipe[1].closed) def test_ping(self): with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0): self.assertEqual(self.sbox.ping(), 1) self._check_all_pipese_closed(pipes) with _mock_os_pipe(['False:ERROR']) as pipes, _mock_sbus(-1): self.assertEqual(self.sbox.ping(), -1) self._check_all_pipese_closed(pipes) with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0): with self.assertRaises(StorletRuntimeException): self.sbox.ping() self._check_all_pipese_closed(pipes) def test_wait(self): with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'time.sleep') as _s: self.sbox.wait() self.assertEqual(_s.call_count, 0) self._check_all_pipese_closed(pipes) with _mock_os_pipe(['False:ERROR', 'True:OK']) as pipes, \ _mock_sbus(0), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'time.sleep') as _s: self.sbox.wait() self.assertEqual(_s.call_count, 1) self._check_all_pipese_closed(pipes) with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'time.sleep') as _s: with self.assertRaises(StorletRuntimeException): self.sbox.wait() self._check_all_pipese_closed(pipes) # TODO(takashi): should test timeout case def test_restart(self): with mock.patch('storlets.gateway.gateways.docker.runtime.' 'RunTimePaths.create_host_pipe_prefix'), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'subprocess.call'): _wait = self.sbox.wait def dummy_wait_success(*args, **kwargs): return 1 self.sbox.wait = dummy_wait_success self.sbox.restart() self.sbox.wait = _wait with mock.patch('storlets.gateway.gateways.docker.runtime.' 'RunTimePaths.create_host_pipe_prefix'), \ mock.patch('storlets.gateway.gateways.docker.runtime.' 'subprocess.call'): _wait = self.sbox.wait def dummy_wait_failure(*args, **kwargs): raise StorletRuntimeException() self.sbox.wait = dummy_wait_failure with self.assertRaises(StorletRuntimeException): self.sbox.restart() self.sbox.wait = _wait def test_get_storlet_classpath(self): storlet_id = 'Storlet.jar' storlet_main = 'org.openstack.storlet.Storlet' dependencies = ['dep1', 'dep2'] self.assertEqual( self.sbox._get_storlet_classpath(storlet_main, storlet_id, dependencies), '/home/swift/org.openstack.storlet.Storlet/Storlet.jar:' '/home/swift/org.openstack.storlet.Storlet/dep1:' '/home/swift/org.openstack.storlet.Storlet/dep2')