def test_invocation_protocol_remote_fds(self): # In default, we have 5 fds in remote_fds storlet_request = DockerStorletRequest(self.storlet_id, {}, {}, iter(StringIO()), options=self.options) protocol = StorletInvocationProtocol(storlet_request, self.pipe_path, self.log_file, 1, self.logger) self.assertEqual(5, len(protocol.remote_fds)) # extra_resources expands the remote_fds storlet_request = DockerStorletRequest(self.storlet_id, {}, {}, iter(StringIO()), options=self.options) protocol = StorletInvocationProtocol(storlet_request, self.pipe_path, self.log_file, 1, self.logger, extra_sources=[storlet_request]) self.assertEqual(6, len(protocol.remote_fds)) # 2 more extra_resources expands the remote_fds storlet_request = DockerStorletRequest(self.storlet_id, {}, {}, iter(StringIO()), options=self.options) protocol = StorletInvocationProtocol(storlet_request, self.pipe_path, self.log_file, 1, self.logger, extra_sources=[storlet_request] * 3) self.assertEqual(8, len(protocol.remote_fds))
def invocation_flow(self, sreq, extra_sources=None): """ Invoke the backend protocl with gateway :param sreq: StorletRequest instance :param extra_sources (WIP): A list of StorletRequest instance to gather as extra resoureces to feed to storlet container as data source :return: StorletResponse instance """ run_time_sbox = RunTimeSandbox(self.scope, self.conf, self.logger) docker_updated = self.update_docker_container_from_cache(sreq) run_time_sbox.activate_storlet_daemon(sreq, docker_updated) self._add_system_params(sreq) slog_path = self.paths.get_host_slog_path(sreq.storlet_main) storlet_pipe_path = \ self.paths.get_host_storlet_pipe(sreq.storlet_main) sprotocol = StorletInvocationProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout, self.logger, extra_sources=extra_sources) sresp = sprotocol.communicate() self._upload_storlet_logs(slog_path, sreq) return sresp
def invocation_flow(self, sreq, extra_sources=None): """ Invoke the backend protocl with gateway :param sreq: StorletRequest instance :param extra_sources (WIP): A list of StorletRequest instance to gather as extra resoureces to feed to storlet container as data source :return: StorletResponse instance """ run_time_sbox = RunTimeSandbox(self.scope, self.conf, self.logger) docker_updated = self.update_docker_container_from_cache(sreq) run_time_sbox.activate_storlet_daemon(sreq, docker_updated) self._add_system_params(sreq) slog_path = self.paths.slog_path(sreq.storlet_main) storlet_pipe_path = self.paths.host_storlet_pipe(sreq.storlet_main) sprotocol = StorletInvocationProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout, self.logger, extra_sources=extra_sources) sresp = sprotocol.communicate() self._upload_storlet_logs(slog_path, sreq) return sresp
def setUp(self): self.pipe_path = tempfile.mktemp() self.log_file = tempfile.mktemp() self.logger = FakeLogger() self.storlet_id = 'Storlet-1.0.jar' self.options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} storlet_request = DockerStorletRequest( self.storlet_id, {}, {}, iter(StringIO()), options=self.options) self.protocol = StorletInvocationProtocol( storlet_request, self.pipe_path, self.log_file, 1, self.logger)
def setUp(self): self.pipe_path = tempfile.mktemp() self.log_file = tempfile.mktemp() self.logger = FakeLogger() storlet_id = 'Storlet-1.0.jar' options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} storlet_request = DockerStorletRequest( storlet_id, {}, {}, iter(StringIO()), options=options) self.protocol = StorletInvocationProtocol( storlet_request, self.pipe_path, self.log_file, 1, self.logger)
class TestStorletInvocationProtocol(unittest.TestCase): def setUp(self): self.pipe_path = tempfile.mktemp() self.log_file = tempfile.mktemp() self.logger = FakeLogger() self.storlet_id = 'Storlet-1.0.jar' self.options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} storlet_request = DockerStorletRequest( self.storlet_id, {}, {}, iter(StringIO()), options=self.options) self.protocol = StorletInvocationProtocol( storlet_request, self.pipe_path, self.log_file, 1, self.logger) def tearDown(self): for path in [self.pipe_path, self.log_file]: try: os.unlink(path) except OSError: pass def test_send_execute_command(self): with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.' 'execute') as execute: execute.return_value = SBusResponse(True, 'OK', 'someid') self.protocol._send_execute_command() self.assertEqual('someid', self.protocol.task_id) with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.' 'execute') as execute: execute.return_value = SBusResponse(True, 'OK') with self.assertRaises(StorletRuntimeException): self.protocol._send_execute_command() with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.' 'execute') as execute: execute.return_value = SBusResponse(False, 'NG', 'someid') with self.assertRaises(StorletRuntimeException): self.protocol._send_execute_command() with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.' 'execute') as execute: execute.side_effect = SBusClientIOError() with self.assertRaises(StorletRuntimeException): self.protocol._send_execute_command() def test_invocation_protocol(self): # os.pipe will be called 3 times pipe_called = 3 with _mock_os_pipe([''] * pipe_called) as pipes: with mock.patch.object(self.protocol, '_wait_for_read_with_timeout'), \ mock.patch.object(self.protocol, '_send_execute_command'): self.protocol._invoke() self.assertEqual(pipe_called, len(pipes)) pipes = iter(pipes) # data write is not directly closed # data read is closed input_data_read_fd, input_data_write_fd = next(pipes) self.assertTrue(input_data_read_fd.closed) self.assertFalse(input_data_write_fd.closed) # data write is closed but data read is still open data_read_fd, data_write_fd = next(pipes) self.assertFalse(data_read_fd.closed) self.assertTrue(data_write_fd.closed) # metadata write fd is closed, metadata read fd is still open. metadata_read_fd, metadata_write_fd = next(pipes) self.assertFalse(metadata_read_fd.closed) self.assertTrue(metadata_write_fd.closed) # sanity self.assertRaises(StopIteration, next, pipes) def test_invocation_protocol_remote_fds(self): # In default, we have 4 fds in remote_fds storlet_request = DockerStorletRequest( self.storlet_id, {}, {}, iter(StringIO()), options=self.options) protocol = StorletInvocationProtocol( storlet_request, self.pipe_path, self.log_file, 1, self.logger) self.assertEqual(4, len(protocol.remote_fds)) # extra_resources expands the remote_fds storlet_request = DockerStorletRequest( self.storlet_id, {}, {}, iter(StringIO()), options=self.options) protocol = StorletInvocationProtocol( storlet_request, self.pipe_path, self.log_file, 1, self.logger, extra_sources=[storlet_request]) self.assertEqual(5, len(protocol.remote_fds)) # 2 more extra_resources expands the remote_fds storlet_request = DockerStorletRequest( self.storlet_id, {}, {}, iter(StringIO()), options=self.options) protocol = StorletInvocationProtocol( storlet_request, self.pipe_path, self.log_file, 1, self.logger, extra_sources=[storlet_request] * 3) self.assertEqual(7, len(protocol.remote_fds)) def test_open_writer_with_invalid_fd(self): invalid_fds = ( (None, TypeError), (-1, ValueError), ('blah', TypeError)) for invalid_fd, expected_error in invalid_fds: with self.assertRaises(expected_error): with self.protocol._open_writer(invalid_fd): pass def _test_writer_with_exception(self, exception_cls): pipes = [os.pipe()] def raise_in_the_context(): with self.protocol._open_writer(pipes[0][1]): raise exception_cls() try: # writer context doesn't suppress any exception self.assertRaises(exception_cls, raise_in_the_context) # since _open_writer closes the write fd, the os.close will fail as # BadFileDescriptor with self.assertRaises(OSError) as os_error: os.close(pipes[0][1]) self.assertEqual(9, os_error.exception.errno) finally: for fd in pipes[0]: try: os.close(fd) except OSError: pass def test_writer_raise_while_in_writer_context(self): # basic storlet timeout self._test_writer_with_exception(StorletTimeout) # unexpected IOError self._test_writer_with_exception(IOError) # else self._test_writer_with_exception(Exception)
class TestStorletInvocationProtocol(unittest.TestCase): def setUp(self): self.pipe_path = tempfile.mktemp() self.log_file = tempfile.mktemp() self.logger = FakeLogger() storlet_id = 'Storlet-1.0.jar' options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} storlet_request = DockerStorletRequest( storlet_id, {}, {}, iter(StringIO()), options=options) self.protocol = StorletInvocationProtocol( storlet_request, self.pipe_path, self.log_file, 1, self.logger) def tearDown(self): for path in [self.pipe_path, self.log_file]: try: os.unlink(path) except OSError: pass def test_invocation_protocol(self): # os.pipe will be called 4 times pipe_called = 4 with _mock_sbus(0), _mock_os_pipe([''] * pipe_called) as pipes: with mock.patch.object( self.protocol, '_wait_for_read_with_timeout'): self.protocol._invoke() self.assertEqual(pipe_called, len(pipes)) pipes = iter(pipes) # data write is not directly closed # data read is closed input_data_read_fd, input_data_write_fd = next(pipes) self.assertTrue(input_data_read_fd.closed) self.assertFalse(input_data_write_fd.closed) # data write is closed but data read is still open data_read_fd, data_write_fd = next(pipes) self.assertFalse(data_read_fd.closed) self.assertTrue(data_write_fd.closed) # both execution str fds are closed execution_read_fd, execution_write_fd = next(pipes) self.assertTrue(execution_read_fd.closed) self.assertTrue(execution_write_fd.closed) # metadata write fd is closed, metadata read fd is still open. metadata_read_fd, metadata_write_fd = next(pipes) self.assertFalse(metadata_read_fd.closed) self.assertTrue(metadata_write_fd.closed) # sanity self.assertRaises(StopIteration, next, pipes) def test_invocation_protocol_remote_fds(self): # In default, we have 5 fds in remote_fds storlet_id = 'Storlet-1.0.jar' options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} storlet_request = DockerStorletRequest( storlet_id, {}, {}, iter(StringIO()), options=options) protocol = StorletInvocationProtocol( storlet_request, self.pipe_path, self.log_file, 1, self.logger) self.assertEqual(5, len(protocol.remote_fds)) # extra_resources expands the remote_fds storlet_request = DockerStorletRequest( storlet_id, {}, {}, iter(StringIO()), options=options) protocol = StorletInvocationProtocol( storlet_request, self.pipe_path, self.log_file, 1, self.logger, extra_sources=[storlet_request]) self.assertEqual(6, len(protocol.remote_fds)) # 2 more extra_resources expands the remote_fds storlet_request = DockerStorletRequest( storlet_id, {}, {}, iter(StringIO()), options=options) protocol = StorletInvocationProtocol( storlet_request, self.pipe_path, self.log_file, 1, self.logger, extra_sources=[storlet_request] * 3) self.assertEqual(8, len(protocol.remote_fds)) def test_open_writer_with_invalid_fd(self): invalid_fds = ( (None, TypeError), (-1, ValueError), ('blah', TypeError)) for invalid_fd, expected_error in invalid_fds: with mock.patch('os.close') as mock_close: with self.assertRaises(expected_error): with self.protocol._open_writer(invalid_fd): pass # writer attempts to close fd via os call self.assertEqual(1, mock_close.call_count) def _test_writer_with_exception(self, exception_cls): mock_writer = mock.MagicMock() with mock.patch('os.fdopen') as mock_fdopen, \ mock.patch('os.close') as mock_close: mock_fdopen.return_value = mock_writer def raise_in_the_context(): with self.protocol._open_writer(1): raise exception_cls() # writer context doesn't suppress any exception self.assertRaises(exception_cls, raise_in_the_context) # sanity self.assertEqual(1, mock_fdopen.call_count) self.assertEqual(0, mock_close.call_count) # writer was closed self.assertEqual(1, mock_writer.close.call_count) def test_writer_raise_while_in_writer_context(self): # basic storlet timeout self._test_writer_with_exception(StorletTimeout) # unexpected IOError self._test_writer_with_exception(IOError) # else self._test_writer_with_exception(Exception)