コード例 #1
0
ファイル: test_runtime.py プロジェクト: scottwedge/storlets
    def test_invocation_protocol_remote_fds(self):
        # In default, we have 5 fds in remote_fds
        storlet_request = DockerStorletRequest(self.storlet_id, {}, {},
                                               iter(StringIO()),
                                               options=self.options)
        protocol = StorletInvocationProtocol(storlet_request, self.pipe_path,
                                             self.log_file, 1, self.logger)
        self.assertEqual(5, len(protocol.remote_fds))

        # extra_resources expands the remote_fds
        storlet_request = DockerStorletRequest(self.storlet_id, {}, {},
                                               iter(StringIO()),
                                               options=self.options)
        protocol = StorletInvocationProtocol(storlet_request,
                                             self.pipe_path,
                                             self.log_file,
                                             1,
                                             self.logger,
                                             extra_sources=[storlet_request])
        self.assertEqual(6, len(protocol.remote_fds))

        # 2 more extra_resources expands the remote_fds
        storlet_request = DockerStorletRequest(self.storlet_id, {}, {},
                                               iter(StringIO()),
                                               options=self.options)
        protocol = StorletInvocationProtocol(storlet_request,
                                             self.pipe_path,
                                             self.log_file,
                                             1,
                                             self.logger,
                                             extra_sources=[storlet_request] *
                                             3)
        self.assertEqual(8, len(protocol.remote_fds))
コード例 #2
0
    def invocation_flow(self, sreq, extra_sources=None):
        """
        Invoke the backend protocl with gateway

        :param sreq: StorletRequest instance
        :param extra_sources (WIP): A list of StorletRequest instance to gather
                                    as extra resoureces to feed to storlet
                                    container as data source
        :return: StorletResponse instance
        """
        run_time_sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
        docker_updated = self.update_docker_container_from_cache(sreq)
        run_time_sbox.activate_storlet_daemon(sreq, docker_updated)
        self._add_system_params(sreq)

        slog_path = self.paths.get_host_slog_path(sreq.storlet_main)
        storlet_pipe_path = \
            self.paths.get_host_storlet_pipe(sreq.storlet_main)

        sprotocol = StorletInvocationProtocol(sreq,
                                              storlet_pipe_path,
                                              slog_path,
                                              self.storlet_timeout,
                                              self.logger,
                                              extra_sources=extra_sources)

        sresp = sprotocol.communicate()

        self._upload_storlet_logs(slog_path, sreq)

        return sresp
コード例 #3
0
ファイル: gateway.py プロジェクト: openstack/storlets
    def invocation_flow(self, sreq, extra_sources=None):
        """
        Invoke the backend protocl with gateway

        :param sreq: StorletRequest instance
        :param extra_sources (WIP): A list of StorletRequest instance to gather
                                    as extra resoureces to feed to storlet
                                    container as data source
        :return: StorletResponse instance
        """
        run_time_sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
        docker_updated = self.update_docker_container_from_cache(sreq)
        run_time_sbox.activate_storlet_daemon(sreq, docker_updated)
        self._add_system_params(sreq)

        slog_path = self.paths.slog_path(sreq.storlet_main)
        storlet_pipe_path = self.paths.host_storlet_pipe(sreq.storlet_main)

        sprotocol = StorletInvocationProtocol(sreq,
                                              storlet_pipe_path,
                                              slog_path,
                                              self.storlet_timeout,
                                              self.logger,
                                              extra_sources=extra_sources)

        sresp = sprotocol.communicate()

        self._upload_storlet_logs(slog_path, sreq)

        return sresp
コード例 #4
0
ファイル: test_runtime.py プロジェクト: mir-17101088/storlets
 def setUp(self):
     self.pipe_path = tempfile.mktemp()
     self.log_file = tempfile.mktemp()
     self.logger = FakeLogger()
     self.storlet_id = 'Storlet-1.0.jar'
     self.options = {'storlet_main': 'org.openstack.storlet.Storlet',
                     'storlet_dependency': 'dep1,dep2',
                     'storlet_language': 'java',
                     'file_manager': FakeFileManager('storlet', 'dep')}
     storlet_request = DockerStorletRequest(
         self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
     self.protocol = StorletInvocationProtocol(
         storlet_request, self.pipe_path, self.log_file, 1, self.logger)
コード例 #5
0
ファイル: test_runtime.py プロジェクト: openstack/storlets
 def setUp(self):
     self.pipe_path = tempfile.mktemp()
     self.log_file = tempfile.mktemp()
     self.logger = FakeLogger()
     storlet_id = 'Storlet-1.0.jar'
     options = {'storlet_main': 'org.openstack.storlet.Storlet',
                'storlet_dependency': 'dep1,dep2',
                'storlet_language': 'java',
                'file_manager': FakeFileManager('storlet', 'dep')}
     storlet_request = DockerStorletRequest(
         storlet_id, {}, {}, iter(StringIO()), options=options)
     self.protocol = StorletInvocationProtocol(
         storlet_request, self.pipe_path, self.log_file, 1, self.logger)
コード例 #6
0
ファイル: test_runtime.py プロジェクト: mir-17101088/storlets
class TestStorletInvocationProtocol(unittest.TestCase):
    def setUp(self):
        self.pipe_path = tempfile.mktemp()
        self.log_file = tempfile.mktemp()
        self.logger = FakeLogger()
        self.storlet_id = 'Storlet-1.0.jar'
        self.options = {'storlet_main': 'org.openstack.storlet.Storlet',
                        'storlet_dependency': 'dep1,dep2',
                        'storlet_language': 'java',
                        'file_manager': FakeFileManager('storlet', 'dep')}
        storlet_request = DockerStorletRequest(
            self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
        self.protocol = StorletInvocationProtocol(
            storlet_request, self.pipe_path, self.log_file, 1, self.logger)

    def tearDown(self):
        for path in [self.pipe_path, self.log_file]:
            try:
                os.unlink(path)
            except OSError:
                pass

    def test_send_execute_command(self):
        with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.'
                        'execute') as execute:
            execute.return_value = SBusResponse(True, 'OK', 'someid')
            self.protocol._send_execute_command()
            self.assertEqual('someid', self.protocol.task_id)

        with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.'
                        'execute') as execute:
            execute.return_value = SBusResponse(True, 'OK')
            with self.assertRaises(StorletRuntimeException):
                self.protocol._send_execute_command()

        with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.'
                        'execute') as execute:
            execute.return_value = SBusResponse(False, 'NG', 'someid')
            with self.assertRaises(StorletRuntimeException):
                self.protocol._send_execute_command()

        with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.'
                        'execute') as execute:
            execute.side_effect = SBusClientIOError()
            with self.assertRaises(StorletRuntimeException):
                self.protocol._send_execute_command()

    def test_invocation_protocol(self):
        # os.pipe will be called 3 times
        pipe_called = 3

        with _mock_os_pipe([''] * pipe_called) as pipes:
            with mock.patch.object(self.protocol,
                                   '_wait_for_read_with_timeout'), \
                    mock.patch.object(self.protocol, '_send_execute_command'):
                self.protocol._invoke()

            self.assertEqual(pipe_called, len(pipes))
            pipes = iter(pipes)

            # data write is not directly closed
            # data read is closed
            input_data_read_fd, input_data_write_fd = next(pipes)
            self.assertTrue(input_data_read_fd.closed)
            self.assertFalse(input_data_write_fd.closed)

            # data write is closed but data read is still open
            data_read_fd, data_write_fd = next(pipes)
            self.assertFalse(data_read_fd.closed)
            self.assertTrue(data_write_fd.closed)

            # metadata write fd is closed, metadata read fd is still open.
            metadata_read_fd, metadata_write_fd = next(pipes)
            self.assertFalse(metadata_read_fd.closed)
            self.assertTrue(metadata_write_fd.closed)

            # sanity
            self.assertRaises(StopIteration, next, pipes)

    def test_invocation_protocol_remote_fds(self):
        # In default, we have 4 fds in remote_fds
        storlet_request = DockerStorletRequest(
            self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
        protocol = StorletInvocationProtocol(
            storlet_request, self.pipe_path, self.log_file, 1, self.logger)
        self.assertEqual(4, len(protocol.remote_fds))

        # extra_resources expands the remote_fds
        storlet_request = DockerStorletRequest(
            self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
        protocol = StorletInvocationProtocol(
            storlet_request, self.pipe_path, self.log_file, 1, self.logger,
            extra_sources=[storlet_request])
        self.assertEqual(5, len(protocol.remote_fds))

        # 2 more extra_resources expands the remote_fds
        storlet_request = DockerStorletRequest(
            self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
        protocol = StorletInvocationProtocol(
            storlet_request, self.pipe_path, self.log_file, 1, self.logger,
            extra_sources=[storlet_request] * 3)
        self.assertEqual(7, len(protocol.remote_fds))

    def test_open_writer_with_invalid_fd(self):
        invalid_fds = (
            (None, TypeError), (-1, ValueError), ('blah', TypeError))

        for invalid_fd, expected_error in invalid_fds:
            with self.assertRaises(expected_error):
                with self.protocol._open_writer(invalid_fd):
                    pass

    def _test_writer_with_exception(self, exception_cls):
        pipes = [os.pipe()]

        def raise_in_the_context():
            with self.protocol._open_writer(pipes[0][1]):
                raise exception_cls()
        try:
            # writer context doesn't suppress any exception
            self.assertRaises(exception_cls, raise_in_the_context)

            # since _open_writer closes the write fd, the os.close will fail as
            # BadFileDescriptor
            with self.assertRaises(OSError) as os_error:
                os.close(pipes[0][1])
            self.assertEqual(9, os_error.exception.errno)

        finally:
            for fd in pipes[0]:
                try:
                    os.close(fd)
                except OSError:
                    pass

    def test_writer_raise_while_in_writer_context(self):
        # basic storlet timeout
        self._test_writer_with_exception(StorletTimeout)
        # unexpected IOError
        self._test_writer_with_exception(IOError)
        # else
        self._test_writer_with_exception(Exception)
コード例 #7
0
ファイル: test_runtime.py プロジェクト: openstack/storlets
class TestStorletInvocationProtocol(unittest.TestCase):
    def setUp(self):
        self.pipe_path = tempfile.mktemp()
        self.log_file = tempfile.mktemp()
        self.logger = FakeLogger()
        storlet_id = 'Storlet-1.0.jar'
        options = {'storlet_main': 'org.openstack.storlet.Storlet',
                   'storlet_dependency': 'dep1,dep2',
                   'storlet_language': 'java',
                   'file_manager': FakeFileManager('storlet', 'dep')}
        storlet_request = DockerStorletRequest(
            storlet_id, {}, {}, iter(StringIO()), options=options)
        self.protocol = StorletInvocationProtocol(
            storlet_request, self.pipe_path, self.log_file, 1, self.logger)

    def tearDown(self):
        for path in [self.pipe_path, self.log_file]:
            try:
                os.unlink(path)
            except OSError:
                pass

    def test_invocation_protocol(self):
        # os.pipe will be called 4 times
        pipe_called = 4

        with _mock_sbus(0), _mock_os_pipe([''] * pipe_called) as pipes:
            with mock.patch.object(
                    self.protocol, '_wait_for_read_with_timeout'):
                self.protocol._invoke()

            self.assertEqual(pipe_called, len(pipes))
            pipes = iter(pipes)

            # data write is not directly closed
            # data read is closed
            input_data_read_fd, input_data_write_fd = next(pipes)
            self.assertTrue(input_data_read_fd.closed)
            self.assertFalse(input_data_write_fd.closed)

            # data write is closed but data read is still open
            data_read_fd, data_write_fd = next(pipes)
            self.assertFalse(data_read_fd.closed)
            self.assertTrue(data_write_fd.closed)

            # both execution str fds are closed
            execution_read_fd, execution_write_fd = next(pipes)
            self.assertTrue(execution_read_fd.closed)
            self.assertTrue(execution_write_fd.closed)

            # metadata write fd is closed, metadata read fd is still open.
            metadata_read_fd, metadata_write_fd = next(pipes)
            self.assertFalse(metadata_read_fd.closed)
            self.assertTrue(metadata_write_fd.closed)

            # sanity
            self.assertRaises(StopIteration, next, pipes)

    def test_invocation_protocol_remote_fds(self):
        # In default, we have 5 fds in remote_fds
        storlet_id = 'Storlet-1.0.jar'
        options = {'storlet_main': 'org.openstack.storlet.Storlet',
                   'storlet_dependency': 'dep1,dep2',
                   'storlet_language': 'java',
                   'file_manager': FakeFileManager('storlet', 'dep')}
        storlet_request = DockerStorletRequest(
            storlet_id, {}, {}, iter(StringIO()), options=options)
        protocol = StorletInvocationProtocol(
            storlet_request, self.pipe_path, self.log_file, 1, self.logger)
        self.assertEqual(5, len(protocol.remote_fds))

        # extra_resources expands the remote_fds
        storlet_request = DockerStorletRequest(
            storlet_id, {}, {}, iter(StringIO()), options=options)
        protocol = StorletInvocationProtocol(
            storlet_request, self.pipe_path, self.log_file, 1, self.logger,
            extra_sources=[storlet_request])
        self.assertEqual(6, len(protocol.remote_fds))

        # 2 more extra_resources expands the remote_fds
        storlet_request = DockerStorletRequest(
            storlet_id, {}, {}, iter(StringIO()), options=options)
        protocol = StorletInvocationProtocol(
            storlet_request, self.pipe_path, self.log_file, 1, self.logger,
            extra_sources=[storlet_request] * 3)
        self.assertEqual(8, len(protocol.remote_fds))

    def test_open_writer_with_invalid_fd(self):
        invalid_fds = (
            (None, TypeError), (-1, ValueError), ('blah', TypeError))

        for invalid_fd, expected_error in invalid_fds:
            with mock.patch('os.close') as mock_close:
                with self.assertRaises(expected_error):
                    with self.protocol._open_writer(invalid_fd):
                        pass
                # writer attempts to close fd via os call
                self.assertEqual(1, mock_close.call_count)

    def _test_writer_with_exception(self, exception_cls):
        mock_writer = mock.MagicMock()
        with mock.patch('os.fdopen') as mock_fdopen, \
                mock.patch('os.close') as mock_close:
            mock_fdopen.return_value = mock_writer

            def raise_in_the_context():
                with self.protocol._open_writer(1):
                    raise exception_cls()

            # writer context doesn't suppress any exception
            self.assertRaises(exception_cls, raise_in_the_context)

        # sanity
        self.assertEqual(1, mock_fdopen.call_count)
        self.assertEqual(0, mock_close.call_count)

        # writer was closed
        self.assertEqual(1, mock_writer.close.call_count)

    def test_writer_raise_while_in_writer_context(self):
        # basic storlet timeout
        self._test_writer_with_exception(StorletTimeout)
        # unexpected IOError
        self._test_writer_with_exception(IOError)
        # else
        self._test_writer_with_exception(Exception)