Exemple #1
0
    def invocation_flow(self, sreq, extra_sources=None):
        """
        Invoke the backend protocl with gateway

        :param sreq: StorletRequest instance
        :param extra_sources (WIP): A list of StorletRequest instance to gather
                                    as extra resoureces to feed to storlet
                                    container as data source
        :return: StorletResponse instance
        """
        run_time_sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
        docker_updated = self.update_docker_container_from_cache(sreq)
        run_time_sbox.activate_storlet_daemon(sreq, docker_updated)
        self._add_system_params(sreq)

        slog_path = self.paths.get_host_slog_path(sreq.storlet_main)
        storlet_pipe_path = \
            self.paths.get_host_storlet_pipe(sreq.storlet_main)

        sprotocol = StorletInvocationProtocol(sreq,
                                              storlet_pipe_path,
                                              slog_path,
                                              self.storlet_timeout,
                                              self.logger,
                                              extra_sources=extra_sources)

        sresp = sprotocol.communicate()

        self._upload_storlet_logs(slog_path, sreq)

        return sresp
Exemple #2
0
    def invocation_flow(self, sreq, extra_sources=None):
        """
        Invoke the backend protocl with gateway

        :param sreq: StorletRequest instance
        :param extra_sources (WIP): A list of StorletRequest instance to gather
                                    as extra resoureces to feed to storlet
                                    container as data source
        :return: StorletResponse instance
        """
        run_time_sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
        docker_updated = self.update_docker_container_from_cache(sreq)
        run_time_sbox.activate_storlet_daemon(sreq, docker_updated)
        self._add_system_params(sreq)

        slog_path = self.paths.slog_path(sreq.storlet_main)
        storlet_pipe_path = self.paths.host_storlet_pipe(sreq.storlet_main)

        sprotocol = StorletInvocationProtocol(sreq,
                                              storlet_pipe_path,
                                              slog_path,
                                              self.storlet_timeout,
                                              self.logger,
                                              extra_sources=extra_sources)

        sresp = sprotocol.communicate()

        self._upload_storlet_logs(slog_path, sreq)

        return sresp
 def setUp(self):
     self.logger = FakeLogger()
     # TODO(takashi): take these values from config file
     self.conf = {'docker_repo': 'localhost:5001'}
     self.scope = '0123456789abc'
     self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
class TestRunTimeSandbox(unittest.TestCase):
    def setUp(self):
        self.logger = FakeLogger()
        # TODO(takashi): take these values from config file
        self.conf = {'docker_repo': 'localhost:5001'}
        self.scope = '0123456789abc'
        self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)

    def test_ping(self):
        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'SBusClient.ping') as ping:
            ping.return_value = SBusResponse(True, 'OK')
            self.assertEqual(self.sbox.ping(), 1)

        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'SBusClient.ping') as ping:
            ping.return_value = SBusResponse(False, 'Error')
            self.assertEqual(self.sbox.ping(), 0)

        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'SBusClient.ping') as ping:
            ping.side_effect = SBusClientSendError()
            self.assertEqual(self.sbox.ping(), -1)

        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'SBusClient.ping') as ping:
            ping.side_effect = SBusClientMalformedResponse()
            self.assertEqual(self.sbox.ping(), -1)

        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'SBusClient.ping') as ping:
            ping.side_effect = SBusClientIOError()
            self.assertEqual(self.sbox.ping(), -1)

    def test_wait(self):
        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'SBusClient.ping') as ping, \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'time.sleep') as sleep:
            ping.return_value = SBusResponse(True, 'OK')
            self.sbox.wait()
            self.assertEqual(sleep.call_count, 0)

        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'SBusClient.ping') as ping, \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'time.sleep') as sleep:
            ping.side_effect = [SBusResponse(False, 'Error'),
                                SBusResponse(True, 'OK')]
            self.sbox.wait()
            self.assertEqual(sleep.call_count, 1)

        # TODO(takashi): should test timeout case

    def test_restart(self):

        class FakeProc(object):
            def __init__(self, stdout, stderr, code):
                self.stdout = stdout
                self.stderr = stderr
                self.returncode = code

            def communicate(self):
                return (self.stdout, self.stderr)

        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'RunTimePaths.create_host_pipe_dir'), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'subprocess.Popen') as popen:
            _wait = self.sbox.wait

            def dummy_wait_success(*args, **kwargs):
                return 1

            self.sbox.wait = dummy_wait_success

            # Test that popen is called successfully
            popen.return_value = FakeProc('Try to restart\nOK', '', 0)
            self.sbox.restart()
            self.assertEqual(1, popen.call_count)
            self.sbox.wait = _wait

        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'RunTimePaths.create_host_pipe_dir'), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'subprocess.Popen') as popen:
            _wait = self.sbox.wait

            def dummy_wait_success(*args, **kwargs):
                return 1

            self.sbox.wait = dummy_wait_success

            # Test double failure to restart the container
            # for both the tenant image and generic image
            popen.return_value = FakeProc('Try to restart', 'Some error', 1)
            with self.assertRaises(StorletRuntimeException):
                self.sbox.restart()
            self.assertEqual(2, popen.call_count)
            self.sbox.wait = _wait

        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'RunTimePaths.create_host_pipe_dir'), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'subprocess.Popen') as popen:
            _wait = self.sbox.wait

            def dummy_wait_success(*args, **kwargs):
                return 1

            self.sbox.wait = dummy_wait_success

            # Test failure to restart the container for the tenant image
            # success for the generic image
            popen.side_effect = [FakeProc('Try to restart', 'Some error', 1),
                                 FakeProc('Try to restart\nOK', '', 0)]
            self.sbox.restart()
            self.assertEqual(2, popen.call_count)
            self.sbox.wait = _wait

        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'RunTimePaths.create_host_pipe_dir'), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'subprocess.Popen') as popen:
            _wait = self.sbox.wait

            def dummy_wait_failure(*args, **kwargs):
                raise StorletTimeout()

            self.sbox.wait = dummy_wait_failure

            popen.return_value = FakeProc('OK', '', 0)
            with self.assertRaises(StorletRuntimeException):
                self.sbox.restart()
            self.sbox.wait = _wait

    def test_get_storlet_classpath(self):
        storlet_id = 'Storlet.jar'
        storlet_main = 'org.openstack.storlet.Storlet'
        dependencies = ['dep1', 'dep2']
        self.assertEqual(
            '/home/swift/org.openstack.storlet.Storlet/Storlet.jar:'
            '/home/swift/org.openstack.storlet.Storlet/dep1:'
            '/home/swift/org.openstack.storlet.Storlet/dep2',
            self.sbox._get_storlet_classpath(storlet_main, storlet_id,
                                             dependencies),)
Exemple #5
0
class TestRunTimeSandbox(unittest.TestCase):
    def setUp(self):
        self.logger = FakeLogger()
        # TODO(takashi): take these values from config file
        self.conf = {'docker_repo': 'localhost:5001'}
        self.scope = '0123456789abc'
        self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)

    def tearDown(self):
        pass

    def test_parse_sandbox_factory_answer(self):
        status, msg = self.sbox._parse_sandbox_factory_answer('True:message')
        self.assertTrue(status)
        self.assertEqual('message', msg)

        status, msg = self.sbox._parse_sandbox_factory_answer('False:message')
        self.assertFalse(status)
        self.assertEqual('message', msg)

        with self.assertRaises(StorletRuntimeException):
            self.sbox._parse_sandbox_factory_answer('Foo')

    def _check_all_pipese_closed(self, pipes):
        for _pipe in pipes:
            self.assertTrue(_pipe[0].closed)
            self.assertTrue(_pipe[1].closed)

    def test_ping(self):
        with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0):
            self.assertEqual(1, self.sbox.ping())
            self._check_all_pipese_closed(pipes)

        with _mock_os_pipe(['False:ERROR']) as pipes, _mock_sbus(-1):
            self.assertEqual(-1, self.sbox.ping())
            self._check_all_pipese_closed(pipes)

        with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0):
            with self.assertRaises(StorletRuntimeException):
                self.sbox.ping()
            self._check_all_pipese_closed(pipes)

    def test_wait(self):
        with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'time.sleep') as _s:
            self.sbox.wait()
            self.assertEqual(0, _s.call_count)
            self._check_all_pipese_closed(pipes)

        with _mock_os_pipe(['False:ERROR', 'True:OK']) as pipes, \
            _mock_sbus(0), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'time.sleep') as _s:
            self.sbox.wait()
            self.assertEqual(1, _s.call_count)
            self._check_all_pipese_closed(pipes)

        with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'time.sleep') as _s:
            with self.assertRaises(StorletRuntimeException):
                self.sbox.wait()
            self._check_all_pipese_closed(pipes)

        # TODO(takashi): should test timeout case

    def test_restart(self):
        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'RunTimePaths.create_host_pipe_prefix'), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'subprocess.call'):
            _wait = self.sbox.wait

            def dummy_wait_success(*args, **kwargs):
                return 1

            self.sbox.wait = dummy_wait_success
            self.sbox.restart()
            self.sbox.wait = _wait

        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'RunTimePaths.create_host_pipe_prefix'), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'subprocess.call'):
            _wait = self.sbox.wait

            def dummy_wait_failure(*args, **kwargs):
                raise StorletRuntimeException()

            self.sbox.wait = dummy_wait_failure
            with self.assertRaises(StorletRuntimeException):
                self.sbox.restart()
            self.sbox.wait = _wait

    def test_get_storlet_classpath(self):
        storlet_id = 'Storlet.jar'
        storlet_main = 'org.openstack.storlet.Storlet'
        dependencies = ['dep1', 'dep2']
        self.assertEqual(
            '/home/swift/org.openstack.storlet.Storlet/Storlet.jar:'
            '/home/swift/org.openstack.storlet.Storlet/dep1:'
            '/home/swift/org.openstack.storlet.Storlet/dep2',
            self.sbox._get_storlet_classpath(storlet_main, storlet_id,
                                             dependencies),
        )
Exemple #6
0
 def setUp(self):
     self.logger = FakeLogger()
     # TODO(takashi): take these values from config file
     self.conf = {'docker_repo': 'localhost:5001'}
     self.scope = '0123456789abc'
     self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
Exemple #7
0
class TestRunTimeSandbox(unittest.TestCase):
    def setUp(self):
        self.logger = FakeLogger()
        # TODO(takashi): take these values from config file
        self.conf = {'docker_repo': 'localhost:5001'}
        self.scope = '0123456789abc'
        self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)

    def tearDown(self):
        pass

    def test_parse_sandbox_factory_answer(self):
        status, msg = self.sbox._parse_sandbox_factory_answer('True:message')
        self.assertTrue(status)
        self.assertEqual(msg, 'message')

        status, msg = self.sbox._parse_sandbox_factory_answer('False:message')
        self.assertFalse(status)
        self.assertEqual(msg, 'message')

        with self.assertRaises(StorletRuntimeException):
            self.sbox._parse_sandbox_factory_answer('Foo')

    def _check_all_pipese_closed(self, pipes):
        for _pipe in pipes:
            self.assertTrue(_pipe[0].closed)
            self.assertTrue(_pipe[1].closed)

    def test_ping(self):
        with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0):
            self.assertEqual(self.sbox.ping(), 1)
            self._check_all_pipese_closed(pipes)

        with _mock_os_pipe(['False:ERROR']) as pipes, _mock_sbus(-1):
            self.assertEqual(self.sbox.ping(), -1)
            self._check_all_pipese_closed(pipes)

        with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0):
            with self.assertRaises(StorletRuntimeException):
                self.sbox.ping()
            self._check_all_pipese_closed(pipes)

    def test_wait(self):
        with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'time.sleep') as _s:
            self.sbox.wait()
            self.assertEqual(_s.call_count, 0)
            self._check_all_pipese_closed(pipes)

        with _mock_os_pipe(['False:ERROR', 'True:OK']) as pipes, \
            _mock_sbus(0), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'time.sleep') as _s:
            self.sbox.wait()
            self.assertEqual(_s.call_count, 1)
            self._check_all_pipese_closed(pipes)

        with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'time.sleep') as _s:
            with self.assertRaises(StorletRuntimeException):
                self.sbox.wait()
            self._check_all_pipese_closed(pipes)

        # TODO(takashi): should test timeout case

    def test_restart(self):
        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'RunTimePaths.create_host_pipe_prefix'), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'subprocess.call'):
            _wait = self.sbox.wait

            def dummy_wait_success(*args, **kwargs):
                return 1

            self.sbox.wait = dummy_wait_success
            self.sbox.restart()
            self.sbox.wait = _wait

        with mock.patch('storlets.gateway.gateways.docker.runtime.'
                        'RunTimePaths.create_host_pipe_prefix'), \
            mock.patch('storlets.gateway.gateways.docker.runtime.'
                       'subprocess.call'):
            _wait = self.sbox.wait

            def dummy_wait_failure(*args, **kwargs):
                raise StorletRuntimeException()

            self.sbox.wait = dummy_wait_failure
            with self.assertRaises(StorletRuntimeException):
                self.sbox.restart()
            self.sbox.wait = _wait

    def test_get_storlet_classpath(self):
        storlet_id = 'Storlet.jar'
        storlet_main = 'org.openstack.storlet.Storlet'
        dependencies = ['dep1', 'dep2']
        self.assertEqual(
            self.sbox._get_storlet_classpath(storlet_main, storlet_id,
                                             dependencies),
            '/home/swift/org.openstack.storlet.Storlet/Storlet.jar:'
            '/home/swift/org.openstack.storlet.Storlet/dep1:'
            '/home/swift/org.openstack.storlet.Storlet/dep2')