def invocation_flow(self, sreq, extra_sources=None):
"""
Invoke the backend protocl with gateway
:param sreq: StorletRequest instance
:param extra_sources (WIP): A list of StorletRequest instance to gather
as extra resoureces to feed to storlet
container as data source
:return: StorletResponse instance
"""
run_time_sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
docker_updated = self.update_docker_container_from_cache(sreq)
run_time_sbox.activate_storlet_daemon(sreq, docker_updated)
self._add_system_params(sreq)
slog_path = self.paths.get_host_slog_path(sreq.storlet_main)
storlet_pipe_path = \
self.paths.get_host_storlet_pipe(sreq.storlet_main)
sprotocol = StorletInvocationProtocol(sreq,
storlet_pipe_path,
slog_path,
self.storlet_timeout,
self.logger,
extra_sources=extra_sources)
sresp = sprotocol.communicate()
self._upload_storlet_logs(slog_path, sreq)
return sresp
def invocation_flow(self, sreq, extra_sources=None):
"""
Invoke the backend protocl with gateway
:param sreq: StorletRequest instance
:param extra_sources (WIP): A list of StorletRequest instance to gather
as extra resoureces to feed to storlet
container as data source
:return: StorletResponse instance
"""
run_time_sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
docker_updated = self.update_docker_container_from_cache(sreq)
run_time_sbox.activate_storlet_daemon(sreq, docker_updated)
self._add_system_params(sreq)
slog_path = self.paths.slog_path(sreq.storlet_main)
storlet_pipe_path = self.paths.host_storlet_pipe(sreq.storlet_main)
sprotocol = StorletInvocationProtocol(sreq,
storlet_pipe_path,
slog_path,
self.storlet_timeout,
self.logger,
extra_sources=extra_sources)
sresp = sprotocol.communicate()
self._upload_storlet_logs(slog_path, sreq)
return sresp
def setUp(self):
self.logger = FakeLogger()
# TODO(takashi): take these values from config file
self.conf = {'docker_repo': 'localhost:5001'}
self.scope = '0123456789abc'
self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
class TestRunTimeSandbox(unittest.TestCase):
def setUp(self):
self.logger = FakeLogger()
# TODO(takashi): take these values from config file
self.conf = {'docker_repo': 'localhost:5001'}
self.scope = '0123456789abc'
self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
def test_ping(self):
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'SBusClient.ping') as ping:
ping.return_value = SBusResponse(True, 'OK')
self.assertEqual(self.sbox.ping(), 1)
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'SBusClient.ping') as ping:
ping.return_value = SBusResponse(False, 'Error')
self.assertEqual(self.sbox.ping(), 0)
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'SBusClient.ping') as ping:
ping.side_effect = SBusClientSendError()
self.assertEqual(self.sbox.ping(), -1)
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'SBusClient.ping') as ping:
ping.side_effect = SBusClientMalformedResponse()
self.assertEqual(self.sbox.ping(), -1)
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'SBusClient.ping') as ping:
ping.side_effect = SBusClientIOError()
self.assertEqual(self.sbox.ping(), -1)
def test_wait(self):
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'SBusClient.ping') as ping, \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'time.sleep') as sleep:
ping.return_value = SBusResponse(True, 'OK')
self.sbox.wait()
self.assertEqual(sleep.call_count, 0)
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'SBusClient.ping') as ping, \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'time.sleep') as sleep:
ping.side_effect = [SBusResponse(False, 'Error'),
SBusResponse(True, 'OK')]
self.sbox.wait()
self.assertEqual(sleep.call_count, 1)
# TODO(takashi): should test timeout case
def test_restart(self):
class FakeProc(object):
def __init__(self, stdout, stderr, code):
self.stdout = stdout
self.stderr = stderr
self.returncode = code
def communicate(self):
return (self.stdout, self.stderr)
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_dir'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.Popen') as popen:
_wait = self.sbox.wait
def dummy_wait_success(*args, **kwargs):
return 1
self.sbox.wait = dummy_wait_success
# Test that popen is called successfully
popen.return_value = FakeProc('Try to restart\nOK', '', 0)
self.sbox.restart()
self.assertEqual(1, popen.call_count)
self.sbox.wait = _wait
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_dir'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.Popen') as popen:
_wait = self.sbox.wait
def dummy_wait_success(*args, **kwargs):
return 1
self.sbox.wait = dummy_wait_success
# Test double failure to restart the container
# for both the tenant image and generic image
popen.return_value = FakeProc('Try to restart', 'Some error', 1)
with self.assertRaises(StorletRuntimeException):
self.sbox.restart()
self.assertEqual(2, popen.call_count)
self.sbox.wait = _wait
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_dir'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.Popen') as popen:
_wait = self.sbox.wait
def dummy_wait_success(*args, **kwargs):
return 1
self.sbox.wait = dummy_wait_success
# Test failure to restart the container for the tenant image
# success for the generic image
popen.side_effect = [FakeProc('Try to restart', 'Some error', 1),
FakeProc('Try to restart\nOK', '', 0)]
self.sbox.restart()
self.assertEqual(2, popen.call_count)
self.sbox.wait = _wait
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_dir'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.Popen') as popen:
_wait = self.sbox.wait
def dummy_wait_failure(*args, **kwargs):
raise StorletTimeout()
self.sbox.wait = dummy_wait_failure
popen.return_value = FakeProc('OK', '', 0)
with self.assertRaises(StorletRuntimeException):
self.sbox.restart()
self.sbox.wait = _wait
def test_get_storlet_classpath(self):
storlet_id = 'Storlet.jar'
storlet_main = 'org.openstack.storlet.Storlet'
dependencies = ['dep1', 'dep2']
self.assertEqual(
'/home/swift/org.openstack.storlet.Storlet/Storlet.jar:'
'/home/swift/org.openstack.storlet.Storlet/dep1:'
'/home/swift/org.openstack.storlet.Storlet/dep2',
self.sbox._get_storlet_classpath(storlet_main, storlet_id,
dependencies),)
class TestRunTimeSandbox(unittest.TestCase):
def setUp(self):
self.logger = FakeLogger()
# TODO(takashi): take these values from config file
self.conf = {'docker_repo': 'localhost:5001'}
self.scope = '0123456789abc'
self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
def tearDown(self):
pass
def test_parse_sandbox_factory_answer(self):
status, msg = self.sbox._parse_sandbox_factory_answer('True:message')
self.assertTrue(status)
self.assertEqual('message', msg)
status, msg = self.sbox._parse_sandbox_factory_answer('False:message')
self.assertFalse(status)
self.assertEqual('message', msg)
with self.assertRaises(StorletRuntimeException):
self.sbox._parse_sandbox_factory_answer('Foo')
def _check_all_pipese_closed(self, pipes):
for _pipe in pipes:
self.assertTrue(_pipe[0].closed)
self.assertTrue(_pipe[1].closed)
def test_ping(self):
with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0):
self.assertEqual(1, self.sbox.ping())
self._check_all_pipese_closed(pipes)
with _mock_os_pipe(['False:ERROR']) as pipes, _mock_sbus(-1):
self.assertEqual(-1, self.sbox.ping())
self._check_all_pipese_closed(pipes)
with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0):
with self.assertRaises(StorletRuntimeException):
self.sbox.ping()
self._check_all_pipese_closed(pipes)
def test_wait(self):
with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'time.sleep') as _s:
self.sbox.wait()
self.assertEqual(0, _s.call_count)
self._check_all_pipese_closed(pipes)
with _mock_os_pipe(['False:ERROR', 'True:OK']) as pipes, \
_mock_sbus(0), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'time.sleep') as _s:
self.sbox.wait()
self.assertEqual(1, _s.call_count)
self._check_all_pipese_closed(pipes)
with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'time.sleep') as _s:
with self.assertRaises(StorletRuntimeException):
self.sbox.wait()
self._check_all_pipese_closed(pipes)
# TODO(takashi): should test timeout case
def test_restart(self):
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_prefix'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.call'):
_wait = self.sbox.wait
def dummy_wait_success(*args, **kwargs):
return 1
self.sbox.wait = dummy_wait_success
self.sbox.restart()
self.sbox.wait = _wait
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_prefix'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.call'):
_wait = self.sbox.wait
def dummy_wait_failure(*args, **kwargs):
raise StorletRuntimeException()
self.sbox.wait = dummy_wait_failure
with self.assertRaises(StorletRuntimeException):
self.sbox.restart()
self.sbox.wait = _wait
def test_get_storlet_classpath(self):
storlet_id = 'Storlet.jar'
storlet_main = 'org.openstack.storlet.Storlet'
dependencies = ['dep1', 'dep2']
self.assertEqual(
'/home/swift/org.openstack.storlet.Storlet/Storlet.jar:'
'/home/swift/org.openstack.storlet.Storlet/dep1:'
'/home/swift/org.openstack.storlet.Storlet/dep2',
self.sbox._get_storlet_classpath(storlet_main, storlet_id,
dependencies),
)
def setUp(self):
self.logger = FakeLogger()
# TODO(takashi): take these values from config file
self.conf = {'docker_repo': 'localhost:5001'}
self.scope = '0123456789abc'
self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
class TestRunTimeSandbox(unittest.TestCase):
def setUp(self):
self.logger = FakeLogger()
# TODO(takashi): take these values from config file
self.conf = {'docker_repo': 'localhost:5001'}
self.scope = '0123456789abc'
self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
def tearDown(self):
pass
def test_parse_sandbox_factory_answer(self):
status, msg = self.sbox._parse_sandbox_factory_answer('True:message')
self.assertTrue(status)
self.assertEqual(msg, 'message')
status, msg = self.sbox._parse_sandbox_factory_answer('False:message')
self.assertFalse(status)
self.assertEqual(msg, 'message')
with self.assertRaises(StorletRuntimeException):
self.sbox._parse_sandbox_factory_answer('Foo')
def _check_all_pipese_closed(self, pipes):
for _pipe in pipes:
self.assertTrue(_pipe[0].closed)
self.assertTrue(_pipe[1].closed)
def test_ping(self):
with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0):
self.assertEqual(self.sbox.ping(), 1)
self._check_all_pipese_closed(pipes)
with _mock_os_pipe(['False:ERROR']) as pipes, _mock_sbus(-1):
self.assertEqual(self.sbox.ping(), -1)
self._check_all_pipese_closed(pipes)
with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0):
with self.assertRaises(StorletRuntimeException):
self.sbox.ping()
self._check_all_pipese_closed(pipes)
def test_wait(self):
with _mock_os_pipe(['True:OK']) as pipes, _mock_sbus(0), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'time.sleep') as _s:
self.sbox.wait()
self.assertEqual(_s.call_count, 0)
self._check_all_pipese_closed(pipes)
with _mock_os_pipe(['False:ERROR', 'True:OK']) as pipes, \
_mock_sbus(0), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'time.sleep') as _s:
self.sbox.wait()
self.assertEqual(_s.call_count, 1)
self._check_all_pipese_closed(pipes)
with _mock_os_pipe(['Foo']) as pipes, _mock_sbus(0), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'time.sleep') as _s:
with self.assertRaises(StorletRuntimeException):
self.sbox.wait()
self._check_all_pipese_closed(pipes)
# TODO(takashi): should test timeout case
def test_restart(self):
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_prefix'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.call'):
_wait = self.sbox.wait
def dummy_wait_success(*args, **kwargs):
return 1
self.sbox.wait = dummy_wait_success
self.sbox.restart()
self.sbox.wait = _wait
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_prefix'), \
mock.patch('storlets.gateway.gateways.docker.runtime.'
'subprocess.call'):
_wait = self.sbox.wait
def dummy_wait_failure(*args, **kwargs):
raise StorletRuntimeException()
self.sbox.wait = dummy_wait_failure
with self.assertRaises(StorletRuntimeException):
self.sbox.restart()
self.sbox.wait = _wait
def test_get_storlet_classpath(self):
storlet_id = 'Storlet.jar'
storlet_main = 'org.openstack.storlet.Storlet'
dependencies = ['dep1', 'dep2']
self.assertEqual(
self.sbox._get_storlet_classpath(storlet_main, storlet_id,
dependencies),
'/home/swift/org.openstack.storlet.Storlet/Storlet.jar:'
'/home/swift/org.openstack.storlet.Storlet/dep1:'
'/home/swift/org.openstack.storlet.Storlet/dep2')