def test_has_range(self):
storlet_id = 'Storlet-1.0.jar'
params = {}
metadata = {}
options = {'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')}
dsreq = DockerStorletRequest(storlet_id, params, metadata,
None, 0, options=options)
self.assertFalse(dsreq.has_range)
options = {'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep'),
'range_start': 1,
'range_end': 6}
dsreq = DockerStorletRequest(storlet_id, params, metadata,
None, 0, options=options)
self.assertTrue(dsreq.has_range)
options = {'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep'),
'range_start': 0,
'range_end': 6}
dsreq = DockerStorletRequest(storlet_id, params, metadata,
None, 0, options=options)
self.assertTrue(dsreq.has_range)
def test_init_with_range(self):
storlet_id = 'Storlet-1.0.jar'
params = {}
metadata = {}
options = {
'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep'),
'range_start': 1,
'range_end': 6
}
dsreq = DockerStorletRequest(storlet_id,
params,
metadata,
None,
0,
options=options)
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
self.assertEqual(['dep1', 'dep2'], dsreq.dependencies)
self.assertEqual('java', dsreq.storlet_language)
self.assertEqual(1, dsreq.start)
self.assertEqual(6, dsreq.end)
options = {
'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep'),
'range_start': 0,
'range_end': 0
}
dsreq = DockerStorletRequest(storlet_id,
params,
metadata,
None,
0,
options=options)
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
self.assertEqual(['dep1', 'dep2'], dsreq.dependencies)
self.assertEqual('java', dsreq.storlet_language)
self.assertEqual(0, dsreq.start)
self.assertEqual(0, dsreq.end)
def setUp(self):
self.pipe_path = tempfile.mktemp()
self.log_file = tempfile.mktemp()
self.logger = FakeLogger()
self.storlet_id = 'Storlet-1.0.jar'
self.options = {'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')}
storlet_request = DockerStorletRequest(
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
self.protocol = StorletInvocationProtocol(
storlet_request, self.pipe_path, self.log_file, 1, self.logger)
def test_docker_gateway_communicate_with_extra_sources(self):
options = {
'generate_log': False,
'scope': 'AUTH_account',
'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')
}
data_sources = []
def generate_extra_st_request():
# This works similarly with build_storlet_request
# TODO(kota_): think of more generarl way w/o
# build_storlet_request
sw_req = Request.blank(self.req_path,
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Run-Storlet': self.sobj})
sw_resp = Response(app_iter=iter(['This is a response body']),
status=200)
st_req = DockerStorletRequest(
storlet_id=sw_req.headers['X-Run-Storlet'],
params=sw_req.params,
user_metadata={},
data_iter=sw_resp.app_iter,
options=options)
data_sources.append(sw_resp.app_iter)
return st_req
extra_request = generate_extra_st_request()
mock_calls = self._test_docker_gateway_communicate(
extra_sources=[extra_request])
self.assertEqual('This is a response body', mock_calls[1][1])
# run all existing eventlet threads
for app_iter in data_sources:
# ensure all app_iters are drawn
self.assertRaises(StopIteration, next, app_iter)
def _test_docker_gateway_communicate(self, extra_sources=None):
extra_sources = extra_sources or []
options = {'generate_log': False,
'scope': 'AUTH_account',
'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')}
st_req = DockerStorletRequest(
storlet_id=self.sobj,
params={},
user_metadata={},
data_iter=iter('body'), options=options)
# TODO(kota_): need more efficient way for emuration of return value
# from SDaemon
value_generator = iter([
# first, we get metadata json
json.dumps({'metadata': 'return'}),
# then we get object data
'something', '',
])
def mock_read(fd, size):
try:
value = next(value_generator)
except StopIteration:
raise Exception('called more then expected')
# NOTE(takashi): Make sure that we return bytes in PY3
return value.encode('utf-8')
def mock_close(fd):
pass
called_fd_and_bodies = []
invocation_protocol = \
'storlets.gateway.gateways.docker.runtime.' \
'StorletInvocationProtocol._write_input_data'
def mock_writer(self, fd, app_iter):
body = ''
for chunk in app_iter:
body += chunk
called_fd_and_bodies.append((fd, body))
# prepare nested mock patch
# SBus -> mock SBus.send() for container communication
# os.read -> mock reading the file descriptor from container
# select.slect -> mock fd communication which can be readable
@mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient')
@mock.patch('storlets.gateway.gateways.docker.runtime.os.read',
mock_read)
@mock.patch('storlets.gateway.gateways.docker.runtime.os.close',
mock_close)
@mock.patch('storlets.gateway.gateways.docker.runtime.select.select',
lambda r, w, x, timeout=None: (r, w, x))
@mock.patch('storlets.gateway.common.stob.os.read', mock_read)
@mock.patch(invocation_protocol, mock_writer)
def test_invocation_flow(client):
client.ping.return_value = SBusResponse(True, 'OK')
client.stop_daemon.return_value = SBusResponse(True, 'OK')
client.start_daemon.return_value = SBusResponse(True, 'OK')
client.execute.return_value = SBusResponse(True, 'OK', 'someid')
sresp = self.gateway.invocation_flow(st_req, extra_sources)
eventlet.sleep(0.1)
file_like = FileLikeIter(sresp.data_iter)
self.assertEqual(b'something', file_like.read())
# I hate the decorator to return an instance but to track current
# implementation, we have to make a mock class for this. Need to fix.
class MockFileManager(object):
def get_storlet(self, req):
return BytesIO(b'mock'), None
def get_dependency(self, req):
return BytesIO(b'mock'), None
st_req.file_manager = MockFileManager()
test_invocation_flow()
# ensure st_req.app_iter is drawn
self.assertRaises(StopIteration, next, st_req.data_iter)
expected_mock_writer_calls = len(extra_sources) + 1
self.assertEqual(expected_mock_writer_calls,
len(called_fd_and_bodies))
self.assertEqual('body', called_fd_and_bodies[0][1])
return called_fd_and_bodies
def test_init(self):
# Java
storlet_id = 'Storlet-1.0.jar'
params = {'Param1': 'Value1', 'Param2': 'Value2'}
metadata = {'MetaKey1': 'MetaValue1', 'MetaKey2': 'MetaValue2'}
# with dependencies
options = {'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')}
dsreq = DockerStorletRequest(storlet_id, params, metadata,
iter(StringIO()), options=options)
self.assertEqual(metadata, dsreq.user_metadata)
self.assertEqual(params, dsreq.params)
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
self.assertEqual(['dep1', 'dep2'], dsreq.dependencies)
self.assertEqual('java', dsreq.storlet_language)
self.assertIsNone(dsreq.storlet_language_version)
# without dependencies
options = {'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')}
dsreq = DockerStorletRequest(storlet_id, params, metadata,
iter(StringIO()), options=options)
self.assertEqual(metadata, dsreq.user_metadata)
self.assertEqual(params, dsreq.params)
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
self.assertEqual([], dsreq.dependencies)
self.assertEqual('java', dsreq.storlet_language)
self.assertIsNone(dsreq.storlet_language_version)
# storlet_language is not given
options = {'storlet_main': 'org.openstack.storlet.Storlet',
'file_manager': FakeFileManager('storlet', 'dep')}
with self.assertRaises(ValueError):
DockerStorletRequest(storlet_id, params, metadata,
iter(StringIO()), options=options)
# storlet_main is not given
options = {'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')}
with self.assertRaises(ValueError):
DockerStorletRequest(storlet_id, params, metadata,
iter(StringIO()), options=options)
# file_manager is not given
options = {'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_language': 'java'}
with self.assertRaises(ValueError):
DockerStorletRequest(storlet_id, params, metadata,
iter(StringIO()), options=options)
# Python
storlet_id = 'storlet.py'
params = {'Param1': 'Value1', 'Param2': 'Value2'}
metadata = {'MetaKey1': 'MetaValue1', 'MetaKey2': 'MetaValue2'}
# without language version
options = {'storlet_main': 'storlet.Storlet',
'storlet_language': 'python',
'file_manager': FakeFileManager('storlet', 'dep')}
dsreq = DockerStorletRequest(storlet_id, params, metadata,
iter(StringIO()), options=options)
self.assertEqual(metadata, dsreq.user_metadata)
self.assertEqual(params, dsreq.params)
self.assertEqual('storlet.py', dsreq.storlet_id)
self.assertEqual('storlet.Storlet', dsreq.storlet_main)
self.assertEqual([], dsreq.dependencies)
self.assertEqual('python', dsreq.storlet_language)
self.assertIsNone(dsreq.storlet_language_version)
# with language version
options = {'storlet_main': 'storlet.Storlet',
'storlet_language': 'python',
'storlet_language_version': '2.7',
'file_manager': FakeFileManager('storlet', 'dep')}
dsreq = DockerStorletRequest(storlet_id, params, metadata,
iter(StringIO()), options=options)
self.assertEqual(metadata, dsreq.user_metadata)
self.assertEqual(params, dsreq.params)
self.assertEqual('storlet.py', dsreq.storlet_id)
self.assertEqual('storlet.Storlet', dsreq.storlet_main)
self.assertEqual([], dsreq.dependencies)
self.assertEqual('python', dsreq.storlet_language)
self.assertEqual('2.7', dsreq.storlet_language_version)
def _test_docker_gateway_communicate(self, extra_sources=None):
extra_sources = extra_sources or []
options = {
'generate_log': False,
'scope': 'AUTH_account',
'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')
}
st_req = DockerStorletRequest(storlet_id=self.sobj,
params={},
user_metadata={},
data_iter=iter('body'),
options=options)
# TODO(kota_): need more efficient way for emuration of return value
# from SDaemon
value_generator = iter([
# Firt is for confirmation for SDaemon running
'True: daemon running confirmation',
# Second is stop SDaemon in activation
'True: stop daemon',
# Third is start SDaemon again in activation
'True: start daemon',
# Forth is return value for invoking as task_id
'This is task id',
# Fifth is for getting meta
json.dumps({'metadata': 'return'}),
# At last return body and EOF
'something',
'',
])
def mock_read(fd, size):
try:
value = next(value_generator)
except StopIteration:
raise Exception('called more then expected')
return value
def mock_close(fd):
pass
called_fd_and_bodies = []
invocation_protocol = \
'storlets.gateway.gateways.docker.runtime.' \
'StorletInvocationProtocol._write_input_data'
def mock_writer(self, fd, app_iter):
body = ''
for chunk in app_iter:
body += chunk
called_fd_and_bodies.append((fd, body))
# prepare nested mock patch
# SBus -> mock SBus.send() for container communication
# os.read -> mock reading the file descriptor from container
# select.slect -> mock fd communication which can be readable
@mock.patch('storlets.gateway.gateways.docker.runtime.SBus', MockSBus)
@mock.patch('storlets.gateway.gateways.docker.runtime.os.read',
mock_read)
@mock.patch('storlets.gateway.gateways.docker.runtime.os.close',
mock_close)
@mock.patch('storlets.gateway.gateways.docker.runtime.select.select',
lambda r, w, x, timeout=None: (r, w, x))
@mock.patch('storlets.gateway.common.stob.os.read', mock_read)
@mock.patch(invocation_protocol, mock_writer)
def test_invocation_flow():
sresp = self.gateway.invocation_flow(st_req, extra_sources)
eventlet.sleep(0.1)
file_like = FileLikeIter(sresp.data_iter)
self.assertEqual('something', file_like.read())
# I hate the decorator to return an instance but to track current
# implementation, we have to make a mock class for this. Need to fix.
class MockFileManager(object):
def get_storlet(self, req):
return StringIO('mock'), None
def get_dependency(self, req):
return StringIO('mock'), None
st_req.file_manager = MockFileManager()
test_invocation_flow()
# ensure st_req.app_iter is drawn
self.assertRaises(StopIteration, next, st_req.data_iter)
expected_mock_writer_calls = len(extra_sources) + 1
self.assertEqual(expected_mock_writer_calls, len(called_fd_and_bodies))
self.assertEqual('body', called_fd_and_bodies[0][1])
return called_fd_and_bodies
def test_init(self):
storlet_id = 'Storlet-1.0.jar'
params = {'Param1': 'Value1', 'Param2': 'Value2'}
metadata = {'MetaKey1': 'MetaValue1', 'MetaKey2': 'MetaValue2'}
options = {
'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')
}
dsreq = DockerStorletRequest(storlet_id,
params,
metadata,
iter(StringIO()),
options=options)
self.assertEqual(metadata, dsreq.user_metadata)
self.assertEqual(params, dsreq.params)
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
self.assertEqual(['dep1', 'dep2'], dsreq.dependencies)
self.assertEqual('java', dsreq.storlet_language)
options = {
'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')
}
dsreq = DockerStorletRequest(storlet_id,
params,
metadata,
iter(StringIO()),
options=options)
self.assertEqual(metadata, dsreq.user_metadata)
self.assertEqual(params, dsreq.params)
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
self.assertEqual([], dsreq.dependencies)
self.assertEqual('java', dsreq.storlet_language)
options = {
'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2'
}
with self.assertRaises(ValueError):
DockerStorletRequest(storlet_id,
params,
metadata,
iter(StringIO()),
options=options)
options = {
'storlet_dependency': 'dep1,dep2',
'file_manager': FakeFileManager('storlet', 'dep')
}
with self.assertRaises(ValueError):
DockerStorletRequest(storlet_id,
params,
metadata,
iter(StringIO()),
options=options)