def test_invocation_protocol_remote_fds(self): # In default, we have 5 fds in remote_fds storlet_request = DockerStorletRequest(self.storlet_id, {}, {}, iter(StringIO()), options=self.options) protocol = StorletInvocationProtocol(storlet_request, self.pipe_path, self.log_file, 1, self.logger) self.assertEqual(5, len(protocol.remote_fds)) # extra_resources expands the remote_fds storlet_request = DockerStorletRequest(self.storlet_id, {}, {}, iter(StringIO()), options=self.options) protocol = StorletInvocationProtocol(storlet_request, self.pipe_path, self.log_file, 1, self.logger, extra_sources=[storlet_request]) self.assertEqual(6, len(protocol.remote_fds)) # 2 more extra_resources expands the remote_fds storlet_request = DockerStorletRequest(self.storlet_id, {}, {}, iter(StringIO()), options=self.options) protocol = StorletInvocationProtocol(storlet_request, self.pipe_path, self.log_file, 1, self.logger, extra_sources=[storlet_request] * 3) self.assertEqual(8, len(protocol.remote_fds))
def test_has_range(self): storlet_id = 'Storlet-1.0.jar' params = {} metadata = {} options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} dsreq = DockerStorletRequest(storlet_id, params, metadata, None, 0, options=options) self.assertFalse(dsreq.has_range) options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep'), 'range_start': 1, 'range_end': 6} dsreq = DockerStorletRequest(storlet_id, params, metadata, None, 0, options=options) self.assertTrue(dsreq.has_range) options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep'), 'range_start': 0, 'range_end': 6} dsreq = DockerStorletRequest(storlet_id, params, metadata, None, 0, options=options) self.assertTrue(dsreq.has_range)
def test_init_with_range(self): storlet_id = 'Storlet-1.0.jar' params = {} metadata = {} options = { 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep'), 'range_start': 1, 'range_end': 6 } dsreq = DockerStorletRequest(storlet_id, params, metadata, None, 0, options=options) self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id) self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main) self.assertEqual(['dep1', 'dep2'], dsreq.dependencies) self.assertEqual('java', dsreq.storlet_language) self.assertEqual(1, dsreq.start) self.assertEqual(6, dsreq.end) options = { 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep'), 'range_start': 0, 'range_end': 0 } dsreq = DockerStorletRequest(storlet_id, params, metadata, None, 0, options=options) self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id) self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main) self.assertEqual(['dep1', 'dep2'], dsreq.dependencies) self.assertEqual('java', dsreq.storlet_language) self.assertEqual(0, dsreq.start) self.assertEqual(0, dsreq.end)
def setUp(self): self.pipe_path = tempfile.mktemp() self.log_file = tempfile.mktemp() self.logger = FakeLogger() self.storlet_id = 'Storlet-1.0.jar' self.options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} storlet_request = DockerStorletRequest( self.storlet_id, {}, {}, iter(StringIO()), options=self.options) self.protocol = StorletInvocationProtocol( storlet_request, self.pipe_path, self.log_file, 1, self.logger)
def generate_extra_st_request(): # This works similarly with build_storlet_request # TODO(kota_): think of more generarl way w/o # build_storlet_request sw_req = Request.blank( self.req_path, environ={'REQUEST_METHOD': 'GET'}, headers={'X-Run-Storlet': self.sobj}) sw_resp = Response( app_iter=iter(['This is a response body']), status=200) st_req = DockerStorletRequest( storlet_id=sw_req.headers['X-Run-Storlet'], params=sw_req.params, user_metadata={}, data_iter=sw_resp.app_iter, options=options) data_sources.append(sw_resp.app_iter) return st_req
def _test_docker_gateway_communicate(self, extra_sources=None): extra_sources = extra_sources or [] options = {'generate_log': False, 'scope': 'AUTH_account', 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} st_req = DockerStorletRequest( storlet_id=self.sobj, params={}, user_metadata={}, data_iter=iter('body'), options=options) # TODO(kota_): need more efficient way for emuration of return value # from SDaemon value_generator = iter([ # first, we get metadata json json.dumps({'metadata': 'return'}), # then we get object data 'something', '', ]) def mock_read(fd, size): try: value = next(value_generator) except StopIteration: raise Exception('called more then expected') # NOTE(takashi): Make sure that we return bytes in PY3 return value.encode('utf-8') def mock_close(fd): pass called_fd_and_bodies = [] invocation_protocol = \ 'storlets.gateway.gateways.docker.runtime.' \ 'StorletInvocationProtocol._write_input_data' def mock_writer(self, fd, app_iter): body = '' for chunk in app_iter: body += chunk called_fd_and_bodies.append((fd, body)) # prepare nested mock patch # SBus -> mock SBus.send() for container communication # os.read -> mock reading the file descriptor from container # select.slect -> mock fd communication which can be readable @mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient') @mock.patch('storlets.gateway.gateways.docker.runtime.os.read', mock_read) @mock.patch('storlets.gateway.gateways.docker.runtime.os.close', mock_close) @mock.patch('storlets.gateway.gateways.docker.runtime.select.select', lambda r, w, x, timeout=None: (r, w, x)) @mock.patch('storlets.gateway.common.stob.os.read', mock_read) @mock.patch(invocation_protocol, mock_writer) def test_invocation_flow(client): client.ping.return_value = SBusResponse(True, 'OK') client.stop_daemon.return_value = SBusResponse(True, 'OK') client.start_daemon.return_value = SBusResponse(True, 'OK') client.execute.return_value = SBusResponse(True, 'OK', 'someid') sresp = self.gateway.invocation_flow(st_req, extra_sources) eventlet.sleep(0.1) file_like = FileLikeIter(sresp.data_iter) self.assertEqual(b'something', file_like.read()) # I hate the decorator to return an instance but to track current # implementation, we have to make a mock class for this. Need to fix. class MockFileManager(object): def get_storlet(self, req): return BytesIO(b'mock'), None def get_dependency(self, req): return BytesIO(b'mock'), None st_req.file_manager = MockFileManager() test_invocation_flow() # ensure st_req.app_iter is drawn self.assertRaises(StopIteration, next, st_req.data_iter) expected_mock_writer_calls = len(extra_sources) + 1 self.assertEqual(expected_mock_writer_calls, len(called_fd_and_bodies)) self.assertEqual('body', called_fd_and_bodies[0][1]) return called_fd_and_bodies
def test_init(self): # Java storlet_id = 'Storlet-1.0.jar' params = {'Param1': 'Value1', 'Param2': 'Value2'} metadata = {'MetaKey1': 'MetaValue1', 'MetaKey2': 'MetaValue2'} # with dependencies options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} dsreq = DockerStorletRequest(storlet_id, params, metadata, iter(StringIO()), options=options) self.assertEqual(metadata, dsreq.user_metadata) self.assertEqual(params, dsreq.params) self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id) self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main) self.assertEqual(['dep1', 'dep2'], dsreq.dependencies) self.assertEqual('java', dsreq.storlet_language) self.assertIsNone(dsreq.storlet_language_version) # without dependencies options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} dsreq = DockerStorletRequest(storlet_id, params, metadata, iter(StringIO()), options=options) self.assertEqual(metadata, dsreq.user_metadata) self.assertEqual(params, dsreq.params) self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id) self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main) self.assertEqual([], dsreq.dependencies) self.assertEqual('java', dsreq.storlet_language) self.assertIsNone(dsreq.storlet_language_version) # storlet_language is not given options = {'storlet_main': 'org.openstack.storlet.Storlet', 'file_manager': FakeFileManager('storlet', 'dep')} with self.assertRaises(ValueError): DockerStorletRequest(storlet_id, params, metadata, iter(StringIO()), options=options) # storlet_main is not given options = {'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} with self.assertRaises(ValueError): DockerStorletRequest(storlet_id, params, metadata, iter(StringIO()), options=options) # file_manager is not given options = {'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_language': 'java'} with self.assertRaises(ValueError): DockerStorletRequest(storlet_id, params, metadata, iter(StringIO()), options=options) # Python storlet_id = 'storlet.py' params = {'Param1': 'Value1', 'Param2': 'Value2'} metadata = {'MetaKey1': 'MetaValue1', 'MetaKey2': 'MetaValue2'} # without language version options = {'storlet_main': 'storlet.Storlet', 'storlet_language': 'python', 'file_manager': FakeFileManager('storlet', 'dep')} dsreq = DockerStorletRequest(storlet_id, params, metadata, iter(StringIO()), options=options) self.assertEqual(metadata, dsreq.user_metadata) self.assertEqual(params, dsreq.params) self.assertEqual('storlet.py', dsreq.storlet_id) self.assertEqual('storlet.Storlet', dsreq.storlet_main) self.assertEqual([], dsreq.dependencies) self.assertEqual('python', dsreq.storlet_language) self.assertIsNone(dsreq.storlet_language_version) # with language version options = {'storlet_main': 'storlet.Storlet', 'storlet_language': 'python', 'storlet_language_version': '2.7', 'file_manager': FakeFileManager('storlet', 'dep')} dsreq = DockerStorletRequest(storlet_id, params, metadata, iter(StringIO()), options=options) self.assertEqual(metadata, dsreq.user_metadata) self.assertEqual(params, dsreq.params) self.assertEqual('storlet.py', dsreq.storlet_id) self.assertEqual('storlet.Storlet', dsreq.storlet_main) self.assertEqual([], dsreq.dependencies) self.assertEqual('python', dsreq.storlet_language) self.assertEqual('2.7', dsreq.storlet_language_version)
def _test_docker_gateway_communicate(self, extra_sources=None): extra_sources = extra_sources or [] options = { 'generate_log': False, 'scope': 'AUTH_account', 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep') } st_req = DockerStorletRequest(storlet_id=self.sobj, params={}, user_metadata={}, data_iter=iter('body'), options=options) # TODO(kota_): need more efficient way for emuration of return value # from SDaemon value_generator = iter([ # Firt is for confirmation for SDaemon running 'True: daemon running confirmation', # Second is stop SDaemon in activation 'True: stop daemon', # Third is start SDaemon again in activation 'True: start daemon', # Forth is return value for invoking as task_id 'This is task id', # Fifth is for getting meta json.dumps({'metadata': 'return'}), # At last return body and EOF 'something', '', ]) def mock_read(fd, size): try: value = next(value_generator) except StopIteration: raise Exception('called more then expected') return value def mock_close(fd): pass called_fd_and_bodies = [] invocation_protocol = \ 'storlets.gateway.gateways.docker.runtime.' \ 'StorletInvocationProtocol._write_input_data' def mock_writer(self, fd, app_iter): body = '' for chunk in app_iter: body += chunk called_fd_and_bodies.append((fd, body)) # prepare nested mock patch # SBus -> mock SBus.send() for container communication # os.read -> mock reading the file descriptor from container # select.slect -> mock fd communication which can be readable @mock.patch('storlets.gateway.gateways.docker.runtime.SBus', MockSBus) @mock.patch('storlets.gateway.gateways.docker.runtime.os.read', mock_read) @mock.patch('storlets.gateway.gateways.docker.runtime.os.close', mock_close) @mock.patch('storlets.gateway.gateways.docker.runtime.select.select', lambda r, w, x, timeout=None: (r, w, x)) @mock.patch('storlets.gateway.common.stob.os.read', mock_read) @mock.patch(invocation_protocol, mock_writer) def test_invocation_flow(): sresp = self.gateway.invocation_flow(st_req, extra_sources) eventlet.sleep(0.1) file_like = FileLikeIter(sresp.data_iter) self.assertEqual('something', file_like.read()) # I hate the decorator to return an instance but to track current # implementation, we have to make a mock class for this. Need to fix. class MockFileManager(object): def get_storlet(self, req): return StringIO('mock'), None def get_dependency(self, req): return StringIO('mock'), None st_req.file_manager = MockFileManager() test_invocation_flow() # ensure st_req.app_iter is drawn self.assertRaises(StopIteration, next, st_req.data_iter) expected_mock_writer_calls = len(extra_sources) + 1 self.assertEqual(expected_mock_writer_calls, len(called_fd_and_bodies)) self.assertEqual('body', called_fd_and_bodies[0][1]) return called_fd_and_bodies
def test_init(self): storlet_id = 'Storlet-1.0.jar' params = {'Param1': 'Value1', 'Param2': 'Value2'} metadata = {'MetaKey1': 'MetaValue1', 'MetaKey2': 'MetaValue2'} options = { 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep') } dsreq = DockerStorletRequest(storlet_id, params, metadata, iter(StringIO()), options=options) self.assertEqual(metadata, dsreq.user_metadata) self.assertEqual(params, dsreq.params) self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id) self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main) self.assertEqual(['dep1', 'dep2'], dsreq.dependencies) self.assertEqual('java', dsreq.storlet_language) options = { 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep') } dsreq = DockerStorletRequest(storlet_id, params, metadata, iter(StringIO()), options=options) self.assertEqual(metadata, dsreq.user_metadata) self.assertEqual(params, dsreq.params) self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id) self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main) self.assertEqual([], dsreq.dependencies) self.assertEqual('java', dsreq.storlet_language) options = { 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2' } with self.assertRaises(ValueError): DockerStorletRequest(storlet_id, params, metadata, iter(StringIO()), options=options) options = { 'storlet_dependency': 'dep1,dep2', 'file_manager': FakeFileManager('storlet', 'dep') } with self.assertRaises(ValueError): DockerStorletRequest(storlet_id, params, metadata, iter(StringIO()), options=options)
def _test_docker_gateway_communicate(self, extra_sources=None): extra_sources = extra_sources or [] options = {'generate_log': False, 'scope': 'AUTH_account', 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} st_req = DockerStorletRequest( storlet_id=self.sobj, params={}, user_metadata={}, data_iter=iter('body'), options=options) # TODO(kota_): need more efficient way for emuration of return value # from SDaemon value_generator = iter([ # Firt is for confirmation for SDaemon running 'True: daemon running confirmation', # Second is stop SDaemon in activation 'True: stop daemon', # Third is start SDaemon again in activation 'True: start daemon', # Forth is return value for invoking as task_id 'This is task id', # Fifth is for getting meta json.dumps({'metadata': 'return'}), # At last return body and EOF 'something', '', ]) def mock_read(fd, size): try: value = next(value_generator) except StopIteration: raise Exception('called more then expected') return value def mock_close(fd): pass called_fd_and_bodies = [] invocation_protocol = \ 'storlets.gateway.gateways.docker.runtime.' \ 'StorletInvocationProtocol._write_input_data' def mock_writer(self, fd, app_iter): body = '' for chunk in app_iter: body += chunk called_fd_and_bodies.append((fd, body)) # prepare nested mock patch # SBus -> mock SBus.send() for container communication # os.read -> mock reading the file descriptor from container # select.slect -> mock fd communication wich can be readable @mock.patch('storlets.gateway.gateways.docker.runtime.SBus', MockSBus) @mock.patch('storlets.gateway.gateways.docker.runtime.os.read', mock_read) @mock.patch('storlets.gateway.gateways.docker.runtime.os.close', mock_close) @mock.patch('storlets.gateway.gateways.docker.runtime.select.select', lambda r, w, x, timeout=None: (r, w, x)) @mock.patch('storlets.gateway.common.stob.os.read', mock_read) @mock.patch(invocation_protocol, mock_writer) def test_invocation_flow(): sresp = self.gateway.invocation_flow(st_req, extra_sources) eventlet.sleep(0.1) file_like = FileLikeIter(sresp.data_iter) self.assertEqual('something', file_like.read()) # I hate the decorator to return an instance but to track current # implementation, we have to make a mock class for this. Need to fix. class MockFileManager(object): def get_storlet(self, req): return StringIO('mock'), None def get_dependency(self, req): return StringIO('mock'), None st_req.file_manager = MockFileManager() test_invocation_flow() # ensure st_req.app_iter is drawn self.assertRaises(StopIteration, next, st_req.data_iter) expected_mock_writer_calls = len(extra_sources) + 1 self.assertEqual(expected_mock_writer_calls, len(called_fd_and_bodies)) self.assertEqual(called_fd_and_bodies[0][1], 'body') return called_fd_and_bodies