def test_validate_dependency_registration(self): # w/o dependency parameter obj = 'dep_file' params = {'Dependency-Version': '1.0'} StorletGatewayDocker.validate_dependency_registration(params, obj) # w/ correct dependency parameter params = { 'Dependency-Permissions': '755', 'Dependency-Version': '1.0'} StorletGatewayDocker.validate_dependency_registration(params, obj) # w/ wrong dependency parameter params = { 'Dependency-Permissions': '400', 'Dependency-Version': '1.0'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_dependency_registration(params, obj) # w/ invalid dependency parameter params = { 'Dependency-Permissions': 'foo', 'Dependency-Version': '1.0'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_dependency_registration(params, obj) params = { 'Dependency-Permissions': '888', 'Dependency-Version': '1.0'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_dependency_registration(params, obj)
def test_check_mandatory_params(self): params = {'keyA': 'valueA', 'keyB': 'valueB', 'keyC': 'valueC'} # all mandatory headers are included StorletGatewayDocker._check_mandatory_params(params, ['keyA', 'keyB']) # some of mandatory headers are missing with self.assertRaises(ValueError): StorletGatewayDocker._check_mandatory_params( params, ['keyA', 'KeyD'])
def test_check_mandatory_params(self): params = {'keyA': 'valueA', 'keyB': 'valueB', 'keyC': 'valueC'} # all mandatory headers are included StorletGatewayDocker._check_mandatory_params( params, ['keyA', 'keyB']) # some of mandatory headers are missing with self.assertRaises(ValueError): StorletGatewayDocker._check_mandatory_params( params, ['keyA', 'KeyD'])
def test_validate_storlet_registration_java(self): # correct name and headers w/ dependency obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} StorletGatewayDocker.validate_storlet_registration(params, obj) # correct name and headers w/o dependency obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} StorletGatewayDocker.validate_storlet_registration(params, obj) # some header keys are missing params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong name obj = 'storlet.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj)
def setUp(self): if six.PY3: self.skipTest( "swift.common.internal_client.InternalClient for testing" "doesn't suppoert py3 yet") # TODO(takashi): take these values from config file self.tempdir = mkdtemp() self.sconf = { 'host_root': self.tempdir, 'swift_dir': self.tempdir, 'storlet_timeout': '9', 'storlet_container': 'storlet', 'storlet_dependency': 'dependency', 'reseller_prefix': 'AUTH' } self.logger = FakeLogger() self.storlet_container = self.sconf['storlet_container'] self.storlet_dependency = self.sconf['storlet_dependency'] self.version = 'v1' self.account = 'AUTH_account' self.container = 'container' self.obj = 'object' self.sobj = 'storlet-1.0.jar' # TODO(kota_): should be 'storlet-internal-client.conf' actually ic_conf_path = os.path.join(self.tempdir, 'storlet-proxy-server.conf') with open(ic_conf_path, 'w') as f: f.write(""" [DEFAULT] [pipeline:main] pipeline = catch_errors proxy-logging cache proxy-server [app:proxy-server] use = egg:swift#proxy [filter:cache] use = egg:swift#memcache [filter:proxy-logging] use = egg:swift#proxy_logging [filter:catch_errors] use = egg:swift#catch_errors """) self.gateway = StorletGatewayDocker(self.sconf, self.logger, self.account)
def test_validate_storlet_registration_python(self): # correct name and headers w/ dependency obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.Storlet'} StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong name obj = 'storlet.pyfoo' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong main class obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'another_storlet.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.foo.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj)
def test_validate_storlet_registration_not_suppoeted(self): # unsupported language obj = 'storlet.foo' params = {'Language': 'bar', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # same name for storlet and dependency obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'storlet-1.0.jar', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # duplicated name in dependencies obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file,dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj)
def setUp(self): if six.PY3: self.skipTest( "swift.common.internal_client.InternalClient for testing" "doesn't suppoert py3 yet") # TODO(takashi): take these values from config file self.tempdir = mkdtemp() self.sconf = { 'host_root': self.tempdir, 'swift_dir': self.tempdir, 'storlet_timeout': '9', 'storlet_container': 'storlet', 'storlet_dependency': 'dependency', 'reseller_prefix': 'AUTH' } self.logger = FakeLogger() self.storlet_container = self.sconf['storlet_container'] self.storlet_dependency = self.sconf['storlet_dependency'] self.version = 'v1' self.account = 'AUTH_account' self.container = 'container' self.obj = 'object' self.sobj = 'storlet-1.0.jar' # TODO(kota_): shoudl be 'storlet-internal-client.conf' actually ic_conf_path = os.path.join(self.tempdir, 'storlet-proxy-server.conf') with open(ic_conf_path, 'w') as f: f.write(""" [DEFAULT] [pipeline:main] pipeline = catch_errors proxy-logging cache proxy-server [app:proxy-server] use = egg:swift#proxy [filter:cache] use = egg:swift#memcache [filter:proxy-logging] use = egg:swift#proxy_logging [filter:catch_errors] use = egg:swift#catch_errors """) self.gateway = StorletGatewayDocker( self.sconf, self.logger, self.account)
def test_validate_storlet_registration_python(self): # correct name and headers w/ dependency obj = 'storlet.py' params = {'Language': 'python', 'Language-Version': '2.7', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.Storlet'} StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong version obj = 'storlet.py' params = {'Language': 'python', 'Language-Version': '1.7', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong name obj = 'storlet.pyfoo' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong main class obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'another_storlet.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.foo.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj)
class TestStorletDockerGateway(unittest.TestCase): def setUp(self): # TODO(takashi): take these values from config file self.tempdir = mkdtemp() self.sconf = { 'host_root': self.tempdir, 'swift_dir': self.tempdir, 'storlet_timeout': '9', 'storlet_container': 'storlet', 'storlet_dependency': 'dependency', 'reseller_prefix': 'AUTH' } self.logger = FakeLogger() self.storlet_container = self.sconf['storlet_container'] self.storlet_dependency = self.sconf['storlet_dependency'] self.version = 'v1' self.account = 'AUTH_account' self.container = 'container' self.obj = 'object' self.sobj = 'storlet-1.0.jar' # TODO(kota_): should be 'storlet-internal-client.conf' actually ic_conf_path = os.path.join(self.tempdir, 'storlet-proxy-server.conf') with open(ic_conf_path, 'wb') as f: f.write(b""" [DEFAULT] [pipeline:main] pipeline = catch_errors proxy-logging cache proxy-server [app:proxy-server] use = egg:swift#proxy [filter:cache] use = egg:swift#memcache [filter:proxy-logging] use = egg:swift#proxy_logging [filter:catch_errors] use = egg:swift#catch_errors """) self.gateway = StorletGatewayDocker( self.sconf, self.logger, self.account) def tearDown(self): rmtree(self.tempdir) @property def req_path(self): return self._create_proxy_path( self.version, self.account, self.container, self.obj) @property def storlet_path(self): return self._create_proxy_path( self.version, self.account, self.storlet_container, self.sobj) def _create_proxy_path(self, version, account, container, obj): return '/'.join(['', version, account, container, obj]) def test_check_mandatory_params(self): params = {'keyA': 'valueA', 'keyB': 'valueB', 'keyC': 'valueC'} # all mandatory headers are included StorletGatewayDocker._check_mandatory_params( params, ['keyA', 'keyB']) # some of mandatory headers are missing with self.assertRaises(ValueError): StorletGatewayDocker._check_mandatory_params( params, ['keyA', 'KeyD']) def test_validate_storlet_registration_java(self): # correct name and headers w/ dependency obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} StorletGatewayDocker.validate_storlet_registration(params, obj) # correct name and headers w/o dependency obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} StorletGatewayDocker.validate_storlet_registration(params, obj) # some header keys are missing params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong name obj = 'storlet.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) def test_validate_storlet_registration_python(self): # correct name and headers w/ dependency obj = 'storlet.py' params = {'Language': 'python', 'Language-Version': '2.7', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.Storlet'} StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong version obj = 'storlet.py' params = {'Language': 'python', 'Language-Version': '1.7', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong name obj = 'storlet.pyfoo' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong main class obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'another_storlet.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.foo.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) def test_validate_storlet_registration_not_suppoeted(self): # unsupported language obj = 'storlet.foo' params = {'Language': 'bar', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # same name for storlet and dependency obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'storlet-1.0.jar', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # duplicated name in dependencies obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file,dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) def test_validate_dependency_registration(self): # w/o dependency parameter obj = 'dep_file' params = {'Dependency-Version': '1.0'} StorletGatewayDocker.validate_dependency_registration(params, obj) # w/ correct dependency parameter params = { 'Dependency-Permissions': '755', 'Dependency-Version': '1.0'} StorletGatewayDocker.validate_dependency_registration(params, obj) # w/ wrong dependency parameter params = { 'Dependency-Permissions': '400', 'Dependency-Version': '1.0'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_dependency_registration(params, obj) # w/ invalid dependency parameter params = { 'Dependency-Permissions': 'foo', 'Dependency-Version': '1.0'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_dependency_registration(params, obj) params = { 'Dependency-Permissions': '888', 'Dependency-Version': '1.0'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_dependency_registration(params, obj) def _test_docker_gateway_communicate(self, extra_sources=None): extra_sources = extra_sources or [] options = {'generate_log': False, 'scope': 'AUTH_account', 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} st_req = DockerStorletRequest( storlet_id=self.sobj, params={}, user_metadata={}, data_iter=iter('body'), options=options) # TODO(kota_): need more efficient way for emuration of return value # from SDaemon value_generator = iter([ # first, we get metadata json json.dumps({'metadata': 'return'}), # then we get object data 'something', '', ]) def mock_read(fd, size): try: value = next(value_generator) except StopIteration: raise Exception('called more then expected') # NOTE(takashi): Make sure that we return bytes in PY3 return value.encode('utf-8') def mock_close(fd): pass called_fd_and_bodies = [] invocation_protocol = \ 'storlets.gateway.gateways.docker.runtime.' \ 'StorletInvocationProtocol._write_input_data' def mock_writer(self, fd, app_iter): body = '' for chunk in app_iter: body += chunk called_fd_and_bodies.append((fd, body)) # prepare nested mock patch # SBus -> mock SBus.send() for container communication # os.read -> mock reading the file descriptor from container # select.slect -> mock fd communication which can be readable @mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient') @mock.patch('storlets.gateway.gateways.docker.runtime.os.read', mock_read) @mock.patch('storlets.gateway.gateways.docker.runtime.os.close', mock_close) @mock.patch('storlets.gateway.gateways.docker.runtime.select.select', lambda r, w, x, timeout=None: (r, w, x)) @mock.patch('storlets.gateway.common.stob.os.read', mock_read) @mock.patch(invocation_protocol, mock_writer) def test_invocation_flow(client): client.ping.return_value = SBusResponse(True, 'OK') client.stop_daemon.return_value = SBusResponse(True, 'OK') client.start_daemon.return_value = SBusResponse(True, 'OK') client.execute.return_value = SBusResponse(True, 'OK', 'someid') sresp = self.gateway.invocation_flow(st_req, extra_sources) eventlet.sleep(0.1) file_like = FileLikeIter(sresp.data_iter) self.assertEqual(b'something', file_like.read()) # I hate the decorator to return an instance but to track current # implementation, we have to make a mock class for this. Need to fix. class MockFileManager(object): def get_storlet(self, req): return BytesIO(b'mock'), None def get_dependency(self, req): return BytesIO(b'mock'), None st_req.file_manager = MockFileManager() test_invocation_flow() # ensure st_req.app_iter is drawn self.assertRaises(StopIteration, next, st_req.data_iter) expected_mock_writer_calls = len(extra_sources) + 1 self.assertEqual(expected_mock_writer_calls, len(called_fd_and_bodies)) self.assertEqual('body', called_fd_and_bodies[0][1]) return called_fd_and_bodies def test_docker_gateway_communicate(self): self._test_docker_gateway_communicate() def test_docker_gateway_communicate_with_extra_sources(self): options = {'generate_log': False, 'scope': 'AUTH_account', 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} data_sources = [] def generate_extra_st_request(): # This works similarly with build_storlet_request # TODO(kota_): think of more generarl way w/o # build_storlet_request sw_req = Request.blank( self.req_path, environ={'REQUEST_METHOD': 'GET'}, headers={'X-Run-Storlet': self.sobj}) sw_resp = Response( app_iter=iter(['This is a response body']), status=200) st_req = DockerStorletRequest( storlet_id=sw_req.headers['X-Run-Storlet'], params=sw_req.params, user_metadata={}, data_iter=sw_resp.app_iter, options=options) data_sources.append(sw_resp.app_iter) return st_req extra_request = generate_extra_st_request() mock_calls = self._test_docker_gateway_communicate( extra_sources=[extra_request]) self.assertEqual('This is a response body', mock_calls[1][1]) # run all existing eventlet threads for app_iter in data_sources: # ensure all app_iters are drawn self.assertRaises(StopIteration, next, app_iter)
class TestStorletDockerGateway(unittest.TestCase): def setUp(self): if six.PY3: self.skipTest( "swift.common.internal_client.InternalClient for testing" "doesn't suppoert py3 yet") # TODO(takashi): take these values from config file self.tempdir = mkdtemp() self.sconf = { 'host_root': self.tempdir, 'swift_dir': self.tempdir, 'storlet_timeout': '9', 'storlet_container': 'storlet', 'storlet_dependency': 'dependency', 'reseller_prefix': 'AUTH' } self.logger = FakeLogger() self.storlet_container = self.sconf['storlet_container'] self.storlet_dependency = self.sconf['storlet_dependency'] self.version = 'v1' self.account = 'AUTH_account' self.container = 'container' self.obj = 'object' self.sobj = 'storlet-1.0.jar' # TODO(kota_): shoudl be 'storlet-internal-client.conf' actually ic_conf_path = os.path.join(self.tempdir, 'storlet-proxy-server.conf') with open(ic_conf_path, 'w') as f: f.write(""" [DEFAULT] [pipeline:main] pipeline = catch_errors proxy-logging cache proxy-server [app:proxy-server] use = egg:swift#proxy [filter:cache] use = egg:swift#memcache [filter:proxy-logging] use = egg:swift#proxy_logging [filter:catch_errors] use = egg:swift#catch_errors """) self.gateway = StorletGatewayDocker( self.sconf, self.logger, self.account) def tearDown(self): rmtree(self.tempdir) @property def req_path(self): return self._create_proxy_path( self.version, self.account, self.container, self.obj) @property def storlet_path(self): return self._create_proxy_path( self.version, self.account, self.storlet_container, self.sobj) def _create_proxy_path(self, version, account, container, obj): return '/'.join(['', version, account, container, obj]) def test_check_mandatory_params(self): params = {'keyA': 'valueA', 'keyB': 'valueB', 'keyC': 'valueC'} # all mandatory headers are included StorletGatewayDocker._check_mandatory_params( params, ['keyA', 'keyB']) # some of mandatory headers are missing with self.assertRaises(ValueError): StorletGatewayDocker._check_mandatory_params( params, ['keyA', 'KeyD']) def test_validate_storlet_registration_java(self): # correct name and headers w/ dependency obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} StorletGatewayDocker.validate_storlet_registration(params, obj) # correct name and headers w/o dependency obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} StorletGatewayDocker.validate_storlet_registration(params, obj) # some header keys are missing params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong name obj = 'storlet.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) def test_validate_storlet_registration_python(self): # correct name and headers w/ dependency obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.Storlet'} StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong name obj = 'storlet.pyfoo' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # wrong main class obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'another_storlet.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) obj = 'storlet.py' params = {'Language': 'python', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'storlet.foo.Storlet'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) def test_validate_storlet_registration_not_suppoeted(self): # unsupported language obj = 'storlet.foo' params = {'Language': 'bar', 'Interface-Version': '1.0', 'Dependency': 'dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # same name for storlet and dependency obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'storlet-1.0.jar', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) # duplicated name in dependencies obj = 'storlet-1.0.jar' params = {'Language': 'java', 'Interface-Version': '1.0', 'Dependency': 'dep_file,dep_file', 'Object-Metadata': 'no', 'Main': 'path.to.storlet.class'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_storlet_registration(params, obj) def test_validate_dependency_registration(self): # w/o dependency parameter obj = 'dep_file' params = {'Dependency-Version': '1.0'} StorletGatewayDocker.validate_dependency_registration(params, obj) # w/ correct dependency parameter params = { 'Dependency-Permissions': '755', 'Dependency-Version': '1.0'} StorletGatewayDocker.validate_dependency_registration(params, obj) # w/ wrong dependency parameter params = { 'Dependency-Permissions': '400', 'Dependency-Version': '1.0'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_dependency_registration(params, obj) # w/ invalid dependency parameter params = { 'Dependency-Permissions': 'foo', 'Dependency-Version': '1.0'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_dependency_registration(params, obj) params = { 'Dependency-Permissions': '888', 'Dependency-Version': '1.0'} with self.assertRaises(ValueError): StorletGatewayDocker.validate_dependency_registration(params, obj) def _test_docker_gateway_communicate(self, extra_sources=None): extra_sources = extra_sources or [] options = {'generate_log': False, 'scope': 'AUTH_account', 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} st_req = DockerStorletRequest( storlet_id=self.sobj, params={}, user_metadata={}, data_iter=iter('body'), options=options) # TODO(kota_): need more efficient way for emuration of return value # from SDaemon value_generator = iter([ # Firt is for confirmation for SDaemon running 'True: daemon running confirmation', # Second is stop SDaemon in activation 'True: stop daemon', # Third is start SDaemon again in activation 'True: start daemon', # Forth is return value for invoking as task_id 'This is task id', # Fifth is for getting meta json.dumps({'metadata': 'return'}), # At last return body and EOF 'something', '', ]) def mock_read(fd, size): try: value = next(value_generator) except StopIteration: raise Exception('called more then expected') return value def mock_close(fd): pass called_fd_and_bodies = [] invocation_protocol = \ 'storlets.gateway.gateways.docker.runtime.' \ 'StorletInvocationProtocol._write_input_data' def mock_writer(self, fd, app_iter): body = '' for chunk in app_iter: body += chunk called_fd_and_bodies.append((fd, body)) # prepare nested mock patch # SBus -> mock SBus.send() for container communication # os.read -> mock reading the file descriptor from container # select.slect -> mock fd communication wich can be readable @mock.patch('storlets.gateway.gateways.docker.runtime.SBus', MockSBus) @mock.patch('storlets.gateway.gateways.docker.runtime.os.read', mock_read) @mock.patch('storlets.gateway.gateways.docker.runtime.os.close', mock_close) @mock.patch('storlets.gateway.gateways.docker.runtime.select.select', lambda r, w, x, timeout=None: (r, w, x)) @mock.patch('storlets.gateway.common.stob.os.read', mock_read) @mock.patch(invocation_protocol, mock_writer) def test_invocation_flow(): sresp = self.gateway.invocation_flow(st_req, extra_sources) eventlet.sleep(0.1) file_like = FileLikeIter(sresp.data_iter) self.assertEqual('something', file_like.read()) # I hate the decorator to return an instance but to track current # implementation, we have to make a mock class for this. Need to fix. class MockFileManager(object): def get_storlet(self, req): return StringIO('mock'), None def get_dependency(self, req): return StringIO('mock'), None st_req.file_manager = MockFileManager() test_invocation_flow() # ensure st_req.app_iter is drawn self.assertRaises(StopIteration, next, st_req.data_iter) expected_mock_writer_calls = len(extra_sources) + 1 self.assertEqual(expected_mock_writer_calls, len(called_fd_and_bodies)) self.assertEqual(called_fd_and_bodies[0][1], 'body') return called_fd_and_bodies def test_docker_gateway_communicate(self): self._test_docker_gateway_communicate() def test_docker_gateway_communicate_with_extra_sources(self): options = {'generate_log': False, 'scope': 'AUTH_account', 'storlet_main': 'org.openstack.storlet.Storlet', 'storlet_dependency': 'dep1,dep2', 'storlet_language': 'java', 'file_manager': FakeFileManager('storlet', 'dep')} data_sources = [] def generate_extra_st_request(): # This works similarly with build_storlet_request # TODO(kota_): think of more generarl way w/o # build_storlet_request sw_req = Request.blank( self.req_path, environ={'REQUEST_METHOD': 'GET'}, headers={'X-Run-Storlet': self.sobj}) sw_resp = Response( app_iter=iter(['This is a response body']), status=200) st_req = DockerStorletRequest( storlet_id=sw_req.headers['X-Run-Storlet'], params=sw_req.params, user_metadata={}, data_iter=sw_resp.app_iter, options=options) data_sources.append(sw_resp.app_iter) return st_req extra_request = generate_extra_st_request() mock_calls = self._test_docker_gateway_communicate( extra_sources=[extra_request]) self.assertEqual(mock_calls[1][1], 'This is a response body') # run all existing eventlet threads for app_iter in data_sources: # ensure all app_iters are drawn self.assertRaises(StopIteration, next, app_iter)