def test_load_gateway_entry_point(self): # existing entry point self.assertEqual( StorletGatewayStub, load_gateway('stub')) self.assertEqual( StorletGatewayDocker, load_gateway('docker')) # If the given entry point does not exist with self.assertRaises(StorletGatewayLoadError): load_gateway('foo')
def filter_factory(global_conf, **local_conf): """ Standard filter factory to use the middleware with paste.deploy """ conf = global_conf.copy() conf.update(local_conf) conf['redis_host'] = conf.get('redis_host', 'controller') conf['redis_port'] = int(conf.get('redis_port', 6379)) conf['redis_db'] = int(conf.get('redis_db', 0)) conf['native_filters_path'] = conf.get('native_filters_path', '/opt/crystal/native_filters') # Add source directory to sys path native_filters_path = conf.get('native_filters_path') sys.path.insert(0, native_filters_path) """ Storlets Configuration """ if STORLETS: """ Load Storlets Gateway class """ from storlets.gateway.loader import load_gateway module_name = conf.get('storlet_gateway_module', 'stub') gateway_class = load_gateway(module_name) conf['storlets_gateway_module'] = gateway_class """ Load Storlets Gateway configuration """ configParser = ConfigParser.RawConfigParser() configParser.read( conf.get('storlet_gateway_conf', '/etc/swift/storlet_stub_gateway.conf')) additional_items = configParser.items("DEFAULT") for key, val in additional_items: conf[key] = val """ Register Lua script to retrieve policies in a single redis call """ r = redis.StrictRedis(conf['redis_host'], conf['redis_port'], conf['redis_db']) lua = """ local t = {} if redis.call('EXISTS', 'pipeline:'..ARGV[1]..':'..ARGV[2])==1 then t = redis.call('HGETALL', 'pipeline:'..ARGV[1]..':'..ARGV[2]) elseif redis.call('EXISTS', 'pipeline:'..ARGV[1])==1 then t = redis.call('HGETALL', 'pipeline:'..ARGV[1]) end t[#t+1] = '@@@@' local t3 = redis.call('HGETALL', 'pipeline:global') for i=1,#t3 do t[#t+1] = t3[i] end return t""" lua_sha = r.script_load(lua) conf['LUA_get_pipeline_sha'] = lua_sha def crystal_filter_handler(app): return CrystalHandlerMiddleware(app, conf) return crystal_filter_handler
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) module_name = conf.get('storlet_gateway_module', 'stub') gateway_class = load_gateway(module_name) conf['gateway_module'] = gateway_class configParser = ConfigParser.RawConfigParser() configParser.read( conf.get('storlet_gateway_conf', '/etc/swift/storlet_stub_gateway.conf')) gateway_conf = dict(configParser.items("DEFAULT")) # TODO(eranr): Add supported storlets languages and # supported storlet API version containers = get_container_names(conf) swift_info = { 'storlet_container': containers['storlet'], 'storlet_dependency': containers['dependency'], 'storlet_gateway_class': gateway_class.__name__ } register_swift_info('storlet_handler', False, **swift_info) def storlet_handler_filter(app): return StorletHandlerMiddleware(app, conf, gateway_conf) return storlet_handler_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) module_name = conf.get('storlet_gateway_module', 'stub') gateway_class = load_gateway(module_name) conf['gateway_module'] = gateway_class configParser = ConfigParser.RawConfigParser() configParser.read(conf.get('storlet_gateway_conf', '/etc/swift/storlet_stub_gateway.conf')) gateway_conf = dict(configParser.items("DEFAULT")) # TODO(eranr): Add supported storlets languages and # supported storlet API version containers = get_container_names(conf) swift_info = {'storlet_container': containers['storlet'], 'storlet_dependency': containers['dependency'], 'storlet_gateway_class': gateway_class.__name__} register_swift_info('storlet_handler', False, **swift_info) def storlet_handler_filter(app): return StorletHandlerMiddleware(app, conf, gateway_conf) return storlet_handler_filter
def test_load_gateway_full_class_path(self): # If the given class path exists self.assertEqual( StorletGatewayStub, load_gateway('storlets.gateway.gateways.stub.StorletGatewayStub')) self.assertEqual( StorletGatewayDocker, load_gateway('storlets.gateway.gateways.docker.' 'StorletGatewayDocker')) # If module does not exist with self.assertRaises(StorletGatewayLoadError): load_gateway('storlets.gateway.gateways.another_stub.' 'StorletGatewayStub') # If class does not exist with self.assertRaises(StorletGatewayLoadError): load_gateway('storlets.gateway.gateways.stub.' 'StorletGatewayAnotherStub')
def filter_factory(global_conf, **local_conf): """Standard filter factory to use the middleware with paste.deploy""" register_swift_info('vertigo') conf = global_conf.copy() conf.update(local_conf) vertigo_conf = dict() vertigo_conf['devices'] = conf.get('devices', '/srv/node') vertigo_conf['execution_server'] = conf.get('execution_server') vertigo_conf['mc_timeout'] = conf.get('mc_timeout', 5) vertigo_conf['mc_pipe'] = conf.get('mc_pipe', 'vertigo_pipe') # vertigo_conf['api_pipe'] = conf.get('mc_pipe', 'api_pipe') vertigo_conf['metadata_visibility'] = conf.get('metadata_visibility', True) vertigo_conf['mc_dir'] = conf.get('mc_dir', '/home/docker_device/vertigo/scopes') vertigo_conf['cache_dir'] = conf.get('cache_dir', '/home/docker_device/cache/scopes') vertigo_conf['mc_container'] = conf.get('mc_container', 'microcontroller') vertigo_conf['mc_dependency'] = conf.get('mc_dependency', 'dependency') ''' Load storlet parameters ''' configParser = RawConfigParser() configParser.read(conf.get('__file__')) storlet_parameters = configParser.items('filter:storlet_handler') for key, val in storlet_parameters: vertigo_conf[key] = val """ Load Storlets Gateway configuration """ configParser = RawConfigParser() configParser.read(vertigo_conf['storlet_gateway_conf']) additional_items = configParser.items("DEFAULT") for key, val in additional_items: vertigo_conf[key] = val """ Load Storlets Gateway class """ module_name = vertigo_conf.get('storlet_gateway_module', 'stub') gateway_class = load_gateway(module_name) vertigo_conf['storlets_gateway_module'] = gateway_class """ Register Lua script to retrieve policies in a single redis call """ vertigo_conf['redis_host'] = conf.get('redis_host', 'controller') vertigo_conf['redis_port'] = int(conf.get('redis_port', 6379)) vertigo_conf['redis_db'] = int(conf.get('redis_db', 0)) if vertigo_conf['execution_server'] == 'proxy': r = redis.StrictRedis(vertigo_conf['redis_host'], vertigo_conf['redis_port'], vertigo_conf['redis_db']) lua = """ local t = {} if redis.call('EXISTS', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2]..':'..ARGV[3])==1 then t = redis.call('HGETALL', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2]..':'..ARGV[3]) elseif redis.call('EXISTS', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2])==1 then t = redis.call('HGETALL', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2]) end return t""" lua_sha = r.script_load(lua) vertigo_conf['LUA_get_mc_sha'] = lua_sha def swift_vertigo(app): return VertigoHandlerMiddleware(app, global_conf, vertigo_conf) return swift_vertigo