def test_load_gateway_entry_point(self):
# existing entry point
self.assertEqual(
StorletGatewayStub,
load_gateway('stub'))
self.assertEqual(
StorletGatewayDocker,
load_gateway('docker'))
# If the given entry point does not exist
with self.assertRaises(StorletGatewayLoadError):
load_gateway('foo')
def filter_factory(global_conf, **local_conf):
"""
Standard filter factory to use the middleware with paste.deploy
"""
conf = global_conf.copy()
conf.update(local_conf)
conf['redis_host'] = conf.get('redis_host', 'controller')
conf['redis_port'] = int(conf.get('redis_port', 6379))
conf['redis_db'] = int(conf.get('redis_db', 0))
conf['native_filters_path'] = conf.get('native_filters_path',
'/opt/crystal/native_filters')
# Add source directory to sys path
native_filters_path = conf.get('native_filters_path')
sys.path.insert(0, native_filters_path)
"""
Storlets Configuration
"""
if STORLETS:
""" Load Storlets Gateway class """
from storlets.gateway.loader import load_gateway
module_name = conf.get('storlet_gateway_module', 'stub')
gateway_class = load_gateway(module_name)
conf['storlets_gateway_module'] = gateway_class
""" Load Storlets Gateway configuration """
configParser = ConfigParser.RawConfigParser()
configParser.read(
conf.get('storlet_gateway_conf',
'/etc/swift/storlet_stub_gateway.conf'))
additional_items = configParser.items("DEFAULT")
for key, val in additional_items:
conf[key] = val
"""
Register Lua script to retrieve policies in a single redis call
"""
r = redis.StrictRedis(conf['redis_host'], conf['redis_port'],
conf['redis_db'])
lua = """
local t = {}
if redis.call('EXISTS', 'pipeline:'..ARGV[1]..':'..ARGV[2])==1 then
t = redis.call('HGETALL', 'pipeline:'..ARGV[1]..':'..ARGV[2])
elseif redis.call('EXISTS', 'pipeline:'..ARGV[1])==1 then
t = redis.call('HGETALL', 'pipeline:'..ARGV[1])
end
t[#t+1] = '@@@@'
local t3 = redis.call('HGETALL', 'pipeline:global')
for i=1,#t3 do
t[#t+1] = t3[i]
end
return t"""
lua_sha = r.script_load(lua)
conf['LUA_get_pipeline_sha'] = lua_sha
def crystal_filter_handler(app):
return CrystalHandlerMiddleware(app, conf)
return crystal_filter_handler
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
module_name = conf.get('storlet_gateway_module', 'stub')
gateway_class = load_gateway(module_name)
conf['gateway_module'] = gateway_class
configParser = ConfigParser.RawConfigParser()
configParser.read(
conf.get('storlet_gateway_conf',
'/etc/swift/storlet_stub_gateway.conf'))
gateway_conf = dict(configParser.items("DEFAULT"))
# TODO(eranr): Add supported storlets languages and
# supported storlet API version
containers = get_container_names(conf)
swift_info = {
'storlet_container': containers['storlet'],
'storlet_dependency': containers['dependency'],
'storlet_gateway_class': gateway_class.__name__
}
register_swift_info('storlet_handler', False, **swift_info)
def storlet_handler_filter(app):
return StorletHandlerMiddleware(app, conf, gateway_conf)
return storlet_handler_filter
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
module_name = conf.get('storlet_gateway_module', 'stub')
gateway_class = load_gateway(module_name)
conf['gateway_module'] = gateway_class
configParser = ConfigParser.RawConfigParser()
configParser.read(conf.get('storlet_gateway_conf',
'/etc/swift/storlet_stub_gateway.conf'))
gateway_conf = dict(configParser.items("DEFAULT"))
# TODO(eranr): Add supported storlets languages and
# supported storlet API version
containers = get_container_names(conf)
swift_info = {'storlet_container': containers['storlet'],
'storlet_dependency': containers['dependency'],
'storlet_gateway_class': gateway_class.__name__}
register_swift_info('storlet_handler', False, **swift_info)
def storlet_handler_filter(app):
return StorletHandlerMiddleware(app, conf, gateway_conf)
return storlet_handler_filter
def test_load_gateway_full_class_path(self):
# If the given class path exists
self.assertEqual(
StorletGatewayStub,
load_gateway('storlets.gateway.gateways.stub.StorletGatewayStub'))
self.assertEqual(
StorletGatewayDocker,
load_gateway('storlets.gateway.gateways.docker.'
'StorletGatewayDocker'))
# If module does not exist
with self.assertRaises(StorletGatewayLoadError):
load_gateway('storlets.gateway.gateways.another_stub.'
'StorletGatewayStub')
# If class does not exist
with self.assertRaises(StorletGatewayLoadError):
load_gateway('storlets.gateway.gateways.stub.'
'StorletGatewayAnotherStub')
def filter_factory(global_conf, **local_conf):
"""Standard filter factory to use the middleware with paste.deploy"""
register_swift_info('vertigo')
conf = global_conf.copy()
conf.update(local_conf)
vertigo_conf = dict()
vertigo_conf['devices'] = conf.get('devices', '/srv/node')
vertigo_conf['execution_server'] = conf.get('execution_server')
vertigo_conf['mc_timeout'] = conf.get('mc_timeout', 5)
vertigo_conf['mc_pipe'] = conf.get('mc_pipe', 'vertigo_pipe')
# vertigo_conf['api_pipe'] = conf.get('mc_pipe', 'api_pipe')
vertigo_conf['metadata_visibility'] = conf.get('metadata_visibility', True)
vertigo_conf['mc_dir'] = conf.get('mc_dir', '/home/docker_device/vertigo/scopes')
vertigo_conf['cache_dir'] = conf.get('cache_dir', '/home/docker_device/cache/scopes')
vertigo_conf['mc_container'] = conf.get('mc_container', 'microcontroller')
vertigo_conf['mc_dependency'] = conf.get('mc_dependency', 'dependency')
''' Load storlet parameters '''
configParser = RawConfigParser()
configParser.read(conf.get('__file__'))
storlet_parameters = configParser.items('filter:storlet_handler')
for key, val in storlet_parameters:
vertigo_conf[key] = val
""" Load Storlets Gateway configuration """
configParser = RawConfigParser()
configParser.read(vertigo_conf['storlet_gateway_conf'])
additional_items = configParser.items("DEFAULT")
for key, val in additional_items:
vertigo_conf[key] = val
""" Load Storlets Gateway class """
module_name = vertigo_conf.get('storlet_gateway_module', 'stub')
gateway_class = load_gateway(module_name)
vertigo_conf['storlets_gateway_module'] = gateway_class
"""
Register Lua script to retrieve policies in a single redis call
"""
vertigo_conf['redis_host'] = conf.get('redis_host', 'controller')
vertigo_conf['redis_port'] = int(conf.get('redis_port', 6379))
vertigo_conf['redis_db'] = int(conf.get('redis_db', 0))
if vertigo_conf['execution_server'] == 'proxy':
r = redis.StrictRedis(vertigo_conf['redis_host'],
vertigo_conf['redis_port'],
vertigo_conf['redis_db'])
lua = """
local t = {}
if redis.call('EXISTS', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2]..':'..ARGV[3])==1 then
t = redis.call('HGETALL', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2]..':'..ARGV[3])
elseif redis.call('EXISTS', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2])==1 then
t = redis.call('HGETALL', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2])
end
return t"""
lua_sha = r.script_load(lua)
vertigo_conf['LUA_get_mc_sha'] = lua_sha
def swift_vertigo(app):
return VertigoHandlerMiddleware(app, global_conf, vertigo_conf)
return swift_vertigo
