예제 #1
0
    def test_load_gateway_entry_point(self):
        # existing entry point
        self.assertEqual(
            StorletGatewayStub,
            load_gateway('stub'))

        self.assertEqual(
            StorletGatewayDocker,
            load_gateway('docker'))

        # If the given entry point does not exist
        with self.assertRaises(StorletGatewayLoadError):
            load_gateway('foo')
def filter_factory(global_conf, **local_conf):
    """
    Standard filter factory to use the middleware with paste.deploy
    """
    conf = global_conf.copy()
    conf.update(local_conf)

    conf['redis_host'] = conf.get('redis_host', 'controller')
    conf['redis_port'] = int(conf.get('redis_port', 6379))
    conf['redis_db'] = int(conf.get('redis_db', 0))

    conf['native_filters_path'] = conf.get('native_filters_path',
                                           '/opt/crystal/native_filters')

    # Add source directory to sys path
    native_filters_path = conf.get('native_filters_path')
    sys.path.insert(0, native_filters_path)
    """
    Storlets Configuration
    """
    if STORLETS:
        """ Load Storlets Gateway class """
        from storlets.gateway.loader import load_gateway
        module_name = conf.get('storlet_gateway_module', 'stub')
        gateway_class = load_gateway(module_name)
        conf['storlets_gateway_module'] = gateway_class
        """ Load Storlets Gateway configuration """
        configParser = ConfigParser.RawConfigParser()
        configParser.read(
            conf.get('storlet_gateway_conf',
                     '/etc/swift/storlet_stub_gateway.conf'))
        additional_items = configParser.items("DEFAULT")

        for key, val in additional_items:
            conf[key] = val
    """
    Register Lua script to retrieve policies in a single redis call
    """
    r = redis.StrictRedis(conf['redis_host'], conf['redis_port'],
                          conf['redis_db'])
    lua = """
        local t = {}
        if redis.call('EXISTS', 'pipeline:'..ARGV[1]..':'..ARGV[2])==1 then
          t = redis.call('HGETALL', 'pipeline:'..ARGV[1]..':'..ARGV[2])
        elseif redis.call('EXISTS', 'pipeline:'..ARGV[1])==1 then
          t = redis.call('HGETALL', 'pipeline:'..ARGV[1])
        end
        t[#t+1] = '@@@@'
        local t3 = redis.call('HGETALL', 'pipeline:global')
        for i=1,#t3 do
          t[#t+1] = t3[i]
        end
        return t"""
    lua_sha = r.script_load(lua)
    conf['LUA_get_pipeline_sha'] = lua_sha

    def crystal_filter_handler(app):
        return CrystalHandlerMiddleware(app, conf)

    return crystal_filter_handler
예제 #3
0
def filter_factory(global_conf, **local_conf):
    conf = global_conf.copy()
    conf.update(local_conf)

    module_name = conf.get('storlet_gateway_module', 'stub')
    gateway_class = load_gateway(module_name)
    conf['gateway_module'] = gateway_class

    configParser = ConfigParser.RawConfigParser()
    configParser.read(
        conf.get('storlet_gateway_conf',
                 '/etc/swift/storlet_stub_gateway.conf'))
    gateway_conf = dict(configParser.items("DEFAULT"))

    # TODO(eranr): Add supported storlets languages and
    #  supported storlet API version
    containers = get_container_names(conf)
    swift_info = {
        'storlet_container': containers['storlet'],
        'storlet_dependency': containers['dependency'],
        'storlet_gateway_class': gateway_class.__name__
    }
    register_swift_info('storlet_handler', False, **swift_info)

    def storlet_handler_filter(app):
        return StorletHandlerMiddleware(app, conf, gateway_conf)

    return storlet_handler_filter
예제 #4
0
def filter_factory(global_conf, **local_conf):
    conf = global_conf.copy()
    conf.update(local_conf)

    module_name = conf.get('storlet_gateway_module', 'stub')
    gateway_class = load_gateway(module_name)
    conf['gateway_module'] = gateway_class

    configParser = ConfigParser.RawConfigParser()
    configParser.read(conf.get('storlet_gateway_conf',
                               '/etc/swift/storlet_stub_gateway.conf'))
    gateway_conf = dict(configParser.items("DEFAULT"))

    # TODO(eranr): Add supported storlets languages and
    #  supported storlet API version
    containers = get_container_names(conf)
    swift_info = {'storlet_container': containers['storlet'],
                  'storlet_dependency': containers['dependency'],
                  'storlet_gateway_class': gateway_class.__name__}
    register_swift_info('storlet_handler', False, **swift_info)

    def storlet_handler_filter(app):
        return StorletHandlerMiddleware(app, conf, gateway_conf)
    return storlet_handler_filter
예제 #5
0
    def test_load_gateway_full_class_path(self):
        # If the given class path exists
        self.assertEqual(
            StorletGatewayStub,
            load_gateway('storlets.gateway.gateways.stub.StorletGatewayStub'))

        self.assertEqual(
            StorletGatewayDocker,
            load_gateway('storlets.gateway.gateways.docker.'
                         'StorletGatewayDocker'))

        # If module does not exist
        with self.assertRaises(StorletGatewayLoadError):
            load_gateway('storlets.gateway.gateways.another_stub.'
                         'StorletGatewayStub')

        # If class does not exist
        with self.assertRaises(StorletGatewayLoadError):
            load_gateway('storlets.gateway.gateways.stub.'
                         'StorletGatewayAnotherStub')
예제 #6
0
def filter_factory(global_conf, **local_conf):
    """Standard filter factory to use the middleware with paste.deploy"""

    register_swift_info('vertigo')

    conf = global_conf.copy()
    conf.update(local_conf)

    vertigo_conf = dict()
    vertigo_conf['devices'] = conf.get('devices', '/srv/node')
    vertigo_conf['execution_server'] = conf.get('execution_server')
    vertigo_conf['mc_timeout'] = conf.get('mc_timeout', 5)
    vertigo_conf['mc_pipe'] = conf.get('mc_pipe', 'vertigo_pipe')
    # vertigo_conf['api_pipe'] = conf.get('mc_pipe', 'api_pipe')
    vertigo_conf['metadata_visibility'] = conf.get('metadata_visibility', True)
    vertigo_conf['mc_dir'] = conf.get('mc_dir', '/home/docker_device/vertigo/scopes')
    vertigo_conf['cache_dir'] = conf.get('cache_dir', '/home/docker_device/cache/scopes')
    vertigo_conf['mc_container'] = conf.get('mc_container', 'microcontroller')
    vertigo_conf['mc_dependency'] = conf.get('mc_dependency', 'dependency')

    ''' Load storlet parameters '''
    configParser = RawConfigParser()
    configParser.read(conf.get('__file__'))
    storlet_parameters = configParser.items('filter:storlet_handler')
    for key, val in storlet_parameters:
        vertigo_conf[key] = val

    """ Load Storlets Gateway configuration """
    configParser = RawConfigParser()
    configParser.read(vertigo_conf['storlet_gateway_conf'])
    additional_items = configParser.items("DEFAULT")
    for key, val in additional_items:
        vertigo_conf[key] = val

    """ Load Storlets Gateway class """
    module_name = vertigo_conf.get('storlet_gateway_module', 'stub')
    gateway_class = load_gateway(module_name)
    vertigo_conf['storlets_gateway_module'] = gateway_class

    """
    Register Lua script to retrieve policies in a single redis call
    """
    vertigo_conf['redis_host'] = conf.get('redis_host', 'controller')
    vertigo_conf['redis_port'] = int(conf.get('redis_port', 6379))
    vertigo_conf['redis_db'] = int(conf.get('redis_db', 0))

    if vertigo_conf['execution_server'] == 'proxy':
        r = redis.StrictRedis(vertigo_conf['redis_host'],
                              vertigo_conf['redis_port'],
                              vertigo_conf['redis_db'])
        lua = """
            local t = {}
            if redis.call('EXISTS', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2]..':'..ARGV[3])==1 then
              t = redis.call('HGETALL', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2]..':'..ARGV[3])
            elseif redis.call('EXISTS', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2])==1 then
              t = redis.call('HGETALL', 'mc_pipeline:'..ARGV[1]..':'..ARGV[2])
            end
            return t"""

        lua_sha = r.script_load(lua)
        vertigo_conf['LUA_get_mc_sha'] = lua_sha

    def swift_vertigo(app):
        return VertigoHandlerMiddleware(app, global_conf, vertigo_conf)

    return swift_vertigo