def memcached_server(docker, session_id, loop, request):
if not request.config.option.no_pull:
docker.pull('memcached:{}'.format('latest'))
container = docker.create_container(
image='memcached:{}'.format('latest'),
name='memcached-test-server-{}-{}'.format('latest', session_id),
ports=[11211],
detach=True,
)
docker.start(container=container['Id'])
inspection = docker.inspect_container(container['Id'])
host = inspection['NetworkSettings']['IPAddress']
delay = 0.001
for i in range(100):
try:
conn = aiomcache.Client(host, 11211, loop=loop)
loop.run_until_complete(conn.set(b'foo', b'bar'))
break
except ConnectionRefusedError as e:
time.sleep(delay)
delay *= 2
else:
pytest.fail("Cannot start memcached server")
container['memcached_params'] = dict(host=host, port=11211)
yield container
docker.kill(container=container['Id'])
docker.remove_container(container['Id'])
def init_gunicorn():
global mc
middlewares = []
middlewares.append(db_handler())
if settings.debug:
middlewares.append(session_middleware(SimpleCookieStorage()))
else:
middlewares.append(
session_middleware(EncryptedCookieStorage(settings.session_key)))
app = web.Application(middlewares=middlewares)
aiohttp_jinja2.setup(app, loader=jinja2.FunctionLoader(load_templ))
# Memcache init
app.mc = aiomcache.Client(settings.memcache['addr'],
settings.memcache['port'])
# Mongo init
db_connect(app)
union_routes(os.path.join(settings.tao_path, 'libs'))
union_routes(os.path.join(settings.root_path, 'apps'))
union_routes(os.path.join(settings.root_path), p=True)
for res in routes:
name = res[3]
if name is None: name = '{}:{}'.format(res[0], res[2])
app.router.add_route(res[0], res[1], res[2], name=name)
path = os.path.join(settings.root_path, 'static')
app.router.add_static('/static/', path, name='static')
return app
async def main():
mc = aiomcache.Client("127.0.0.1", 11211, loop=loop)
futures = []
for i in range(1,pow(10,5)):
futures.append(mc.set(str(i).encode('utf-8'), uuids[i % len(uuids)]))
for f in futures:
await f
async def setup_db(app, loop):
global client
await init_db()
client = aiomcache.Client(config.MEMCACHED_HOST, config.MEMCACHED_PORT, loop=loop) # noqa
session.init_app(app, interface=MemcacheSessionInterface(client))
app.async_session = aiohttp.ClientSession()
Path(config.UPLOAD_FOLDER).mkdir(parents=True, exist_ok=True)
def __init__(
self,
host=os.environ.get("MEMCACHED_HOST", "memcached"),
port=os.environ.get("MEMCACHED_PORT", 11211),
loop=None,
):
self.client = aiomcache.Client(host=host, port=port, loop=loop)
def __init__(self, loop, id=None):
self.mc = aiomcache.Client(settings.MC['HOST'],
settings.MC['PORT'],
loop=loop)
if not id:
id = str(uuid4())
self.id = id
def middleware(request):
if app.get('memcache') is None:
memcache = aiomcache.Client('127.0.0.1',
pool_size=64,
loop=app.loop)
app['memcache'] = memcache
return (yield from handler(request))
def __init__(
self,
config: Config,
*,
orm_base=None,
using_box: Box = None,
) -> None:
"""Initialize"""
logging.config.dictConfig(config.LOGGING)
logger = logging.getLogger(f'{__name__}.Bot.__init__')
logger.info('start')
Namespace._bot = self
self.process_pool_executor = ProcessPoolExecutor()
self.thread_pool_executor = ThreadPoolExecutor()
logger.info('connect to DB')
config.DATABASE_ENGINE = get_database_engine(config)
logger.info('connect to memcache')
self.mc = aiomcache.Client(
host=config.CACHE['HOST'],
port=config.CACHE['PORT'],
)
self.cache = Cache(self.mc, config.CACHE.get('PREFIX', 'YUI_'))
logger.info('import apps')
for app_name in config.APPS:
logger.debug('import apps: %s', app_name)
importlib.import_module(app_name)
self.config = config
self.orm_base = orm_base or Base
self.box = using_box or box
self.queue: asyncio.Queue = asyncio.Queue()
self.api = SlackAPI(self)
self.channels: List[PublicChannel] = []
self.ims: List[DirectMessageChannel] = []
self.groups: List[PrivateChannel] = []
self.users: List[User] = []
self.restart = False
self.is_ready = False
self.config.check(
self.box.config_required,
self.box.channel_required,
self.box.channels_required,
self.box.user_required,
self.box.users_required,
)
if self.config.REGISTER_CRONTAB:
logger.info('register crontab')
self.register_tasks()
def __getattr__(self, key):
_aiomcache_clients = AiomcacheProxy._aiomcache_clients
loop_key = id(asyncio.get_event_loop())
if loop_key not in _aiomcache_clients:
_aiomcache_clients[loop_key] = aiomcache.Client('127.0.0.1', 11211)
client = _aiomcache_clients[loop_key]
return getattr(client, key)
def __init__(self, endpoint=None, port=None, loop=None):
self.endpoint = endpoint or "127.0.0.1"
self.port = port or 11211
self._loop = loop or asyncio.get_event_loop()
self.client = aiomcache.Client(self.endpoint,
self.port,
loop=self._loop)
self.encoding = "utf-8"
async def connect_memcached_async(**rd_config) -> aiomcache.Client:
from privex.helpers.common import extract_settings
cf = extract_settings('MEMCACHED_',
settings,
merge_conf=dict(rd_config))
# if 'loop' not in cf:
# cf['loop'] = asyncio.get_event_loop()
return aiomcache.Client(**cf)
def hello_aiomcache():
mc = aiomcache.Client("127.0.0.1", 11211, loop=loop)
yield from mc.set(b"some_key", b"Some value")
value = yield from mc.get(b"some_key")
print(value)
values = yield from mc.multi_get(b"some_key", b"other_key")
print(values)
yield from mc.delete(b"another_key")
def create_memcached(addr: str,
log: logging.Logger) -> Optional[aiomcache.Client]:
"""Create the memcached client, if possible."""
if not addr:
return None
host, port = addr.split(":")
port = int(port)
log.info("memcached: %s", addr)
return aiomcache.Client(host, port)
def create(self, app: Quart) -> None:
if self.backend is None:
import aiomcache
loop = asyncio.get_running_loop()
#self.backend = aiomcache.Client("127.0.0.1", 11211, loop=loop)
self.backend = aiomcache.Client(
self._config.get('SESSION_MEMCACHED_HOST', '127.0.0.1'),
self._config.get('SESSION_MEMCACHED_PORT', 11211),
loop=loop)
def __init__(self, endpoint=None, port=None, loop=None, **kwargs):
super().__init__(**kwargs)
self.endpoint = get_cache_value_with_fallbacks(
endpoint, from_config="endpoint",
from_fallback=self.DEFAULT_ENDPOINT, cls=self.__class__)
self.port = get_cache_value_with_fallbacks(
port, from_config="port",
from_fallback=self.DEFAULT_PORT, cls=self.__class__)
self._loop = loop or asyncio.get_event_loop()
self.client = aiomcache.Client(self.endpoint, self.port, loop=self._loop)
self.encoding = "utf-8"
def __init__(self, token, name):
super(EvernoteBot, self).__init__(token, name)
config = settings.EVERNOTE['basic_access']
self.evernote = EvernoteClient(config['key'],
config['secret'],
config['oauth_callback'],
sandbox=settings.DEBUG)
self.evernote_api = AsyncEvernoteApi()
self.cache = aiomcache.Client("127.0.0.1", 11211)
for cmd_class in get_commands():
self.add_command(cmd_class)
def test_aiomcache():
import aiomcache
client = aiomcache.Client('127.0.0.1', 11211)
base = [0]
@ring.func.aiomcache(client, 'ring-test !@#')
@asyncio.coroutine
def cached_function(a, b):
return str(base[0] + a * 100 + b).encode()
yield from common_test(cached_function, base)
async def connect(cls):
log.info('Connecting to memcache')
cls.mc = aiomcache.Client(host=settings.CACHE_HOST,
port=int(settings.CACHE_PORT),
loop=asyncio.get_event_loop())
if settings.DEBUG:
log.info('Debug mode for cache: dummyfying all methods.')
cls.set = lambda x, y, z: None
cls.get = lambda x: None
cls.add = lambda x, y, z: None
cls.delete = lambda x: None
log.info('Connected to memcache')
def _initialize_storage(cls):
app_config = get_app_config()
if DEV_MODE:
from aiohttp_session.cookie_storage import EncryptedCookieStorage
import base64
secret_key = base64.urlsafe_b64decode(
str.encode(app_config['secret_key'], "utf-8"))
cls._session_storage = EncryptedCookieStorage(secret_key)
else:
import aiomcache
from aiohttp_session.memcached_storage import MemcachedStorage
mc = aiomcache.Client("memcached", 11211)
# we assume an HTTPS SSL secured site, hence 'secure=True'
cls._session_storage = MemcachedStorage(mc, secure=True)
def memcached_server(
docker: DockerClient,
session_id: str,
loop: asyncio.AbstractEventLoop,
) -> Generator[_TContainerInfo, None, None]:
image = 'memcached:{}'.format('latest')
if sys.platform.startswith('darwin'):
port = unused_port()
else:
port = None
container = docker.containers.run(
image=image,
detach=True,
name='memcached-test-server-{}-{}'.format('latest', session_id),
ports={
'11211/tcp': port,
},
environment={
'http.host': '0.0.0.0',
'transport.host': '127.0.0.1',
},
)
if sys.platform.startswith('darwin'):
host = '0.0.0.0'
else:
inspection = docker.api.inspect_container(container.id)
host = inspection['NetworkSettings']['IPAddress']
port = 11211
delay = 0.1
for i in range(20):
try:
conn = aiomcache.Client(host, port, loop=loop)
loop.run_until_complete(conn.set(b'foo', b'bar'))
break
except ConnectionRefusedError:
time.sleep(delay)
delay *= 2
else:
pytest.fail("Cannot start memcached server")
yield {'host': host, 'port': port, 'container': container}
container.kill(signal=9)
container.remove(force=True)
def __init__(self,
endpoint="127.0.0.1",
port=11211,
pool_size=2,
loop=None,
**kwargs):
super().__init__(**kwargs)
self.endpoint = endpoint
self.port = port
self.pool_size = int(pool_size)
self._loop = loop
self.client = aiomcache.Client(self.endpoint,
self.port,
loop=self._loop,
pool_size=self.pool_size)
async def get_memcache():
global _memcache
if _memcache is not None:
return _memcache
try:
memcache = memcache_var.get()
except LookupError:
# Hack for debug mode
memcache = None
if memcache is None:
loop = asyncio.get_event_loop()
memcache = aiomcache.Client(config.MEMCACHED_HOST,
config.MEMCACHED_PORT,
loop=loop)
_memcache = memcache
return memcache
def application(loop=None):
middlewares = [
minify_middleware,
error_middleware,
flash_middleware,
]
if getattr(config, 'test', None):
storage = SimpleCookieStorage()
else:
storage = EncryptedCookieStorage(config.session_key)
middlewares.append(session_middleware(storage))
app = web.Application(loop=loop,
middlewares=middlewares,
debug=config.debug)
app.mcache = aiomcache.Client(*config.memcache['server'], loop=loop)
if config.debug:
import aiohttp_debugtoolbar
aiohttp_debugtoolbar.setup(app, intercept_redirects=False)
app.config = config
engine.setup(
app,
context_processors=[context_processor, engine.request_processor],
loader=jinja2.FileSystemLoader('./template'))
app.router \
.add_resource('/', name='home') \
.add_route('GET', home)
app.router \
.add_resource('/contact', name='contact') \
.add_route('POST', contact)
app.router \
.add_resource('/callback', name='callback') \
.add_route('POST', callback)
app.router \
.add_resource('/ws', name='chat') \
.add_route('GET', ws)
app.router.add_route('GET', '/wsh', ws_handler)
return app
async def connect(self):
self._logger.debug("AsyncMcache: Connecting to {}".format(
self._params))
try:
self._pool = aiomcache.Client(pool_size=self._max_queries,
loop=self._loop,
**self._params)
except aiomcache.exceptions.ClientException as err:
raise ProviderError("Unable to connect to Memcache: {}".format(
str(err)))
except Exception as err:
raise ProviderError("Unknown Error: {}".format(str(err)))
return False
# is connected
if self._pool:
self._connected = True
self._initialized_on = time.time()
async def connection(self):
"""
__init async Memcache initialization
"""
self._logger.debug("AsyncMcache: Connecting to {}".format(
self._params))
try:
self._connection = aiomcache.Client(loop=self._loop,
**self._params)
except (aiomcache.exceptions.ValidationException) as err:
raise ProviderError("Invalid Connection Parameters: {}".format(
str(err)))
except (aiomcache.exceptions.ClientException) as err:
raise ProviderError("Connection Error: {}".format(str(err)))
except Exception as err:
raise ProviderError("Unknown Memcache Error: {}".format(str(err)))
return False
# is connected
if self._connection:
self._connected = True
self._initialized_on = time.time()
def __init__(self, config: Config = None) -> None:
if config is None:
config = Config(**DEFAULT, TOKEN='asdf', CHANNELS={}, USERS={})
Namespace._bot = self
self.loop = asyncio.get_event_loop()
self.call_queue: List[Call] = []
self.api = SlackAPI(self)
self.channels: List[PublicChannel] = []
self.ims: List[DirectMessageChannel] = []
self.groups: List[PrivateChannel] = []
self.mc = aiomcache.Client(
host=config.CACHE['HOST'],
port=config.CACHE['PORT'],
)
self.cache: CacheMock = CacheMock(self.mc, 'YUI_TEST_')
self.users: List[User] = [User(id='U0', team_id='T0', name='system')]
self.responses: Dict[str, Callable] = {}
self.config = config
self.process_pool_executor = ProcessPoolExecutor()
self.thread_pool_executor = ThreadPoolExecutor()
def create(self, app: Quart) -> None:
if self.backend is None:
import aiomcache
loop = asyncio.get_running_loop()
self.backend = aiomcache.Client("127.0.0.1", 11211, loop=loop)
async def dbsetup():
app.mc = aiomcache.Client("127.0.0.1", 11211, loop=app.loop)
async def create(cls, app):
host = getattr(app.cfg, 'MEMCACHE_HOST', cls.DEFAULT_MEMCACHE_HOST)
port = getattr(app.cfg, 'MEMCACHE_PORT', cls.DEFAULT_MEMCACHE_PORT)
memcache = aiomcache.Client(host, port)
return cls(app, memcache)
def memcached(loop, memcached_params):
conn = aiomcache.Client(loop=loop, **memcached_params)
yield conn
conn.close()