def test_cache_set_error():
store = RedisStore(port=9999, max_retry_count=2, retry_delay=2)
start_time = time.time()
with pytest.raises(redis.ConnectionError):
store.cache_set('key', 'value', 60)
excec_time = time.time() - start_time
assert excec_time >= 4
def setUp(self):
self.redis_storage = RedisStore(host=os.getenv('REDIS_HOST',
'localhost'),
port=os.getenv('REDIS_PORT', 6379))
self.store = Store(self.redis_storage)
self.store.connect()
self.key = 'sample_key'
self.value = 'sample_value'
def test_cache_get(self, value):
self.redis_base.flushall()
store = Store(RedisStore())
key, val = value
self.redis_base.set(key, val)
self.assertEqual(val, store.cache_get(key))
self.assertIsInstance(store.cache_get(key), basestring)
def st():
if store == 'redis':
rs = RedisStore(Config.get('store', 'redis_host'), Config.getint('store', 'redis_port'), db='11')
# override the configured expiration time
rs.ex = 1
return rs
else:
ms = MongoStore(Config.get('store', 'mongo_host'), Config.getint('store', 'mongo_port'), db='11')
# we want to recreate the index each time so we start off trying to drop it, but the first time we run
# after the db is instantiated the index won't exist.
try:
ms._db.drop_index('exp_1')
except Exception:
pass
ms._db.create_index('exp', expireAfterSeconds=1)
return ms
def main(opts):
settings, secur = configs_load(opts)
store = RedisStore()
con, curs = connect_bd(db=opts.bdata, passw=secur['security']['pswdbd'])
try:
create_tab(
con, curs, "base_comm", {
"id": "INT PRIMARY KEY",
"name": "VARCHAR(256)",
"classif": "VARCHAR(80)"
})
create_tab(
con, curs, "loc_comm", {
"id": "INT PRIMARY KEY",
'country': "VARCHAR(80)",
'city': "VARCHAR(80)"
})
create_tab(con, curs, "aud_comm", {
"id": "INT PRIMARY KEY",
'nmembers': "INT",
'age': "VARCHAR(4)"
})
data_db_base = fetch_data(con, curs, "base_comm")
except Exception as e:
log.error(e.args[0])
raise e
list_ids_db = [data_db_base[ind][0] for ind in range(len(data_db_base))]
queue = multiprocessing.Queue()
queue_store = multiprocessing.Queue()
taskq = CreateTaskQueue(queue,
int(opts.numpackco),
start_ids=int(opts.startid),
num_ids_chunk=int(opts.packreq))
taskq.start()
num_corr = 4
filds = settings['paramreq']["filds"]
tasks = [
TaskVkPars(queue,
secur['security']['token_vk'],
filds,
store,
queue_store,
num_cor_tasks=num_corr)
for _ in range(NUM_PROC_TASK_VK_PARS)
]
for t in tasks:
t.start()
taskclsf = multiprocessing.Process(target=classif_comm,
args=(settings['train'], queue_store,
store, con, curs, list_ids_db))
taskclsf.start()
taskq.join()
for _ in range(len(tasks) * num_corr):
queue.put(None)
for t in tasks:
t.join()
queue_store.put(None)
taskclsf.join()
log.info('Scripts finished working')
def test_cache_set_expire(self, value):
store = Store(RedisStore())
key = hashlib.md5("".join(value) + time.ctime()).hexdigest()
is_set = store.cache_set(key, value[1], 1)
self.assertTrue(is_set)
time.sleep(1.1)
self.assertIs(None, self.redis_base.get(key))
def test_cache_set(self, value):
self.redis_base.flushall()
store = Store(RedisStore())
key, val = value
is_set = store.cache_set(key, val, 10)
self.assertTrue(is_set)
self.assertEqual(val, self.redis_base.get(key))
self.assertIsInstance(self.redis_base.get(key), basestring)
def get_session(session_id):
"""
获取session
:param session_id:
:return:
"""
conn = RedisStore.connection()
session = json.loads(conn.get(session_id))
return session
def set_session(session):
"""
保存session，存储1小时
:param session:
:return:
"""
conn = RedisStore.connection()
conn.set(session.session_id, session.json())
conn.expire(session.session_id, 3600)
return session
def test_redis_timeout_error(self):
redis_store = RedisStore()
redis_store.get = MagicMock(side_effect=TimeoutError())
redis_store.set = MagicMock(side_effect=TimeoutError())
store = Store(redis_store)
self.assertEqual(store.cache_get("key"), None)
self.assertEqual(store.cache_set("key", "value"), None)
self.assertEqual(redis_store.get.call_count, Store.MAX_ATTEMPT)
self.assertEqual(redis_store.set.call_count, Store.MAX_ATTEMPT)
with self.assertRaises(TimeoutError):
redis_store.get('key')
with self.assertRaises(TimeoutError):
redis_store.set('key', 'value', 10)
class MainHTTPHandler(BaseHTTPRequestHandler):
router = {"method": method_handler}
store = Store(
RedisStore(host=os.getenv('REDIS_HOST', 'localhost'),
port=os.getenv('REDIS_PORT', 6379)))
def get_request_id(self, headers):
'''
Return random request_id that replicate the production
'''
return headers.get('HTTP_X_REQUEST_ID', uuid.uuid4().hex)
def do_POST(self):
response, code = {}, OK
context = {"request_id": self.get_request_id(self.headers)}
request = None
try:
data_string = self.rfile.read(int(self.headers['Content-Length']))
request = json.loads(data_string.decode())
except:
code = BAD_REQUEST
if request:
# extract path from the api
path = self.path.strip("/")
logging.info("%s: %s %s" %
(self.path, data_string, context["request_id"]))
if path in self.router:
try:
response, code = self.router[path]({
"body": request,
"headers": self.headers
}, context, self.store)
except Exception as e:
logging.exception("Unexpected error: %s" % e)
code = INTERNAL_ERROR
else:
code = NOT_FOUND
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.end_headers()
if code not in ERRORS:
r = {"response": response, "code": code}
else:
r = {
"error": response or ERRORS.get(code, "Unknown Error"),
"code": code
}
context.update(r)
logging.info(context)
json_res = json.dumps(r)
self.wfile.write(json_res.encode())
return
class TestStore(unittest.TestCase):
def setUp(self):
self.redis_storage = RedisStore(host=os.getenv('REDIS_HOST',
'localhost'),
port=os.getenv('REDIS_PORT', 6379))
self.store = Store(self.redis_storage)
self.store.connect()
self.key = 'sample_key'
self.value = 'sample_value'
def test_connect_to_store(self):
self.assertTrue(self.store.ping())
def test_connect_to_redis(self):
self.assertTrue(self.redis_storage.ping())
def test_redis_set_value(self):
key = self.key + '_redis'
self.assertTrue(self.redis_storage.set(key, self.value, 30))
self.assertEqual(self.store.cache_get(key), self.value)
def test_redis_connection_error(self):
self.redis_storage.client.disconnect()
self.assertRaises(ConnectionError, self.store.cache_get, self.key)
def test_cache_set_value(self):
key = self.key + '_cache'
self.assertTrue(self.store.cache_set(key, self.value, 30))
self.assertEqual(self.store.cache_get(key), self.value)
def test_cache_connection_error(self):
self.redis_storage.client.disconnect()
self.assertRaises(ConnectionError, self.store.cache_get, self.key)
self.assertRaises(ConnectionError, self.store.cache_set, self.key,
self.value)
def test_store_set_value(self):
key = self.key + '_store'
self.assertTrue(self.store.set(key, self.value, 30))
self.assertEqual(self.store.get(key), self.value)
class MainHTTPHandler(BaseHTTPRequestHandler):
router = {"method": method_handler}
store = Store(RedisStore(), 3, 2, (TimeoutError, ConnectionError))
def get_request_id(self, headers):
return headers.get('HTTP_X_REQUEST_ID', uuid.uuid4().hex)
def do_POST(self):
response, code = {}, OK
context = {"request_id": self.get_request_id(self.headers)}
request = None
try:
data_string = self.rfile.read(int(self.headers['Content-Length']))
request = json.loads(data_string)
except:
code = BAD_REQUEST
if request:
path = self.path.strip("/")
logging.info("%s: %s %s" %
(self.path, data_string, context["request_id"]))
if path in self.router:
try:
response, code = self.router[path]({
"body": request,
"headers": self.headers
}, context, self.store)
except Exception as e:
logging.exception("Unexpected error: %s" % e)
code = INTERNAL_ERROR
else:
code = NOT_FOUND
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.end_headers()
if code not in ERRORS:
r = {"response": response, "code": code}
else:
r = {
"error": response or ERRORS.get(code, "Unknown Error"),
"code": code
}
context.update(r)
logging.info(context)
self.wfile.write(json.dumps(r).encode())
return
class TestStore(unittest.TestCase):
def setUp(self):
self.store = Store(RedisStore())
self.redis_store = RedisStore()
self.redis_start()
def redis_start(self):
subprocess.Popen(shlex.split('redis-server'), shell=False, env=None, stdout=subprocess.PIPE)
time.sleep(0.1)
@cases([['key_one','value_one'],['key_two','value_two']])
def test_ok_store_get(self, value):
key, val = value
self.store.cache_set(key, val, 1)
self.assertEqual(val, self.store.get(key))
self.assertIsInstance(val, basestring)
@cases([['key_one','value_one'],['key_two','value_two']])
def test_ok_store_cache_get(self, value):
key, val = value
self.store.cache_set(key, val, 1)
self.assertEqual(val, self.store.cache_get(key))
self.assertIsInstance(val, basestring)
@cases(['key_one','key_two'])
def test_nonexistent_key_store_cache_get(self, key):
self.assertIsNone(self.store.cache_get(key))
@cases(['key_one','key_two'])
def test_nonexistent_key_store_get(self, key):
self.assertIsNone(self.store.get(key))
@cases([['key_one','value_one'],['key_two','value_two']])
def test_ok_redis_store_get(self, value):
key, val = value
self.redis_store.set(key, val, 1)
self.assertEqual(val, self.redis_store.get(key))
self.assertIsInstance(val, basestring)
@cases(['key_one','key_two'])
def test_nonexistent_key_redis_store_get(self, key):
self.redis_store.redis_base.flushall()
self.assertIsNone(self.redis_store.get(key))
def test_redis_connection_error(self):
redis_store = RedisStore()
redis_store.get = MagicMock(side_effect=ConnectionError())
redis_store.set = MagicMock(side_effect=ConnectionError())
store = Store(redis_store)
self.assertEqual(store.cache_get("key"), None)
self.assertEqual(store.cache_set("key", "value"), None)
self.assertEqual(redis_store.get.call_count, Store.MAX_ATTEMPT)
self.assertEqual(redis_store.set.call_count, Store.MAX_ATTEMPT)
with self.assertRaises(ConnectionError):
redis_store.get('key')
with self.assertRaises(ConnectionError):
redis_store.set('key', 'value', 10)
def test_redis_timeout_error(self):
redis_store = RedisStore()
redis_store.get = MagicMock(side_effect=TimeoutError())
redis_store.set = MagicMock(side_effect=TimeoutError())
store = Store(redis_store)
self.assertEqual(store.cache_get("key"), None)
self.assertEqual(store.cache_set("key", "value"), None)
self.assertEqual(redis_store.get.call_count, Store.MAX_ATTEMPT)
self.assertEqual(redis_store.set.call_count, Store.MAX_ATTEMPT)
with self.assertRaises(TimeoutError):
redis_store.get('key')
with self.assertRaises(TimeoutError):
redis_store.set('key', 'value', 10)
import asyncio
import json
import logging
import os
import aiohttp
import uvloop
from store import RedisStore
from app import App
from schemas import StreamKlineSchema
app = App()
redis = RedisStore()
@app.startup()
async def startup():
await redis.setup()
@app.shutdown()
async def shutdown():
await redis.close()
async def listen_ws(url, session):
async with session.ws_connect(url) as ws:
async for msg in ws:
if msg.data == 'close':
await ws.close()
def set_interests(self):
intersts = ['books, hi-tech', 'cinema, geek', 'travel, music', 'run']
redis_store = RedisStore()
for i, interest in enumerate(intersts, start=0):
redis_store.set('i:%s' % i, '["%s"]' % interest, 60)
def setUp(self):
self.context = {}
self.headers = {}
self.store = Store(RedisStore())
self.redis_start()
def setUp(self):
self.context = {}
self.headers = {}
self.store = Store(RedisStore(), 3, 2, (TimeoutError, ConnectionError))
def test_get_store_not_available(self, value):
store = Store(RedisStore(db=1))
key = hashlib.md5("".join(value) + time.ctime()).hexdigest()
self.assertIsNone(store.get(key))
def store():
store = RedisStore()
store.cache_set('key', 'value', 60 * 60)
return store
def test_cache_get_nonexistent_key(self, value):
store = Store(RedisStore())
key = hashlib.md5("".join(value) + time.ctime()).hexdigest()
self.assertIsNone(store.cache_get(key))
def connect(self, db=0):
self.store = [RedisStore(db), FileStore(ROOT_DIR)]
from store import RedisStore
if __name__ == '__main__':
n = RedisStore.connection().set('a', 's')
print(n)
def test_set_get_key(self):
store = Store(RedisStore(), 3, 2, (TimeoutError, ConnectionError))
store.set('key1', 'val1')
self.assertEqual(store.cache_get("key1"), 'val1')
def setUp(self):
self.store = Store(RedisStore())
self.redis_store = RedisStore()
self.redis_start()
def test_failed_store(self):
store = Store(RedisStore(port=99999999), 3, 2,
(TimeoutError, ConnectionError))
with self.assertRaises(StoreCacheError):
store.get("key1")
def test_get_not_exists_key(self):
store = Store(RedisStore(), 3, 2, (TimeoutError, ConnectionError))
self.assertEqual(store.cache_get("key2"), None)