def test_token(redis_server):
conn = StrictRedis(unix_socket_path=UDS_PATH)
lock = Lock(conn, "foobar-tok")
tok = lock.id
assert conn.get(lock._name) is None
lock.acquire(blocking=False)
assert conn.get(lock._name) == tok
def app(environ, start_response):
ret = {}
qs = dict(parse_qsl(environ["QUERY_STRING"]))
red = StrictRedis(host="localhost", port=6379, db=0)
root_key = red.get("root")
if root_key is None: #Load trucks into redis if there is no tree already in it
inp_file = "Mobile_Food_Facility_Permit.csv"
load_trucks(inp_file, red)
root_key = red.get("root")
try:
lat = float(qs["latitude"])
lon = float(qs["longitude"])
rad = float(qs["radius"])
except KeyError: #Return error if required fields aren't present
start_response("400 Bad Request", [("Content-type", "text/plain")])
ret["error"] = "latitude, longitude, and radius query parameters are required"
return [dumps(ret)]
ret["latitude"] = lat
ret["longitude"] = lon
ret["radius"] = rad
food = qs.get("food", "").upper()
if food:
ret["food"] = food
ret["trucks"] = [str(t)
for t in get_trucks(lat, lon, rad, red, root_key) if food in t.food]
else:
trucks = []
foods = set()
for t in get_trucks(lat, lon, rad, red, root_key):
trucks.append(str(t))
foods |= set(t.food)
ret["trucks"] = trucks
ret["foods"] = list(foods)
start_response("200 OK", [("Content-type", "text/plain")])
return [dumps(ret)]
class Redis:
"""
To store and query local/remote Redis server.
"""
def __init__(self, host='localhost',
port=6379,
db=0,
password=None):
self.r = StrictRedis(host, port, db, password)
def __del__(self):
del(self.r)
def update(self, track):
self.r.set(track.key, track.stats())
def lookup(self, keyword):
# TODO: add option for caseless search
return self.r.keys('*' + keyword + '*')
def retrieve(self, key):
if isinstance(key, list):
return self.get_all(key)
return Track.from_redis(key, self.r.get(key))
def get_all(self, keys):
tracks = []
for key in keys:
tracks.append(Track.from_redis(key, self.r.get(key)))
return tracks
class RedisDataStore(DataStore):
"""Redis-backed datastore object."""
def __init__(self, number=0):
redis_host = os.environ.get('REDIS_PORT_6379_TCP_ADDR')
redis_port = os.environ.get('REDIS_PORT_6379_TCP_PORT')
self.redis_conn = StrictRedis(host=redis_host, port=redis_port,
db=number)
def __setitem__(self, k, v):
self.redis_conn.set(k, v)
def __getitem__(self, k):
return self.redis_conn.get(k)
def __delitem__(self, k):
self.redis_conn.delete(k)
def get(self, k):
return self.redis_conn.get(k)
def __contains__(self, k):
return self.redis_conn.exists(k)
def todict(self):
#TODO(tvoran): use paginate
#TODO(tvoran): do something besides multiple gets
data = {}
for key in self.redis_conn.keys():
data[key] = self.get(key)
return data
def clear_all(self):
self.redis_conn.flushdb()
class RedisRequestLogger(RequestLoggerBase):
"""A faster alternative to the default logger, but requiring more configuration"""
def __init__(self):
from redis import StrictRedis
super(RedisRequestLogger, self).__init__()
self.db = StrictRedis(
host = getattr(settings, 'API_REDIS_HOST', 'localhost'),
port = getattr(settings, 'API_REDIS_PORT', 6379),
db = getattr(settings, 'API_REDIS_DB', 0),
password = getattr(settings, 'API_REDIS_PASSWORD', '')
)
def get_key(self, app):
return 'bambu-api-requests-%s' % app.key
def log_request(self, app):
"""Saves the number of requests within the given timeframe to the Redis database"""
timestamp = self.get_timestamp()
key = self.get_key(app)
values = self.db.get(key)
if values:
try:
values = pickle.loads(values)
except:
values = {}
else:
values = {}
if values.has_key(timestamp):
values = {
timestamp: values[timestamp] + 1
}
else:
values = {
timestamp: 1
}
self.db.set(key, pickle.dumps(values))
def get_request_count(self, app, timestamp):
"""Returns the number of requests by the specified app, within the specified timeframe"""
values = self.db.get(
self.get_key(app)
)
if values:
try:
values = pickle.loads(values)
except:
values = {}
else:
values = {}
return values.get(timestamp) or 0
class BearerAuth(BasicAuth):
""" Overrides Eve's built-in basic authorization scheme and uses Redis to
validate bearer token
"""
def __init__(self):
super(BearerAuth, self).__init__()
self.redis = StrictRedis()
def check_auth(self, token, allowed_roles, resource, method):
""" Check if API request is authorized.
Examines token in header and checks Redis cache to see if token is
valid. If so, request is allowed.
:param token: OAuth 2.0 access token submitted.
:param allowed_roles: Allowed user roles.
:param resource: Resource being requested.
:param method: HTTP method being executed (POST, GET, etc.)
"""
if not token:
return False
user_id = self.redis.get(token)
if not user_id:
return False
# now switching to the user-reserved mongo instance.
mongo_prefix = 'MONGO%s' % user_id
# TODO remove defaulting to localhost so exception is raised
# if db host is not available. Right now, unless redis hodls a
# key for the user, all dbs are hosted on localhost.
host = self.redis.get(user_id) or 'localhost'
if not host:
raise ConfigException('Cannot locate host for user database %s' %
user_id)
uri = 'mongodb://%s/%s' % (host, user_id)
current_app.config['%s_URI' % mongo_prefix] = uri
self.set_mongo_prefix(mongo_prefix)
return True
def authorized(self, allowed_roles, resource, method):
""" Validates the the current request is allowed to pass through.
:param allowed_roles: allowed roles for the current request, can be a
string or a list of roles.
:param resource: resource being requested.
"""
try:
token = request.headers.get('Authorization').split(' ')[1]
except:
token = None
return self.check_auth(token, allowed_roles, resource, method)
def get(cls, name):
db = StrictRedis()
value = None
while True:
ver = db.get(name+'::v')
value = cls._get(db, name)
if ver == db.get(name+'::v'):
break
return value
class RedisRequestLogger(RequestLoggerBase):
def __init__(self):
from redis import StrictRedis
super(RedisRequestLogger, self).__init__()
self.db = StrictRedis(
host = getattr(settings, 'API_REDIS_HOST', 'localhost'),
port = getattr(settings, 'API_REDIS_PORT', 6379),
db = getattr(settings, 'API_REDIS_DB', 0),
password = getattr(settings, 'API_REDIS_PASSWORD', '')
)
def get_key(self, app):
return 'bambu-api-requests-%s' % app.key
def log_request(self, app):
timestamp = self.get_timestamp()
key = self.get_key(app)
values = self.db.get(key)
if values:
try:
values = pickle.loads(values)
except:
values = {}
else:
values = {}
if values.has_key(timestamp):
values = {
timestamp: values[timestamp] + 1
}
else:
values = {
timestamp: 1
}
self.db.set(key, pickle.dumps(values))
def get_request_count(self, app, timestamp):
values = self.db.get(
self.get_key(app)
)
if values:
try:
values = pickle.loads(values)
except:
values = {}
else:
values = {}
return values.get(timestamp) or 0
def check_health_status(self):
'''
Verify that all necessary systems are available and running correctly.
'''
status = {'result': 'error', 'message': 'failure'}
self.log.info("check_health_status(): starting health check")
# Check Redis. Fail right-away if Redis is down - verifies infrastructure
try:
cache = StrictRedis(db=config.health_cache_redis_db)
cache.set('test', 'test')
rv = cache.get('test')
if rv <> 'test':
return {'result': 'error', 'message': 'redis is not responding correctly'}
except Exception as ex:
self.log.critical("check_health_status() redis is not responding: " + str(ex))
return {'result': 'error', 'message': 'redis is not responding'}
# Check if down for maintenance
cache = StrictRedis(db=config.health_cache_redis_db)
is_maintenance = cache.get('is_maintenance')
if is_maintenance == 'true':
return {'result': 'error', 'message': 'system is down for maintenance'}
# Do a web request for all buildings - verifies data quality
try:
cache = StrictRedis(db=config.tokens_cache_redis_db)
token = config.test_token
token_hash = auth.calc_hash(token)
cache.set(token_hash, 'test@test')
r = requests.get('http://localhost:8888/org/v1/buildings', auth=(token, ''))
buildings = r.json()
if len(buildings) < 65 or len(buildings) > 90:
self.log.critical("check_health_status(): building data failure")
return {'result': 'error', 'message': 'building data failure'}
#cache.delete(token_hash)
except Exception as ex:
self.log.critical("check_health_status(): building data failure: " + str(ex))
return {'result': 'error', 'message': 'building data failure: ' + str(ex)}
# Check db -- if down set flag to not expire data. Do not fail if db is down
status = {'result': 'success', 'message': "success"}
return status
def store(self):
from simplekv.memory.redisstore import RedisStore
r = StrictRedis()
try:
r.get('anything')
except ConnectionError:
pytest.skip('Could not connect to redis server')
r.flushdb()
yield RedisStore(r)
r.flushdb()
class TaskCache(ITaskCache):
"""
`self.user_store': {'user_id': <list of task_ids>}
`self.task_store`: {'task_id`: <AsyncResult>}
`self.progress_store`: {`task_id`: `progress_value(int)`}
"""
def __init__(self, user_kwargs, task_kwargs ,progress_kwargs):
self.user_store = StrictRedis(**user_kwargs)
self.task_store = StrictRedis(**task_kwargs)
self.progress_store = LingvodocRedisClient(**progress_kwargs)
def get(self, user, remove_finished=False):
result = dict()
tasks = self.user_store.get(user.id)
if tasks is None:
return {}
tasks = pickle.loads(tasks)
remained_tasks = list()
for t in tasks:
val = self.task_store.get(t)
if val is None:
continue
async_result = pickle.loads(val)
progress = self.progress_store.get(t)
# Redis client returns byte array. We need to decode it
if progress is not None:
progress = int(progress.decode())
result[t] = {'finished': async_result.ready(),
'progress': progress}
if remove_finished:
if async_result.ready():
self.task_store.delete(t)
else:
remained_tasks.append(t)
if remove_finished:
self.user_store.set(user.id, pickle.dumps(remained_tasks))
return result
# TODO: add try/catch handlers.
# we should remove the task from caches (and queue?) if exception is raised
def set(self, user, task_key, async_task):
self.task_store.set(task_key, pickle.dumps(async_task))
cached = self.user_store.get(user.id)
if cached is None:
tmp_tasks = [task_key]
else:
tmp_tasks = pickle.loads(cached)
tmp_tasks.append(task_key)
self.user_store.set(user.id, pickle.dumps(tmp_tasks))
def polite_get(url, ttl=TTL, db=0, port=6379):
"""
Don't hammer the remote servers.
1. They don't update that often anyway.
2. We don't want to get throttled or banned.
3. It's polite.
Accepts kwargs for ttl, db, and port; otherwise
uses Redis defaults and a one-hour ttl.
"""
db = StrictRedis(db=db, port=port)
key = "url_cache::{0}".format(url)
result = db.get(key)
if result is None:
page = requests.get(url)
result = page.text
db.setex(key, ttl, result.encode('utf-8'))
else:
result = result.decode('utf-8')
return result
class CacheHelper(object):
"""A thin wrapper on top of Redis."""
def __init__(self, host="localhost", port=6379, db=0):
self.host = host
self.port = port
self.db = db
def __enter__(self):
self.r = StrictRedis(host=self.host, port=self.port, db=self.db)
return self
def __exit__(self, exc_type, exc_value, traceback):
try:
if any((exc_type, exc_value, traceback,)):
raise
finally:
pass
def get(self, key):
return self.r.get(key) if key in self.r.keys() else None
def set(self, key, json):
self.r.set(key, json)
def keys(self):
return self.r.keys()
def reset(self):
for key in self.keys():
self.r.delete(key)
def sync_get(self, identity, *args, **kwargs):
"""
For getting data from cache
:param identity: Unique Integer for the data
:param args: Args for the sync function. (Default: None)
"""
redis = StrictRedis(connection_pool=self.redis_pool)
key = key_generator(self.key, identity)
try:
if redis.exists(key):
data = self.get_func(redis.get(key))
else:
data = self.sync_func(identity, *args, **kwargs)
if self.expire:
self._setex(redis, key, self.set_func(data))
else:
redis.set(key, self.set_func(data))
if data is not None or data != "":
return data
return None
except RedisError as re:
self.log.error("[REDIS] %s", str(re))
data = self.sync_func(identity, args)
return data
finally:
del redis
class JSONRedisBackend(BaseBackend):
def __init__(self):
self.redis = StrictRedis()
def serialize(self, value):
return json.dumps(value)
def deserialize(self, value):
return value and json.loads(value) or None
def get(self, key):
value = self.redis.get(key)
result = self.deserialize(value)
return result
def set(self, key, value):
product = self.serialize(value)
return self.redis.set(key, product)
def rpush(self, key, value):
product = self.serialize(value)
return self.redis.rpush(key, product)
def lpush(self, key, value):
product = self.serialize(value)
return self.redis.lpush(key, product)
def lpop(self, key):
value = self.redis.lpop(key)
result = self.deserialize(value)
return result
def llen(self, key):
return self.redis.llen(key)
def rpop(self, key):
value = self.redis.rpop(key)
result = self.deserialize(value)
return result
def report_steps(self, name, consumers, producers):
pipeline = self.redis.pipeline()
producers_key = ':'.join([name, 'producers'])
consumers_key = ':'.join([name, 'consumers'])
for consumer in consumers:
pipeline.sadd(consumers_key, consumer)
for producer in producers:
pipeline.sadd(producers_key, producer)
pipeline.smembers(consumers_key)
pipeline.smembers(producers_key)
result = pipeline.execute()
all_consumers = result[-2]
all_producers = result[-1]
return all_consumers, all_producers
def __admin_handler(self, endpoint: bytes):
"""
Handle Admin Request
:param bytes endpoint: Endpoint (in bytes!)
:return: jsonified answer data
"""
json_data = request.get_json()
if json_data is None:
raise BadRequest()
if 'admin_token' not in json_data:
raise BadRequest()
admin_token = json_data['admin_token']
if not isinstance(admin_token, str):
raise BadRequest()
if not RULE_TOKEN.match(admin_token):
raise BadRequest()
redis = StrictRedis(connection_pool=self.api_pool)
ep_key = 'ADMIN_TOKEN:{:s}'.format(admin_token)
should_endpoint = redis.get(ep_key)
if should_endpoint is None:
raise BadRequest()
redis.delete(ep_key)
if should_endpoint != endpoint:
raise BadRequest()
if 'data' not in json_data:
raise BadRequest()
data = json_data['data']
if not isinstance(data, dict):
raise BadRequest()
return jsonify(self.queue_dispatcher({
'_': 'admin:{:s}'.format(endpoint.decode('utf-8')),
'data': data,
}))
class RedisProducer(object):
def __init__(self, hostname = 'localhost', port = 6379):
log.debug("Initializing RedisProducer with hostname of %s and port %s" % (hostname, port))
self.r = StrictRedis(host = hostname, port = port)
def send(self, message):
tries = 0
next_index_key = get_next_index_for_topic_key(message.topic)
next_index = 1
result = None
log.debug("Sending message on topic %s" % message.topic)
while result is None and tries < TRIES_LIMIT:
if self.r.exists(next_index_key):
next_index = long(self.r.get(next_index_key)) + 1
message_key = get_message_key(message.topic, next_index)
try:
pl = self.r.pipeline()
pl.watch(next_index_key, message_key)
pl.multi()
pl.incr(next_index_key).set(message_key, message.payload)
result = pl.execute()
except WatchError:
# Should probably log something here, but all it means is we're
# retrying
pass
if result is None:
log.error("Could not send message, retry amount exceeded")
raise RuntimeError("Attempted to send message %s times and failed" % TRIES_LIMIT)
class MispRedisConnector(object):
def __init__(self):
self.r = StrictRedis(unix_socket_path=redis_socket)
def search(self, authkey, values=None, hash_values=None, return_eid=False, quiet=False):
if isinstance(values, list):
hash_values = [SHA256.new(v.lower()).hexdigest() for v in values]
elif values:
hash_values = [SHA256.new(values.lower()).hexdigest()]
elif not isinstance(hash_values, list):
hash_values = [hash_values]
if not hash_values:
raise Exception('No value to search.')
org = self.__get_org_by_auth(authkey)
if not org:
raise Exception('Invalid authkey')
if quiet:
return [(self.r.exists(h) or self.r.exists(org + ':' + h)) for h in hash_values]
uuid_by_hashes = [self.r.smembers(h).union(self.r.smembers(org + ':' + h)) for h in hash_values]
if not return_eid:
to_return = uuid_by_hashes
else:
to_return = []
for h in uuid_by_hashes:
to_return.append([self.r.hget('uuid_id', uuid) for uuid in h])
return to_return
def __get_org_by_auth(self, authkey):
return self.r.get(authkey)
class BearerAuth(BasicAuth):
""" Overrides Eve's built-in basic authorization scheme and uses Redis to
validate bearer token
"""
def __init__(self):
super(BearerAuth, self).__init__()
self.redis = StrictRedis()
def check_auth(self, token, allowed_roles, resource, method):
""" Check if API request is authorized.
Examines token in header and checks Redis cache to see if token is
valid. If so, request is allowed.
:param token: OAuth 2.0 access token submitted.
:param allowed_roles: Allowed user roles.
:param resource: Resource being requested.
:param method: HTTP method being executed (POST, GET, etc.)
"""
return token and self.redis.get(token)
def authorized(self, allowed_roles, resource, method):
""" Validates the the current request is allowed to pass through.
:param allowed_roles: allowed roles for the current request, can be a
string or a list of roles.
:param resource: resource being requested.
"""
try:
token = request.headers.get("Authorization").split(" ")[1]
except:
token = None
return self.check_auth(token, allowed_roles, resource, method)
class DbRedis(Db):
def __init__(self):
super().__init__()
self.redis = StrictRedis(**dbSettings)
def generateKey(self):
return self.redis.incr("id")
def store(self, key, objSerial):
self.redis.setex(key, TTL, objSerial)
def retrieve(self, key):
return self.redis.get(key)
def lenList(self, name):
return self.redis.llen(name)
def listAppend(self, name, val):
self.redis.lpush(name, val)
def listPopLeft(self, name):
return self.redis.lpop(name)
def retrieveList(self, name):
return self.redis.lrange(name, 0, -1)
def removeFromList(self, name, item):
self.redis.lrem(name, item, 0)
def keyExists(self, key):
return self.redis.exists(key)
def archive_events():
print("Starting to archive events.")
redis = StrictRedis(host=REDIS_HOST, port=REDIS_PORT, db=0)
s3 = tinys3.Connection(AWS_ACCESS_KEY_ID,
AWS_SECRET_ACCESS_KEY,
default_bucket=S3_BUCKET,
endpoint=S3_ENDPOINT,
tls=True)
now = datetime.utcnow()
nowstring = now.strftime("%Y-%m-%d")
# 1. We collect all events objects from the query time span
# and group them by day.
# keep everything in RAM, keyed by day
events = {}
count = 0
for event in fetch_events():
event_datetime = datetime.fromtimestamp(event["created"])
daystring = event_datetime.strftime("%Y-%m-%d")
if daystring == nowstring:
continue
if daystring not in events:
events[daystring] = []
count += 1
events[daystring].append(event)
print("Fetched %d events." % count)
# 2. We go through day by day and see where new entries have been added
# since the last run. For these we create a backup file.
redis_retention = REDIS_RETENTION_DAYS * 24 * 60 * 60
for daystring in sorted(events.keys()):
print("Processing day %s with %d events" % (daystring, len(events[daystring])))
# possibly skip days already archived
redis_value = redis.get(daystring)
if redis_value is not None:
num_entries_before = int(redis_value)
if len(events[daystring]) <= num_entries_before:
print("Skipping day %s, already archived" % daystring)
continue
try:
target_path = upload_dump(daystring, events[daystring], s3)
print("Uploaded %s" % target_path)
# write number of entries per day to redis
redis.setex(daystring, redis_retention, str(len(events[daystring])))
except Exception as ex:
sys.stderr.write("ERROR: No backup created for %s\n" % daystring)
sys.stderr.write(traceback.format_exc() + "\n")
del redis
del s3
del events
print("Done for today.")
def search():
query = request.args["q"]
ands = request.args["and"] if "and" in request.args else ""
ors = request.args["or"] if "or" in request.args else ""
nots = request.args["not"] if "not" in request.args else ""
try:
page = int(request.args["page"])
except:
page = 1
# cache server
redis = StrictRedis(host="localhost")
# start measuring elapsed time
start = time.time()
# try to load result from cache
key = json.dumps({
"q": query,
"a": ands,
"o": ors,
"n": nots
})
cache_result = redis.get(key)
items = cPickle.loads(cache_result) if cache_result else None
if not items:
items = SearchEngine(get_db(), query, ands, nots, ors) \
.search()
# store results in cache
redis.set(key, cPickle.dumps(items))
elapsed_time = (time.time() - start) * 1000
items_count = len(items)
pages_count = items_count // 10
if pages_count < items_count / 10.0:
pages_count += 1
if page > pages_count:
page = pages_count
# convert to index based
page -= 1
# calculate last item within this page
last_item_index = (page * 10) + 10
json_data = json.dumps({
"items": items[page * 10: last_item_index if last_item_index < items_count else items_count],
"total_count": items_count,
"elapsed_time": int(elapsed_time)
}, cls=SuperListJsonEncoder)
return Response(response=json_data,
status=200,
mimetype="application/json")
def view_user_json(username):
"""The twitter user JSON view"""
username = username.lower()
redis_key = "%s.user.%s" % (REDIS_PREFIX, username)
redis = StrictRedis()
cache = redis.get(redis_key)
if not cache:
cache = dict(status='queued', header='Queued',
message="Your request will be processed shortly",
code=200)
redis.set(redis_key, dumps(cache))
redis.expire(redis_key, CACHE_HOURS*60*60)
load_tweets.delay(username)
sleep(.5)
cache = loads(redis.get(redis_key))
return jsonify(cache)
class RedisBackend(Backend):
def __init__(self, config):
super(RedisBackend, self).__init__(config)
self.redis = StrictRedis(host=config.get("host", "localhost"),
port=config.get("port", 6379),
db=config.get("db", 0))
self.namespace = config.get("namespace", "short:")
def furl(self, name):
return self.namespace + "url:" + name
def fvisits(self, name):
return self.namespace + "visits:" + name
def next_name(self):
name = None
while 1:
name = hashids.encrypt(
self.redis.incr(self.namespace + "meta:num"))
if not self.exists(name):
break
return name
def exists(self, name):
return self.redis.exists(self.furl(name))
def set(self, link):
if self.redis.exists(self.furl(link.name)):
raise NameUnavailableError(link.name)
self.redis.set(self.furl(link.name), link.url)
self.redis.set(self.fvisits(link.name), 0)
def get(self, name):
rawlink = self.redis.get(self.furl(name))
if not rawlink:
raise NotFoundError(name)
link = Link(name=name,
url=rawlink.decode("utf-8"),
visits=int(
self.redis.get(self.fvisits(name)) or 0
))
return link
def visit(self, name):
self.redis.incr(self.fvisits(name))
class RedisCache(object):
def __init__(self, params={}):
self._validate(params)
if not self.server:
raise Exception('Redis Server Not Defined')
try:
log.debug('Connecting to redis at [%s]?[%s]' % (self.server, self.database))
self.cache = StrictRedis(self.server, port=self.port, db=self.database)
except ConnectionError as ex:
raise Exception("Unable to connect to Redis", ex)
def get(self, key):
"""
Fetch a given key from the cache. If the key does not exist, return
default, which itself defaults to None.
"""
ckey = self._create_key(key)
log.debug("Getting the cache key [%s]" % ckey)
return self.cache.get(ckey)
def ping(self):
"""
This command is often used to test if the cache is still alive, or to measure latency.
"""
log.debug("Ping to the cache")
return self.cache.ping()
def store(self, key, value, expires=None):
"""
Set a value in the cache. If timeout is given, that timeout will be
used for the key; otherwise the default cache timeout will be used.
"""
ckey = self._create_key(key)
log.debug("Storing the cache key [%s]" % ckey)
return self.cache.set(ckey, value, ex=expires)
def delete(self, key):
"""
Delete a key from the cache, failing silently.
"""
ckey = self._create_key(key)
log.debug("Deleting the cache key [%s]" % ckey)
return self.cache.delete(ckey)
def _validate(self, params):
"""
Initialize all the needed parameters
"""
self.server = params.get('server', 'localhost')
self.port = params.get('port', 6379)
self.database = params.get('database', 2)
self.key_prefix = params.get('key_prefix', 'domus')
def _create_key(self, key):
return "%s.%s" % (self.key_prefix, key)
class RedisConsumer(object):
def __init__(self, timeout, group_id, hostname = 'localhost', port = 6379):
self.group_id = group_id
self.timeout = timeout
log.debug("Initializing RedisConsumer with hostname of %s and port %s" % (hostname, port))
self.r = StrictRedis(host = hostname, port = port)
def poll(self, topic):
result = None
current_index_key = get_next_index_for_topic_key(topic)
end_millis = time.time() * 1000 + self.timeout
log.debug("Polling topic %s" % topic)
while time.time() * 1000 < end_millis:
if self.r.exists(current_index_key):
current_index = long(self.r.get(current_index_key))
consumer_index_key = get_next_index_for_group_id_key(topic, self.group_id)
pl = self.r.pipeline()
pl.watch(consumer_index_key)
consumer_index = 0
if self.r.exists(consumer_index_key):
consumer_index = long(self.r.get(consumer_index_key))
if current_index > consumer_index:
try:
pl.multi()
pl.incr(consumer_index_key)
incr_result = pl.execute()
if not incr_result is None and len(incr_result) > 0:
consumer_index = long(incr_result[0])
key = get_message_key(topic, consumer_index)
if self.r.exists(key):
result = self.r.get(key)
break
except WatchError:
log.debug("Redis keys changed for topic %s and group %s, trying again" % (topic, self.group_id))
pass
return result
def unsubscribe_from_topic(self, topic):
self.r.delete(get_next_index_for_group_id_key(topic, self.group_id))
class TestTransactionOperations(unittest.TestCase):
def setUp(self):
video_data = {
'duration' : 22,
'random' : 0.31461311114729994,
'vimeoid' : 24879869,
'description' : 'The story of a cartoon character',
'title' : 'Overcast'
}
self.database = connection()
self.video_id = str(self.database.videos.insert(video_data))
test_user_name = 'test user: {0}'.format(repr(self))
test_user = dict(name=test_user_name, chances=222)
uid = self.database.accounts.save(test_user)
self.test_user_id = str(uid)
redis_url = os.environ.get('REDISTOGO_URL')
url = urlparse(redis_url)
self.redis = StrictRedis(host=url.hostname, port=url.port,
password=url.password)
self.prefix = 'watch-transaction:'
def tearDown(self):
uid = ObjectId(self.test_user_id)
self.database.accounts.remove(uid)
self.database.videos.remove(ObjectId(self.video_id))
self.database.screening.remove({'user.id': uid})
def test_begin(self):
user = users.find(self.test_user_id)
video = videos.find(self.video_id)
watchtran.begin(user, video)
key = self.prefix + self.test_user_id
val = self.redis.get(key)
self.assertIsNotNone(val)
def test_commit_without_begin(self):
user = users.find(self.test_user_id)
video = videos.find(self.video_id)
result = watchtran.commit(user, video)
self.assertIsNone(result)
def test_commit_too_early(self):
user = users.find(self.test_user_id)
video = videos.find(self.video_id)
watchtran.begin(user, video)
result = watchtran.commit(user, video)
self.assertIsNone(result)
def test_commit(self):
user = users.find(self.test_user_id)
video = videos.find(self.video_id)
video._duration = 0
watchtran.begin(user, video)
result = watchtran.commit(user, video)
self.assertIsNotNone(result)
class testTaggedCache(unittest.TestCase):
def setUp(self):
self.redis = StrictRedis(db=DATABASE)
self.redis.flushdb()
self.cache = TaggedCache(self.redis)
def testEverything(self):
cache = self.cache
# Set
cache.set("User:1", "mitch", tags=["User", "PremiumUser"])
cache.set("User:2", "foo", tags=["User"])
cache.set("Post:1", "Hello World!", tags=["Post"])
cache.set("Post:2", "Hello World, again!", tags=["Post"])
self.assertEquals(cache.get("Post:1"), "Hello World!")
self.assertEquals(cache.get_keys("Post"), set(["Post:1", "Post:2"]))
self.assertEquals(cache.get_keys("User"), set(["User:1", "User:2"]))
self.assertEquals(cache.get_tags("User:1"), set(["User", "PremiumUser"]))
self.assertEquals(cache.get_tags("User:2"), set(["User"]))
# Delete all post cache entries
cache.clear_tag("Post")
self.assertEquals(cache.get("Post:1"), None, "Post:1 still exists")
self.assertEquals(cache.get("Post:2"), None, "Post:2 still exists")
# Delete User 2 from cache
cache.clear("User:2")
self.assertEquals(cache.get_tags("User:2"), set())
# Clear everything else
cache.clear_all()
cache.gc()
self.assertEquals(self.redis.get("CacheKeys"), None)
self.assertEquals(self.redis.get("Tags"), None)
self.assertEquals(len(self.redis.keys("*")), 0, "Some keys were not gc'ed")
def testExpireTtl(self):
self.cache.set("ExpireMe", "foo", ttl=1)
time.sleep(2)
self.assertEquals(self.cache.get("ExpireMe"), None)
def stat_maint(self):
self.log.info("stat_maint(): checking maintenance status")
cache = StrictRedis(db=config.health_cache_redis_db)
is_maintenance = cache.get('is_maintenance')
if is_maintenance == 'true':
self.log.info("stat_maint(): in maintenance mode")
return True
else:
self.log.info("stat_maint(): not in maintenance mode")
return False
def app_redis_api(app_id):
'''
Simple REST api to redis DBs.
GET, PUT, DELETE are key operations, and POST allows for any command
method = 'GET': ../redis?key=key
return r.get('key')
method = 'PUT': ../redis?key=key&val=val
return r.set('key', 'val')
method = 'DELETE': ../redis?key=key
return r.delete('key')
method = 'POST': ../redis?cmd=hset&args=key,hashkey,hashval
Request args
return getattr(r,cmd)(*args.split(','))
aka r.hset(key, hashkey, hashval)
'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
app_id = str(app_id)
if not app_id in monaco.app_ids:
abort(404)
userapp = schema.App(app_id=app_id)
userapp.refresh(r)
master_host = None
for node_id, role in userapp.nodes.iteritems():
if role == 'master':
master_host = monaco.hostnames_by_node_id[node_id]
break
assert master_host != None
r = StrictRedis(master_host, userapp.port)
if request.method == 'GET':
if 'key' not in request.args:
abort(400)
return r.get(request.args['key'])
if request.method == 'PUT':
if 'key' not in request.args or 'val' not in request.args:
abort(400)
return r.set(request.args['key'], request.args['val'])
if request.method == 'DELETE':
if 'key' not in request.args:
abort(400)
return r.delete(request.args['key'])
if request.method == 'POST':
if 'cmd' not in request.args or not hasattr(r, request.args['cmd']):
abort(400)
if 'args' in request.args:
args = request.args['args'].split(',')
else:
args = []
return getattr(r, request.args['cmd'])(*args)
abort(400)
class RedisClient:
def __init__(self, host='127.0.0.1', port=6379):
self.r = StrictRedis(host=host, port=port, db=0)
def get(self, key):
return self.r.get(key)
def put(self, key, value):
return self.r.set(key, value, nx=True)
def update(self, key, value):
return self.r.set(key, value, xx=True)
def remove(self, key):
return self.r.delete(key)
def remove_all(self):
return self.r.flushdb()
def get_dict_from_redis(
redis_client: redis.StrictRedis,
key_format: str,
key: str,
) -> Optional[Dict[str, Any]]:
# This function requires inputting the intended key_format to validate
# that the key fits it, as an additionally security measure. This protects
# against bugs where a caller requests a key based on user input and doesn't
# validate it - which could potentially allow users to poke around arbitrary redis keys.
if len(key) > MAX_KEY_LENGTH:
error_msg = "Requested key too long in get_dict_from_redis: %s"
raise ZulipRedisKeyTooLongError(error_msg % (key, ))
validate_key_fits_format(key, key_format)
data = redis_client.get(key)
if data is None:
return None
return orjson.loads(data)
class Redis():
def __init__(self, host, port):
self.redis = StrictRedis(host=host, port=port, db=0, password=None)
def addKey(self, key, value):
self.redis.set(key, value)
def setExpireTime(self, key, expireTime) -> bool:
return self.redis.expire(key, expireTime)
def deleteKey(self, key):
self.redis.delete(key)
def containsKey(self, key):
return self.redis.exists(key)
def getValue(self, key):
return self.redis.get(key)
def get_q(self, redis: StrictRedis):
""" Get a queue object with this ID for a given ``redis`` connection. """
if (self._buffer is None or self._serializer is None):
res = redis.get(self._key + OPEN_APPEND)
if res:
chid = pickle.loads(res)
self._buffer = chid.buffer
self._serializer = chid.serializer
self.opened = chid.opened
if self._buffer is not None and self._serializer is not None:
return Ch(redis,
self._key,
buffer=self._buffer,
serializer=self._serializer,
opened=self.opened)
return None
class RedisBrain(object):
def __init__(self, redis_url):
try:
self.redis = StrictRedis(host=redis_url, port=7777, db=0)
except:
self.redis = None
def set(self, key, value):
if self.redis:
self.redis.set(key, value)
return True
else:
return False
def get(self, key):
if self.redis:
return self.redis.get(key)
return None
class RedisCache(BaseCache):
def __init__(self, redis_host, redis_port, default_timeout):
super(RedisCache, self).__init__(default_timeout=default_timeout)
self.redis_connection = StrictRedis(host=redis_host,
port=redis_port,
db=1)
def _put(self, context, params, item, timeout):
key = RedisCache._make_key(context, params)
self.redis_connection.set(key, item, ex=timeout)
context_key = RedisCache._make_context_key(context)
self.redis_connection.sadd(context_key, key)
self.redis_connection.expire(context_key, self.timeout())
def _get(self, context, params):
item = None
key = RedisCache._make_key(context, params)
context_key = RedisCache._make_context_key(context)
if self.redis_connection.sismember(context_key, key):
item = self.redis_connection.get(key)
if item is None:
self.redis_connection.srem(context_key, key)
else:
self.redis_connection.expire(key, self.timeout())
self.redis_connection.expire(context_key, self.timeout())
return item
def _clear(self, context):
context_key = RedisCache._make_context_key(context)
pipe = self.redis_connection.pipeline()
item = self.redis_connection.spop(context_key)
while item is not None:
pipe.delete(item)
item = self.redis_connection.spop(context_key)
pipe.execute(raise_on_error=True)
@staticmethod
def _make_key(context, params):
params = json.dumps(params, ensure_ascii=True, sort_keys=True)
return '{}:{}'.format(context, params)
@staticmethod
def _make_context_key(context):
return RedisCache._make_key('ctx', context)
class redis_session:
prefix = 'was:session_key:' # Redis key 앞에 넣을 값
server_ip = 'localhost' # Redis ip
port = 6379
timeout = 3600
def __init__(self):
self.db = StrictRedis(self.server_ip, self.port)
# 세션이 있으면 타임아웃 만큼 다시 연장해주고 없으면 False 있으면 사용자id 리턴
def get_session(self, session_key):
inst = self.db.get(self.prefix+session_key)
if inst:
self.db.expire(self.prefix+session_key, self.timeout)
return inst
# 신규 세션 요청 시 세션 값을 만들어서 리턴
def save_session(self, user_name):
session_key = str(uuid4())
self.db.setex(self.prefix+session_key, user_name, self.timeout)
return session_key
class RateLimiter(cherrypy.Tool):
def __init__(self, limit=100, window=60):
cherrypy.Tool.__init__(self, 'before_handler',
self.process_request, priority=10)
cherrypy.log("Creating rate limiter with limit={} and window={}".format(limit, window))
self.limit = limit
self.window = window
self.redis = StrictRedis(host='redis', port=6379)
def process_request(self):
print(cherrypy.request)
print(cherrypy.request.remote)
requester = cherrypy.request.remote.ip
print("remote:", requester)
# un-comment if you want to ignore calls from localhost
# if requester == '127.0.0.1':
# return
key = "{0}: {1}".format(requester, cherrypy.request.path_info)
print('Key: {0}'.format(key))
try:
remaining = self.limit - int(self.redis.get(key))
except (ValueError, TypeError):
remaining = self.limit
self.redis.set(key, 0)
expires_in = self.redis.ttl(key)
if expires_in == -1:
self.redis.expire(key, self.window)
expires_in = self.window
cherrypy.request.headers.update({
'X-RateLimit-Remaining: ': str(remaining - 1),
'X-RateLimit-Limit: ': str(self.limit),
'X-RateLimit-Reset: ': str(time.time() + expires_in)
})
if remaining > 0:
self.redis.incr(key, 1)
else:
raise cherrypy.HTTPError(429, 'Blocked: Too many requests!')
class Store:
def __init__(self):
self.redis = StrictRedis(host="localhost",
db=0,
socket_connect_timeout=2,
socket_timeout=2)
def get(self, name, id):
if int(id) == 0:
return None
item = self.redis.lindex(name, int(id) - 1)
if item:
return json.loads(item)
def get_all(self, name):
return [json.loads(item) for item in self.redis.lrange(name, 0, -1)]
def add(self, value):
new_id = self.redis.llen(value.path()) + 1
value.set_id(new_id)
self.redis.rpush(value.path(), value.serialize())
return new_id
def update(self, value):
items = self.get_all(value.path())
for idx, item in enumerate(items):
if value.equal(item):
value.set_id(idx + 1)
self.redis.lset(value.path(), idx, value.serialize())
return idx + 1
return self.add(value)
def set_item(self, key, value):
self.redis.set(key, json.dumps(value))
def get_item(self, key):
item = self.redis.get(key)
if item:
return json.loads(item)
def clear(self):
self.redis.flushdb()
def execute_redis_db(key):
"""
:param key: 要查的key值
:return:查到的结果
"""
value = None
if key is None:
return value
try:
host, port, db, password = get_redis_config()
redis_pool = StrictRedis(host=host,
port=port,
db=db,
password=password)
log.error(redis_pool.keys())
value = redis_pool.get(key)
except Exception as e:
log.error(e)
return value
def login():
username = request.json.get('username') or request.json.get('mail')
password = request.json.get('password')
if not username or not password:
abort(404)
accounts = app.data.driver.db['accounts']
user = accounts.find_one({'mail': username})
#print user
if not user:
status = "error"
message = "user not exist"
return jsonify(status=status, messag=message)
if not verify_password(password, user['hash_password']):
status = "error"
message = "password invalid"
return jsonify(status=status, messag=message)
status = "success"
message = "login ok"
redis = StrictRedis()
#token = generate_auth_token(mail=username)
payload = {
"mail": username,
"iss": app.config['JWT_ISSUER'],
'aud': app.config['JWT_AUDIENCES']
}
token = jwt.encode(payload, key=app.config['JWT_SECRET'])
print token
redis = StrictRedis()
redis.set(username, token)
token = redis.get(username)
# detoken = jwt.decode(token, key=app.config['JWT_SECRET'])
# print "decode token"
# print detoken
out = jsonify(status=status, messag=message, token=token)
out.set_cookie('jwttoken', token)
#return jsonify(status=status,messag=message, token=token)
return out
class RedisCache:
def __init__(self, expires=timedelta(hours=10), encoding='utf-8'):
self.client = StrictRedis(host=os.environ.get('REDIS_HOST',
'localhost'),
port=6379,
db=0)
self.expires = expires
self.encoding = encoding
def __getitem__(self, url):
record = self.client.get(url)
if record:
return json.loads(record.decode(self.encoding))
else:
raise KeyError(url + ' does not exist')
def __setitem__(self, url, result):
data = bytes(json.dumps(result), self.encoding)
self.client.setex(url, self.expires, data)
class RedisBrain(object):
def __init__(self):
try:
self.redis = StrictRedis(host=REDIS_URL)
except Exception as e:
logger.error(e)
self.redis = None
def set(self, key, value):
if self.redis:
self.redis.set(key, value)
return True
else:
return False
def get(self, key):
if self.redis:
return self.redis.get(key)
return None
def get_q(self, redis: StrictRedis):
""" Get a queue object with this ID for a given ``redis`` connection. """
if (self._buffer is None or self._serializer is None):
res = redis.get(self._key + OPEN_APPEND)
if res:
# Q is open, so apply the existing qid props to this obj.
qid = pickle.loads(res)
self._buffer = qid.buffer
self._serializer = qid.serializer
self.opened = qid.opened
if self._buffer is not None and self._serializer is not None:
return Q(redis,
self._key,
buffer=self._buffer,
serializer=self._serializer,
opened=self.opened)
return None
class RedisUse(object):
def __init__(self):
self.sr = StrictRedis(host='localhost',
port=6379,
decode_responses=True)
def insertTokenOpenid(self, token, openid):
res = self.sr.set(token, openid)
res_time = self.sr.expire(token, 7200)
return res
def getTokenOpenid(self, token):
res = self.sr.get(token)
return res
def insertOpenidData(self, openid, data):
res = self.sr.hmset(openid, data)
res_time = self.sr.expire(openid, 604800)
return res
def selectOpenidNature(self, openid):
res = self.sr.hkeys(openid)
return res
def getOpenidNature(self, openid, nature):
res = self.sr.hget(openid, nature)
return res
def getOpenidNatureAll(self, openid):
res = self.sr.hgetall(openid)
return res
def deleteOpenidNature(self, openid, keys):
res = self.sr.hdel(openid, keys)
return res
class Cache(object):
def __init__(self):
self.redis_cache = StrictRedis(host=CACHE_HOST)
self.ex = CACHE_TIMEOUT
def get(self, key, start=None, end=None):
""" Get an object from cache by its key
Args:
key (str): cache key
start (int): when querying a redis list,\
starting range of the list.
end (int): when querying a redis list, ending range of the list.
Returns:
str: cached object
"""
if not (start is None or end is None):
result = self.redis_cache.lrange(key, start=start, end=end)
else:
result = self.redis_cache.get(key)
return result
def set(self, key, value):
""" Set an object in cache by its key
Args:
key (str): cache key
value (str): object to store in cache
"""
return self.redis_cache.set(key, value, ex=self.ex)
def append(self, key, value):
""" Append to a redis list
Args:
key (str): cache key
value (str): object to store in cache
"""
return self.redis_cache.lpush(key, value)
class SearchEngine(object):
def __init__(self):
self.client = StrictRedis()
def search(self, content):
# 根据内容查询对应的数据 ID
names = self.client.zrange(name=content, start=0, end=-1, desc=True)
result = []
# 根据 ID 拿数据
for name in names:
value = self.client.get(name=name)
result.append(value)
return result
def add(self, name=None, value=""):
# 存储
self.client.set(name=name, value=value)
# 分词
self.analyze(name, value)
return True
# 分词
def analyze(self, name, value):
if type(value) == dict:
for v in value.values():
analyzed_list = jieba.cut(v, cut_all=False)
self.generate_analyze_score(name, analyzed_list)
elif type(value) == str:
analyzed_list = jieba.cut(value, cut_all=False)
self.generate_analyze_score(name, analyzed_list)
elif isinstance(value, Iterable):
for v in value:
analyzed_list = jieba.cut(v, cut_all=False)
self.generate_analyze_score(name, analyzed_list)
else:
raise Exception("数据格式错误")
# 对分词后的结果计算 score
def generate_analyze_score(self, name, analyzed_list):
for e in analyzed_list:
self.client.zincrby(name=e, value=name)
class RedisDB(DBSpec):
def __init__(self, port=12000, prefix=None, use_compression=True):
self._server = StrictRedis(port=port)
self._prefix = "" if prefix is None else prefix
self.index = 0
self._use_compression = use_compression and LZ4_ENABLED
if self._use_compression:
self._pack = compress
self._unpack = decompress
else:
self._pack = serialize
self._unpack = deserialize
@property
def num_trajectories(self) -> int:
num_trajectories = self._server.llen("trajectories") - 1
return num_trajectories
def push_trajectory(self, trajectory):
trajectory = self._pack(trajectory)
self._server.rpush("trajectories", trajectory)
def get_trajectory(self, index=None):
index = index if index is not None else self.index
trajectory = self._server.lindex("trajectories", index)
if trajectory is not None:
trajectory = self._unpack(trajectory)
self.index = index + 1
return trajectory
def dump_weights(self, weights, prefix):
weights = self._pack(weights)
self._server.set(f"{self._prefix}_{prefix}_weights", weights)
def load_weights(self, prefix):
weights = self._server.get(f"{self._prefix}_{prefix}_weights")
if weights is not None:
weights = self._unpack(weights)
return weights
def parse(self, response):
datas = response.xpath('//div[@class="info clear"]')
for data in datas:
data_dict = {
'selectData': [],
}
data_dict['title'] = data.xpath(
'./div[@class="title"]/a/text()').extract_first()
data_dict['url'] = data.xpath(
'./div[@class="title"]/a/@href').extract_first()
focus = data.xpath('./div[@class="followInfo"]/text()'
).extract_first().split('/')[0]
vide = data.xpath('./div[@class="followInfo"]/text()'
).extract_first().split('/')[1]
# area = data.xpath('.//div[@class="houseInfo"]/text()[2]').extract_first()
houseYear = data.xpath(
'.//div[@class="positionInfo"]/text()[2]').extract_first()
data_dict['buil_year'] = houseYear
data_dict['vide'] = int(re.findall(r'\d+', vide)[0])
data_dict['focus'] = int(re.findall(r'\d+', focus)[0])
yield scrapy.Request(url=data_dict['url'],
callback=self.parse_detail,
meta=data_dict)
con = StrictRedis(host='192.168.177.138', port=6379, db=1)
page = con.get("page")
page = int(page.decode())
if (page < 100):
time.sleep(1)
next_url = "https://zz.lianjia.com/ershoufang/pg{}/".format(page)
print("++++++++++++++++++++%s+++++++++++++++" % next_url)
page = page + 1
con.set("page", page)
yield scrapy.Request(url=next_url, callback=self.parse)
else:
con.set("num", 1)
class MispRedisConnector(object):
def __init__(self):
self.r = StrictRedis(unix_socket_path=redis_socket)
def search(self,
authkey,
values=None,
hash_values=None,
return_eid=False,
quiet=False):
if isinstance(values, list):
hash_values = [SHA256.new(v.lower()).hexdigest() for v in values]
elif values:
hash_values = [SHA256.new(values.lower()).hexdigest()]
elif not isinstance(hash_values, list):
hash_values = [hash_values]
if not hash_values:
raise Exception('No value to search.')
org = self.__get_org_by_auth(authkey)
if not org:
raise Exception('Invalid authkey')
if quiet:
return [(self.r.exists(h) or self.r.exists(org + ':' + h))
for h in hash_values]
uuid_by_hashes = [
self.r.smembers(h).union(self.r.smembers(org + ':' + h))
for h in hash_values
]
if not return_eid:
to_return = uuid_by_hashes
else:
to_return = []
for h in uuid_by_hashes:
to_return.append([self.r.hget('uuid_id', uuid) for uuid in h])
return to_return
def __get_org_by_auth(self, authkey):
return self.r.get(authkey)
def get_photo(mid, lvid, pid):
print("Get Photossss")
result = None
with open('tmp.jpg', 'wb+') as image:
# try to get from redis first
print("This is from get_photo pid = %s" % pid)
redis_client = StrictRedis(host='128.2.100.176', port=6379)
result = redis_client.get(pid)
if not result: # cache miss
print('cache missed for pid ' + pid)
else:
print("Cache hit for %s!!!" % pid)
image.write(base64.decodebytes(result))
# Response(image, content_type=result.headers['content-type'])
if result:
return send_file('tmp.jpg', mimetype="image/jpeg")
with open('tmp.jpg', 'wb+') as image:
# try to get from cassandra if not found in redis
#if mid == '1':
cluster = Cluster(["128.2.100.174"], port=9337)
#else:
#cluster = Cluster([mid], port = 9337)
db_session = cluster.connect('photostore')
query = db_session.prepare("SELECT * FROM photo WHERE pid=?")
try:
result = db_session.execute(query, [pid])
if not result:
abort(404)
else:
image.write(base64.b64decode(result[0].payload))
except IOError:
log.exception("Query error")
print("This is from Cassandra!!!\n")
value = {'payload': result[0].payload, 'mimetype': result[0].mimetype}
tempImage = open("tmp.jpg", "rb")
#tempImage = open(result[0].pid + "."+ result[0].mimetype.split('/')[1].replace('e',''), 'rb')
imagestr = base64.b64encode(tempImage.read())
redis_client.setex(result[0].pid, 20, imagestr)
tempImage.close()
return send_file('tmp.jpg', mimetype=value['mimetype'])
class Reader(BaseReader):
"""
Redis settings Reader
A simple redis getter
"""
_default_conf = {'host': 'localhost', 'port': 6379}
def __init__(self, conf):
super(Reader, self).__init__(conf)
self.redis = StrictRedis(host=self.conf['host'],
port=self.conf['port'])
def _get(self, key):
result = self.redis.get(key)
if isinstance(result, six.binary_type):
result = result.decode('utf-8')
return result
def _set(self, key, value):
self.redis.set(key, value)
class redis_session:
prefix = 'was:session_key:' # Redis key 앞에 넣을 값
server_ip = 'localhost' # Redis ip
port = 6379
def __init__(self):
self.db = StrictRedis(host=self.server_ip, port=self.port, db=0)
# 세션이 있으면 타임아웃 만큼 다시 연장해주고 없으면 False 있으면 사용자id 리턴
def open_session(self, session_key):
user_name = self.db.get(self.prefix + session_key)
return user_name
# 신규 세션 요청 시 세션 값을 만들어서 리턴
def save_session(self, user_name):
session_key = str(uuid4())
try:
self.db.set(self.prefix + session_key, user_name)
except:
print('error')
return session_key
def get_shops_list(etsy_api_key: str,
max_stores: int,
redis_store: StrictRedis,
page: int = 1) -> List[str]:
"""TODO: Implement cache invalidation."""
cache_key = f'etsy_store_list:page-{page}-num-{max_stores}'
cache_hit = redis_store.get(cache_key)
if cache_hit:
return json.loads(cache_hit)
api_base_url = 'https://openapi.etsy.com/v2'
query = {'api_key': etsy_api_key, 'limit': max_stores, page: page}
resp = requests.get(f'{api_base_url}/shops', params=query)
if resp.status_code != 200:
return []
stores = resp.json()['results']
shop_names = [store['shop_name'] for store in stores]
redis_store.set(cache_key, json.dumps(shop_names))
return shop_names
class RedisCache:
def __init__(self, client=None, expires=timedelta(days=30), encoding='utf-8', compress=True):
self.client = StrictRedis(host='localhost', port=6379, db=0) if client is None else client
self.expires = expires
self.encoding = encoding
self.compress = compress
def __getitem__(self, url):
record = self.client.get(url)
if record:
if self.compress:
record = zlib.decompress(record)
return json.loads(record.decode(self.encoding))
else:
raise KeyError(url + ' does not exist')
def __setitem__(self, url, result):
data = bytes(json.dumps(result), self.encoding)
if self.compress:
data = zlib.compress(data)
self.client.setex(url, self.expires, data)
class RedisCache:
def __init__(self, client = None, encoding = 'utf-8', db = 0, compress = True):
# if a client object is not passed then try
# connecting to redis at the default localhost port
self.client = StrictRedis(host = 'localhost', port = 6379, db = db) if client is None else client
# self.expires = expires
self.encoding = encoding
self.compress = compress
def __getitem__(self, url):
'''
Load value from Redis for the given URL
'''
record = self.client.get(url)
if record:
if self.compress:
record = zlib.decompress(record)
try:
rec = record.decode(self.encoding)
except UnicodeDecodeError:
rec = bytes(json.dumps({'html' : None, 'code' : 403}), self.encoding)
return json.loads(rec)
else:
raise KeyError(url + ' does not exist.')
def __setitem__(self, url, result):
'''
Save value in Redis for the given URL
'''
data = bytes(json.dumps(result), self.encoding, errors = 'ignore')
if self.compress:
data = zlib.compress(data)
self.client.set(url, data)
def __len__(self):
return self.client.dbsize()
def erase(self):
self.client.flushdb()
class GraphStorage(object):
def __init__(self, link_file='links.txt', host='localhost', port=6379):
self.db = StrictRedis(host, port)
self.link_file = link_file
def add_nodes(self, graphDict, nameDict):
pipe = self.db.pipeline()
with open(self.link_file) as link_file:
for _ in xrange(1):
for _ in xrange(570607):
node, links = self.create_link(next(link_file))
name = linecache.getline('titles.txt', node)
pipe.rpush('links-{0}'.format(node), *links)
pipe.append('name-{0}'.format(node), name)
pipe.append('id-{0}'.format(name), node)
pipe.execute()
def create_link(self, link):
start, targets = link.rstrip('\n').split(': ')
return int(start), map(int, targets.split())
def get(self, kind, value):
key = '{0}-{1}'.format(kind, value)
if kind in ('name', 'id'):
return self.db.get(key)
elif kind == 'links':
return self.db.lrange(key, 0, -1)
raise ValueError
def is_leaf(self, value):
return self.db.sismember('leaves', value)
def flush(self):
self.db.flushall()
def __len__(self):
return self.db.dbsize()
# 5706070
class RedisCache(AbstractCache, Loggable):
"""A cache backed by Redis
Use as a dictionary,
>>> cache = RedisCache(host="localhost", port=6379)
>>> cache['hello'] = 'world'
>>> cache['hello'] # 'world'
>>> 'hello' in cache # True
>>> 'goodbye' in cache # False
or as a function memoizer,
>>> @cache.memoize
>>> def hello(name):
... return "Hello, " + name
Parameters
----------
same as `redis.StrictRedis`
"""
def __init__(self, *args, **kwargs):
AbstractCache.__init__(self, kwargs.get('timeout', datetime.timedelta(days=1)))
Loggable.__init__(self)
if 'timeout' in kwargs:
del kwargs['timeout']
self.redis = StrictRedis(*args, **kwargs)
def get(self, key):
# value will be None if key is missing, but this is ambiguous
value = self.redis.get(key)
if not self.redis.exists(key):
raise KeyError()
else:
return pickle.loads(value)
def set(self, key, value, timeout=None):
self.redis.set(key, pickle.dumps(value))
self.redis.expire(key, datetime.timedelta(seconds=timeout) or self.timeout)
def login_fun():
data = request.json
response = make_response()
response.content_type = 'application/json'
status = login_sql(data['username'], data['password'])
if status == 1:
# 获取SECRET_KEY
redis = StrictRedis(host='localhost',
port=6379,
db=0,
password='******')
secret_key = redis.get('SECRET_KEY')
expiration = 3600
s = Serializer(secret_key, expires_in=expiration) # expiration是过期时间
token = s.dumps({'username': data['username']})
token = str(token, 'utf-8')
redis.set(data['username'], token)
redis.expire(data['username'], 3600)
post_data = {
'info': '登录成功',
'token': token,
'username': data['username']
}
response = make_response(json.dumps(post_data))
response.content_type = 'application/json'
response.status_code = 200
return response
elif status == 0:
post_data = {'info': '密码错误'}
response = make_response(json.dumps(post_data))
response.content_type = 'application/json'
response.status_code = 401
return response
else:
post_data = {"info": "此用户不存在"}
response = make_response(json.dumps(post_data))
response.content_type = 'application/json'
response.status_code = 403
return response
class ExpiringDatasetCache(object):
"""
Cache with expiring values to keep track of recently created replicas.
"""
def __init__(self, redis_host, redis_port, timeout=1, prefix='expiring_did_cache'):
self._redis = StrictRedis(host=redis_host, port=redis_port)
self._prefix = prefix + '_' + str(uuid4()).split('-')[0]
self._timeout = timeout
def add_dataset(self, dataset):
""" Adds a datasets to cache with lifetime """
key = ':'.join((self._prefix, dataset))
self._redis.set(key, 1)
self._redis.expire(key, self._timeout)
def check_dataset(self, dataset):
""" Checks if dataset is still in cache """
key = ':'.join((self._prefix, dataset))
if self._redis.get(key) is None:
return False
return True
