class RedisManager(NoSqlManager):
def __init__(self,
namespace,
url=None,
data_dir=None,
lock_dir=None,
**params):
self.db = params.pop('db', None)
self.dbpass = params.pop('password', None)
self.connection_pool = params.get('redis_connection_pool', None)
self.expires = params.get('expires', params.get('expiretime', None))
NoSqlManager.__init__(self,
namespace,
url=url,
data_dir=data_dir,
lock_dir=lock_dir,
**params)
def open_connection(self, host, port, **params):
if not self.connection_pool:
self.connection_pool = ConnectionPool(host=host, port=port, db=self.db,
password=self.dbpass)
self.db_conn = StrictRedis(connection_pool=self.connection_pool, **params)
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
# beaker.container.Value.set_value calls NamespaceManager.set_value
# however it (until version 1.6.4) never sets expiretime param.
#
# Checking "type(value) is tuple" is a compromise
# because Manager class can be instantiated outside container.py (See: session.py)
if (expiretime is None) and (type(value) is tuple):
expiretime = value[1]
# If the machinery above fails, then pickup the expires time from the
# init params.
if not expiretime and self.expires is not None:
expiretime = self.expires
# Set or setex, according to whether we got an expires time or not.
if expiretime:
self.db_conn.setex(key, expiretime, pickle.dumps(value, 2))
else:
self.db_conn.set(key, pickle.dumps(value, 2))
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:%s:%s' % (self.namespace, key.replace(' ', '\302\267'))
def _format_pool_key(self, host, port, db):
return '{0}:{1}:{2}'.format(host, port, self.db)
def do_remove(self):
self.db_conn.flush()
def keys(self):
return self.db_conn.keys('beaker:%s:*' % self.namespace)
class MispRedisConnector(object):
def __init__(self):
self.r = StrictRedis(unix_socket_path=redis_socket)
def search(self, authkey, values=None, hash_values=None, return_eid=False, quiet=False):
if isinstance(values, list):
hash_values = [SHA256.new(v.lower()).hexdigest() for v in values]
elif values:
hash_values = [SHA256.new(values.lower()).hexdigest()]
elif not isinstance(hash_values, list):
hash_values = [hash_values]
if not hash_values:
raise Exception('No value to search.')
org = self.__get_org_by_auth(authkey)
if not org:
raise Exception('Invalid authkey')
if quiet:
return [(self.r.exists(h) or self.r.exists(org + ':' + h)) for h in hash_values]
uuid_by_hashes = [self.r.smembers(h).union(self.r.smembers(org + ':' + h)) for h in hash_values]
if not return_eid:
to_return = uuid_by_hashes
else:
to_return = []
for h in uuid_by_hashes:
to_return.append([self.r.hget('uuid_id', uuid) for uuid in h])
return to_return
def __get_org_by_auth(self, authkey):
return self.r.get(authkey)
class RedisProducer(object):
def __init__(self, hostname = 'localhost', port = 6379):
log.debug("Initializing RedisProducer with hostname of %s and port %s" % (hostname, port))
self.r = StrictRedis(host = hostname, port = port)
def send(self, message):
tries = 0
next_index_key = get_next_index_for_topic_key(message.topic)
next_index = 1
result = None
log.debug("Sending message on topic %s" % message.topic)
while result is None and tries < TRIES_LIMIT:
if self.r.exists(next_index_key):
next_index = long(self.r.get(next_index_key)) + 1
message_key = get_message_key(message.topic, next_index)
try:
pl = self.r.pipeline()
pl.watch(next_index_key, message_key)
pl.multi()
pl.incr(next_index_key).set(message_key, message.payload)
result = pl.execute()
except WatchError:
# Should probably log something here, but all it means is we're
# retrying
pass
if result is None:
log.error("Could not send message, retry amount exceeded")
raise RuntimeError("Attempted to send message %s times and failed" % TRIES_LIMIT)
class RedisManager(NoSqlManager):
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
self.connection_pool = params.pop('connection_pool', None)
NoSqlManager.__init__(self, namespace, url=url, data_dir=data_dir, lock_dir=lock_dir, **params)
def open_connection(self, host, port, **params):
self.db_conn = StrictRedis(host=host, port=int(port), connection_pool=self.connection_pool, **params)
def __contains__(self, key):
log.debug('%s contained in redis cache (as %s) : %s'%(key, self._format_key(key), self.db_conn.exists(self._format_key(key))))
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
#XXX: beaker.container.Value.set_value calls NamespaceManager.set_value
# however it(until version 1.6.3) never sets expiretime param. Why?
if expiretime:
self.db_conn.setex(key, expiretime, pickle.dumps(value))
else:
self.db_conn.set(key, pickle.dumps(value))
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:%s:%s' % (self.namespace, key.replace(' ', '\302\267'))
def do_remove(self):
self.db_conn.flush()
def keys(self):
return self.db_conn.keys('beaker:%s:*' % self.namespace)
class DbRedis(Db):
def __init__(self):
super().__init__()
self.redis = StrictRedis(**dbSettings)
def generateKey(self):
return self.redis.incr("id")
def store(self, key, objSerial):
self.redis.setex(key, TTL, objSerial)
def retrieve(self, key):
return self.redis.get(key)
def lenList(self, name):
return self.redis.llen(name)
def listAppend(self, name, val):
self.redis.lpush(name, val)
def listPopLeft(self, name):
return self.redis.lpop(name)
def retrieveList(self, name):
return self.redis.lrange(name, 0, -1)
def removeFromList(self, name, item):
self.redis.lrem(name, item, 0)
def keyExists(self, key):
return self.redis.exists(key)
class RedisDataStore(DataStore):
"""Redis-backed datastore object."""
def __init__(self, number=0):
redis_host = os.environ.get('REDIS_PORT_6379_TCP_ADDR')
redis_port = os.environ.get('REDIS_PORT_6379_TCP_PORT')
self.redis_conn = StrictRedis(host=redis_host, port=redis_port,
db=number)
def __setitem__(self, k, v):
self.redis_conn.set(k, v)
def __getitem__(self, k):
return self.redis_conn.get(k)
def __delitem__(self, k):
self.redis_conn.delete(k)
def get(self, k):
return self.redis_conn.get(k)
def __contains__(self, k):
return self.redis_conn.exists(k)
def todict(self):
#TODO(tvoran): use paginate
#TODO(tvoran): do something besides multiple gets
data = {}
for key in self.redis_conn.keys():
data[key] = self.get(key)
return data
def clear_all(self):
self.redis_conn.flushdb()
def sync_get(self, identity, *args, **kwargs):
"""
For getting data from cache
:param identity: Unique Integer for the data
:param args: Args for the sync function. (Default: None)
"""
redis = StrictRedis(connection_pool=self.redis_pool)
key = key_generator(self.key, identity)
try:
if redis.exists(key):
data = self.get_func(redis.get(key))
else:
data = self.sync_func(identity, *args, **kwargs)
if self.expire:
self._setex(redis, key, self.set_func(data))
else:
redis.set(key, self.set_func(data))
if data is not None or data != "":
return data
return None
except RedisError as re:
self.log.error("[REDIS] %s", str(re))
data = self.sync_func(identity, args)
return data
finally:
del redis
class RedisManager(NoSqlManager):
def __init__(self,
namespace,
url=None,
data_dir=None,
lock_dir=None,
**params):
self.db = params.pop('db', None)
self.connection_pools = {}
NoSqlManager.__init__(self,
namespace,
url=url,
data_dir=data_dir,
lock_dir=lock_dir,
**params)
def open_connection(self, host, port, **params):
pool_key = self._format_pool_key(host, port, self.db)
if pool_key not in self.connection_pools:
self.connection_pools[pool_key] = ConnectionPool(host=host,
port=port,
db=self.db)
self.db_conn = StrictRedis(connection_pool=self.connection_pools[pool_key],
**params)
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
#
# beaker.container.Value.set_value calls NamespaceManager.set_value
# however it (until version 1.6.4) never sets expiretime param.
#
# Checking "type(value) is tuple" is a compromise
# because Manager class can be instantiated outside container.py (See: session.py)
#
if (expiretime is None) and (type(value) is tuple):
expiretime = value[1]
if expiretime:
self.db_conn.setex(key, expiretime, pickle.dumps(value, 2))
else:
self.db_conn.set(key, pickle.dumps(value, 2))
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:%s:%s' % (self.namespace, key.replace(' ', '\302\267'))
def _format_pool_key(self, host, port, db):
return '{0}:{1}:{2}'.format(host, port, self.db)
def do_remove(self):
self.db_conn.flush()
def keys(self):
return self.db_conn.keys('beaker:%s:*' % self.namespace)
def new_group(request):
groupname = request.POST.get("groupname")
totwords = request.POST.get("totwords")
totmembers = request.POST.get('totmembers')
pref = settings.MY_PREFIX
prefg = pref+":"+groupname
user = str(request.user)
rd = StrictRedis()
# the above statements are self explanatory
exists = rd.exists(pref+":"+groupname)
if exists:
# return error, can't create the group with that name
# don't do that, just add this user to the already existing group
# if group size < totmembers
d = rd.hgetall(prefg+":hash")
response = {'facility':""}
if int(d['totmembers'])>int(d['curmembers']):
rd.hincrby(prefg+":hash", 'curmembers')
#d = rd.hgetall(prefg+":hash")
response['facility'] = groupname
response['new_group_created'] = False
rd.sadd(prefg, user)
#now notify the others that this user joined the group!
redis_publisher = RedisPublisher(facility = pref, broadcast = True)
mydict = {"type":"new_join", 'who':user, 'name':groupname}
msgstring = json.dumps(mydict)
print "broadcast message:"+msgstring
message = RedisMessage(msgstring)
redis_publisher.publish_message(message)
# now check if the competition is ready to start
if int(d['totmembers'])-1 == int(d['curmembers']):
start_competition(request,groupname)
return JsonResponse(response)
# so the group doesn't exist
rd.sadd(prefg, user)
# add this user to the set of all the users that are part of this group
hashdict = {'totwords':totwords, 'totmembers':totmembers, "owner":user, "name":groupname, 'curmembers':1}
# adding group name is redundant but it simplifies things a lot
rd.hmset(prefg+":hash", hashdict)
# using hash greatly simplifies the things(dict interconversion hgetall()), though i feel that the
# naming convention used is horrible
redis_publisher = RedisPublisher(facility = pref, broadcast = True)
mydict = {"type":"new_group"}
mydict.update(hashdict)
msgstring = json.dumps(mydict)
print msgstring
message = RedisMessage(msgstring)
redis_publisher.publish_message(message)
# notify the others about this new group that was created
rd.sadd(pref+":groups", groupname)
return JsonResponse({'facility':groupname, 'new_group_created':True})
class RedisBackend(Backend):
def __init__(self, config):
super(RedisBackend, self).__init__(config)
self.redis = StrictRedis(host=config.get("host", "localhost"),
port=config.get("port", 6379),
db=config.get("db", 0))
self.namespace = config.get("namespace", "short:")
def furl(self, name):
return self.namespace + "url:" + name
def fvisits(self, name):
return self.namespace + "visits:" + name
def next_name(self):
name = None
while 1:
name = hashids.encrypt(
self.redis.incr(self.namespace + "meta:num"))
if not self.exists(name):
break
return name
def exists(self, name):
return self.redis.exists(self.furl(name))
def set(self, link):
if self.redis.exists(self.furl(link.name)):
raise NameUnavailableError(link.name)
self.redis.set(self.furl(link.name), link.url)
self.redis.set(self.fvisits(link.name), 0)
def get(self, name):
rawlink = self.redis.get(self.furl(name))
if not rawlink:
raise NotFoundError(name)
link = Link(name=name,
url=rawlink.decode("utf-8"),
visits=int(
self.redis.get(self.fvisits(name)) or 0
))
return link
def visit(self, name):
self.redis.incr(self.fvisits(name))
class RedisConsumer(object):
def __init__(self, timeout, group_id, hostname = 'localhost', port = 6379):
self.group_id = group_id
self.timeout = timeout
log.debug("Initializing RedisConsumer with hostname of %s and port %s" % (hostname, port))
self.r = StrictRedis(host = hostname, port = port)
def poll(self, topic):
result = None
current_index_key = get_next_index_for_topic_key(topic)
end_millis = time.time() * 1000 + self.timeout
log.debug("Polling topic %s" % topic)
while time.time() * 1000 < end_millis:
if self.r.exists(current_index_key):
current_index = long(self.r.get(current_index_key))
consumer_index_key = get_next_index_for_group_id_key(topic, self.group_id)
pl = self.r.pipeline()
pl.watch(consumer_index_key)
consumer_index = 0
if self.r.exists(consumer_index_key):
consumer_index = long(self.r.get(consumer_index_key))
if current_index > consumer_index:
try:
pl.multi()
pl.incr(consumer_index_key)
incr_result = pl.execute()
if not incr_result is None and len(incr_result) > 0:
consumer_index = long(incr_result[0])
key = get_message_key(topic, consumer_index)
if self.r.exists(key):
result = self.r.get(key)
break
except WatchError:
log.debug("Redis keys changed for topic %s and group %s, trying again" % (topic, self.group_id))
pass
return result
def unsubscribe_from_topic(self, topic):
self.r.delete(get_next_index_for_group_id_key(topic, self.group_id))
class RedisManager(NoSqlManager):
connection_pools = {}
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
self.db = params.pop("db", None)
self.dbpass = params.pop("password", None)
NoSqlManager.__init__(self, namespace, url=url, data_dir=data_dir, lock_dir=lock_dir, **params)
def open_connection(self, host, port, **params):
pool_key = self._format_pool_key(host, port, self.db)
if pool_key not in self.connection_pools:
self.connection_pools[pool_key] = ConnectionPool(host=host, port=port, db=self.db, password=self.dbpass)
self.db_conn = StrictRedis(connection_pool=self.connection_pools[pool_key], **params)
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
#
# beaker.container.Value.set_value calls NamespaceManager.set_value
# however it (until version 1.6.4) never sets expiretime param.
#
# Checking "type(value) is tuple" is a compromise
# because Manager class can be instantiated outside container.py (See: session.py)
#
if (expiretime is None) and (type(value) is tuple):
expiretime = value[1]
if self.serializer == "json":
serialized_value = json.dumps(value, ensure_ascii=True)
else:
serialized_value = pickle.dumps(value, 2)
if expiretime:
self.db_conn.setex(key, expiretime, serialized_value)
else:
self.db_conn.set(key, serialized_value)
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return "beaker:%s:%s" % (self.namespace, key.replace(" ", "\302\267"))
def _format_pool_key(self, host, port, db):
return "{0}:{1}:{2}".format(host, port, self.db)
def do_remove(self):
for key in self.keys():
self.db_conn.delete(key)
def keys(self):
return self.db_conn.keys("beaker:%s:*" % self.namespace)
class RedisDB(object):
def __init__(self):
if not hasattr(RedisDB, 'pool'):
RedisDB.createPool()
self.r = StrictRedis(connection_pool = RedisDB.pool)
@staticmethod
def createPool():
RedisDB.pool = redis.ConnectionPool(
host = '127.0.0.1',
port = 6379,
db = 0
)
def saveToRedis(self, receiver, to_station_ab, to_station_name, from_station_ab, from_station_name, querydate, purpose_code, noticetime, publishtime):
'''将需要抓取的信息存入redis'''
uid = self.r.incr('uid')
tickets_info = {'uid':uid, 'receiver':receiver, 'to_station_ab':to_station_ab, 'to_station_name':to_station_name, 'from_station_ab':from_station_ab,'from_station_name':from_station_name, 'querydate':querydate, 'purpose_code':purpose_code, 'noticetime':noticetime, 'publishtime': publishtime}
self.r.zadd('email_que_set_all', uid, str(tickets_info))
if noticetime == '9am':
self.r.zadd('email_que_set_9am', uid, str(tickets_info))
elif noticetime == '11am':
self.r.zadd('email_que_set_11am', uid, str(tickets_info))
elif noticetime == '3pm':
self.r.zadd('email_que_set_3pm', uid, str(tickets_info))
elif noticetime == '5pm':
self.r.zadd('email_que_set_5pm', uid, str(tickets_info))
self.r.save()
def getStation(self, set, name):
return self.r.hget(set, name)
def zrevrange(self, set, begin, end):
return self.r.zrevrange(set, begin, end)
def zremrangebyscore(self, queue, uid):
return self.r.zremrangebyscore(queue, uid, uid)
def station_validate(self, form, field):
'''
表单tostation和tostation验证函数
'''
if not self.r.getStation(field.data):
raise ValidationError(u'木有这个站')
def saveJSONToSet(self, setName, json):
if not self.r.exists(setName):
for i, name in enumerate(json):
self.r.hset(setName, name, json[name])
print 'insert'+name
self.r.save()
else:
pass
class WatchmanBlacklist(object):
def __init__(self, config_xml = None):
if config_xml is None:
config_xml = ET.parse(CONFIG_FILE)
#elif not isinstance(config_xml,ET.Element) and not isinstance(config_xml,ET.ElementTree):
# raise TypeError("config_xml must be either None or an ElementTree element")
try:
password = config_xml.find('/global/password').text
except StandardError as e:
password = ""
try:
redishost = config_xml.find('/global/redis').text
except StandardError as e:
redishost = "localhost"
try:
expire = config_xml.find('/global/expire').text
self.expire = int(expire)
except StandardError as e:
logging.warning("No <expire> setting in the <global> section of config. Defaulting to 360s.")
self.expire = 360
try:
dbnum = config_xml.find('/global/blacklistdb').text
self._dbnum = int(dbnum)
except StandardError as e:
logging.warning("No blacklistdb setting in the <global> section of config. Defaulting to Redis database 2.")
dbnum = 2
self._conn = StrictRedis(host=redishost, password=password, db=dbnum)
def get(self,filepath,update=True,value="(locked)"):
"""
Check if the given path is in the blacklist, and optionally update the lock whether or not it exists
:param filepath: file path to check
:param update: if True, then add the filepath to the blacklist and reset the expiry counter - even if it already exists.
:param value: value to store against the file path (typically the mtime)
:return: value of the key or None
"""
rtn = self._conn.get(filepath)
#if update:
# self._conn.setnx(filepath, value)
# self._conn.expire(filepath, self.expire)
if not self._conn.exists(filepath):
logging.debug("{0} does not exist in the blacklist. Attempting to add it.".format(filepath))
self._conn.setnx(filepath, value)
self._conn.expire(filepath, self.expire)
return rtn
def factory(request):
# note: will raise ConnectionError if connection is not established
redis = getattr(request.registry, '_redis_sessions', None)
if redis is None: # pragma no cover
redis = StrictRedis(host=host, port=port, db=db, password=password,
socket_timeout=socket_timeout,
connection_pool=connection_pool, charset=charset,
errors=errors, unix_socket_path=unix_socket_path)
setattr(request.registry, '_redis_sessions', redis)
def add_cookie(session_id):
if not cookie_on_exception:
exc = getattr(request, 'exception', None)
if exc is None: # don't set cookie during exceptions
return
def set_cookie_callback(request, response):
cookieval = sign_session_id(session_id, secret)
response.set_cookie(
cookie_name,
value = cookieval,
max_age = cookie_max_age,
domain = cookie_domain,
secure = cookie_secure,
httponly = cookie_httponly,
)
request.add_response_callback(set_cookie_callback)
return
def delete_cookie():
def set_cookie_callback(request, response):
response.delete_cookie(cookie_name)
request.add_response_callback(set_cookie_callback)
return
cookieval = request.cookies.get(cookie_name)
session_id = None
if cookieval is not None:
try:
session_id = unsign_session_id(cookieval, secret)
except ValueError:
pass
if session_id and redis.exists(session_id):
session = RedisSession(redis, session_id, timeout, delete_cookie)
else:
new_id = new_session_id(redis, timeout)
add_cookie(new_id)
session = RedisSession(redis, new_id, timeout, delete_cookie)
session._v_new = True
return session
class CachedIDTClient(IDTClient):
def __init__(self, REDIS_HOST='localhost', REDIS_PORT=6379, REDIS_DB=0):
super(CachedIDTClient, self).__init__()
self.r = StrictRedis(REDIS_HOST, REDIS_PORT, REDIS_DB)
def get_info(self, sequence, oligo=2, na=40, mg=2, dntp=0.2):
if not self.r.exists(sequence):
tm = super(CachedIDTClient, self).get_melting_temp(sequence, oligo, na, mg, dntp)
dimer = super(CachedIDTClient, self).self_dimer_check(sequence)
info = {'tm': tm.tm, 'dimer': dimer.self_dimer, 'compPercent': dimer.compPercent}
self.r.hmset(sequence, info)
else:
info = self.r.hgetall(sequence)
return info
class RedisManager(NoSqlManager):
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
self.expiretime = params.pop('expiretime', None)
NoSqlManager.__init__(self, namespace, url=url, data_dir=data_dir, lock_dir=lock_dir, **params)
def open_connection(self, host, port, **params):
self.db_conn = StrictRedis(host=host, port=int(port), **params)
def __getitem__(self, key):
return pickle.loads(self.db_conn.hget(self._format_key(key), 'data'))
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
#
# beaker.container.Value.set_value calls NamespaceManager.set_value
# however it (until version 1.6.4) never sets expiretime param.
#
# Checking "type(value) is tuple" is a compromise
# because Manager class can be instantiated outside container.py (See: session.py)
#
if (expiretime is None) and (type(value) is tuple):
expiretime = value[1]
self.db_conn.hset(key, 'data', pickle.dumps(value))
self.db_conn.hset(key, 'accessed', datetime.now())
self.db_conn.hsetnx(key, 'created', datetime.now())
if expiretime or self.expiretime:
self.db_conn.expire(key, expiretime or self.expiretime)
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:%s:%s' % (self.namespace, key.replace(' ', '\302\267'))
def do_remove(self):
self.db_conn.flushdb()
def keys(self):
return self.db_conn.keys('beaker:%s:*' % self.namespace)
def ginfo(request):
print(str(request.user)+' requested groupinfo')
rd = StrictRedis()
pref = settings.MY_PREFIX
groupinfo = []
response = {}
if rd.exists(pref+":groups"):
# there is atleast 1 group already created
groupnames = rd.smembers(pref+":groups")
# groupnames is a set
for groupname in groupnames:
groupdict = rd.hgetall(pref+":"+groupname+":hash")
groupinfo.append(groupdict)
#print groupdict
# time to serialize!
response['group_list'] = groupinfo
response.update( {"success":True, "group_count":len(groupinfo)})
#print response
return JsonResponse(response)
def index(lang, degree, loc):
r = StrictRedis(host='localhost', port=6379, db=0)
hash_sum = "h" + md5("{0}/{1}/{2}".format(lang, degree, loc)).hexdigest()
if r.exists(hash_sum):
return r.get(hash_sum)
weather, weather_dict = weather_data(lang, degree, loc)
temp = temp_now = float(weather_dict["weather_now"]["temp"])
if degree == "F":
temp_now = (temp_now - 32)*5.0/9
result = dumps({
"weather": weather,
"links": links(lang),
"surfaces": surfaces(lang, temp_now),
"feedback": {"current_temp": str(int(temp)),
"sensation_desc": vote_sensation_list(lang),
"list_surfaces": all_surface(lang)}})
r.set(hash_sum, result)
r.expire(hash_sum, 600)
return result
class RedisCache(AbstractCache, Loggable):
"""A cache backed by Redis
Use as a dictionary,
>>> cache = RedisCache(host="localhost", port=6379)
>>> cache['hello'] = 'world'
>>> cache['hello'] # 'world'
>>> 'hello' in cache # True
>>> 'goodbye' in cache # False
or as a function memoizer,
>>> @cache.memoize
>>> def hello(name):
... return "Hello, " + name
Parameters
----------
same as `redis.StrictRedis`
"""
def __init__(self, *args, **kwargs):
AbstractCache.__init__(self, kwargs.get('timeout', datetime.timedelta(days=1)))
Loggable.__init__(self)
if 'timeout' in kwargs:
del kwargs['timeout']
self.redis = StrictRedis(*args, **kwargs)
def get(self, key):
# value will be None if key is missing, but this is ambiguous
value = self.redis.get(key)
if not self.redis.exists(key):
raise KeyError()
else:
return pickle.loads(value)
def set(self, key, value, timeout=None):
self.redis.set(key, pickle.dumps(value))
self.redis.expire(key, datetime.timedelta(seconds=timeout) or self.timeout)
class RedisManager(NoSqlManager):
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
self.db = params.pop('db', None)
NoSqlManager.__init__(self, namespace, url=url, data_dir=data_dir, lock_dir=lock_dir, **params)
def open_connection(self, host, port, **params):
self.db_conn = StrictRedis(host=host, port=int(port), db=self.db, **params)
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
#XXX: beaker.container.Value.set_value calls NamespaceManager.set_value
# however it(until version 1.6.3) never sets expiretime param. Why?
# Fortunately we can access expiretime through value.
# >>> value = list(storedtime, expire_argument, real_value)
if expiretime is None:
expiretime = value[1]
if expiretime:
self.db_conn.setex(key, expiretime, pickle.dumps(value))
else:
self.db_conn.set(key, pickle.dumps(value))
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:%s:%s' % (self.namespace, key.replace(' ', '\302\267'))
def do_remove(self):
self.db_conn.flush()
def keys(self):
return self.db_conn.keys('beaker:%s:*' % self.namespace)
redis = StrictRedis(connection_pool=pool)
# 或则
'''
redis://[:password]@host:port/db
rediss://[:password]@host:port/db
unix://[:password]@/path/to/socket.sock?db=db
'''
url = 'redis://:foobared@localhost:6379/0'
pool = ConnectionPool.from_url(url)
redis = StrictRedis(connection_pool=pool)
redis.set('name', 'Bob')
print(redis.get('name'))
"""键操作"""
# 判断是否存在
redis.exists('name')
# 删除
# redis.delete('name')
# 获取键的类型
redis.type('name')
# 正则匹配
redis.keys('n*')
# 随机一个键
redis.randomkey()
# 重命名
redis.rename('name', 'nickname')
# 获取键的数目
redis.dbsize()
# 设置键的过期时间 2秒
redis.expire('name', 2)
# 获取键的过期时间 -1表示永远不过期
def exists(cls, rc: StrictRedis, task_id: str):
"""Check if a given task_id exists in Redis"""
task_hname = cls._generate_hname(task_id)
return rc.exists(task_hname)
class Uptime(object):
jobs = Queue() # TODO: Are the consequences of this intended?
def __init__(self, config):
self.running = False
self.config = config
self.checks = []
self.pool = gevent.pool.Group()
self.config_path = 'uptime_config:'
self.results_path = 'uptime_results:' + self.config.source + ':'
self.stats_path = 'uptime_stats:'
if self.config.slack_url and self.config.slack_channel:
self.notifier = SlackNotifier(self.config.slack_url,
self.config.slack_channel)
else:
log.warn('No notifiers configured')
self.notifier = None
self.redis = StrictRedis(host=self.config.redis_host, port=self.config.redis_port)
if not self.redis.exists(self.stats_path):
self.redis.set(self.stats_path + 'total_checks', 0)
self.start()
def start(self):
self.running = True
workers = [gevent.spawn(self._check_worker) for _ in range(1, self.config.concurrency)]
workers.append(gevent.spawn(self._controller))
t = Thread(target=self._watcher)
t.daemon = True
t.start()
gevent.signal(signal.SIGQUIT, gevent.killall, workers)
gevent.signal(signal.SIGINT, gevent.killall, workers)
gevent.joinall(workers)
def _watcher(self):
"""
Worker to poll redis for key changes, updating accordingly
"""
while self.running:
configs = self._get_configs()
# add all checks
for c in configs:
self._add_check(c)
# cleanup removed checks
config_ids = [str(c['check_id']) for c in configs]
for c in self.checks:
if c.check_id not in config_ids:
self._remove_check(c.check_id)
sleep(5)
def _get_configs(self):
pattern = self.config_path + '*'
return [json.loads(self.redis.get(k).decode(self.config.encoding)) for
k in self.redis.keys(pattern)]
def _controller(self):
"""
Controller worker. Submits any overdue checks to queue.
"""
while self.running:
now = datetime.utcnow()
[self.jobs.put_nowait(c) for c in self.checks if
(now - c.last).seconds > c.interval]
gevent.sleep(0)
def _check_worker(self):
"""
Worker to perform url checks
"""
logging.info('[{}] worker started'.format(id(self)))
while self.running:
while not self.jobs.empty():
check = self.jobs.get()
log.info('checking %s' % check.url)
result = self._check_url(check.url, check.content)
self.redis.incr(self.stats_path + 'total_checks')
check.response_time = result['elapsed']
if result['ok']:
check.last = datetime.utcnow()
check.ok()
else:
check.failures += 1
if check.failures > 3 and not check.notified and self.notifier:
log.info('sending notification for failed check')
if self.notifier:
self.notifier.notify(
'url check failure for %s -- %s' %
(check.url, result['reason'])
)
check.notified = True
# after 10 failures, return to normal interval to prevent
# excessive checks
if check.failures > 10:
check.last = datetime.utcnow()
key = self.results_path + check.check_id
self.redis.set(key, check.dump_json())
gevent.sleep(0)
def _add_check(self, check):
"""
Internal method to add a check for the controller to schedule,
if it exists
params:
- check(dict): dictionary of check config
"""
check_id = check['check_id']
if check_id not in [str(c.check_id) for c in self.checks]:
self.checks.append(Check(**check))
def _remove_check(self, id):
# remove loaded check
[self.checks.remove(c) for c in self.checks if c.check_id == id]
# remove check from results in redis
key = self.results_path + id
self.redis.delete(key)
def _check_url(self, url, content, timeout=5):
try:
r = requests.get(url, timeout=timeout)
except ConnectionError as e:
log.warn('unable to reach %s:\n%s' % (url, e))
return {'ok': False, 'reason': e, 'elapsed': 0}
except Timeout as e:
log.warn('connection timed out checking %s:\n%s' % (url, e))
return {'ok': False, 'reason': e, 'elapsed': timeout}
log.debug('%s returned %s' % (url, r.status_code))
if not r.ok:
return {'ok': False,
'reason': r.status_code,
'elapsed': r.elapsed.total_seconds()}
if content and content not in r.text:
return {'ok': False,
'reason': 'content check failure',
'elapsed': r.elapsed.total_seconds()}
return {'ok': True, 'elapsed': r.elapsed.total_seconds()}
def new_group(request):
groupname = request.POST.get("groupname")
totwords = request.POST.get("totwords")
totmembers = request.POST.get('totmembers')
pref = settings.MY_PREFIX
prefg = pref + ":" + groupname
user = str(request.user)
rd = StrictRedis()
# the above statements are self explanatory
exists = rd.exists(pref + ":" + groupname)
if exists:
# return error, can't create the group with that name
# don't do that, just add this user to the already existing group
# if group size < totmembers
d = rd.hgetall(prefg + ":hash")
response = {'facility': ""}
if int(d['totmembers']) > int(d['curmembers']):
rd.hincrby(prefg + ":hash", 'curmembers')
#d = rd.hgetall(prefg+":hash")
response['facility'] = groupname
response['new_group_created'] = False
rd.sadd(prefg, user)
#now notify the others that this user joined the group!
redis_publisher = RedisPublisher(facility=pref, broadcast=True)
mydict = {"type": "new_join", 'who': user, 'name': groupname}
msgstring = json.dumps(mydict)
print "broadcast message:" + msgstring
message = RedisMessage(msgstring)
redis_publisher.publish_message(message)
# now check if the competition is ready to start
if int(d['totmembers']) - 1 == int(d['curmembers']):
start_competition(request, groupname)
return JsonResponse(response)
# so the group doesn't exist
rd.sadd(prefg, user)
# add this user to the set of all the users that are part of this group
hashdict = {
'totwords': totwords,
'totmembers': totmembers,
"owner": user,
"name": groupname,
'curmembers': 1
}
# adding group name is redundant but it simplifies things a lot
rd.hmset(prefg + ":hash", hashdict)
# using hash greatly simplifies the things(dict interconversion hgetall()), though i feel that the
# naming convention used is horrible
redis_publisher = RedisPublisher(facility=pref, broadcast=True)
mydict = {"type": "new_group"}
mydict.update(hashdict)
msgstring = json.dumps(mydict)
print msgstring
message = RedisMessage(msgstring)
redis_publisher.publish_message(message)
# notify the others about this new group that was created
rd.sadd(pref + ":groups", groupname)
return JsonResponse({'facility': groupname, 'new_group_created': True})
# @Time : 2020/1/16 9:16
# @Author : 结尾！！
# @FileName: 从redis中取出数据保存到Excel.py
# @Software: PyCharm
import json
import redis
import pandas as pd
from redis import StrictRedis
#方法一
redis_conn = StrictRedis(
host='192.168.31.104', port=6379, db=0,
decode_responses=True) #decode_responses=True解决出数据带b的问题
redis_len = redis_conn.llen('comments_info:items')
print(redis_len)
print(redis_conn.exists('USA_Detail:items'))
data = []
for each in redis_conn.lrange('USA_Detail:items', 0, redis_len):
print(each)
each = json.loads(each) # 将json对象转换为Python对象
data.append(each)
df1 = pd.DataFrame(data)
print(df1.shape)
df1.to_excel('./detail_page/fanny_pack.xlsx',
header=True,
index=None,
encoding='utf-8')
#方法二 以删除的形式
# redis_conn=redis.Redis(host='192.168.31.104',port=6379,db=0,) #decode_responses=True解决出数据带b的问题
# redis_len=redis_conn.llen('comments_info:items')
async def test_throttle_removes_garbage_token(redis: StrictRedis):
throttle = AsyncThrottle('test', 1, redis)
garbage_token = await throttle.wait(1.0)
assert redis.zscore('redis_gt:test', garbage_token)
await throttle.run(asyncio.sleep(0.01))
assert not redis.exists('redis_gt:test')
class SnapshotConnector(object):
def __init__(self):
self.r = StrictRedis(unix_socket_path=redis_socket, decode_responses=True)
# ##### Helpers web interface #####
def get_groups(self, groups=None):
if not groups:
grps = sorted(self.r.smembers('groups'))
else:
grps = sorted(groups)
return [(g, self.get_events(self.r.smembers(g))) for g in grps]
def get_events(self, events=None):
if events is None:
eids = sorted(self.r.smembers('events'), key=int, reverse=True)
else:
eids = sorted(events, key=int, reverse=True)
return [self.get_event_digest(eid) for eid in eids]
def hashes_eids(self, hashes):
eids = set()
for h in hashes:
eids.update(self.r.smembers('{}:eids'.format(h)))
return eids
def rebuild_eid_cache(self):
for sha256 in self.r.smembers('hashes_sha256'):
sha1, md5 = self.r.hmget(sha256, ['sha1', 'md5'])
eids = search('{} {} {}'.format(sha256, sha1, md5), 'value')
if eids:
all_eids = [e for e, f in eids.most_common()]
self.r.sadd('{}:eids'.format(sha256), *all_eids)
# ##### Values functions #####
def make_hashed_value(self, value):
'''
Hash the value to search
'''
return SHA256.new(value.strip().lower()).hexdigest()
def get_value_details(self, hashed_value):
'''
Returns all attributes of a value
'''
attributes_ids = self.r.smembers('{}:attrs'.format(hashed_value))
return [self.r.hgetall('attribute:{}'.format(attrid)) for attrid in attributes_ids]
def get_all_value_digest(self):
p = self.r.pipeline(False)
attrs = [self.r.hmget('attribute:{}'.format(attrid), 'event_id', 'value1', 'value2', 'comment')
for attrid in self.r.smembers('attributes')]
p.execute()
return attrs
def get_value_digest(self, hashed_value):
'''
Returns value1 & 2 and comment, deduplicate
'''
attrids = self.r.smembers('{}:attrs'.format(hashed_value))
digest = [self.r.hmget('attribute:{}'.format(aid), 'value1', 'value2', 'comment') for aid in attrids]
values = set()
comments = set()
for v1, v2, comment in digest:
values.add(v1)
if v2:
values.add(v2)
if comment:
comments.add(comment)
return values, comments
def get_events_digest_from_value(self, hashed_value):
'''
Returns digests of events the value is listed in.
'''
return [self.get_event_digest(eid) for eid in self.r.smembers(hashed_value)]
# ##### Keys functions #####
def key_values_digests(self, key):
'''
Returns value digests of all values in a key
'''
return [self.get_value_digest(hashed_value) for hashed_value in self.r.smembers(key)]
# ##### Event functions #####
def get_event_digest(self, eid):
'''
Returns info and date of the event
'''
to_return = {'eid': eid, 'tags': self.r.smembers('event:{}:tags'.format(eid))}
to_return.update(dict(zip(['info', 'date'], self.r.hmget('event:{}'.format(eid), 'info', 'date'))))
return to_return
def merge(self, events):
'''
Merge a list of events into one set.
The key of the set is <event1>|<event2>|<event3>|...
'''
events = sorted(events, key=int)
out_key = '|'.join(map(str, events))
if not self.r.exists(out_key):
p = self.r.pipeline(False)
p.sunionstore(out_key, *['event_vals:{}'.format(eid) for eid in events])
p.expire(out_key, 300)
p.execute()
return out_key
def intersection(self, events):
'''
Keeps only the values in *all* the sets
The key of the set is <event1>&<event2>&<event3>&...
'''
events = sorted(events, key=int)
out_key = '&'.join(map(str, events))
if not self.r.exists(out_key):
p = self.r.pipeline(False)
p.sinterstore(out_key, *['event_vals:{}'.format(eid) for eid in events])
p.expire(out_key, 300)
p.execute()
return out_key
def events_similarities(self, *events):
'''
Returns the intersection and the total amount of values in multiple events
'''
return self.r.scard(self.intersection(events)), self.r.scard(self.merge(events))
# ##### Group functions #####
def get_events_in_group(self, name):
return self.r.smembers(name)
def make_group(self, name, *events):
'''
Create a group of events
'''
if not self.r.exists(name):
self.r.sadd(name, *events)
self.r.sadd('groups', name)
else:
raise Exception('Group name already exists, maybe you want to update.')
def update_group(self, name, *events):
'''
Update a group of events
'''
self.r.sadd(name, *events)
self.r.sadd('groups', name)
def delete_all_groups(self):
for g in self.r.smembers('groups'):
self.del_group(g)
def del_group(self, name):
'''
Delete a group of events
'''
self.r.delete(name)
self.r.srem('groups', name)
def merge_groups(self, group_names):
'''
Merge groups of events
The key of the set is <group1>|<group2>|<group3>|...
'''
groups = sorted([self.merge(self.r.smembers(group_name)) for group_name in group_names])
out_key = '|'.join(groups)
if not self.r.exists(out_key):
p = self.r.pipeline(False)
p.sunionstore(out_key, *groups)
p.expire(out_key, 300)
p.execute()
return out_key
def intersection_groups(self, group_names):
'''
Keeps only the values in *all* the sets
The key of the set is <group1>&<group2>&<group3>&...
'''
if len(group_names) == 1:
return self.intersection(self.r.smembers(group_names[0]))
groups = sorted([self.merge(self.r.smembers(group_name)) for group_name in group_names])
out_key = '&'.join(groups)
if not self.r.exists(out_key):
p = self.r.pipeline(False)
p.sinterstore(out_key, *groups)
p.expire(out_key, 300)
p.execute()
return out_key
def groups_similarities(self, *group_names):
'''
Returns the intersection and the total amount of values in multiple groups
'''
return self.r.scard(self.intersection_groups(group_names)), self.r.scard(self.merge_groups(group_names))
class Cache:
"""A class that implements a most-recently-used cache algorithm."""
def __init__(self,
expire=600,
service_name='redis',
port=None,
server=None):
# TODO: Use the coordinates from the configuration server
if server is None:
service = configuration().service(service_name)
self.expire = expire
self.connection = StrictRedis()
def contains_key(self, key):
"""Tests if the key is fresh in the cache."""
try:
return self.connection.exists(self.__hash_key__(key))
except RedisError as error:
self.__log_cache_error__(error)
return False
def get(self, key):
"""Get the contents of a key from the cache. Can return None if not present."""
try:
return self.connection.get(self.__hash_key__(key))
except RedisError as error:
self.__log_cache_error__(error)
return None
def put(self, key, value):
"""Store a value in the cache with an appropriate timeout."""
serialized_value = json.dumps(value)
try:
self.connection.set(self.__hash_key__(key),
serialized_value,
ex=self.expire,
nx=True)
except RedisError as error:
self.__log_cache_error__(error)
def delete(self, key):
"""Delete a specified key from the cache. Returns if the key was deleted."""
try:
keys_deleted = self.connection.delete(self.__hash_key__(key))
return keys_deleted > 0
except RedisError as error:
self.__log_cache_error__(error)
return False
@staticmethod
def __hash_key__(key):
return hashlib.sha256(key.encode('utf-8')).hexdigest()
@staticmethod
def __log_cache_error__(error):
logging.warning(error)
# coding=utf-8
from redis import StrictRedis, ConnectionPool
# redis存储
# 正常连接
redis = StrictRedis(password='******')
redis.set('name', 'Bob')
print(redis.get('name'))
# 键操作
# 判断一个键是否存在
print(redis.exists('name'))
# 删除一个键
print(redis.delete('name'))
# 判断键类型
print(redis.type('name'))
# 获取所有符合规则的键
print(redis.keys('n*'))
# 获取随机的一个键
print(redis.randomkey())
# 重命名键
print(redis.rename('name', 'nickname'))
# 获取当前数据库中键的数目
print(redis.dbsize())
# 设置键的过期时间，单位为秒
print(redis.expire('name', 2))
# 获取键的过期时间，单位为秒，-1表示永久不过期
print(redis.ttl('name'))
# 将键移动到其他数据库，db:数据库代号
print(redis.move('name', 2))
# 删除当前选择数据库中的所有键
print(redis.flushdb())
class RedisOp:
def __init__(self, obj, data_base_config):
# if hasattr(obj, "Log") and obj.Log.get("output_console", True):
# self.log = Logger(level="info")
# else:
# self.log = None
self.log = obj.log if hasattr(obj, "log") else None
self.host = data_base_config["host"]
self.port = data_base_config["port"]
self.user = data_base_config["user"]
self.password = data_base_config["password"]
self.db = data_base_config["database"]
self.charset = data_base_config.get("charset", "UTF-8")
self.decode_responses = data_base_config.get("decode_responses", True)
self.pool = ConnectionPool(host=self.host,
port=self.port,
password=self.password,
db=self.db,
decode_responses=self.decode_responses)
# 获取连接
self.connection = StrictRedis(connection_pool=self.pool)
def close(self):
# 关闭连接池所有连接，PS：慎用
self.connection.connection_pool.disconnect()
"""
string 类型 redis 操作：{"key": "value"}
"""
@log_wrapper
def set(self, key, value, time=None):
"""
单条插入 key_value
:param key:
:param value:
:param time: 单位为秒
:return:
"""
if isinstance(value, dict):
value = json.dumps(value, ensure_ascii=False)
if time:
ret = self.connection.setex(key, time, value)
else:
ret = self.connection.set(key, value)
return ret
@log_wrapper
def setnx(self, key, value):
"""
key 不存在时 插入数据
:param key:
:param value:
:return:
"""
return self.connection.setnx(key, value)
@log_wrapper
def psetex(self, name, time_ms, value):
"""
插入含过期时间的 key_value
:param name:
:param time_ms: 单位为毫秒
:param value:
:return:
"""
return self.connection.psetex(name, time_ms, value)
@log_wrapper
def mset(self, key_value_dict):
"""
批量插入 key_value
:param key_value_dict:
:return:
"""
for key, value in key_value_dict.items():
if isinstance(value, dict):
key_value_dict[key] = json.dumps(value, ensure_ascii=False)
return self.connection.mset(key_value_dict)
@log_wrapper
def msetnx(self, key_value_dict):
"""
key 均不存在时才插入
:param key_value_dict:
:return:
"""
return self.connection.msetnx(key_value_dict)
@log_wrapper
def get(self, key):
"""
获取 key 的 value
:param key:
:return:
"""
return self.connection.get(key)
@log_wrapper
def mget(self, key_list):
"""
回多个 key 对应的 value
:param key_list: 格式为 列表
:return:
"""
return self.connection.mget(key_list)
@log_wrapper
def getset(self, key):
"""
给数据库中 key 赋予值 value 并返回上次的 value
:param key:
:return:
"""
return self.connection.getset(key)
@log_wrapper
def keys(self, key):
"""
获取所有符合规则的 key
:param key: eg: "n*"
:return:
"""
return self.connection.keys(key)
"""
redis key 操作
"""
@log_wrapper
def exists(self, key):
"""
判断 key 是否存在
:param key:
:return:
"""
return self.connection.exists(key)
@log_wrapper
def expire(self, key, time):
"""
设定key的过期时间，单位秒
:param key:
:param time: 单位秒
:return:
"""
return self.connection.expire(key, time)
@log_wrapper
def delete(self, key):
"""
删除一个 key
:param key:
:return:
"""
return self.connection.delete(key)
@log_wrapper
def mdelete(self, key_list):
"""
删除多个指定的 key
:param key_list:
:return:
"""
for key in key_list:
self.connection.delete(key)
"""
hash 类型 redis 操作：{"name":{"key": "value"}}
"""
# TODO hash 类型 redis 操作
"""
class RedisDatabase:
"""
docstring for RedisDatabase
"""
def __init__(self, expire_time = 1800):
self.expire_time = max(expire_time, 1800)
self.start_redis()
def is_redis_running(self):
try:
if sys.platform == 'win32':
process = len(os.popen('tasklist | findstr ' + "redis-server.exe").readlines())
if process >= 1:
return True
else:
return False
elif sys.platform == 'darwin':
# macOS
return True
else:
# other platform
return True
except Exception as e:
raise Exception('Unable to check redis running staate，error message: ' + str(e))
def start_redis(self, password=''):
try:
if not self.is_redis_running():
if sys.platform == 'win32':
os.system("e:/redis/redis-server --service-start")
elif sys.platform == 'darwin':
# macOS
pass
else:
pass
except Exception as e:
raise Exception('Unble to start redis, error message: ' + str(e))
try:
self.datadb = StrictRedis(host='localhost', port=6379, db=0)
self.cachedb = StrictRedis(host='localhost', port=6379, db=1)
self.hashdb = StrictRedis(host='localhost', port=6379, db=2)
except Exception as e:
raise Exception('Redis connection failed，error message:' + str(e))
def stop_redis(self):
try:
if self.is_redis_running():
# self.flushall()
if sys.platform == 'win32':
os.system("e:/redis/redis-server --service-stop")
elif sys.platform == 'darwin':
pass
else:
pass
except Exception as e:
raise Exception('Unble to stop redis，error message:' + str(e))
def set_value(self, obj, data_source, atlas, feature, window_length=0, step_size=0):
"""
Using a dictionary, a Mongdb object, a Net class, a Attr class, a DynamicNet class or a DynamicAttr class
to set a new entry in Redis.
"""
if type(obj) is dict:
key = self.generate_static_key(data_source, obj['scan'], atlas, feature, obj['comment'])
self.datadb.set(key, obj['value'], ex=self.expire_time)
return self.trans_netattr(obj['scan'], atlas, feature, pickle.loads(obj['value']))
elif type(obj) is list:
value = []
scan = obj[0]['scan']
comment = obj[0]['comment']
key_all = self.generate_dynamic_key(data_source, scan, atlas, feature, window_length, step_size, comment)
pipe = self.datadb.pipeline()
length = len(obj)
try:
pipe.multi()
pipe.set(key_all + ':0', length, ex=self.expire_time - 200)
for i in range(length): # 使用查询关键字保证升序
pipe.set(key_all + ':' + str(i + 1), (obj[i]['value']), ex=self.expire_time)
value.append(pickle.loads(obj[i]['value']))
pipe.execute()
except Exception as e:
raise Exception('An error occur when tring to set value in redis, error message: ' + str(e))
return self.trans_dynamic_netattr(scan, atlas, feature, window_length, step_size, np.array(value))
elif type(obj) is netattr.Net or type(obj) is netattr.Attr:
key = self.generate_static_key(data_source, obj.scan, obj.atlasobj.name, obj.feature_name, {})
self.datadb.set(key, pickle.dumps(obj.data))
elif type(obj) is netattr.DynamicNet or type(obj) is netattr.DynamicAttr:
key_all = self.generate_dynamic_key(data_source, obj.scan, obj.atlasobj.name, obj.feature_name, obj.window_length, obj.step_size, {})
length=obj.data.shape[2]
pipe = self.datadb.pipeline()
if type(obj) is netattr.DynamicNet:
flag = True
else:
flag = False
try:
pipe.multi()
pipe.set(key_all + ':0', length, ex=self.expire_time - 200)
for i in range(length): # 使用查询关键字保证升序
if flag:
pipe.set(key_all + ':' + str(i + 1), pickle.dumps(obj.data[:, :, i]), ex=self.expire_time)
else:
pipe.set(key_all + ':' + str(i + 1), obj.data[:, i], ex=self.expire_time)
pipe.execute()
except Exception as e:
raise Exception('An error occur when tring to set value in redis, error message: ' + str(e))
def generate_static_key(self, data_source, subject_scan, atlas_name, feature_name, comment):
key = data_source + ':' + subject_scan + ':' + atlas_name + ':' + feature_name + ':0'
if comment is not None:
key += ':' + str(comment)
return key
def generate_dynamic_key(self, data_source, subject_scan, atlas_name, feature_name, window_length, step_size, comment):
key = data_source + ':' + subject_scan + ':' + atlas_name + ':' + feature_name +':1:'+ str(window_length) + ':' + str(step_size)
if comment is not None:
key += ':' + str(comment)
return key
def get_static_value(self, data_source, subject_scan, atlas_name, feature_name, comment = {}):
"""
Using data source, scan name, altasobj name, feature name to query static networks and attributes from Redis.
If the query succeeds, return a Net or Attr class, if not, return none.
"""
key = self.generate_static_key(data_source, subject_scan, atlas_name, feature_name, comment)
res = self.datadb.get(key)
self.datadb.expire(key, self.expire_time)
if res is not None:
return self.trans_netattr(subject_scan, atlas_name, feature_name, pickle.loads(res))
else:
return None
def trans_netattr(self,subject_scan, atlas_name, feature_name, value):
if value.ndim == 1: # 这里要改一下
arr = netattr.Attr(value, atlas.get(atlas_name),subject_scan, feature_name)
return arr
else:
net = netattr.Net(value, atlas.get(atlas_name), subject_scan, feature_name)
return net
def get_dynamic_value(self, data_source, subject_scan, atlas_name, feature_name, window_length, step_size, comment = {}):
"""
Using data source, scan name, altasobj name, feature name, window length, step size to query dynamic
networks and attributes from Redis.
If the query succeeds, return a DynamicNet or DynamicAttr class, if not, return none.
"""
key_all = self.generate_dynamic_key(data_source, subject_scan, atlas_name, feature_name, window_length, step_size, comment)
if self.datadb.exists(key_all + ':0'):
pipe = self.datadb.pipeline()
try:
pipe.multi()
length = int(self.datadb.get(key_all + ':0').decode())
for i in range(1,length + 1,1):
pipe.get(key_all + ':' + str(i))
res = pipe.execute()
except Exception as e:
raise Exception('An error occur when tring to get value in redis, error message: ' + str(e))
try:
pipe.multi()
value = []
for i in range(length):
value.append(pickle.loads(res[i]))
pipe.expire(key_all + ':' + str(i+1), self.expire_time)
pipe.expire(key_all + ':0', self.expire_time - 200)
pipe.execute()
except Exception as e:
raise Exception('An error occur when tring to update expiration time in redis, error message: ' + str(e))
return self.trans_dynamic_netattr(subject_scan, atlas_name, feature_name, window_length, step_size, np.array(value))
else:
return None
def trans_dynamic_netattr(self, subject_scan, atlas_name, feature_name, window_length, step_size, value):
if value.ndim == 2: # 这里要改一下
arr = netattr.DynamicAttr(value.swapaxes(0,1), atlas.get(atlas_name), window_length, step_size, subject_scan, feature_name)
return arr
else:
net = netattr.DynamicNet(value.swapaxes(0,2).swapaxes(0,1), atlas.get(atlas_name), window_length, step_size, subject_scan, feature_name)
return net
def exists_key(self,data_source, subject_scan, atlas_name, feature_name, isdynamic = False, window_length = 0, step_size = 0, comment ={}):
"""
Using data source, scan name, atlas name, feature name to check the existence of an static entry in Redis.
You can add isdynamic(True), window length, step size to check the existence of an dynamic entry in Redis.
"""
if isdynamic is False:
return self.datadb.exists(self.generate_static_key(data_source, subject_scan, atlas_name, feature_name,comment))
else:
return self.datadb.exists(self.generate_dynamic_key(data_source, subject_scan, atlas_name, feature_name, window_length, step_size, comment) + ':0')
"""
Redis supports storing and querying list as cache.
Note: the items in list must be int or float.
"""
def set_list_all_cache(self,key,value):
"""
Store a list to Redis as cache with cache_key.
Note: please check the existence of the cache_key, or it will cover the origin entry.
"""
self.cachedb.delete(key)
for i in value:
self.cachedb.rpush(key, pickle.dumps(i))
#self.cachedb.save()
return self.cachedb.llen(key)
def set_list_cache(self,key,value):
"""
Append value to a list as the last one in Redis with cache_key.
If the given key is empty in Redis, a new list will be created.
"""
self.cachedb.rpush(key, pickle.dumps(value))
#self.cachedb.save()
return self.cachedb.llen(key)
def get_list_cache(self, key, start = 0, end = -1):
"""
Return a list with given cache_key in Redis.
"""
res = self.cachedb.lrange(key, start, end)
lst=[]
for x in res:
lst.append(pickle.loads(x))
return lst
def exists_key_cache(self, key):
"""
Check the existence of a list in Redis by cache_key.
"""
return self.cachedb.exists(key)
def delete_key_cache(self, key):
"""
Delete an entry in Redis by cache_key.
If the given key is empty in Redis, do nothing.
"""
value = self.cachedb.delete(key)
#self.cachedb.save()
return value
def clear_cache(self):
"""
Delete all the entries in Redis.
"""
self.cachedb.flushdb()
"""
Redis supports storing and querying hash.
Note: the keys in hash must be string.
"""
def set_hash_all(self,name,hash):
"""
Store a hash to Redis with hash_name and a hash.
Note: please check the existence of the hash_name, or it will cover the origin hash.
"""
self.hashdb.delete(name)
for i in hash:
hash[i]=pickle.dumps(hash[i])
self.hashdb.hmset(name,hash)
def set_hash(self,name, item1, item2=''):
"""
Append an entry/entries to a hash in Redis with hash_name.
If the given name is empty in Redis, a new hash will be created.
The input format should be as follows:
1.A hash
2.A key and a value
"""
if type(item1) is dict:
for i in item1:
item1[i] = pickle.dumps(item1[i])
self.hashdb.hmset(name,item1)
else:
self.hashdb.hset(name, item1, pickle.dumps(item2))
def get_hash(self,name,keys=[]):
"""
Support three query functions:
1.Return a hash with a given hash_name in Redis.
2.Return a value_list with a given hash_name and a key_list in Redis,
the value_list is the same sequence as key_list.
3.Return a value with a given hash_name and a key in Redis.
"""
if not keys:
res = self.hashdb.hgetall(name)
hash={}
for i in res:
hash[i.decode()]=pickle.loads(res[i])
return hash
else:
if type(keys) is list:
res = self.hashdb.hmget(name, keys)
for i in range(len(res)):
res[i]=pickle.loads(res[i])
return res
else:
return pickle.loads(self.hashdb.hget(name, keys))
def exists_hash(self,name):
"""
Check the existence of a hash in Redis by hash_name.
"""
return self.hashdb.exists(name)
def exists_hash_key(self,name,key):
"""
Check the existence of a key in a given hash by key_name and hash_name.
"""
return self.hashdb.hexists(name, key)
def delete_hash(self,name):
"""
Delete a hash in Redis by hash_name.
"""
self.hashdb.delete(name)
def delete_hash_key(self,name,key):
"""
Delete a key in a given hash by key_name and hash_name.
"""
self.hashdb.hdel(name,key)
def clear_hash(self):
"""
Delete all the hashes in Redis by hash_name.
"""
self.hashdb.flushdb()
def flushall(self):
self.datadb.flushall()
class RedisEvalParallelSampler(Sampler):
"""
Redis based low latency sampler.
This sampler is well performing in distributed environments.
It is usually faster than the
:class:`pyabc.sampler.DaskDistributedSampler` for
short model evaluation runtimes. The longer the model evaluation times,
the less the advantage becomes. It requires a running Redis server as
broker.
This sampler requires workers to be started via the command
``abc-redis-worker``.
An example call might look like
``abc-redis-worker --host=123.456.789.123 --runtime=2h``
to connect to a Redis server on IP ``123.456.789.123`` and to terminate
the worker after finishing the first population which ends after 2 hours
since worker start. So the actual runtime might be longer than 2h.
See ``abc-redis-worker --help`` for its options.
Use the command ``abc-redis-manager`` to retrieve info and stop the running
workers.
Start as many workers as you wish. Workers can be dynamically added
during the ABC run.
Parameters
----------
host: str, optional
IP address or name of the Redis server.
Default is "localhost".
port: int, optional
Port of the Redis server.
Default is 6379.
password: str, optional
Password for a protected server. Default is None (no protection).
batch_size: int, optional
Number of model evaluations the workers perform before contacting
the REDIS server. Defaults to 1. Increase this value if model
evaluation times are short or the number of workers is large
to reduce communication overhead.
"""
def __init__(self,
host: str = "localhost",
port: int = 6379,
password: str = None,
batch_size: int = 1):
super().__init__()
logger.debug(f"Redis sampler: host={host} port={port}")
# handles the connection to the redis-server
self.redis = StrictRedis(host=host, port=port, password=password)
self.batch_size = batch_size
# check if there is still something going on
if (self.redis.exists(N_WORKER)
and int(self.redis.get(N_WORKER).decode()) > 0) or (
self.redis.exists(QUEUE) and self.redis.llen(QUEUE) > 0):
raise ValueError(
"This server seems to be still in use. A redis server cannot "
"be used for more than one pyABC inference at a time.")
def n_worker(self):
"""
Get the number of connected workers.
Returns
-------
Number of workers connected.
"""
return self.redis.pubsub_numsub(MSG)[0][-1]
def sample_until_n_accepted(self,
n,
simulate_one,
max_eval=np.inf,
all_accepted=False):
# open pipeline
pipeline = self.redis.pipeline()
# write initial values to pipeline
self.redis.set(SSA,
cloudpickle.dumps((simulate_one, self.sample_factory)))
pipeline.set(N_EVAL, 0)
pipeline.set(N_ACC, 0)
pipeline.set(N_REQ, n)
pipeline.set(ALL_ACCEPTED, int(all_accepted)) # encode as int
pipeline.set(N_WORKER, 0)
pipeline.set(BATCH_SIZE, self.batch_size)
# delete previous results
pipeline.delete(QUEUE)
# execute all commands
pipeline.execute()
id_results = []
# publish start message
self.redis.publish(MSG, START)
# wait until n acceptances
with jabbar(total=n, enable=self.show_progress, keep=False) as bar:
while len(id_results) < n:
# pop result from queue, block until one is available
dump = self.redis.blpop(QUEUE)[1]
# extract pickled object
particle_with_id = pickle.loads(dump)
# append to collected results
id_results.append(particle_with_id)
bar.inc()
# wait until all workers done
while int(self.redis.get(N_WORKER).decode()) > 0:
sleep(SLEEP_TIME)
# make sure all results are collected
while self.redis.llen(QUEUE) > 0:
id_results.append(pickle.loads(self.redis.blpop(QUEUE)[1]))
# set total number of evaluations
self.nr_evaluations_ = int(self.redis.get(N_EVAL).decode())
# delete keys from pipeline
pipeline = self.redis.pipeline()
pipeline.delete(SSA)
pipeline.delete(N_EVAL)
pipeline.delete(N_ACC)
pipeline.delete(N_REQ)
pipeline.delete(ALL_ACCEPTED)
pipeline.delete(BATCH_SIZE)
pipeline.execute()
# avoid bias toward short running evaluations (for
# dynamic scheduling)
id_results.sort(key=lambda x: x[0])
id_results = id_results[:n]
results = [res[1] for res in id_results]
# create 1 to-be-returned sample from results
sample = self._create_empty_sample()
for j in range(n):
sample += results[j]
return sample
else:
item = rclient.brpoplpush('sendmail', sendmailbackup)
logger.info("Getting Mails from {}".format('sendmail'))
item = item.decode()
#Get the smtp msg from redis
logger.info("Item : {}".format(item))
msgtuplepickle = rclient.get(item)
if msgtuplepickle:
msgtuple = pickle.loads(msgtuplepickle)
#Get the inbound json obj from redis
logger.info('item is {} '.format(item))
keys = item.split(',')
sendmail(msgtuple[1], msgtuple[0], logger)
if len(keys) == 2:
evKey = keys[1]
if evKey and rclient.exists(evKey):
rclient.delete(evKey)
rclient.delete(item)
else:
pass
#No need to remove from redis .. it will be removed after expiry
if (backmail == False):
logger.info('len of {} is : {}'.format(
sendmailbackup, rclient.llen(sendmailbackup)))
rclient.lrem(sendmailbackup, 0, item)
logger.info('len of {} is : {}'.format(
sendmailbackup, rclient.llen(sendmailbackup)))
class Channel():
""" Communication Channel, via Redis
A channel is defined by a (blocking) list on a redis server. Messages are strings. """
redis = None
def __init__(self, host, channel):
""" A channel is defined by a redis host and a channel name
"""
self.host = host
self.channel = channel
self.redis = StrictRedis(host, socket_timeout=None)
#logging.debug(" CH init %s %s" % (host, channel))
if not self.redis.exists(self.channel):
if config.verbose:
logging.debug(" CH write, new channel %s" % self.channel)
def write(self, message):
""" writes a message to the channel. The channel is created automatically """
if config.verbose:
logging.debug(" CH write: channel: %s message: %s" % (str(self.channel), str(message)))
while(True):
pipe = self.redis.pipeline()
l1,ret,l2 = pipe.llen(self.channel).rpush(self.channel, message).llen(self.channel).execute()
if not ret:
logging.error("not ret: %s" % self.channel)
continue
if not l2>0:
logging.error("not l2>0 %s" % self.channel)
continue
if l1 and not l2 == l1 +1:
logging.error("l1 and not l2 == l1 +1: %s" % self.channel)
continue
break
def read(self, blocking=False, timeout=0):
""" reads a message from the underlining channel. This method can be blocking or it could timeout in a while
"""
ret = None
time_start = time.time()
if blocking:
while True:
try:
# set a pipe that performs on a channel: len, pop, len.
pipe = self.redis.pipeline()
retup = pipe.llen(self.channel).blpop(self.channel, timeout).llen(self.channel).execute()
l1,ret,l2 = retup
if ret == None:
if config.verbose:
logging.debug("None in blpop: %s" % self.channel)
if timeout and (time.time() - time_start) > timeout:
logging.exception(" CH TIMEOUT server explicit")
return None
time.sleep(5)
continue
else:
assert l1>=0
assert l2>=0
assert l1 == l2 + 1, "l1: %s l2: %s" %(l1,l2)
break;
except ConnectionError, e:
logging.exception(" CH TIMEOUT server")
ret = None
if not ret and timeout:
logging.debug(" CH TIMEOUT read")
return None
ch, message = ret
else:
class BaseRedis:
def __init__(self, conf: dict, cluster=False):
'''
:param conf: 连接配置.
普通版:
conf = dict(
host='127.0.0.1', # 必须参数
port=6379, # 必须参数
db=6, # 必须参数
# max_connections=10,
# decode_responses=True,
)
集群版:
conf = dict(
startup_nodes=[
dict(
host='127.0.0.1', # 必须参数
port=6379, # 必须参数
),
],
# decode_responses=True, # 非必须参数
)
:param cluster: 要连接的是不是redis集群
'''
self.redis = Redis(**conf)
# 删除键
def delete_name(self, name):
return self.redis.delete(name)
# 键是否存在
def exists_name(self, name):
return self.redis.exists(name)
# 是否需要返回bytes
def need_tytes(self,
value,
need_bytes,
encoding='utf-8') -> str or bytes or None:
if isinstance(value, bytes):
if need_bytes:
result = value
else:
result = value.decode(encoding=encoding)
else:
if need_bytes and isinstance(value, str):
result = value.encode(encoding=encoding)
else:
result = value
return result
# 获取string
def str_get(self,
key,
need_bytes=False,
encoding='utf-8') -> str or bytes or None:
value = self.redis.get(key)
return self.need_tytes(value, need_bytes, encoding)
# 设置string
def str_set(self, key, value, overtime=60):
return self.redis.set(key, value, overtime)
# 不存在则设置
def setnx(self, key, value, overtime=60):
if self.redis.setnx(key, value):
return self.expire(key, overtime)
else:
return False
# 更新过期时间
def expire(self, name, overtime=60):
return self.redis.expire(name, overtime)
# 获取hash单个键的值
def hash_get(self,
name,
key,
need_bytes=False,
encoding='utf-8') -> str or bytes or None:
value = self.redis.hget(name, key)
return self.need_tytes(value, need_bytes, encoding)
# 获取hash所有键值
def hash_getall(self, name, need_bytes=False, encoding='utf-8') -> dict:
value = self.redis.hgetall(name)
result_dict = {}
if isinstance(value, dict):
for k, v in result_dict.items():
k = self.need_tytes(k, False)
v = self.need_tytes(v, need_bytes, encoding)
result_dict[k] = v
return result_dict
# 获取hash指定键的值
def hash_hmget(self,
name,
keys: list,
need_bytes=False,
encoding='utf-8') -> list:
value = self.redis.hmget(name, keys)
result = []
for va in value:
val = self.need_tytes(va, need_bytes, encoding)
result.append(val)
return result
# 设置hash单个键值
def hash_set(self, name, key, value):
return self.redis.hset(name, key, value)
# 设置hash多个键值
def hash_hmset(self, name, mapping: dict):
return self.redis.hmset(name, mapping)
# 删除hash单个键值
def hash_del(self, name, key):
return self.redis.hdel(name, key)
# 弹出集合中的值
def sets_pop(self,
name,
need_bytes=False,
encoding='utf-8') -> str or bytes or None:
value = self.redis.spop(name)
return self.need_tytes(value, need_bytes, encoding)
# 增加到集合
def sets_add(self, name, value):
return self.redis.sadd(name, value)
# 是否存在于集合
def sets_exists(self, name, value) -> bool:
return self.redis.sismember(name, value)
# 从集合中删除
def sets_del(self, name, value):
return self.redis.srem(name, value)
# 集合长度
def sets_len(self, name) -> int:
return self.redis.scard(name)
# 加入list头部(与rpop配合, 实现先进先出队列)
def list_lpush(self, name, value):
return self.redis.lpush(name, value)
# 加入list底部(基于lpush与rpop配合实现先进先出队列, 此方法用于先进先出队列的插队到最前面)
def list_rpush(self, name, value):
return self.redis.rpush(name, value)
# 插队到指定的下标(会替换原值)
def list_lset(self, name, value, index):
return self.redis.lset(name, index, value)
# 从list底部弹出(与lpush配合, 实现先进先出队列)
def list_rpop(self,
name,
need_bytes=False,
encoding='utf-8') -> str or bytes or None:
value = self.redis.rpop(name)
return self.need_tytes(value, need_bytes, encoding)
# list长度
def list_len(self, name) -> int:
return self.redis.llen(name)
for labels_entry in labels_response.json():
all_labels.add(labels_entry['name'])
if labels_paging_url is None:
break
page = 1
params = {
'page': page,
}
# If no radars were added, skip PAGES_TO_SKIP - 1
# If already skipped, and no radars were added, don't skip and add 1 to PAGES_TO_SKIP
if r.exists(PAGES_TO_SKIP_KEY):
pages_to_skip = int(r.get(PAGES_TO_SKIP_KEY)) - 1
if pages_to_skip < 0:
pages_to_skip = 0
else:
pages_to_skip = 0
pages_skipped = False
while True:
try:
print params, OPENRADAR_API_ENDPOINT
openradar_response = requests.get(OPENRADAR_API_ENDPOINT,
params=params,
headers=OPENRADAR_HEADERS)
except requests.exceptions.ConnectionError:
import json
import os
import logging
from redis import StrictRedis, ConnectionPool
app = Flask(__name__)
app.config['CELERY_BROKER_URL'] = 'redis://123.206.71.208:6379/0'
app.config['CELERY_BACKEND_URL'] = 'redis://123.206.71.208:6379/0'
app.config['FILE_TMP_PATH'] = r'tmp'
REDIS_CONF = {'host': '123.206.71.208', 'port': 6379, 'db': 1}
COUNTER_KEY = 'boss_download_count'
REDIS_POOL = ConnectionPool(**REDIS_CONF)
REDIS = StrictRedis(connection_pool=REDIS_POOL)
if not REDIS.exists(COUNTER_KEY):
REDIS.set(COUNTER_KEY, 0)
handler = logging.FileHandler('log/flask.log', encoding='UTF-8')
handler.setLevel(logging.DEBUG)
logging_format = logging.Formatter(
'%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(lineno)s - %(message)s'
)
handler.setFormatter(logging_format)
app.logger.addHandler(handler)
celery = Celery(app.name,
broker=app.config['CELERY_BROKER_URL'],
backend=app.config['CELERY_BACKEND_URL'])
celery.conf.update(app.config)
class Link(object):
def __init__(self):
self.r = StrictRedis(host=settings.LINKDB_REDIS_HOST,
port=settings.LINKDB_REDIS_PORT,
db=settings.LINKDB_REDIS_DB)
def shorten(self, link):
"""
create a new link.
"""
n = 3
i = 0
while True:
hash = self._hash(n)
if self.r.hsetnx(hash, 'link', link):
break
i = i + 1
# if we didn't find an available hash after this number of tries,
# increment hash length.
if i > 100:
n = n + 1
i = 0
uuid = self._uuid()
date = self._date()
self.r.hset(hash, 'uuid', uuid)
self.r.hset(hash, 'hits', 0)
self.r.hset(hash, 'mods', 0)
self.r.hset(hash, 'created_at', date)
self.r.hset(hash, 'lasthit_at', '')
self.r.hset(hash, 'changed_at', '')
return hash, uuid
def modify(self, hash, uuid, link):
"""
update hash to point to a new long url.
"""
if not self.exists(hash, uuid):
raise AuthFailure
self.r.hset(hash, 'link', link)
self.r.hincrby(hash, 'mods', 1)
self.r.hset(hash, 'changed_at', self._date())
def resolve(self, hash):
"""
resolve hash -> long url, and update counters.
"""
link = self.r.hget(hash, 'link')
if link is None:
raise NotFound
self.r.hincrby(hash, 'hits', 1)
self.r.hset(hash, 'lasthit_at', self._date())
return link
def exists(self, hash, uuid=None):
"""
check if a hash or a (hash + uuid) combination exists.
"""
if uuid:
return self.r.hget(hash, 'uuid') == uuid.lower()
return self.r.exists(hash)
def _hash(self, length=7):
"""
random base62 string
"""
return ''.join(choice(letters + digits) for _ in range(length))
def _uuid(self):
"""
random uuid string
"""
return str(uuid4())
def _date(self):
"""
`YYYY-MM-DD HH:MM:SS`
"""
return str(datetime.utcnow().replace(microsecond=0))
class Link(object):
def __init__(self):
self.r = StrictRedis(
host=settings.LINKDB_REDIS_HOST,
port=settings.LINKDB_REDIS_PORT,
db=settings.LINKDB_REDIS_DB)
def shorten(self, link):
"""
create a new link.
"""
n = 3
i = 0
while True:
hash = self._hash(n)
if self.r.hsetnx(hash, 'link', link):
break
i = i + 1
# if we didn't find an available hash after this number of tries,
# increment hash length.
if i > 100:
n = n + 1
i = 0
uuid = self._uuid()
date = self._date()
self.r.hset(hash, 'uuid', uuid)
self.r.hset(hash, 'hits', 0)
self.r.hset(hash, 'mods', 0)
self.r.hset(hash, 'created_at', date)
self.r.hset(hash, 'lasthit_at', '')
self.r.hset(hash, 'changed_at', '')
return hash, uuid
def modify(self, hash, uuid, link):
"""
update hash to point to a new long url.
"""
if not self.exists(hash, uuid):
raise AuthFailure
self.r.hset(hash, 'link', link)
self.r.hincrby(hash, 'mods', 1)
self.r.hset(hash, 'changed_at', self._date())
def resolve(self, hash):
"""
resolve hash -> long url, and update counters.
"""
link = self.r.hget(hash, 'link')
if link is None:
raise NotFound
self.r.hincrby(hash, 'hits', 1)
self.r.hset(hash, 'lasthit_at', self._date())
return link
def exists(self, hash, uuid=None):
"""
check if a hash or a (hash + uuid) combination exists.
"""
if uuid:
return self.r.hget(hash, 'uuid') == uuid.lower()
return self.r.exists(hash)
def _hash(self, length=7):
"""
random base62 string
"""
return ''.join(choice(letters + digits) for _ in range(length))
def _uuid(self):
"""
random uuid string
"""
return str(uuid4())
def _date(self):
"""
`YYYY-MM-DD HH:MM:SS`
"""
return str(datetime.utcnow().replace(microsecond=0))
class Querying():
def __init__(self, loglevel: int = logging.DEBUG):
self.__init_logger(loglevel)
self.storage = StrictRedis(unix_socket_path=get_socket_path('storage'),
decode_responses=True)
self.ranking = StrictRedis(unix_socket_path=get_socket_path('storage'),
db=1)
self.asn_meta = StrictRedis(
unix_socket_path=get_socket_path('storage'),
db=2,
decode_responses=True)
self.cache = StrictRedis(unix_socket_path=get_socket_path('cache'),
db=1,
decode_responses=True)
def __init_logger(self, loglevel: int):
self.logger = logging.getLogger(f'{self.__class__.__name__}')
self.logger.setLevel(loglevel)
def __normalize_date(self, date: Dates):
if isinstance(date, datetime.datetime):
return date.date().isoformat()
elif isinstance(date, datetime.date):
return date.isoformat()
elif isinstance(date, str):
try:
return parse(date).date().isoformat()
except ValueError:
raise InvalidDateFormat(
'Unable to parse the date. Should be YYYY-MM-DD.')
def _ranking_cache_wrapper(self, key):
if not self.cache.exists(key):
if self.ranking.exists(key):
key_dump = self.ranking.dump(key)
# Cache for 10 hours
self.cache.restore(key, 36000, key_dump, True)
def asns_global_ranking(self,
date: Dates = datetime.date.today(),
source: Union[list, str] = '',
ipversion: str = 'v4',
limit: int = 100):
'''Aggregated ranking of all the ASNs known in the system, weighted by source.'''
to_return = {
'meta': {
'ipversion': ipversion,
'limit': limit
},
'source': source,
'response': set()
}
d = self.__normalize_date(date)
to_return['meta']['date'] = d
if source:
if isinstance(source, list):
keys = []
for s in source:
key = f'{d}|{s}|asns|{ipversion}'
self._ranking_cache_wrapper(key)
keys.append(key)
# union the ranked sets
key = '|'.join(sorted(source)) + f'|{d}|asns|{ipversion}'
if not self.cache.exists(key):
self.cache.zunionstore(key, keys)
else:
key = f'{d}|{source}|asns|{ipversion}'
else:
key = f'{d}|asns|{ipversion}'
self._ranking_cache_wrapper(key)
to_return['response'] = self.cache.zrevrange(key,
start=0,
end=limit,
withscores=True)
return to_return
def asn_details(self,
asn: int,
date: Dates = datetime.date.today(),
source: Union[list, str] = '',
ipversion: str = 'v4'):
'''Aggregated ranking of all the prefixes anounced by the given ASN, weighted by source.'''
to_return = {
'meta': {
'asn': asn,
'ipversion': ipversion,
'source': source
},
'response': set()
}
d = self.__normalize_date(date)
to_return['meta']['date'] = d
if source:
if isinstance(source, list):
keys = []
for s in source:
key = f'{d}|{s}|{asn}|{ipversion}|prefixes'
self._ranking_cache_wrapper(key)
keys.append(key)
# union the ranked sets
key = '|'.join(sorted(source)) + f'|{d}|{asn}|{ipversion}'
if not self.cache.exists(key):
self.cache.zunionstore(key, keys)
else:
key = f'{d}|{source}|{asn}|{ipversion}|prefixes'
else:
key = f'{d}|{asn}|{ipversion}'
self._ranking_cache_wrapper(key)
to_return['response'] = self.cache.zrevrange(key,
start=0,
end=-1,
withscores=True)
return to_return
def asn_rank(self,
asn: int,
date: Dates = datetime.date.today(),
source: Union[list, str] = '',
ipversion: str = 'v4',
with_position: bool = False):
'''Get the rank of a single ASN, weighted by source.'''
to_return = {
'meta': {
'asn': asn,
'ipversion': ipversion,
'source': source,
'with_position': with_position
},
'response': 0.0
}
d = self.__normalize_date(date)
to_return['meta']['date'] = d
if source:
to_return['meta']['source'] = source
if isinstance(source, list):
keys = []
for s in source:
key = f'{d}|{s}|{asn}|{ipversion}'
self._ranking_cache_wrapper(key)
keys.append(key)
r = sum(
float(self.cache.get(key)) for key in keys
if self.cache.exists(key))
else:
key = f'{d}|{source}|{asn}|{ipversion}'
self._ranking_cache_wrapper(key)
r = self.cache.get(key)
else:
key = f'{d}|asns|{ipversion}'
self._ranking_cache_wrapper(key)
r = self.cache.zscore(key, asn)
if not r:
r = 0
if with_position and not source:
position = self.cache.zrevrank(key, asn)
if position is not None:
position += 1
to_return['response'] = {
'rank': float(r),
'position': position,
'total_known_asns': self.cache.zcard(key)
}
else:
to_return['response'] = float(r)
return to_return
def get_sources(self, date: Dates = datetime.date.today()):
'''Get the sources availables for a specific day (default: today).'''
to_return = {'meta': {}, 'response': set()}
d = self.__normalize_date(date)
to_return['meta']['date'] = d
key = f'{d}|sources'
to_return['response'] = self.storage.smembers(key)
return to_return
def get_asn_descriptions(self, asn: int, all_descriptions=False):
to_return = {
'meta': {
'asn': asn,
'all_descriptions': all_descriptions
},
'response': []
}
descriptions = self.asn_meta.hgetall(f'{asn}|descriptions')
if all_descriptions or not descriptions:
to_return['response'] = descriptions
else:
to_return['response'] = descriptions[sorted(descriptions.keys(),
reverse=True)[0]]
return to_return
def get_prefix_ips(self,
asn: int,
prefix: str,
date: Dates = datetime.date.today(),
source: Union[list, str] = '',
ipversion: str = 'v4'):
to_return = {
'meta': {
'asn': asn,
'prefix': prefix,
'ipversion': ipversion,
'source': source
},
'response': defaultdict(list)
}
d = self.__normalize_date(date)
to_return['meta']['date'] = d
if source:
to_return['meta']['source'] = source
if isinstance(source, list):
sources = source
else:
sources = [source]
else:
sources = self.get_sources(d)['response']
for source in sources:
ips = set([
ip_ts.split('|')[0] for ip_ts in self.storage.smembers(
f'{d}|{source}|{asn}|{prefix}')
])
[to_return['response'][ip].append(source) for ip in ips]
return to_return
def get_asn_history(self,
asn: int,
period: int = 100,
source: Union[list, str] = '',
ipversion: str = 'v4',
date: Dates = datetime.date.today()):
to_return = {
'meta': {
'asn': asn,
'period': period,
'ipversion': ipversion,
'source': source
},
'response': []
}
if isinstance(date, str):
date = parse(date).date()
if date + timedelta(days=period / 3) > datetime.date.today():
# the period to display will be around the date passed at least 2/3 before the date, at most 1/3 after
# FIXME: That is not doing what it is supposed to...
date = datetime.date.today()
to_return['meta']['date'] = date.isoformat()
for i in range(period):
d = date - timedelta(days=i)
rank = self.asn_rank(asn, d, source, ipversion)
if 'response' not in rank:
rank = 0
to_return['response'].insert(0, (d.isoformat(), rank['response']))
return to_return
def country_rank(self,
country: str,
date: Dates = datetime.date.today(),
source: Union[list, str] = '',
ipversion: str = 'v4'):
to_return = {
'meta': {
'country': country,
'ipversion': ipversion,
'source': source
},
'response': []
}
d = self.__normalize_date(date)
to_return['meta']['date'] = d
ripe = StatsRIPE()
response = ripe.country_asns(country, query_time=d, details=1)
if (not response.get('data') or not response['data'].get('countries')
or not response['data']['countries'][0].get('routed')):
logging.warning(f'Invalid response: {response}')
# FIXME: return something
return 0, [(0, 0)]
routed_asns = response['data']['countries'][0]['routed']
ranks = [
self.asn_rank(asn, d, source, ipversion)['response']
for asn in routed_asns
]
to_return['response'] = [sum(ranks), zip(routed_asns, ranks)]
return to_return
def country_history(self,
country: Union[list, str],
period: int = 30,
source: Union[list, str] = '',
ipversion: str = 'v4',
date: Dates = datetime.date.today()):
to_return = {}
to_return = {
'meta': {
'country': country,
'ipversion': ipversion,
'source': source
},
'response': defaultdict(list)
}
if isinstance(date, str):
date = parse(date).date()
if date + timedelta(days=period / 3) > datetime.date.today():
# the period to display will be around the date passed at least 2/3 before the date, at most 1/3 after
date = datetime.date.today()
if isinstance(country, str):
country = [country]
for c in country:
for i in range(period):
d = date - timedelta(days=i)
rank, details = self.country_rank(c, d, source,
ipversion)['response']
if rank is None:
rank = 0
to_return['response'][c].insert(
0, (d.isoformat(), rank, list(details)))
return to_return
def get_source_config(self):
pass
def get_sources_configs(self):
config_dir = get_config_path() / 'modules'
loaded = []
for modulepath in config_dir.glob('*.json'):
with open(modulepath) as f:
loaded.append(json.load(f))
return {
'{}-{}'.format(config['vendor'], config['name']): config
for config in loaded
}
class MediaWiki(object):
def __init__(self, host="https://en.wikipedia.org", path="/w/api.php",
access_token=None, redis_channel=None):
self.api_url = host + path
self.user_agent = \
"crosswatch (https://tools.wmflabs.org/crosswatch;" +\
"[email protected]) python-requests/" +\
requests.__version__
self.headers = {'User-Agent': self.user_agent}
if access_token:
# Construct an auth object with the consumer and access tokens
access_token = json.loads(access_token)
self.auth = OAuth1(config.consumer_token.key,
client_secret=config.consumer_token.secret,
resource_owner_key=access_token['key'],
resource_owner_secret=access_token['secret'])
else:
self.auth = None
self.redis_channel = redis_channel
self.redis = StrictRedis(
host=config.redis_server,
port=config.redis_port,
db=config.redis_db,
decode_responses=True
)
self.ores_url = 'http://ores.wmflabs.org/scores'
def publish(self, message):
if not self.redis_channel:
raise Exception("No redis channel set to publish to")
self.redis.publish(self.redis_channel, json.dumps(message))
@staticmethod
def timestamp(daysdelta=0):
"""
:param daysdelta: calculate timestamp in ´daysdelta´ days
:return: MediaWIki timestamp format
"""
now = datetime.utcnow()
delta = timedelta(days=daysdelta)
time = now + delta
return time.strftime("%Y%m%d%H%M%S")
def _handle_response(self, response):
if 'error' in response:
logger.error(response['error'])
if self.redis_channel:
if response['error']['code'] == \
'mwoauth-invalid-authorization':
self.publish({'msgtype': 'loginerror',
'errorinfo': response['error']['info']})
else:
self.publish({'msgtype': 'apierror',
'errorcode': response['error']['code'],
'errorinfo': response['error']['info']})
raise Exception(response['error']['code'], str(response))
if 'warnings' in response:
logger.warn("API-request warning: " + str(response['warnings']))
def query(self, params):
params['format'] = "json"
response = requests.get(self.api_url, params=params, auth=self.auth,
headers=self.headers)
response.raise_for_status()
response = response.json()
self._handle_response(response)
return response
def query_gen(self, params):
params['action'] = "query"
last_continue = {'continue': ""}
while True:
p = params.copy()
p.update(last_continue)
response = self.query(p)
if 'query' in response:
yield response['query']
if 'continue' not in response:
break
last_continue = response['continue']
def post(self, params, payload, token_type='csrf'):
params['format'] = "json"
token = self.get_token(token_type)
payload['token'] = token
response = requests.post(self.api_url, params=params, data=payload,
auth=self.auth, headers=self.headers)
self._handle_response(json.loads(response.text))
def get_token(self, type='csrf'):
params = {'action': "query",
'meta': "tokens",
'type': type}
r = self.query(params)
token = r['query']['tokens'][type + 'token']
return token
def user_info(self):
params = {
'action': "query",
'meta': "userinfo",
'uiprop': "rights"
}
response = self.query(params)
response = response['query']['userinfo']
User = namedtuple('user', ['name', 'rights'])
user = User(response['name'], response['rights'])
return user
def user_rights(self, username):
"""
User rights for a given username
:param username:
:return: list of rights
"""
params = {
'action': "query",
'list': "users",
'usprop': "rights",
'ususers': username
}
response = self.query(params)
return response['query']['users'][0]['rights']
def diff(self, pageid, old_revid, new_revid, uselang=""):
params = {
'action': "query",
'prop': "revisions",
'rvstartid': old_revid,
'rvendid': old_revid,
'rvdiffto': new_revid,
'pageids': pageid,
'uselang': uselang,
'formatversion': 2
}
response = self.query(params)
diff = response['query']['pages'][0]['revisions'][0]['diff']['body']
return diff
def wikis(self, use_cache=True):
key = config.redis_prefix + 'cached_wikis'
wikis = self.redis.get(key)
if use_cache and wikis:
wikis = json.loads(wikis)
else:
# Cache miss, do api request and fill cache
wikis = self._get_wikis()
self.redis.setex(key, 172800, json.dumps(wikis)) # 2 days exp.
return wikis
def _get_wikis(self):
params = {'action': "sitematrix"}
data = self.query(params)
blacklist_wikis = ['loginwiki']
flaggedrevs_wikis = self._flaggedrevs_wikis()
wikis = {}
for key, val in data['sitematrix'].items():
if key == 'count':
continue
if 'code' in val:
for site in val['site']:
wikis[site['dbname']] = self._create_wiki(
site, val['code'], val['name'],
flaggedrevs_wikis, blacklist_wikis)
else:
for site in val:
wikis[site['dbname']] = self._create_wiki(
site, '', '', flaggedrevs_wikis, blacklist_wikis)
return wikis
def _create_wiki(self, site, langcode, langname, flaggedrevs_wikis,
blacklist_wikis):
wiki = {
'lang': langcode,
'langname': langname,
'url': site['url'].replace("http://", "https://"),
'dbname': site['dbname'],
'group': site['code']
}
if wiki['group'] == 'wiki':
wiki['group'] = 'wikipedia'
inactive_codes = ['closed', 'private', 'fishbowl']
if any([key in site for key in inactive_codes]):
wiki['closed'] = True
if site['dbname'] in blacklist_wikis:
wiki['closed'] = True
if site['dbname'] in flaggedrevs_wikis:
wiki['flaggedrevs'] = True
return wiki
def _flaggedrevs_wikis(self):
url = "https://noc.wikimedia.org/conf/flaggedrevs.dblist"
response = requests.get(url, headers=self.headers)
response.raise_for_status()
return response.text.splitlines()
def ores_context_exists(self, dbname, model='reverted', cached=True):
"""Checks if ORES context for a wiki exists"""
key = config.redis_prefix + 'ores' + model
if not self.redis.exists(key) or not cached:
self._ores_contexts(model)
return self.redis.sismember(key, dbname)
def _ores_contexts(self, model):
"""Fill cache"""
pipe = self.redis.pipeline()
key = config.redis_prefix + 'ores' + model
pipe.delete(key)
contexts = requests.get(self.ores_url, headers=self.headers)
contexts.raise_for_status()
contexts = contexts.json()['contexts']
for context in contexts:
models = requests.get('{}/{}/'.format(self.ores_url, context),
headers=self.headers)
models.raise_for_status()
models = models.json()['models']
if model in models:
pipe.sadd(key, context)
pipe.expire(key, 172800) # 2 days exp.
pipe.execute()
def ores_scores(self, dbname, revids, model='reverted'):
"""Get ORES scores for revision ids"""
url = '{}/{}/{}/'.format(self.ores_url, dbname, model)
revids = '|'.join([str(id) for id in revids])
params = {'revids': revids}
response = requests.get(url, params=params, headers=self.headers)
if response.status_code != requests.codes.ok:
raise Exception('ORES error code {} for {} with params {}'.format(
response.status_code, dbname, str(params)), response.text)
return response.json()
def was_reverted(self, revision):
"""
Checks if a revision was reverted
:param revision: a revision dict containing ‘revid’ and ‘pageid’
"""
session = RevertsSession(self.api_url, user_agent=self.user_agent)
return reverts.check_rev(session, revision)
# Make parser.py work with airmsc.models
# (do not move import at the beginning of file!)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "airmsc.settings")
path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if path not in sys.path:
sys.path.append(path)
django.setup()
from airmsc_main.models import Member, MemberData
Q = Queue(connection=Redis())
r = StrictRedis()
r.setnx("sent_today", 0)
if r.exists("today"):
old_date = r.getset("today", str(datetime.date.today()))
else:
r.set("today", str(datetime.date.today()))
old_date = ""
if old_date != datetime.date.today():
r.set("sent_today", 0)
STATIONS_LINKS = [
("Мещанский", "http://mosecom.ru/air/air-today/station/suhar/table.html", "meschansky"),
("Басманный (Казакова)", "http://mosecom.ru/air/air-today/station/kazak/table.html", "basmanniykazakova"),
("Пресненский", "http://mosecom.ru/air/air-today/station/spirid/table.html", "presnenskiy"),
(
"Басманный (Спартаковская)",
"http://mosecom.ru/air/air-today/station/spartakovskaya/table.html",
"basmanniyspartak",
class DatabaseHandler(object):
def __init__(self, db):
self.redis = StrictRedis(host='localhost',
port=6379,
password=password,
charset='utf-8',
decode_responses=True,
db=db
)
'''find names for argument in data base '''
def _find(self, search):
cursor = None
names = []
while cursor != 0:
if cursor is None:
cursor = 0
fined = self.redis.scan(cursor, str(search))
cursor = fined[0]
names.extend(fined[1])
return sorted(set(names))
def find_names(self, find):
search = '*' + find + '*'
return self._find(search)
def exist_name(self, name):
return True if self.redis.exists(name) else False
def exist_key(self, name, key):
return self.redis.hexists(name, key)
def get_value_name_key(self, name, key):
return self.redis.hget(name, key)
def get_all_keys_for_name(self, name):
keys = self.redis.hgetall(name)
return keys if keys else None
def get_keys_for_name(self, name, *args):
keys = self.redis.hmget(name, *args)
return keys if keys else None
'''set a name and key on database'''
def set_name_key(self, name, mapping: dict):
self.redis.hset(name=name, mapping=mapping)
return bool(self.exist_name(name))
'''set a name and key on database'''
def del_names(self, names: list):
return [self.redis.delete(name) for name in names]
'''check if group exist'''
def exist_group(self, chat_id):
name = 'group:' + str(chat_id)
return self.exist_name(name)
def update_owner(self, chat_id, user_id):
names = self._find('user_url:*' + str(chat_id) + '*')
for name in names:
name_update = name.split(':')
name_update[1] = str(user_id)
name_update = ';'.join(name_update)
self.redis.rename(name, name_update)
'''register or update a url with las_url and last_update'''
def update_group(self, chat_id, chat_name, chat_title, user_id, update_owner=None):
name = 'group:' + str(chat_id)
mapping = {'chat_adm': str(user_id),
'chat_id': str(chat_id),
'chat_lock': 'True',
'chat_name': chat_name,
'chat_quiet': 'True',
'chat_title': str(chat_title)}
if update_owner:
self.update_owner(chat_id, user_id)
return True if self.set_name_key(name=name, mapping=mapping) else False
'''check if url exist'''
def exist_url(self, url):
name = 'url:^' + str(url) + '^'
return self.exist_name(name)
'''register or update a url with las_url and last_update'''
def update_url(self, url, last_update='2000-01-01 00:00:00+00:00', last_url='http://www.exemplo.com'):
name = 'url:^' + str(url) + '^'
mapping = {'last_update': str(last_update), 'last_url': last_url}
return True if self.set_name_key(name=name, mapping=mapping) else False
'''check if url exist in chat'''
def exist_url_to_chat(self, user_id, chat_id, url):
name = 'user_url:' + str(user_id) + ':chat_id:' + str(chat_id) + ':^' + str(url) + '^'
return self.exist_name(name)
'''register a url for user or group'''
def set_url_to_chat(self, chat_id, chat_name, url, user_id):
name_url = self.exist_url(url)
if not name_url:
self.update_url(url=url)
name_url_chat = self.exist_url_to_chat(user_id, chat_id, url)
if not name_url_chat:
name = 'user_url:' + str(user_id) + ':chat_id:' + str(chat_id) + ':^' + str(url) + '^'
mapping = {'chat_id': str(chat_id), 'chat_name': chat_name, 'user_id': str(user_id), 'disable': 'False'}
return True if self.set_name_key(name=name, mapping=mapping) else False
else:
return False
'''extract url for name'''
@staticmethod
def extract_url_from_names(names):
if names:
uncompress_name = [name.split('^') for name in names]
urls = sorted(set(['{}'.format(url[1]) for url in uncompress_name]))
return urls
return ()
'''return all url for a chat_id'''
def get_chat_urls(self, user_id):
names = self._find('user_url:' + str(user_id) + ':*')
chat_urls = []
for name in names:
keys = self.get_all_keys_for_name(name)
chat_id = keys.get('chat_id')
chat_name = keys.get('chat_name')
user_id = keys.get('user_id')
url = self.extract_url_from_names([name])[0]
mapping = {'user_id': str(user_id), 'chat_name': chat_name, 'url': url, 'chat_id': str(chat_id)}
chat_urls.append(mapping)
return chat_urls
'''return info about last update url'''
def get_update_url(self, url):
name = 'url:^' + str(url) + '^'
if self.exist_name(name):
keys = self.get_all_keys_for_name(name)
last_update = keys.get('last_update')
last_url = keys.get('last_url')
return {'last_update': last_update, 'last_url': last_url}
return False
'''return all url activated'''
def get_urls_activated(self):
names = self._find('user_url*')
active_keys = sorted(set([name for name in names if not self.get_value_name_key(name, 'disable') == 'True']))
return self.extract_url_from_names(active_keys)
'''return all url deactivated'''
def get_urls_deactivated(self):
names = self._find('user_url*')
return sorted(set([name for name in names if self.get_value_name_key(name, 'disable') == 'True']))
'''activated all url'''
def activated_all_urls(self):
names = self._find('user_url*')
for name in names:
self.set_name_key(name, {'disable': 'False'})
return True
'''return names for key 'disable' = 'True' from url'''
def get_names_for_user_activated(self, url):
names = self._find('user_url*' + url + '*')
return sorted(set([name for name in names if self.get_value_name_key(name, 'disable') == 'False']))
'''return all url activated'''
def get_chat_id_for_chat_name(self, user_id, chat_name):
names = self._find('user_url:*' + str(user_id) + '*')
for name in names:
chat_name_db = self.get_value_name_key(name, 'chat_name')
chat_id_db = self.get_value_name_key(name, 'chat_id')
if chat_name_db == chat_name and chat_id_db:
return chat_id_db
return None
'''disable url for chat'''
def disable_url_chat(self, chat_id):
names = self._find('user_url:*chat_id:' + str(chat_id) + '*')
mapping = {'disable': 'True'}
disables = [self.set_name_key(name=name, mapping=mapping) for name in names] if names else []
return disables
'''del url for chat'''
def del_url_for_chat(self, chat_id, url):
names = self._find('user_url:*' + str(chat_id) + '*' + url + '*')
result = self.del_names(names)
return True if result[0] == 1 else None
def list_admins(self):
return self.redis.lrange('admins', 0, self.redis.llen('admins'))
def backup(self):
now = DateHandler.get_datetime_now()
last_backup = self.get_value_name_key('backup', 'last_backup')
last_backup = DateHandler.parse_datetime(last_backup)
date_last_backup = DateHandler.date(last_backup)
hour_last_backup = DateHandler.time(last_backup)
if date_last_backup < DateHandler.date(now):
if hour_last_backup <= DateHandler.time(now):
mapping = {'last_backup': str(now)}
self.set_name_key('backup', mapping=mapping)
self.redis.save()
return True
else:
return False
class RedisBackendTest(TestCase):
def setUp(self):
self.backend = RedisBackend()
self.redis = StrictRedis()
def test_get_source_key(self):
self.assertEqual(self.backend.get_source_key('a.jpg'),
'djthumbs-test:sources:a.jpg')
def test_get_thumbnail_key(self):
self.assertEqual(self.backend.get_thumbnail_key('a.jpg'),
'djthumbs-test:thumbnails:a.jpg')
def test_add_delete_source(self):
source_name = 'test-thumbnail.jpg'
source_key = self.backend.get_source_key(source_name)
self.backend.add_source(source_name)
self.assertTrue(self.redis.hexists(source_key, source_name))
self.backend.delete_source(source_name)
self.assertFalse(self.redis.hexists(source_key, source_name))
def test_get_source(self):
source_name = 'test-thumbnail.jpg'
source_key = self.backend.get_source_key(source_name)
self.redis.hset(source_key, source_name, source_name)
self.assertEqual(self.backend.get_source(source_name), source_name)
# Delete Source
self.redis.hdel(source_key, source_name)
def test_add_delete_thumbnail(self):
source_name = 'test-thumbnail.jpg'
size = 'small'
thumbnail_key = self.backend.get_thumbnail_key(source_name)
self.backend.add_source(source_name)
self.backend.add_thumbnail(source_name, size,
'test-thumbnail_small.jpg')
self.assertTrue(self.redis.hexists(thumbnail_key, size))
self.backend.delete_thumbnail(source_name, size)
self.assertFalse(self.redis.hexists(thumbnail_key, size))
# Delete Source
self.redis.hdel(self.backend.get_source_key(source_name), source_name)
def test_get_thumbnail(self):
source_name = 'test-thumbnail.jpg'
self.backend.add_source(source_name)
self.backend.add_thumbnail(source_name, 'small',
'test-thumbnail_small.jpg')
self.assertEqual(
self.backend.get_thumbnail(source_name, 'small'),
ImageMeta(source_name, 'test-thumbnail_small.jpg', 'small'))
self.backend.add_thumbnail(source_name, 'large',
'test-thumbnail_large.jpg')
expected = ['test-thumbnail_large.jpg', 'test-thumbnail_small.jpg']
result = [
image_meta.name
for image_meta in self.backend.get_thumbnails(source_name)
]
# sort is replacing the variable in place, not returning new value, it will always return None
result.sort()
expected.sort()
self.assertEqual(result, expected)
# Delete Source & Thumbnails
thumbnail_key = self.backend.get_thumbnail_key(source_name)
self.redis.hdel(self.backend.get_source_key(source_name), source_name)
self.redis.hdel(thumbnail_key, 'small')
self.redis.hdel(thumbnail_key, 'large')
def test_flush_thumbnails(self):
source_name = 'test-thumbnail.jpg'
thumbnail_key = self.backend.get_thumbnail_key(source_name)
self.backend.add_source(source_name)
self.backend.add_thumbnail(source_name, "small",
'test-thumbnail_small.jpg')
self.backend.add_thumbnail(source_name, "default",
'test-thumbnail_default.jpg')
self.backend.flush_thumbnails(source_name)
self.assertFalse(self.redis.exists(thumbnail_key))
class ImageEmbeddings:
def __init__(self,
mdl_path,
verbose=False,
load_in_memory=False,
layer=13,
forceRecal=False):
self.inform = print if verbose else lambda *a, **k: None
self.r = StrictRedis()
self.key_name = key_name
self.load_in_memory = load_in_memory
self.ico_path = icons_path
self.mdl_path = mdl_path
self.layer = layer
self.model = load_model(mdl_path)
self.output_function = K.function(
[self.model.layers[0].input,
K.learning_phase()], [self.model.layers[layer].output])
self.layer_info = self.model.layers[layer].get_config()
self.distances = None
self.icons = []
for i in range(5):
with open('datasets/LLD-icon/LLD-icon_data_' + str(i) + '.pkl',
'rb') as f:
self.icons.extend(load(f, encoding="bytes"))
print('File', i + 1, 'of 5 loaded')
if not self.r.exists(self.key_name) or forceRecal:
self.calculateVectors()
self.db_len = self.r.llen(self.key_name)
self.inform('\nReady to compare images with model:', mdl_path,
' Layer', layer, '- Name:', self.layer_info['name'])
def calculateVectors(self):
in_size = len(self.icons)
print(in_size, 'icons loaded, calculating vectors.')
self.r.delete(key_name)
step = 100 / in_size
for i, icon in enumerate(self.icons):
vector = self.getOutputFrom(icon)
self.r.rpush(key_name, dumps(vector))
stdout.write('\r[IMG-EMB]\tCalculating Vectors {:2.2f}%'.format(
step * (i + 1)))
def getOutputFrom(self, image, mode=0):
return self.output_function([[image], mode])[0][0]
def getDBIterator(self):
if self.load_in_memory:
return enumerate(self.r.lrange(self.key_name, 0, -1))
return enumerate(range(self.db_len))
def calculateDistances(self, img_array, use_euc_dist=True):
img_vector = self.getOutputFrom(img_array)
self.inform('Image ready, now calculating distances')
dists = zeros((self.db_len, 2))
step = 100 / self.db_len
for i, vector in self.getDBIterator():
if not self.load_in_memory:
vector = self.r.lindex(self.key_name, i)
if use_euc_dist:
dists[i] = i, euclideanDistance(img_vector, loads(vector))
else:
dists[i] = i, cosineDistance(img_vector, loads(vector))
stdout.write('\r[IMG-EMB]\tCalculating Distances {:2.2f}%'.format(
step * (i + 1)))
self.inform('\nFinished calculating distances to image')
self.distances = dists[dists[:, 1].argsort()]
def viewCollageComparison(self, res_size=10):
coll = zeros((32, res_size * 32, 3), dtype=uint8)
self.inform('Distances')
for i, (idx, dist) in enumerate(self.distances[:res_size]):
coll[0:32, 0 + i * 32:32 + i * 32] = self.openImage(idx)
self.inform('Index:', int(idx), 'Distance:', dist)
return coll
def openImage(self, idx):
return self.icons[int(idx)]
class RedisManager(NoSqlManager):
def __init__(self,
namespace,
url=None,
data_dir=None,
lock_dir=None,
**params):
self.db = params.pop('db', None)
self.dbpass = params.pop('password', None)
self.connection_pools = {}
if url is not None and ':' not in url:
url = url + ':6379'
self.localnamespace = params.pop('localnamespace', None)
if self.localnamespace is None:
raise InvalidCacheBackendError("Cannot initialize session. 'localnamespace' config value must be set")
self.namespaces = params.pop('namespaces', '').split('\n')
log.debug('Session namespaces: %s, %s' % (self.localnamespace, self.namespaces))
self.expiretime = int(params.pop('expiretime', '43200'), 10)
NoSqlManager.__init__(self,
namespace,
url=url,
data_dir=data_dir,
lock_dir=lock_dir,
**params)
def open_connection(self, host, port, **params):
pool_key = self._format_pool_key(host, port, self.db)
if pool_key not in self.connection_pools:
self.connection_pools[pool_key] = ConnectionPool(host=host,
port=port,
db=self.db,
password=self.dbpass)
self.db_conn = StrictRedis(connection_pool=self.connection_pools[pool_key],
**params)
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def __getitem__(self, key):
key = self._format_key(key)
log.debug('Fetching key %s' % key)
try:
authNamespace = json.loads(self.db_conn.hget(key, 'Auth') or '{}')
localNamespace = json.loads(self.db_conn.hget(key, self.localnamespace) or '{}')
if authNamespace is None:
authNamespace = {}
if localNamespace is None:
localNamespace = {}
for otherNamespace in self.namespaces:
ns = json.loads(self.db_conn.hget(key, otherNamespace) or '{}')
localNamespace[otherNamespace] = ns
localNamespace['Auth'] = authNamespace
log.debug(localNamespace)
except Exception, e:
log.error(e)
return None
return localNamespace
class MysqlPipeline:
def __init__(self, mysql_host, mysql_database, mysql_user, mysql_password,
mysql_port, reids_host, reids_database, reids_password,
reids_port):
self.mysql_host = mysql_host
self.mysql_database = mysql_database
self.mysql_user = mysql_user
self.mysql_password = mysql_password
self.mysql_port = mysql_port
self.reids_host = reids_host
self.reids_database = reids_database
self.reids_password = reids_password
self.reids_port = reids_port
@classmethod
def from_crawler(cls, crawler):
return cls(mysql_host=crawler.settings.get('MYSQL_HOST'),
mysql_database=crawler.settings.get('MYSQL_DATABASE'),
mysql_user=crawler.settings.get('MYSQL_USER'),
mysql_password=crawler.settings.get('MYSQL_PASSWORD'),
mysql_port=crawler.settings.get('MYSQL_PORT'),
reids_host=crawler.settings.get('REDIS_HOST'),
reids_database=crawler.settings.get('REDIS_DATABASE'),
reids_password=crawler.settings.get('REDIS_PASSWORD'),
reids_port=crawler.settings.get('REDIS_PORT'))
def open_spider(self, spider):
self.db = pymysql.connect(self.mysql_host,
self.mysql_user,
self.mysql_password,
self.mysql_database,
charset='utf8',
port=self.mysql_port)
self.cursor = self.db.cursor()
self.redis = StrictRedis(host=self.reids_host,
port=self.reids_port,
db=self.reids_database,
password=self.reids_password)
def close_spider(self, spider):
self.db.close()
# 先在redis缓存100条数据，后批量插入mysql
# 不适用分布式爬虫
def process_item(self, item, spider):
if not self.redis.exists(item['table']):
self.redis.rpush(item['table'], item)
return item
if self.redis.rpush(item['table'], item) >= 100:
imgs = self.redis.lrange(item['table'], 0,
self.redis.llen(item['table']))
self.redis.delete(item['table'])
for img in imgs:
data = dict(json.loads(
img.decode('utf-8').replace("\'", "\"")))
data.pop('table')
keys = ', '.join(data.keys())
values = ', '.join(['%s'] * len(data))
sql = 'insert into %s (%s) values (%s)' % (item['table'], keys,
values)
self.cursor.execute(sql, tuple(data.values()))
self.db.commit()
return item
async def test_throttle_raises_running_timeout_error_when_running_has_timeout(
redis: StrictRedis):
throttle = AsyncThrottle('test', 1, redis)
with pytest.raises(RunningTimeoutError):
await throttle.run(asyncio.sleep(0.1), running_timeout=0.05)
assert not redis.exists('redis_gt:test')
try:
if m.pjuu.posts.insert(post):
r.delete(COMMENT.format(post_id))
except (pymongo.errors.DuplicateKeyError):
# At the time this script was written uuid1().hex is how the
# `pjuu.lib.get_uuid()` works.
post['_id'] = uuid1().hex
# We won't try again. If this fails its bricked.
if m.pjuu.posts.insert(post):
r.delete(COMMENT.format(post_id))
# Comments are now posts so we need to change the names of the
# voting keys. We will also change it to match the _id in the case
# that it was changed above
if r.exists(COMMENT_VOTES.format(post_id)):
r.rename(COMMENT_VOTES.format(post_id),
POST_VOTES.format(post['_id']))
# Iterate ALL user keys, we can't do this above because we need all posts
# migrated before the user migration will work
for key in r.keys('{user:*}'):
user_match = USER_RE.match(key)
if user_match is not None:
print 'Converting user:'******'_id'] = user.get('uid')
del user['uid'] # Remove the old 'uid' field
class WikiController(object):
def __init__(self, settings):
"""Initialize the controler and preload basic metadata"""
self.redis = Redis(host=settings.redis.bind_address, port=settings.redis.port)
self.store = Store(settings.content.path)
self.get_all_pages() # page modification times
self.get_all_aliases() # page aliases
def get_page(self, path):
"""Returns a single page"""
if path in self.store.pages:
return self.store.get_page(path)
raise KeyError
def resolve_alias(self, path):
"""Attempts to resolve an alias to a page"""
# Check locally first, to save overhead
if path in self.store.aliases:
return self.store.aliases[path]
# Check if there's been an update in Redis
alias = self.redis.hget(META_ALIASES, path)
if alias:
self.store.aliases[path] = alias
return alias
return None
def get_all_pages(self):
"""Returns a hash of all known pages and mtimes"""
if not len(self.store.pages):
if self.redis.exists(META_PAGES):
self.store.pages = self.redis.hgetall(META_PAGES)
else:
# force filesystem scan and alias generation
pages = self.store.get_all_pages()
log.debug(pages)
self.redis.hmset(META_PAGES,self.store.get_all_pages())
return self.store.pages
def get_all_aliases(self):
"""Returns a hash of all known page aliases"""
if not len(self.store.aliases):
if self.redis.exists(META_ALIASES):
self.store.aliases = self.redis.hgetall(META_ALIASES)
else:
# force filesystem scan and alias generation
self.store.get_all_pages()
self.redis.hmset(META_ALIASES, self.store.aliases)
return self.store.aliases
def get_close_matches_for_page(self, path):
"""Get a list of close matches for a given page/path"""
pages = self.get_all_pages()
return get_close_matches(path, pages.keys())
for labels_entry in labels_response.json():
all_labels.add(labels_entry['name'])
if labels_paging_url is None:
break
page = 1
params = {
'page': page,
}
# If no radars were added, skip PAGES_TO_SKIP - 1
# If already skipped, and no radars were added, don't skip and add 1 to PAGES_TO_SKIP
if r.exists(PAGES_TO_SKIP_KEY):
pages_to_skip = int(r.get(PAGES_TO_SKIP_KEY)) - 1
else:
pages_to_skip = 0
pages_skipped = False
while True:
try:
print params, OPENRADAR_API_ENDPOINT
openradar_response = requests.get(OPENRADAR_API_ENDPOINT, params=params)
except requests.exceptions.ConnectionError:
print "Oops. Connection error"
break
else:
radars_added = False
class Feedback(object):
def __init__(self, config=None):
if config is None:
config = {}
self.redis = StrictRedis(**config['redis'])
def get_allocation_key(self, site_id: str, experiment: str, recommender_id: str, allocation_time: datetime=None) -> str:
if allocation_time is None:
allocation_time = datetime.utcnow()
long_key = '/'.join([site_id, experiment, recommender_id])
short_key = '{:08x}'.format(mmh3.hash(long_key) & INT_MAX)
timestamp = allocation_time.strftime(HOUR_FORMAT)
return short_key + ':' + timestamp
def insert(self, allocation_key: str, event_type: str, rec_id: str, ttl=timedelta(days=60)):
full_key = event_type + ':' + allocation_key
key_exists = self.redis.exists(full_key)
self.redis.execute_command('PFADD', full_key, rec_id)
if not key_exists:
timestamp = allocation_key.rsplit(':', 1)[-1]
allocation_time = datetime.strptime(timestamp, HOUR_FORMAT)
expire_time = allocation_time + ttl
self.redis.expireat(full_key, expire_time)
def count_distinct(self, allocation_key: str, event_type: str) -> int:
return self.redis.execute_command('PFCOUNT', event_type + ':' + allocation_key)
def increment_arm(self, event: Dict):
key = self.get_allocation_key(site_id=event['site_id'],
experiment=event['experiment'],
recommender_id=event['recommender_id'],
allocation_time=event['allocation_time']
)
rec_id = event.get('recset', 'NA') + ':' + event.get('rec_group', 'NA') + ':' + event.get('rec_position', 'NA')
self.insert(key, event['event_type'], rec_id)
def flush_redis(self):
self.redis.flushall()
def event_listener(self):
'''TODO: integrate with kafka --currently stubbed'''
event_stream = iter([
dict(site_id='666', event_type='rec_viewed', rec_time=datetime(2016, 11, 1, 12),
recset='abc', rec_group='a', rec_position='1', recommender_id='fmv1', experiment='default'),
dict(site_id='666', event_type='rec_viewed', rec_time=datetime(2016, 11, 1, 12),
recset='abc', rec_group='a', rec_position='1', recommender_id='fmv1', experiment='default'),
dict(site_id='666', event_type='rec_viewed', rec_time=datetime(2016, 11, 1, 12),
recset='abc', rec_group='a', rec_position='1', recommender_id='fmv1', experiment='default'),
dict(site_id='666', event_type='rec_clicked', rec_time=datetime(2016, 11, 1, 12),
recset='abc', rec_group='a', rec_position='1', recommender_id='fmv1', experiment='default'),
dict(site_id='666', event_type='rec_clicked', rec_time=datetime(2016, 11, 1, 12),
recset='abc', rec_group='a', rec_position='1', recommender_id='fmv1', experiment='default'),
dict(site_id='666', event_type='rec_clicked', rec_time=datetime(2016, 11, 1, 12),
recset='abc', rec_group='a', rec_position='1', recommender_id='fmv1', experiment='default'),
dict(site_id='238', event_type='rec_viewed', rec_time=datetime(2016, 11, 2, 8),
recset='abc', rec_group='a', rec_position='1', recommender_id='collb', experiment='default'),
dict(site_id='238', event_type='rec_clicked', rec_time=datetime(2016, 11, 2, 8),
recset='abc', rec_group='a', rec_position='1', recommender_id='collb', experiment='default')
])
for event in event_stream:
if event['event_type'] in ['rec_viewed', 'rec_clicked']:
self.increment_arm(event)
def perdelta(self, start: datetime, end: datetime, delta: timedelta) -> List[datetime]:
output = []
curr = start
while curr < end:
curr = curr + delta
output.append(curr)
return output
def get_recent_hours(self, days_ago: int=30) -> List[datetime]:
end_hour = datetime.now().replace(minute=0, second=0, microsecond=0)
start_hour = end_hour - timedelta(days=days_ago)
return self.perdelta(start_hour, end_hour, timedelta(hours=1))
def enumerate_keys(self, site_ids: List[str], experiments: List[str], recommender_ids: List[str]) -> pd.DataFrame:
header = ['site_id', 'experiment', 'recommender_id', 'date_hour', 'trials', 'successes']
rows = []
for site_id in site_ids:
for experiment in experiments:
for recommender_id in recommender_ids:
for date_hour in self.get_recent_hours():
key = self.get_allocation_key(site_id, experiment, recommender_id, date_hour)
view_count = self.count_distinct(key, 'rec_viewed')
click_count = self.count_distinct(key, 'rec_clicked')
rows.append([site_id, experiment, recommender_id, date_hour, view_count, click_count])
df = pd.DataFrame(data=rows, columns=header)
# TODO: date filter
return df.groupby(['site_id', 'experiment', 'recommender_id']).sum().reset_index()
class RedisManager(NamespaceManager):
connection_pools = {}
def __init__(self,
namespace,
url=None,
data_dir=None,
lock_dir=None,
expire=None,
**params):
self.db = params.pop('db', None)
self.dbpass = params.pop('password', None)
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_tcd_lock"
if hasattr(self, 'lock_dir'):
verify_directory(self.lock_dir)
# Specify the serializer to use (pickle or json?)
self.serializer = params.pop('serializer', 'pickle')
self._expiretime = int(expire) if expire else None
conn_params = {}
parts = url.split('?', 1)
url = parts[0]
if len(parts) > 1:
conn_params = dict(p.split('=', 1) for p in parts[1].split('&'))
host, port = url.split(':', 1)
self.open_connection(host, int(port), **conn_params)
def open_connection(self, host, port, **params):
pool_key = self._format_pool_key(host, port)
if pool_key not in self.connection_pools:
self.connection_pools[pool_key] = ConnectionPool(host=host,
port=port,
db=self.db,
password=self.dbpass)
self.db_conn = StrictRedis(connection_pool=self.connection_pools[pool_key],
**params)
def get_creation_lock(self, key):
return file_synchronizer(identifier="tccontainer/funclock/%s" % self.namespace,
lock_dir=self.lock_dir)
def __getitem__(self, key):
if self.serializer == 'json':
payload = self.db_conn.get(self._format_key(key))
if isinstance(payload, bytes):
return json.loads(payload.decode('utf-8'))
else:
return json.loads(payload)
else:
return pickle.loads(self.db_conn.get(self._format_key(key)))
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def has_key(self, key):
return key in self
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
#
# beaker.container.Value.set_value calls NamespaceManager.set_value
# however it (until version 1.6.4) never sets expiretime param.
#
# Checking "type(value) is tuple" is a compromise
# because Manager class can be instantiated outside container.py (See: session.py)
#
if (expiretime is None) and (type(value) is tuple):
expiretime = value[1]
if self.serializer == 'json':
serialized_value = json.dumps(value, ensure_ascii=True)
else:
serialized_value = pickle.dumps(value, 2)
if expiretime:
self.db_conn.setex(key, expiretime, serialized_value)
else:
self.db_conn.set(key, serialized_value)
def __setitem__(self, key, value):
self.set_value(key, value, self._expiretime)
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:{0}:{1}'.format(self.namespace, key.replace(' ', '\302\267'))
def _format_pool_key(self, host, port):
return '{0}:{1}:{2}'.format(host, port, self.db)
def do_remove(self):
self.db_conn.flushdb()
def keys(self):
return self.db_conn.keys('beaker:{0}:*'.format(self.namespace))
