feedback_mail = readdress_configs.ConfigSectionMap(
'FEEDBACK')['FEEDBACK_MAIL']
contact_mail = readdress_configs.ConfigSectionMap(
'CONTACT')['CONTACT_MAIL']
supportlist = [support_mail, feedback_mail, contact_mail]
backupmail = False
if (rclient.llen(mailhandlerBackUp)):
evt = rclient.brpop(mailhandlerBackUp)
backupmail = True
ev = pickle.loads(evt[1])
#pickledEv = pickle.dumps(ev)
pickledEv = evt[1]
logger.info("Getting events from {}".format(mailhandlerBackUp))
else:
pickledEv = rclient.brpoplpush('mailhandler', mailhandlerBackUp)
ev = pickle.loads(pickledEv)
logger.info("Getting events from {}".format('mailhandler'))
#mail handler
origmsg, msg = emailHandler(ev, pickledEv)
del msg
del origmsg
del ev
if (not backupmail):
logger.info('len of {} is : {}'.format(
mailhandlerBackUp, rclient.llen(mailhandlerBackUp)))
rclient.lrem(mailhandlerBackUp, 0, pickledEv)
logger.info('len of {} is : {}'.format(
logger.info("genBounceMailHandleBackUp ListName : {} ".format(
genBounceMailHandleBackUp))
while True:
del readdress_configs
readdress_configs = ReConfig()
backupmail = False
if (rclient.llen(genBounceMailHandleBackUp)):
evt = rclient.brpop(genBounceMailHandleBackUp)
backupmail = True
pickledData = evt[1]
dict_data = pickle.loads(pickledData)
logger.info(
"Getting events from {}".format(genBounceMailHandleBackUp))
else:
pickledData = rclient.brpoplpush('genBounceMailHandle',
genBounceMailHandleBackUp)
dict_data = pickle.loads(pickledData)
logger.info("Getting events from {}".format('genBounceMailHandle'))
#mail handler
genBounceEmail_handler(dict_data)
if (not backupmail):
logger.info('len of {} is : {}'.format(
genBounceMailHandleBackUp,
rclient.llen(genBounceMailHandleBackUp)))
rclient.lrem(genBounceMailHandleBackUp, 0, pickledData)
logger.info('len of {} is : {}'.format(
genBounceMailHandleBackUp,
rclient.llen(genBounceMailHandleBackUp)))
class RedisChannel(object):
"""A channel for transporting metrics using Redis.
:type host: string
:param host: The host of the Redis server.
:type port: int
:param port: The port of the Redis server.
:type db: int
:param db: The database to use on the Redis server. This should be used
exclusively for Kadabra to prevent collisions with keys that
might be used by your application.
:type logger: string
:param logger: The name of the logger to use.
"""
#: Default arguments for the Redis channel. These will be used by the
#: client and agent to initialize this channel if custom configuration
#: values are not provided.
DEFAULT_ARGS = {
"host": "localhost",
"port": 6379,
"db": 0,
"logger": "kadabra.channel",
"queue_key": "kadabra_queue",
"inprogress_key": "kadabra_inprogress"
}
def __init__(self, host, port, db, logger, queue_key, inprogress_key):
from redis import StrictRedis
self.client = StrictRedis(host=host, port=port, db=db)
self.logger = logging.getLogger(logger)
self.queue_key = queue_key
self.inprogress_key = inprogress_key
def send(self, metrics):
"""Send metrics to a Redis list, which will act as queue for pending
metrics to be received and published.
:type metrics: ~kadabra.Metrics
:param metrics: The metrics to be sent.
"""
to_push = metrics.serialize()
self.logger.debug("Sending %s" % to_push)
self.client.lpush(self.queue_key, json.dumps(to_push))
self.logger.debug("Successfully sent %s" % to_push)
def receive(self):
"""Receive metrics from the queue so they can be published. Once
received, the metrics will be moved into a temporary "in progress"
queue until they have been acknowledged as published (by calling
:meth:`~kadabra.channels.RedisChannel.complete`). This method will
block until there are metrics available on the queue or after 10
seconds.
:rtype: ~kadabra.Metrics
:returns: The metrics to be published, or None if there were no metrics
received after the timeout.
"""
self.logger.debug("Receiving metrics")
raw = self.client.brpoplpush(self.queue_key,
self.inprogress_key,
timeout=10)
if raw:
rv = json.loads(raw)
self.logger.debug("Got metrics: %s" % rv)
return Metrics.deserialize(rv)
self.logger.debug("No metrics received")
return None
def receive_batch(self, max_batch_size):
"""Receive a list of metrics from the queue so they can be published.
Once received, all metrics will be moved into a temporary "in progress"
queue until they have been acknowledged as published (by calling
:meth:`~kadabra.channels.RedisChannel.complete`). The number of metrics
that are received is less than or equal to the ``max_batch_size``, and
possibly empty.
:type max_batch_size: int
:param max_batch_size: The maximum number of metrics to receive in the
batch.
:rtype: list
:returns: The list of metrics to be published. The size of the list is
less than or equal to the ``max_batch_size``, and possibly
empty if there are no metrics in the queue.
"""
self.logger.debug("Receiving batch of metrics")
pipeline = self.client.pipeline()
for i in range(max_batch_size):
pipeline.rpoplpush(self.queue_key, self.inprogress_key)
return [
Metrics.deserialize(json.loads(m)) for m in pipeline.execute()
if m is not None
]
def complete(self, metrics):
"""Mark a list of metrics as completed by removing them from the
in-progress queue.
:type metrics: list
:param metrics: The list of :class:`~kadabra.Metrics` to mark as
complete.
"""
if len(metrics) > 0:
pipeline = self.client.pipeline()
for m in metrics:
pipeline.lrem(self.inprogress_key, 1,
json.dumps(m.serialize()))
pipeline.execute()
def in_progress(self, query_limit):
"""Return a list of the metrics that are in_progress.
:type query_limit: int
:param query_limit: The maximum number of items to get from the in
progress queue.
:rtype: list
:returns: A list of :class:`Metric`\s that are in progress.
"""
in_progress = self.client.lrange(self.inprogress_key, 0,\
query_limit - 1)
self.logger.debug("Found %s in progress metrics" % len(in_progress))
return [Metrics.deserialize(json.loads(m))\
for m in in_progress]
class RedisConn(object):
"""docstring for RedisConn"""
def __init__(self, startup_nodes=None, host="localhost",
port=6379, db=0, password=None, encoding='utf-8',
socket_keepalive=False, connection_pool=None,
max_connections=None, project="", decode_responses=True, **kwargs):
if project:
project = f'{project}:'
self.cluster_flag = False
self.project = project
if startup_nodes:
from rediscluster import StrictRedisCluster
if isinstance(startup_nodes, (str, bytes)):
startup_nodes = _normalize_startup_nodes(startup_nodes)
self._redis = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=decode_responses,
skip_full_coverage_check=True, **kwargs)
self.cluster_flag = True
else:
self._redis = StrictRedis(host=host, port=port, db=db, password=password,
socket_keepalive=socket_keepalive, connection_pool=connection_pool,
max_connections=max_connections, **kwargs)
def add_head(self, key):
return f'{self.project}{key}'
def format_key():
def make_wrapper(func):
def wrapper(self, key, *args, **kwargs):
new_key = self.add_head(key)
return func(self, new_key, *args, **kwargs)
return wrapper
return make_wrapper
def format_key_keys():
def make_wrapper(func):
def wrapper(self, key, keys, *args, **kwargs):
new_key = self.add_head(key)
new_keys = list(map(self.add_head, keys))
return func(self, new_key, new_keys, *args, **kwargs)
return wrapper
return make_wrapper
def format_args():
def make_wrapper(func):
def wrapper(self, *args, **kwargs):
new_args = list(map(self.add_head, list(args)))
return func(self, *new_args, **kwargs)
return wrapper
return make_wrapper
def format_two_key():
def make_wrapper(func):
def wrapper(self, src, dst, *args, **kwargs):
new_src = self.add_head(src)
new_dst = self.add_head(dst)
return func(self, new_src, new_dst, *args, **kwargs)
return wrapper
return make_wrapper
def format_keys():
def make_wrapper(func):
def wrapper(self, keys, *args):
new_keys = list(map(self.add_head, keys))
return func(self, new_keys, *args)
return wrapper
return make_wrapper
def format_dicts():
def make_wrapper(func):
def wrapper(self, mapping, *args):
new_mapping = {}
for key in mapping.keys():
new_key = self.add_head(key)
new_mapping[new_key] = mapping[key]
return func(self, new_mapping, *args)
return wrapper
return make_wrapper
@format_args()
def unlink(self, *keys):
"""
time complexity O(1)
redis异步删除keys
"""
return self._redis.unlink(*keys)
def pipeline(self, transaction=True, shard_hint=None):
"""
返回一个pipe对象
"""
return self._redis.pipeline(transaction, shard_hint)
"""===============================string-start=========================="""
# }
@format_key()
def set(self, key, value, ex=None, px=None, nx=False, xx=False):
"""
time complexity O(1)
Set the value at key ``key`` to ``value``
Arguments:
key (str): key key
value (str): key value
ex(int): 过期时间(秒)
px(int): 过期时间(豪秒)
nx(bool): 如果设置为True,则只有key不存在时,当前set操作才执行(新建)
xx(bool): 如果设置为True,则只有key存在时,当前set操作才执行 （修改）
Returns:
result(bool): 是否成功成功是True失败可能是None
"""
return self._redis.set(key, value, ex, px, nx, xx)
@format_key()
def get(self, key):
"""
time complexity O(1)
Return the value at ``key``, or None if the key doesn't exist
Arguments:
key (str): key
Returns:
value (str):返回value
"""
return self._redis.get(key)
@format_key()
def getset(self, key, value):
"""
time complexity O(1)
设置新值并获取原来的值
"""
return self._redis.getset(key, value)
@format_key()
def strlen(self, key):
"""
time complexity O(1)
获得key对应的value长度
"""
return self._redis.strlen(key)
@format_key()
def getrange(self, key, start, end):
"""
time complexity O(1)
获得key对应的value的start到end长度字符返回
"""
return self._redis.getrange(key, start, end)
@format_key()
def setrange(self, key, offset, value):
"""
time complexity O(1)
设置key对应的value从offset地方用新value替换
"""
return self._redis.setrange(key, offset, value)
@format_key()
def setbit(self, key, offset, value):
"""
time complexity O(1)
value值只能是1或0
设置key对应的value二进制在offset位用value替换
"""
return self._redis.setbit(key, offset, value)
@format_key()
def getbit(self, key, offset):
"""
time complexity O(1)
获取key对应的value二进制在offset位的值
"""
return self._redis.getbit(key, offset)
@format_key()
def expire(self, key, time):
"""
time complexity O(1)
设置key的过期时间s
"""
return self._redis.expire(key, time)
@format_key()
def pexpire(self, key, time):
"""
time complexity O(1)
设置key的过期时间ms
"""
return self._redis.pexpire(key, time)
@format_key()
def pexpireat(self, key, when):
"""
time complexity O(1)
设置key的过期时间(在什么时候过期)
when是uninx的时间戳ms
"""
return self._redis.pexpireat(key, when)
@format_key()
def pttl(self, key):
"""
time complexity O(1)
获得key过期时间(ms),没有设置过期时间返回-1
"""
return self._redis.pttl(key)
@format_key()
def ttl(self, key):
"""
time complexity O(1)
获得name过期时间(s),没有设置过期时间返回-1
"""
return self._redis.ttl(key)
@format_dicts()
def mset(self, mapping):
"""
time complexity O(n)
Arguments:
mapping (dict): {name: value,name1: value1}
Returns:
return ok
"""
return self._redis.mset(mapping)
@format_dicts()
def msetnx(self, mapping):
"""
time complexity O(n)
Arguments:
mapping (dict): {name: value,name1: value1}
Returns:
return (bool): 与mset区别是指定的key中有任意一个已存在,则不进行任何操作,返回错误
"""
return self._redis.msetnx(mapping)
@format_keys()
def mget(self, keys, *args):
"""
time complexity O(n)
Arguments:
keys (list): [name, name1]
Returns:
return (list): 返回对应keys的value, name在数据库不存在返回None
Mind!:
一次性取多个key确实比get提高了性能,但是mget的时间复杂度O(n),
实际使用过程中测试当key的数量到大于100之后性能会急剧下降,
建议mget每次key数量不要超过100。在使用前根据实列的redis吞吐量可能会不一样。
"""
return self._redis.mget(keys, *args)
@format_key()
def incr(self, key, amount=1):
"""
time complexity O(1)
将key对应的value值自增amount，并返回自增后的值。只对可以转换为整型的String数据起作用。
用于统计sql型数据库大表里面的数据量
"""
return self._redis.incr(key, amount)
@format_key()
def incrbyfloat(self, key, amount=1.0):
"""
time complexity O(1)
amount 可以为负数代表减法
将key对应的value值自增amount，并返回自增后的值。只对可以转换为float的String数据起作用。
用于统计sql型数据库大表里面的数据量
"""
return self._redis.incrbyfloat(key, amount)
@format_key()
def decr(self, key, amount=1):
"""
time complexity O(1)
将key对应的value值自减amount，并返回自减后的值。只对可以转换为整型的String数据起作用。
用于统计sql型数据库大表里面的数据量
"""
return self._redis.decr(key, amount)
def keys(self, pattern='*'):
"""
time complexity O(n)
获取匹配pattern的所有key.实际项目中慎用
"""
return self._redis.keys(pattern)
@format_key()
def move(self, key, db):
"""
time complexity O(1)
移动key到其他db
"""
return self._redis.move(key, db)
def randomkey(self):
"""
time complexity O(1)
随机返回一个key
"""
return self._redis.randomkey()
@format_args()
def rename(self, src, dst):
"""
time complexity O(1)
重命名key src to dst
"""
return self._redis.rename(src, dst)
@format_args()
def exists(self, *keys):
"""
time complexity O(1)
查看keys是否存在返回存在的key数量
"""
return self._redis.exists(*keys)
@format_args()
def delete(self, *keys):
"""
time complexity O(1)
删除keys
"""
return self._redis.delete(*keys)
@format_key()
def type(self, key):
"""
time complexity O(1)
查看key对应value类型
"""
return self._redis.type(key)
# {
"""===============================string-end============================"""
"""===============================list-start============================"""
# }
@format_keys()
def blpop(self, keys, timeout=0):
"""
如果keys里面有list为空要求整个服务器被阻塞以保证块执行时的原子性，
该行为阻止了其他客户端执行 LPUSH 或 RPUSH 命令
阻塞的一个命令，用来做轮询和会话配合使用
Arguments:
keys(list): [keys, keys]
timeout(int): S
"""
return self._redis.blpop(keys, timeout)
@format_keys()
def brpop(self, keys, timeout=0):
"""
同上,取数据的方向不同
"""
return self._redis.brpop(keys, timeout)
@format_two_key()
def brpoplpush(self, src, dst, timeout=0):
"""
从src表尾取一个数据插入dst表头。同上src为空阻塞
"""
return self._redis.brpoplpush(src, dst, timeout)
@format_key()
def lpush(self, key, *values):
"""
time complexity O(n)
Set the value at key ``key`` to ``value``
Arguments:
key (str): key key
value (list): key value
Returns:
result(int): 插入成功之后list长度
"""
return self._redis.lpush(key, *values)
@format_key()
def lpushx(self, key, *values):
"""
time complexity O(n)
only key not exists
Arguments:
key (str): key
value (list): key value
Returns:
result(int): 插入成功之后list长度
"""
return self._redis.lpushx(key, *values)
@format_key()
def lpop(self, key):
"""
time complexity O(1)
移除并返回列表 key 的头元素。
"""
return self._redis.lpop(key)
@format_key()
def rpush(self, key, *values):
"""
time complexity O(n)
Set the value at key ``key`` to ``value``
Arguments:
key (str): key key
value (list): key value
Returns:
result(int): 插入成功之后list长度
"""
return self._redis.rpush(key, *values)
@format_key()
def rpushx(self, key, *values):
"""
time complexity O(n)
only key not exists
Arguments:
key (str): key
value (list): key value
Returns:
result(int): 插入成功之后list长度
"""
return self._redis.rpushx(key, *values)
@format_key()
def rpop(self, key):
"""
time complexity O(1)
移除并返回列表 key尾元素。
"""
return self._redis.rpop(key)
@format_key()
def lrange(self, key, start, end):
"""
time complexity O(n)
获取list数据包含start,end.在不清楚list的情况下尽量不要使用lrange(key, 0, -1)操作
应尽可能控制一次获取的元素数量
"""
return self._redis.lrange(key, start, end)
@format_args()
def rpoplpush(self, src, dst):
"""
从src表尾取一个数据插入dst表头
"""
return self._redis.rpoplpush(src, dst)
@format_key()
def llen(self, key):
"""
time complexity O(1)
获取list长度,如果key不存在返回0,如果key不是list类型返回错误
"""
return self._redis.llen(key)
@format_key()
def lindex(self, key, index):
"""
time complexity O(n) n为经过的元素数量
返回key对应list的index位置的value
"""
return self._redis.lindex(key, index)
@format_key()
def linsert(self, key, where, refvalue, value):
"""
time complexity O(n) n为经过的元素数量
key或者refvalue不存在就不进行操作
Arguments:
where(str): BEFORE|AFTER 后|前
refvalue(str): list里面的值
"""
return self._redis.linsert(key, where, refvalue, value)
@format_key()
def lrem(self, key, count, value):
"""
time complexity O(n)
删除count数量的value
Arguments:
count(int): count>0 表头开始搜索
count<0 表尾开始搜索
count=0 删除所有与value相等的数值
Returns:
result(int): 删除的value的数量
"""
if self.cluster_flag:
return self._redis.lrem(key, value, count)
return self._redis.lrem(key, count, value)
@format_key()
def lset(self, key, index, value):
"""
time complexity O(n)
设置list的index位置的值，没有key和超出返回错误
"""
return self._redis.lset(key, index, value)
@format_key()
def ltrim(self, key, start, end):
"""
time complexity O(n) n为被删除的元素数量
裁剪让列表只保留指定区间内的元素，不在指定区间之内的元素都将被删除。
"""
return self._redis.ltrim(key, start, end)
@format_key()
def sort(self, key, start=None, num=None, by=None, get=None,
desc=False, alpha=False, store=None, groups=False):
"""
time complexity O(n)
O(N+M*log(M))， N 为要排序的列表或集合内的元素数量， M 为要返回的元素数量。
删除count数量的value
Arguments:
by(str): 让排序按照外部条件排序,
可以先将权重插入redis然后再作为条件进行排序如(user_level_*)
get(str): redis有一组user_name_*然后*是按照list里面的值，
按照排序取一个个key的value
store(str): 保留sort之后的结果，可以设置expire过期时间作为结果缓存
alpha: 按照字符排序
desc: 逆序
Returns:
result(list): 排序之后的list
"""
return self._redis.sort(key, start, num, by, get, desc, alpha, store, groups)
def scan(self, cursor=0, match=None, count=None):
"""
time complexity O(1) 单次
增量迭代返回redis数据库里面的key,因为是增量迭代过程中返回可能会出现重复
Arguments:
cursor(int): 游标
match(str): 匹配
count(int): 每次返回的key数量
Returns:
result(set): 第一个是下次scan的游标,后面是返回的keys(list)当返回的游标为0代表遍历完整个redis
"""
return self._redis.scan(cursor, match, count)
# {
"""===============================list-end===================================="""
"""===============================hash-start==================================="""
# }
@format_key()
def hdel(self, key, *names):
"""
time complexity O(n) n为names长度
Return the value at ``key``, or None if the key doesn't exist
Arguments:
key (str): key
names(list): hash里面的域
Returns:
result (int): 成功删除的个数
"""
return self._redis.hdel(key, *names)
@format_key()
def hexists(self, key, name):
"""
time complexity O(1)
判断key中是否有name域
"""
return self._redis.hexists(key, name)
@format_key()
def hget(self, key, name):
"""
time complexity O(1)
"""
return self._redis.hget(key, name)
@format_key()
def hgetall(self, key):
"""
time complexity O(n)
"""
return self._redis.hgetall(key)
@format_key()
def hincrby(self, key, name, amount=1):
"""
time complexity O(1)
amount可以为负数,且value值为整数才能使用否则返回错误
"""
return self._redis.hincrby(key, name, amount)
@format_key()
def hincrbyfloat(self, key, name, amount=1.0):
"""
time complexity O(1)
"""
return self._redis.hincrbyfloat(key, name, amount)
@format_key()
def hkeys(self, key):
"""
time complexity O(n)
"""
return self._redis.hkeys(key)
@format_key()
def hlen(self, key):
"""
time complexity O(1)
"""
return self._redis.hlen(key)
@format_key()
def hset(self, key, name, value):
"""
time complexity O(1)
"""
return self._redis.hset(key, name, value)
@format_key()
def hsetnx(self, key, name, value):
"""
time complexity O(1)
"""
return self._redis.hsetnx(key, name, value)
@format_key()
def hmset(self, key, mapping):
"""
time complexity O(n)
"""
return self._redis.hmset(key, mapping)
@format_key()
def hmget(self, key, names, *args):
"""
time complexity O(n)
"""
return self._redis.hmget(key, names, *args)
@format_key()
def hvals(self, key):
"""
time complexity O(n)
返回hash表所有的value
"""
return self._redis.hvals(key)
@format_key()
def hstrlen(self, key, name):
"""
time complexity O(1)
"""
return self._redis.hstrlen(key, name)
# {
"""=================================hash-end==================================="""
"""=================================set-start================================="""
# }
@format_key()
def sadd(self, key, *values):
"""
time complexity O(n) n为values长度
"""
return self._redis.sadd(key, *values)
@format_key()
def scard(self, key):
"""
time complexity O(n) set长度
返回set大小
"""
return self._redis.scard(key)
@format_args()
def sdiff(self, key, *args):
"""
time complexity O(n) N 是所有给定集合的成员数量之和
返回差集成员的列表。
"""
return self._redis.sdiff(key, *args)
@format_args()
def sdiffstore(self, dest, keys, *args):
"""
time complexity O(n) N 是所有给定集合的成员数量之和
返回差集成员的数量。并将结果保存到dest这个set里面
"""
return self._redis.sdiffstore(dest, keys, *args)
@format_args()
def sinter(self, key, *args):
"""
time complexity O(N * M)， N 为给定集合当中基数最小的集合， M 为给定集合的个数。
返回交集数据的list
"""
return self._redis.sinter(key, *args)
@format_args()
def sinterstore(self, dest, keys, *args):
"""
time complexity O(n) N 是所有给定集合的成员数量之和
返回交集成员的数量。并将结果保存到dest这个set里面
"""
return self._redis.sinterstore(dest, keys, *args)
@format_key()
def sismember(self, key, name):
"""
time complexity O(1)
判断name是否在key中
"""
return self._redis.sismember(key, name)
@format_key()
def smembers(self, key):
"""
time complexity O(n)
返回set里面所有成员
"""
return self._redis.smembers(key)
@format_two_key()
def smove(self, src, dst, value):
"""
time complexity O(1)
将value从src移动到dst原子性操作
"""
return self._redis.smove(src, dst, value)
@format_key()
def spop(self, key, count=None):
"""
time complexity O(n) n
默认随机删除一条, 删除count条
"""
return self._redis.spop(key, count)
@format_key()
def srandmember(self, key, number=None):
"""
time complexity O(n) n
默认随机返回一条, 返回number条
"""
return self._redis.srandmember(key, number)
@format_key()
def srem(self, key, *values):
"""
time complexity O(n) n为values长度
移除key里面values
"""
return self._redis.srem(key, *values)
@format_args()
def sunion(self, keys, *args):
"""
time complexity O(N)， N 是所有给定集合的成员数量之和
返回并集
"""
return self._redis.sunion(keys, *args)
@format_args()
def sunionstore(self, dest, keys, *args):
"""
time complexity O(N)， N 是所有给定集合的成员数量之和。
求并集并保存
"""
return self._redis.sunionstore(dest, keys, *args)
@format_key()
def sscan(self, key, cursor=0, match=None, count=None):
"""
time complexity O(1)
同scan只是这个是set使用
"""
return self._redis.sscan(key, cursor, match, count)
# {
"""==================================set-end=================================="""
"""===============================SortedSet-start============================="""
# }
@format_key()
def zadd(self, key, mapping, nx=False, xx=False, ch=False, incr=False):
"""
time complexity O(M*log(N)), N 是有序集的基数， M 为成功添加的新成员的数量。
Arguments:
mapping(dict): (value:score)
XX(bool): 仅仅更新存在的成员，不添加新成员。
NX(bool): 不更新存在的成员。只添加新成员。
CH(bool): 修改返回值为发生变化的成员总数，原始是返回新添加成员的总数 (CH 是 changed 的意思)。
更改的元素是新添加的成员，已经存在的成员更新分数。 所以在命令中指定的成员有相同的分数将不被计算在内。
注：在通常情况下，ZADD返回值只计算新添加成员的数量。
INCR(bool): 当ZADD指定这个选项时，成员的操作就等同ZINCRBY命令，对成员的分数进行递增操作。
Returns:
result(int): 成功插入数量
"""
if self.cluster_flag:
return self._redis.zadd(key, **mapping)
return self._redis.zadd(key, mapping, nx, xx, ch, incr)
@format_key()
def zcard(self, key):
"""
time complexity O(1)
返回zset()基数
"""
return self._redis.zcard(key)
@format_key()
def zcount(self, key, minz, maxz):
"""
time complexity O(log(N)), N 为有序集的基数。
返回score在min和max之间的value的个数
"""
return self._redis.zcount(key, minz, maxz)
@format_key()
def zincrby(self, key, amount, value):
"""
time complexity O(log(N)), N 为有序集的基数。
amount 可以为负数
"""
if self.cluster_flag:
return self._redis.zincrby(key, value, amount)
return self._redis.zincrby(key, amount, value)
@format_key_keys()
def zinterstore(self, dest, keys, aggregate=None):
"""
time complexity O(N*K)+O(M*log(M))， N 为给定 key 中基数最小的有序集， K 为给定有序集的数量， M 为结果集的基数。
求交集并按照aggregate做处理之后保存到dest。默认是求和
Arguments:
aggregate(str):sum 和, min 最小值, max 最大值
返回新zset里面的value个数
"""
return self._redis.zinterstore(dest, keys, aggregate)
@format_key()
def zrange(self, key, start, end, desc=False, withscores=False,
score_cast_func=float):
"""
time complexity O(log(N)+M)， N 为有序集的基数，而 M 为结果集的基数。
Arguments:
start,有序集合索引起始位置（非分数）
end,有序集合索引结束位置（非分数）
desc,排序规则，默认按照分数从小到大排序
withscores,是否获取元素的分数，默认只获取元素的值
score_cast_func,对分数进行数据转换的函数
"""
return self._redis.zrange(key, start, end, desc, withscores, score_cast_func)
@format_key()
def zrevrange(self, key, start, end, withscores=False, score_cast_func=float):
"""
time complexity O(log(N)+M)， N 为有序集的基数，而 M 为结果集的基数。
Arguments:
start,有序集合索引起始位置（非分数）
end,有序集合索引结束位置（非分数）
withscores,是否获取元素的分数，默认只获取元素的值
score_cast_func,对分数进行数据转换的函数
"""
return self._redis.zrevrange(key, start, end, withscores, score_cast_func)
@format_key()
def zrangebyscore(self, key, minz, maxz, start=None, num=None, withscores=False, score_cast_func=float):
"""
time complexity O(log(N)+M)， N 为有序集的基数，而 M 为结果集的基数。
有序集成员按 score 值递增(从小到大)次序排列。
"""
return self._redis.zrangebyscore(key, minz, maxz, start, num, withscores, score_cast_func)
@format_key()
def zrevrangebyscore(self, key, minz, maxz, start=None, num=None, withscores=False, score_cast_func=float):
"""
time complexity O(log(N)+M)， N 为有序集的基数，而 M 为结果集的基数。
有序集成员按 score 值递减(从大到小)次序排列。
"""
return self._redis.zrevrangebyscore(key, minz, maxz, start, num, withscores, score_cast_func)
@format_key()
def zrangebylex(self, key, minz, maxz, start=None, num=None):
"""
time complexity O(log(N)+M)， N 为有序集的基数，而 M 为结果集的基数。
有序集成员按 value 字典序递增(从小到大)次序排列。
"""
return self._redis.zrangebylex(key, minz, maxz, start, num)
@format_key()
def zrevrangebylex(self, key, minz, maxz, start=None, num=None):
"""
time complexity O(log(N)+M)， N 为有序集的基数，而 M 为结果集的基数。
有序集成员按 value 字典序递减(从大到小)次序排列。
"""
return self._redis.zrevrangebylex(key, minz, maxz, start, num)
@format_key()
def zrank(self, key, value):
"""
time complexity O(log(N))
查找zset里面这个value的rank排名从0开始
"""
return self._redis.zrank(key, value)
@format_key()
def zrevrank(self, key, value):
"""
time complexity O(log(N))
查找zset里面这个value的rank排名从0开始
"""
return self._redis.zrevrank(key, value)
@format_key()
def zrem(self, key, *values):
"""
time complexity O(M*log(N))， N 为有序集的基数， M 为被成功移除的成员的数量
删除zset里面单个或者多个成员
"""
return self._redis.zrem(key, *values)
@format_key()
def zremrangebylex(self, key, minz, maxz):
"""
time complexity O(log(N)+M)， N 为有序集的基数，而 M 为被移除成员的数量。
按照字典增序范围删除
"""
return self._redis.zremrangebylex(key, minz, maxz)
@format_key()
def zremrangebyrank(self, key, minz, maxz):
"""
time complexity O(log(N)+M)， N 为有序集的基数，而 M 为被移除成员的数量。
按照rank范围删除
"""
return self._redis.zremrangebyrank(key, minz, maxz)
@format_key()
def zremrangebyscore(self, key, minz, maxz):
"""
time complexity O(log(N)+M)， N 为有序集的基数，而 M 为被移除成员的数量。
按照score范围删除
"""
return self._redis.zremrangebyscore(key, minz, maxz)
@format_key()
def zscore(self, key, value):
"""
time complexity O(log(N))
查找zset里面这个value的score排名从0开始
"""
return self._redis.zscore(key, value)
@format_key_keys()
def zunionstore(self, dest, keys, aggregate=None):
"""
time complexity O(N)+O(M log(M))， N 为给定有序集基数的总和， M 为结果集的基数。
求并集保存
"""
return self._redis.zunionstore(dest, keys, aggregate)
@format_key()
def zscan(self, key, cursor=0, match=None, count=None, score_cast_func=float):
"""
time complexity O(1)
同SCAN
"""
return self._redis.zscan(key, cursor, match, count, score_cast_func)
def zlexcount(self, key, minz, maxz):
"""
time complexity O(log(N))，其中 N 为有序集合包含的元素数量。
min -负无限 [闭空间不包括自己 (开空间包括自己
max +正无限 [a, (c
"""
return self._redis.zlexcount(key, minz, maxz)
# {
"""===============================SortedSet-end================================="""
"""===============================HyperLogLog-start==============================="""
# }
@format_key()
def pfadd(self, key, *values):
"""
time complexity O(n)
"""
return self._redis.pfadd(key, *values)
@format_args()
def pfcount(self, *sources):
"""
time complexity O(1)
计算key的基数
"""
return self._redis.pfcount(*sources)
@format_args()
def pfmerge(self, dest, *sources):
"""
time complexity O(n) 其中 N 为被合并的 HyperLogLog 数量,不过这个命令的常数复杂度比较高
合并HyperLogLog
"""
return self._redis.pfmerge(dest, *sources)
# {
"""===============================HyperLogLog-end================================="""
"""==================================GEO-start===================================="""
# }
@format_key()
def geoadd(self, key, *values):
"""
time complexity O(log(N)) 每添加一个元素的复杂度为 O(log(N)) ， 其中 N 为键里面包含的位置元素数量。
"""
return self._redis.geoadd(key, *values)
@format_key()
def geopos(self, key, *values):
"""
time complexity O(log(N))
从键里面返回所有给定位置元素的位置（经度和纬度）。
"""
return self._redis.geopos(key, *values)
@format_key()
def geohash(self, key, *values):
"""
time complexity O(log(N))
命令返回的 geohash 的位置与用户给定的位置元素的位置一一对应
"""
return self._redis.geohash(key, *values)
@format_key()
def geodist(self, key, place1, place2, unit=None):
"""
time complexity O(log(N))
返回两个给定位置之间的距离。
Argument:
unit : m: 米,km: 千米,mi: 英里,ft: 英尺
"""
return self._redis.geodist(key, place1, place2, unit)
@format_key()
def georadius(self, key, longitude, latitude, radius, unit=None,
withdist=False, withcoord=False, withhash=False, count=None,
sort=None, store=None, store_dist=None):
"""
time complexity O(N+log(M))， 其中 N 为指定半径范围内的位置元素数量， 而 M 则是被返回位置元素的数量。
以给定的经纬度为中心， 返回键包含的位置元素当中， 与中心的距离不超过给定最大距离的所有位置元素。
Argument:
longitude: 经度
latitude: 纬度
radius: 距离
unit: 距离单位
withdist: 在返回位置元素的同时， 将位置元素与中心之间的距离也一并返回。 距离的单位和用户给定的范围单位保持一致。
withcoord: 将位置元素的经度和维度也一并返回
withhash: 以 52 位有符号整数的形式， 返回位置元素经过原始 geohash 编码的有序集合分值。
这个选项主要用于底层应用或者调试， 实际中的作用并不大。
sort: 根据中心的位置排序 ASC,DESC
count: 取前多少个
store: 保存
store_dist: 存储地名和距离
Return:
list(list)
[['Foshan', 109.4922], ['Guangzhou', 105.8065]]
"""
return self._redis.georadius(key, longitude, latitude, radius, unit, withdist, withcoord,
withhash, count, sort, store, store_dist)
@format_key()
def georadiusbymember(self, key, member, radius, unit=None,
withdist=False, withcoord=False, withhash=False, count=None,
sort=None, store=None, store_dist=None):
"""
time complexity O(N+log(M))， 其中 N 为指定半径范围内的位置元素数量， 而 M 则是被返回位置元素的数量。
以给定的经纬度为中心， 返回键包含的位置元素当中， 与中心的距离不超过给定最大距离的所有位置元素。
Argument:
member: 位置元素
radius: 距离
unit: 距离单位
withdist: 在返回位置元素的同时， 将位置元素与中心之间的距离也一并返回。 距离的单位和用户给定的范围单位保持一致。
withcoord: 将位置元素的经度和维度也一并返回
withhash: 以 52 位有符号整数的形式， 返回位置元素经过原始 geohash 编码的有序集合分值。 这个选项主要用于底层应用或者调试， 实际中的作用并不大。
sort: 根据中心的位置排序 ASC,DESC
count: 取前多少个
store: 保存
store_dist: 存储地名和距离
Return:
list(list)
[['Foshan', 109.4922], ['Guangzhou', 105.8065]]
"""
return self._redis.georadiusbymember(key, member, radius, unit, withdist, withcoord,
withhash, count, sort, store, store_dist)
# {
"""==================================GEO-end======================================"""
message = pickle.loads(item[1])
mailcontent = pystache.render(
mailtemplates[message['template_name']]['template'],
message.get('global_merge_vars', ''))
msg = msg = MIMEText(mailcontent, 'html')
msg['Subject'] = mailtemplates[message['template_name']]['subject']
From = mailtemplates[message['template_name']]['from_email']
if From == 'noreply':
From = readdress_configs.get_formatted_noreply()
msg['From'] = From
msg['To'] = message['email']
s = smtplib.SMTP('localhost')
s.send_message(msg)
s.quit()
else:
item = rclient.brpoplpush('mailer', mailerbackup)
message = pickle.loads(item)
mailcontent = pystache.render(
mailtemplates[message['template_name']]['template'],
message.get('global_merge_vars', ''))
msg = msg = MIMEText(mailcontent, 'html')
msg['Subject'] = mailtemplates[message['template_name']]['subject']
From = mailtemplates[message['template_name']]['from_email']
if From == 'noreply':
From = readdress_configs.get_formatted_noreply()
msg['From'] = From
msg['To'] = message['email']
s = smtplib.SMTP('localhost')
s.send_message(msg)
s.quit()
logger.info('len of {} is : {}'.format(mailerbackup,
from FileReaderExampleH5py import createLUT, readImage
import h5py
from PIL import Image
from os.path import basename, dirname
from redis import StrictRedis
r = StrictRedis()
while True:
masterFilename = r.brpoplpush('copied:masterfiles', 'converted:masterfiles')
try:
directory = dirname(masterFilename)
filebase = basename(masterFilename).rsplit('_', 1)[0]
hdf5File = h5py.File(masterFilename, 'r')
detector = hdf5File['entry']['instrument']['detector']
print "entries in detector: ", list(detector)
LUT = createLUT(hdf5File)
for imgNr in range(1, len(LUT)+1):
npimage = readImage(imgNr, LUT, hdf5File)
tifimage = Image.fromarray(npimage, 'I')
tifimage.save("{}/tiff/{}_{}.tiff".format(directory, filebase, str(imgNr).zfill(6)))
except:
pass
while True:
del readdress_configs
readdress_configs = ReConfig()
valids.re_readconfig()
backupmail = False
if (rclient.llen(mailDeregisterhandlerBackUp)):
evt = rclient.brpop(mailDeregisterhandlerBackUp)
backupmail = True
ev = pickle.loads(evt[1])
pickledEv = pickle.dumps(ev)
logger.info(
"Getting events from {}".format(mailDeregisterhandlerBackUp))
else:
pickledEv = rclient.brpoplpush('mailDeregisterhandler',
mailDeregisterhandlerBackUp)
ev = pickle.loads(pickledEv)
logger.info(
"Getting events from {}".format('mailDeregisterhandler'))
#mail handler
emailDeregisterHandler(ev)
if (not backupmail):
logger.info('len of {} is : {}'.format(
mailDeregisterhandlerBackUp,
rclient.llen(mailDeregisterhandlerBackUp)))
rclient.lrem(mailDeregisterhandlerBackUp, 0, pickledEv)
logger.info('len of {} is : {}'.format(
mailDeregisterhandlerBackUp,
rclient.llen(mailDeregisterhandlerBackUp)))
':%(asctime)s-[%(filename)s:%(lineno)s]-%(levelname)s - %(message)s')
logging.basicConfig(stream=sys.stdout,
level=logging.DEBUG,
format=formatter)
logger = logging.getLogger('emailarchiver' + instance)
rclient = StrictRedis()
mailarchivebackup = 'mailarchivebackup_' + instance
db = dbops.MongoORM()
logger.info("Starting email archiver")
while True:
if (rclient.llen(mailarchivebackup)):
item = rclient.brpop(mailarchivebackup)
message = pickle.loads(item[1])
logger.info("Getting Mails from {}".format(mailarchivebackup))
utc_timestamp = datetime.datetime.utcnow() + datetime.timedelta(
days=30)
db.dumpmail(message)
else:
item = rclient.brpoplpush('mailarchive', mailarchivebackup)
message = pickle.loads(item)
logger.info("Getting Mails from {}".format('mailarchive'))
db.dumpmail(message)
logger.info('len of {} is : {}'.format(
mailarchivebackup, rclient.llen(mailarchivebackup)))
rclient.lrem(mailarchivebackup, 0, item)
logger.info('len of {} is : {}'.format(
mailarchivebackup, rclient.llen(mailarchivebackup)))
redrmailhandlerBackup = 'redrredrmailhandlerBackup_' + instance
logger.info(
"MailHandlerBackUp ListName : {} ".format(redrmailhandlerBackup))
while True:
backupmail = False
if (rclient.llen(redrmailhandlerBackup)):
logger.info(
"len of (" + redrmailhandlerBackup +
" ) is {} ".format(rclient.llen(redrmailhandlerBackup)))
evt = rclient.brpop(redrmailhandlerBackup)
backupmail = True
ev = pickle.loads(evt[1])
pickledEv = pickle.dumps(ev)
logger.info("Getting events from {}".format(redrmailhandlerBackup))
else:
pickledEv = rclient.brpoplpush('redrmailhandler',
redrmailhandlerBackup)
ev = pickle.loads(pickledEv)
logger.info("Getting events from {}".format('redrmailhandler'))
emailHandler(ev)
if (not backupmail):
logger.info('len of {} is : {}'.format(
redrmailhandlerBackup, rclient.llen(redrmailhandlerBackup)))
rclient.lrem(redrmailhandlerBackup, 0, pickledEv)
logger.info('len of {} is : {}'.format(
redrmailhandlerBackup, rclient.llen(redrmailhandlerBackup)))
args = parser.parse_args()
argsdict = vars(args)
instance = argsdict['instance']
FILESIZE=1024*1024*1024 #1MB
handler=('LI ARCHIVER-['+instance+']')
formatter = ('\n'+handler+':%(asctime)s-[%(filename)s:%(lineno)s]-%(levelname)s - %(message)s')
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format=formatter)
logger = logging.getLogger('liarchiver'+instance)
rclient = StrictRedis()
liarchivebackup = 'liarchivebackup_'+instance
db = dbops.MongoORM()
while True:
if (rclient.llen(liarchivebackup)):
item = rclient.brpop (liarchivebackup)
logger.info("Getting Mails from {}".format(liarchivebackup))
itemlist = pickle.loads(item[1])
if (len(itemlist) == 2):
db.lidump(itemlist, Binary(str(itemlist[1]).encode(), 128))
else:
item = rclient.brpoplpush('liarchive', liarchivebackup)
itemlist = pickle.loads(item)
if (len(itemlist) == 2):
db.lidump(itemlist, Binary(str(itemlist[1]).encode(), 128))
logger.info ('len of {} is : {}'.format(liarchivebackup, rclient.llen(liarchivebackup)))
rclient.lrem(liarchivebackup, 0, item)
logger.info ('len of {} is : {}'.format(liarchivebackup, rclient.llen(liarchivebackup)))
while True:
#Read config changes only while processing the mesage
readdress_configs = ReConfig()
valids.re_readconfig()
backupmail = False
if (rclient.llen(supportChannelBackUp)):
evt = rclient.brpop(supportChannelBackUp)
backupmail = True
ev = pickle.loads(evt[1])
#pickledEv = pickle.dumps(ev)
pickledEv = evt[1]
logger.info("Getting events from {}".format(supportChannelBackUp))
else:
pickledEv = rclient.brpoplpush('supportChannel',
supportChannelBackUp)
ev = pickle.loads(pickledEv)
logger.info("Getting events from {}".format('supportChannel'))
#mail handler
origmsg, msg = supportMailHandler(ev, pickledEv)
del origmsg
del msg
if (not backupmail):
logger.info('len of {} is : {}'.format(
supportChannelBackUp, rclient.llen(supportChannelBackUp)))
rclient.lrem(supportChannelBackUp, 0, pickledEv)
logger.info('len of {} is : {}'.format(
supportChannelBackUp, rclient.llen(supportChannelBackUp)))
rclient = StrictRedis()
if 'debug' in argsdict and argsdict['debug'] is not None:
logger.info("Loegger testing\n")
exit()
sendmailbackup = 'sendmailbackup_' + instance
while True:
backmail = False
if (rclient.llen(sendmailbackup)):
tupitem = rclient.brpop(sendmailbackup)
logger.info("Getting Mails from {}".format(sendmailbackup))
backmail = True
item = (tupitem[1]).decode()
else:
item = rclient.brpoplpush('sendmail', sendmailbackup)
logger.info("Getting Mails from {}".format('sendmail'))
item = item.decode()
#Get the smtp msg from redis
logger.info("Item : {}".format(item))
msgtuplepickle = rclient.get(item)
if msgtuplepickle:
msgtuple = pickle.loads(msgtuplepickle)
#Get the inbound json obj from redis
logger.info('item is {} '.format(item))
keys = item.split(',')
sendmail(msgtuple[1], msgtuple[0], logger)
if len(keys) == 2:
evKey = keys[1]
from FileReaderExampleH5py import createLUT, readImage
import h5py
from PIL import Image
from os.path import basename, dirname
from redis import StrictRedis
r = StrictRedis()
while True:
masterFilename = r.brpoplpush('copied:masterfiles',
'converted:masterfiles')
try:
directory = dirname(masterFilename)
filebase = basename(masterFilename).rsplit('_', 1)[0]
hdf5File = h5py.File(masterFilename, 'r')
detector = hdf5File['entry']['instrument']['detector']
print "entries in detector: ", list(detector)
LUT = createLUT(hdf5File)
for imgNr in range(1, len(LUT) + 1):
npimage = readImage(imgNr, LUT, hdf5File)
tifimage = Image.fromarray(npimage, 'I')
tifimage.save("{}/tiff/{}_{}.tiff".format(directory, filebase,
str(imgNr).zfill(6)))
except: