def pop_highest_buy_order(r: redis.client.Redis) -> Tuple[Order, bool]:
highest_buy = r.zrange(redis_buy_prices_name,
0,
0,
desc=True,
withscores=True)
if len(highest_buy) == 0:
return (None, False)
else:
(buy_order_id, buy_price) = highest_buy[0]
buy_order_size = int(
r.hget(redis_order_sizes_name, buy_order_id).decode('utf-8'))
buy_order_account_name = r.hget(redis_account_names_name, buy_order_id)
successful_remove = remove_price_and_size(r, redis_buy_prices_name,
redis_order_sizes_name,
redis_account_names_name,
buy_order_id)
if successful_remove:
buy_order_id = buy_order_id.decode('utf-8')
buy_order_account_name = buy_order_account_name.decode('utf-8')
order = Order(buy_order_account_name, buy_order_id, "Buy",
buy_order_size, buy_price)
return (order, True)
else:
return (None, False)
def pop_lowest_sell_order(r: redis.client.Redis) -> Tuple[Order, bool]:
lowest_sell = r.zrange(redis_sell_prices_name,
0,
0,
desc=False,
withscores=True)
if len(lowest_sell) == 0:
return (None, False)
else:
(sell_order_id, sell_price) = lowest_sell[0]
sell_order_size = int(
r.hget(redis_order_sizes_name, sell_order_id).decode('utf-8'))
sell_order_account_name = r.hget(redis_account_names_name,
sell_order_id)
successful_remove = remove_price_and_size(r, redis_sell_prices_name,
redis_order_sizes_name,
redis_account_names_name,
sell_order_id)
if successful_remove:
sell_order_id = sell_order_id.decode('utf-8')
sell_order_account_name = sell_order_account_name.decode('utf-8')
order = Order(sell_order_account_name, sell_order_id, "Sell",
sell_order_size, sell_price)
return (order, True)
else:
return (None, False)
def add_remove_groups(conn: redis.client.Redis,
article: str,
to_add=[],
to_remove=[]) -> None:
"""
组别设置
"""
for group in to_add:
conn.sadd(GROUP_NAME + group, article)
for group in to_remove:
conn.srem(GROUP_NAME + group, article)
def vote_for(voted_letter: str, redis_instance: redis.client.Redis,
hashset_name: str) -> Json:
if redis_instance.hget(hashset_name, voted_letter) == None:
abort(400, 'This value does not exist')
for key in get_db_keys(redis_instance, hashset_name):
key = key.decode("utf-8")
if key == voted_letter:
redis_instance.hincrby(hashset_name, voted_letter, 1)
return get_whole_db(redis_instance, hashset_name)
def get_articles(conn: redis.client.Redis,
page: int,
order=SCORE_NAME) -> list:
"""
获取文章（排序）
"""
start = (page - 1) * ARTICLES_PER_PAGE
end = start + ARTICLES_PER_PAGE - 1
ids = conn.zrevrange(order, start, end)
articles = []
for id in ids:
article_data = conn.hgetall(id)
article_data['id'] = id
articles.append(article_data)
return articles
def get_group_articles(conn: redis.client.Redis,
group: str,
page: int,
order=SCORE_NAME) -> list:
"""
获取组内文章（排序）
"""
key = order + group
if not conn.exists(key):
conn.zinterstore(
key,
[GROUP_NAME + group, order],
aggregate='max',
)
conn.expire(key, 60)
return get_articles(conn, page, key)
def zpop(r: redis.client.Redis, name, desc):
try:
r.watch(name)
zrange_result = r.zrange(name, 0, 0, desc=desc)
if len(zrange_result) == 0:
return (None, False)
result = zrange_result[0]
p = r.pipeline()
p.watch(name)
p.multi()
p.zrem(name, result)
p.execute()
return (result, True)
except redis.WatchError as e:
print(e)
return (None, False)
def sub(rds: redis.client.Redis, latest_timestamps: Dict[str, str]):
pubsub = rds.pubsub()
pubsub.psubscribe("sentry.*.updates")
for message in pubsub.listen():
if message.get("type") == "message":
# get the sentry name by inspecting the channel
sentry_name = message.get("channel")[6]
latest_timestamps[sentry_name] = str(message.get("data"))
def __init__(self, redis_client: redis.client.Redis):
timees = redis_client.get('redditUpdated').decode('UTF-8')
print(timees)
self.date_updated_reddit = float(timees)
self.date_updated_twitter = False
self.date_updated_coindesk = False
self.redis_client = redis_client
#self.preprocessor = Preprocessor(self.redis_client)
self.analyzer = Analyzer(self.redis_client)
def __init__(self,
r: redis.client.Redis,
expire=7 * 24 * 60 * 60,
keys_key='bucket_keys'):
super().__init__()
self.r = r
self.pipe = r.pipeline()
self.expire = expire
self.bucket_keys = set()
self.keys_key = keys_key
def get_whole_db(redis_instance: redis.client.Redis,
hashset_name: str) -> Json:
db_set = dict()
for key in get_db_keys(redis_instance, hashset_name):
key = key.decode("utf-8")
value = redis_instance.hget(hashset_name, key)
value = value.decode("utf-8")
db_set[key] = value
return json.dumps(db_set)
def zrem_all(r: redis.client.Redis, ddict):
try:
p = r.pipeline()
p.watch(*list(ddict.keys()))
p.multi()
for (topic, name) in ddict.items():
p.zrem(topic, name)
p.execute()
return True
except redis.WatchError as e:
print(e)
return False
def get_all_orders(r: redis.client.Redis):
# Order sizes and prices are stored in different sorted set/ hashset
# May need to investigate if this can handle in high-frequency situation
order_sizes = r.hgetall(redis_order_sizes_name)
user_account_names = r.hgetall(redis_account_names_name)
sell_prices = r.zrange(redis_sell_prices_name, 0, -1, withscores=True)
buy_prices = r.zrange(redis_buy_prices_name, 0, -1, withscores=True)
orders = [
Order(username=user_account_names[order_id].decode('utf-8'),
order_id=order_id.decode('utf-8'),
side="Sell",
size=int(order_sizes[order_id].decode('utf-8')),
price=price) for order_id, price in sell_prices
] + [
Order(username=user_account_names[order_id].decode('utf-8'),
order_id=order_id.decode('utf-8'),
side="Buy",
size=int(order_sizes[order_id].decode('utf-8')),
price=price) for order_id, price in buy_prices
]
return orders
def add_order(r: redis.client.Redis, order: Order):
if order.side == "Buy":
topic_name = redis_buy_prices_name
else:
topic_name = redis_sell_prices_name
r.zadd(topic_name, {order.order_id: order.price})
r.hset(redis_order_sizes_name, key=order.order_id, value=order.size)
r.hset(redis_account_names_name, key=order.order_id, value=order.username)
def remove_price_and_size(r: redis.client.Redis, price_topic, order_topic,
account_name_topic, order_id):
try:
p = r.pipeline()
p.watch(price_topic, order_topic)
p.multi()
p.zrem(price_topic, order_id)
p.hdel(order_topic, order_id)
p.hdel(account_name_topic, order_id)
p.execute()
return True
except redis.WatchError as e:
print(e)
return False
def article_vote(conn: redis.client.Redis, user: str, article: str) -> None:
"""
用户投票
"""
cutoff = time.time() - ONE_WEEK_IN_SECONDS
a = conn.zscore(TIME_NAME, article)
if conn.zscore(TIME_NAME, article) < cutoff:
return
article_id = article.partition(SEP)[:
-1] # diff between split and partition
if conn.sadd(f'{VOTED_NAME}{article_id}', user):
# before
# zincrby(self, name, value, amount)
# now
# zincrby(self, name, amount, value)
conn.zincrby(SCORE_NAME, VOTE_SCORE, article) # zset score increase
conn.hincrby(article, 'votes', 1) # hash value increase
def detection_worker(rds: redis.client.Redis, latest_timestamps: Dict[str, str]):
whose_turn = "A"
# Load the TFLite model and allocate tensors.
interpreter = tflite.Interpreter(model_path="tflite/model.tflite")
interpreter.allocate_tensors()
_, height, width, _ = interpreter.get_input_details()[0]['shape']
while True:
# rotate turn
whose_turn = "B" if whose_turn == "A" else "A"
# get image from A using redis
latest_timestamp = latest_timestamps[whose_turn]
image_bytes = rds.get(f"sentry.{whose_turn}.raw_images.{latest_timestamp}")
if image_bytes is None:
continue
image = Image.open(io.BytesIO(image_bytes)).convert("RGB").resize((width, height), Image.ANTIALIAS)
output_details = interpreter.get_output_details()
tensor_index = interpreter.get_input_details()[0]["index"]
input_tensor = interpreter.tensor(tensor_index)()[0]
input_tensor[:, :] = image
interpreter.invoke()
boxes = interpreter.get_tensor(output_details[0]['index'])
classes = interpreter.get_tensor(output_details[1]['index'])
scores = interpreter.get_tensor(output_details[2]['index'])
try:
font = ImageFont.truetype("/usr/share/fonts/truetype/liberation/LiberationSansNarrow-Regular.ttf",
25)
except IOError:
print("Font not found, using default font.")
font = ImageFont.load_default()
draw_bounding_box_on_image(image,*boxes[0], list(ImageColor.colormap.values())[0], font, display_str_list=[str(round(scores[0]*10))+"%"])
await rds.set(f"sentry.{whose_turn}.processed_image", image.tobytes())
def redis_set(name: Any, value: Any, connector: redis.client.Redis) -> bool:
if not (type(value) in {str, int, float, bool}):
value = pickle.dumps(value)
result = connector.set(name, value)
return result
def get_db_keys(redis_instance: redis.client.Redis,
hashset_name: str) -> List[str]:
return redis_instance.hgetall(hashset_name)
def initialize_db(redis_instance: redis.client.Redis, cfg_keys: List[str],
default_value: int, hashset_name: str) -> None:
for key in cfg_keys:
redis_instance.hset(hashset_name, key, default_value)
def post_article(conn: redis.client.Redis, user: str, title: str,
link: str) -> str:
"""
添加文章
"""
article_id = str(conn.incr(ARTICLE_NAME))
voted = f'{VOTED_NAME}{article_id}'
conn.sadd(voted, user)
conn.expire(voted, ONE_WEEK_IN_SECONDS) # 设置过去时间
article = f'{ARTICLE_NAME}{article_id}'
now = time.time()
conn.hmset(article,
dict(title=title, link=link, poster=user, time=now, votes=1))
# before
# zadd(name,key1,value1,key2,value2)
# now
# zadd(name,{key1:value1,key2:value2})
conn.zadd(SCORE_NAME, {article: now + VOTE_SCORE})
conn.zadd(TIME_NAME, {article: now})
return article_id
def show_key(conn: redis.client.Redis, key: str, count=100):
return list(conn.scan_iter(key, count))
def clear_all_orders(r: redis.client.Redis):
r.delete(redis_sell_prices_name, redis_buy_prices_name,
redis_order_sizes_name, redis_account_names_name)
def redis_get(name: Any, connector: redis.client.Redis) -> Any:
value = connector.get(name)
if type(value) is bytes:
value = pickle.loads(value)
return value