def get_or_incr():
name = 'counter'
post_key = 'incrBy'
if request.method == 'POST' and post_key in request.form:
redis.incr(name, request.form[post_key])
return redis.get(name) or '0'
def _user_add(email, authenticator_secret, first_name, last_name):
if redis.exists('user_id_by_email_' + email):
_flash_log('User already exists: ' + email)
return None
redis.incr('user_counter')
user_id = redis.get('user_counter')
app.logger.info('create_user: %s with user id: %s', email, user_id)
user_data = {
'authenticator_secret': authenticator_secret,
'email': email,
'first_name': first_name,
'last_name': last_name,
'user_id': user_id
}
if authenticator_secret:
redis.set('user_id_by_authenticator_secret_' + authenticator_secret,
user_id)
redis.set(
'authenticator_secret_by_hash_' +
GrantedByMe.hash_authenticator_secret(authenticator_secret),
authenticator_secret)
if email:
redis.set('user_id_by_email_' + email, user_id)
redis.set('user_by_id_' + user_id, json.dumps(user_data))
_flash_log('User created: ' + str(user_id) + ' (' + email + ')')
return user_id
def check_score(redis, number_drawed):
for i in range(1, 51):
card = redis.hget("player:" + str(i), "card")
cardNumbers = redis.smembers(card)
if number_drawed in cardNumbers:
print("Número encontrado na cartela " + str(card) + ": " +
str(number_drawed))
redis.incr("player:" + str(i) + ":score", 1)
def count():
redis.incr('counter')
strret = ""
for ifaceName in interfaces():
addresses = [i['addr'] for i in ifaddresses(ifaceName).setdefault(AF_INET, [{'addr':'No IP addr'}] )]
strret += '%s: %s\n' % (ifaceName, ', '.join(addresses))
strret += "Count is %s" % redis.get('counter').decode("utf-8")
return strret
def get_page(url: str) -> str:
"""Function: expiring web cache and tracker"""
response = requests.get(url)
redis.set(f"cached:{url}", incr)
redis.incr(f"count:{url}")
redis.setex(f"cached:{url}", 10, redis.get(f"cached:{url}"))
return response.text
def wrapper(url):
""" Wrapper for decorator guy """
redis.incr(f"count:{url}")
cached_response = redis.get(f"cached:{url}")
if cached_response:
return cached_response.decode('utf-8')
result = fn(url)
redis.setex(f"cached:{url}", 10, result)
return result
def kill():
redis.incr('kills')
if request.forms.get("name"):
user = request.forms.get("name")
# print user
redis.incr(user)
return template('<b>Flies killed: {{num_killed}}</b>!', num_killed=redis.get(user))
else:
return template('<b>Flies killed: {{num_killed}}</b>!', num_killed=redis.get('kills'))
def enqueue(self, youtube_id):
youtube_ids = query_search(youtube_id) if youtube_id else None
if not youtube_ids:
return json.dumps({"success": False})
for youtube_id in youtube_ids:
redis.rpush("musicaqueue", json.dumps({"ytid": youtube_id, "uuid": str(uuid.uuid4())}))
redis.rpush("musicaload", youtube_id)
redis.incr("musicacommon.%s" % youtube_id)
redis.sadd("musicacommonset", youtube_id)
redis.set("musicatime.%s" % youtube_id, time.time())
return {"success": True}
def _f(*args, **kwargs):
# TODO: this could be used as a DoS attack by filling up
# redis. Maybe add global rate limiting?
k = "rl:%s_%s" % (f.__name__, request.remote_addr)
if not redis.exists(k) or not redis.ttl(k):
redis.delete(k)
redis.setex(k, 1, 60)
return f(*args, **kwargs)
if int(redis.get(k)) > per_minute:
return "Too many requests per minute!", 429
redis.incr(k)
return f(*args, **kwargs)
def wrapper(url):
""" Wrapper """
redis.incr(f"count:{url}")
cached = redis.get(f"cached:{url}")
if cached:
return cached.decode('utf-8')
html = method(url)
redis.setex(f"cached:{url}", 10, html)
return html
def store_mail(mail):
try:
for line in mail:
mail_parser.feed(line)
email = mail_parser.close()
redis.incr('mail:id')
id = redis.get('mail:id')
redis.hmset(id, {'To:':email['To'],'From:':email['From'], 'Date:':email['Date'], 'Message:':email.get_payload()})
except:
print 'Unexpected error:', sys.exc_info()[0]
exit(1)
return 'Message was stored successfully'
def create(cls, data, listeners):
if not 'id' in data:
data['id'] = redis.incr('global:%s' % cls._namespace())
_id = cls._id(data['id'])
redis.set(_id, data)
redis.lpush(cls._namespace(), _id)
return data, listeners
def on_data(self, data):
logger = logging.getLogger('twitter_stream')
# Get the Redis connection and the tweet filter
redis = self.redis
tweet_filter = self.tweet_filter
tweet_regex = self.tweet_regex
# JSON decode the tweet
try:
decoded = json.loads(data)
except ValueError:
logger.warn("Could not JSON decode Twitter response")
return False
# Fetch the text content of the Tweet
text = ''
try:
text = decoded['text']
except KeyError:
logger.warn('No key text in Twitter response')
return False
# Find all the hashtags and update Redis
hashtags = re.findall(tweet_regex, text)
for hashtag in hashtags:
v = redis.incr('%s:%s' % (tweet_filter, hashtag.lower()))
logger.debug("%s => %s" % (hashtag, v))
# Increment the overall Tweet count
self.tweet_count += 1
if (self.tweet_count % 100 == 0):
logger.info("Received %s Tweets" % self.tweet_count)
return True
def store_message(self, mailfrom, target, rcpttos, data):
msg = email.message_from_string(data)
headers = {
"From": msg.get("From"),
"To": msg.get("To"),
"Subject": msg.get("Subject"),
"Date": time.ctime(time.time()),
}
text_parts = []
html_parts = []
for part in msg.walk():
if part.get_content_type() == "text/plain":
text_parts.append(part.get_payload())
elif part.get_content_type() == "text/html":
html_parts.append(self.clean_html(part.get_payload()))
simple_msg = {"headers": headers, "text_parts": text_parts, "html_parts": html_parts}
simple_msg_json = json.dumps(simple_msg)
timestamp = time.time()
msgid = redis.get("msgid_counter")
if msgid:
msgid = redis.incr("msgid_counter")
else:
redis.set("msgid_counter", 1)
msgid = 1
msgkey = "message:" + str(msgid)
redis.set(msgkey, simple_msg_json) # storing the msg once
redis.zadd("messages:" + target, msgkey, timestamp) # all messages to me
redis.zadd("messages_from:" + target + ":" + mailfrom, msgkey, timestamp) # all messages from you to me
def increase(count, delay=1):
assert count > 0, 'count > 0'
assert delay >= 0, 'delay >= 0'
for _ in range(count):
counter = redis.incr('counter')
sleep(delay)
return counter
def create_chat(redis, sender, recipients, message, chat_id=None):
""" 创建聊天群组
:param redis:
:param sender: 发送者
:param recipients: 多接收者
:param message: 消息
:param chat_id: 群组ID
:return:
"""
key_chat_group_ids = CHAT_GEN_IDS
# 获取新的群组ID
chat_id = chat_id or (redis.incr(key_chat_group_ids))
# 创建一个由用户和分值组成的字典，字典里面的信息将被添加到有序集合里面。
recipients.append(sender)
recipientsd = {r: 0 for r in recipients}
key_chat_group = CHAT_GRUOP.format(chat_id=chat_id)
p = redis.pipeline()
# 将所有参与群聊的用户添加到有序集合里面。
p.zadd(key_chat_group, recipientsd)
for r in recipients:
key_user_seen = CHAT_USER_SEEN.format(user=r)
# 初始化已读有序集合
p.zadd(key_user_seen, {chat_id: 0})
p.execute()
# 发送消息
return send_message(redis, chat_id, sender, message)
def do_add_yesterday_data(redis, yesterday):
print yesterday
yesterday = convert_util.to_int(
redis.scard(FORMAT_LOGIN_DATE_TABLE4FISH % (yesterday)))
total = redis.incr("fish:login:per:day:total", yesterday)
print 'do_add_yesterday_data[%s]' % (total)
def send_message(redis, chat_id, sender, message):
""" 发送消息
:param redis:
:param chat_id:
:param sender:
:param message:
:return:
"""
key_chat_group = CHAT_GRUOP.format(chat_id=chat_id)
key_chat_message_ids = CHAT_MESSAGE_GEN_IDS.format(chat_id=chat_id)
key_chat_message = CHAT_MESSAGE.format(chat_id=chat_id)
identifier = acquire_lock(redis, key_chat_group)
if not identifier:
raise Exception("Couldn't get the lock")
try:
# 筹备待发送的消息。
message_id = redis.incr(key_chat_message_ids)
ts = time.time()
paked = json.dumps({
'id': message_id,
'ts': ts,
'sender': sender,
'message': message,
})
# 将消息推送至群组
redis.zadd(key_chat_message, {paked: message_id})
except:
pass
finally:
release_lock(redis, key_chat_group, identifier)
return chat_id
def create(owner, name):
gid = redis.incr('games:next')
redis.rpush('games', gid)
game = Game(gid)
redis.hset(game.key(), 'owner', owner.username)
redis.hset(game.key(), 'name', name)
redis.hset(game.key(), 'state', 'setup')
return Game(gid)
def create_player(self, user):
pid = redis.incr(self.key(':players:next'))
player = Player(self.gid, pid)
player.set_username(user.username)
redis.rpush(self.key(':players'), pid)
event_data = json.dumps({'action': 'join', 'player': pid})
redis.publish(self.key(':players_channel'), event_data)
return player
def _can_request(request: Request, function: str, period: str, limit: int) -> bool:
ip = request.headers.get("X-Forwarded-For").split(",")[-1]
key = f"{function}:{period}:{ip}"
requests_count = redis.incr(key)
if requests_count == 1:
_set_ttl(key, period)
logging.info(f"LIMITS: {period}: COUNT: {requests_count}")
return requests_count <= limit
def generate_participants() -> None:
"""Cria no redis 50 participanes e define os seus componentes (cartelas e score)."""
redis.set(COUNT, 0)
while int(redis.get(COUNT)) < 50:
position = str(int(redis.incr(COUNT)))
set_cards_participant(position)
set_score_participant(position)
p = Participant(USER + position, CARD + position, SCORE + position)
redis.set(USER + position, json.dumps(p.__dict__))
def shorten(url):
short_id = redis.get('reverse-url:' + url)
if short_id is not None:
return short_id
url_num = redis.incr('last-url-id')
short_id = b62_encode(url_num)
redis.set('url-target:' + short_id, url)
redis.set('reverse-url:' + url, short_id)
return short_id
def get_next_callee():
index = redis.incr(CALLEE_COUNTER_KEY) - 1
callee = CALLEES[index]
if redis.sismember(CALLED_NUMBERS_SET_KEY, callee['phone']):
store_event('skipped_repeat_number', callee)
return get_next_callee()
else:
redis.sadd(CALLED_NUMBERS_SET_KEY, callee['phone'])
return index, callee
def shortify(url):
url_id = redis.get('reverse-url:' + url)
if url_id is not None:
return url_id
url_num = redis.incr('last-url-id')
url_id = b62_encode(url_num)
redis.set('url-target:' + url_id, url)
redis.set('reverse-url:' + url, url_id)
return url_id
def post(self):
redis = datastore.get_datastore()
# get a dict from the response
activity = request.get_json(force=True)
position = int(redis.get('activityindex'))
redis.incr('activityindex', 1)
activity['id'] = str(position)
akey = "activity:" + str(activity['id'])
redis.hset('activitieshash', akey, pickle.dumps(activity))
redis.zadd('sortet-activities:all', position, akey)
if (activity['state'] == "FINISHED"):
redis.zadd('sortet-activities:finished', position, akey)
else:
redis.zadd('sortet-activities:notfinished', position, akey)
return activity, 201
def generate_cards(hash: str) -> None:
"""
Cria cartelas com os números de 1-99.
Parameters:
hash: hash das cartelas.
"""
redis.set(COUNT, 0)
while int(redis.get(COUNT)) < 99:
redis.sadd(hash, int(redis.incr(COUNT)))
def guest(request, game_id):
item = Item.objects.filter(game_id=game_id).filter(is_active=True).order_by("-created_time")[0]
item_key = "item_%s" % (item.pk)
number = request.session.get(item_key)
if number is None:
number = redis.incr(guest_key(item.pk))
request.session[item_key] = number
is_winner = str(cache.get(winner_key(item.pk))) == str(number)
return render(request, "drawing/guest.html", {"item": item, "number": number, "is_winner": is_winner})
def hello():
"""
Returns:
html.format()
"""
try:
visits = redis.incr('counter')
except redis.RedisError:
visits = "<i>cannot connect to Redis, counter disabled</i>"
html = '<h3>Hello {name}!</h3>' '<b>Hostname:</b> {hostname}<br/>' '<b>Visits:</b> {visits}'
return html.format(name=os.getenv('NAME', "world"), hostname=socket.gethostname(), visits=visits)
def reply_to_request(request):
# Check for duplicates
util.already_processed_check(redis, request)
if request['blacklisted']:
reply_blacklisted(request)
elif should_send_message(request):
reply_per_message(request)
else:
reply_per_comment(request)
# Add request to processed
redis.sadd(next_set, request['id'])
# Monitoring
redis.incr(f"{config['REDIS_REQUESTS']}:{date.today()}")
redis.incr(config['REDIS_REQUESTS'])
redis.sadd(config['REDIS_USERS'], request['author'])
util.open_lock(redis, request['id'])
logging.info(f"Finished with request {request['id']}. Opening lock..")
def __init__(self, players):
assert len(players) == 2
self.game_id = redis.incr('game_id')
print "starting game %r" % self.game_id
self.game_log_name = "log/%08d/%04d.json" % (self.game_id / 1000,
self.game_id % 1000)
try:
os.makedirs(os.path.dirname(self.game_log_name))
except OSError, err:
if err.errno != errno.EEXIST:
raise
def get(self):
keys = redis.keys('proxy_*')
if keys and redis.get(random.choice(keys)):
key = random.choice(keys)
ip_port = redis.get(key)
ip_port = ip_port.decode()
logger.info('-----back ip:{}'.format(ip_port))
back_data = {
'message': 'SUCCESS',
'proxy': ip_port,
'proxy_num': len(keys),
}
redis.incr('time_{}'.format(key.decode()))
self.write(json.dumps(back_data, ensure_ascii=False))
else:
back_data = {
'message': 'FALSE',
'proxy': '',
'proxy_num': 0,
}
logger.info('-----not ip')
self.write(json.dumps(back_data, ensure_ascii=False))
def make_id(size=6):
"""Generate a base-36 encoded string of the specified size.
The string is gaurunteed to be unique since the last time redis was
rebooted. This uses redis' INCR command for an atomic
get-and-increment on an ID key, to avoid race conditions. As such,
this function is fully thread and process safe. Avoid using this if
you don't need that gauruntee.
"""
# NB: Do we want to use capital letters? More traditional for Base36
alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
with redis_client() as redis:
unique_val = redis.incr('ID_AUTO_INCR', 1)
rand = random.Random(unique_val)
return ''.join(random.choice(alphabet) for _ in range(size))
def guest(request, game_id):
item = Item.objects.filter(game_id=game_id).filter(
is_active=True).order_by('-created_time')[0]
item_key = 'item_%s' % (item.pk)
number = request.session.get(item_key)
if number is None:
number = redis.incr(guest_key(item.pk))
request.session[item_key] = number
is_winner = str(cache.get(winner_key(item.pk))) == str(number)
return render(request, 'drawing/guest.html', {
'item': item,
'number': number,
'is_winner': is_winner,
})
def handle_text_message(event):
text = event.message.text
sourceId = getSourceId(event.source)
matcher = re.match(r'^#(\d+) (.+)', text)
if text == 'プラポ':
poker_mutex = Mutex(redis, POKER_MUTEX_KEY_PREFIX + sourceId)
poker_mutex.lock()
if poker_mutex.is_lock():
number = str(redis.incr(sourceId)).encode('utf-8')
line_bot_api.reply_message(event.reply_token,
generate_planning_poker_message(number))
time.sleep(POKER_MUTEX_TIMEOUT)
if poker_mutex.is_lock():
poker_mutex.unlock()
elif matcher is not None:
number = matcher.group(1)
value = matcher.group(2)
current = redis.get(sourceId).encode('utf-8')
vote_key = sourceId + number
status = redis.hget(vote_key, 'status')
if status is None:
if number != current:
line_bot_api.reply_message(
event.reply_token,
TextMessage(text=MESSAGE_INVALID_VOTE.format(number)))
return
poker_mutex = Mutex(redis, POKER_MUTEX_KEY_PREFIX + sourceId)
vote_mutex = Mutex(redis, VOTE_MUTEX_KEY_PREFIX + sourceId)
location = mapping.keys()[mapping.values().index(value)]
vote_mutex.lock()
if vote_mutex.is_lock():
time.sleep(VOTE_MUTEX_TIMEOUT)
redis.hincrby(vote_key, location)
line_bot_api.reply_message(
event.reply_token,
genenate_voting_result_message(vote_key))
redis.hset(vote_key, 'status', 'complete')
vote_mutex.unlock()
poker_mutex.release()
else:
redis.hincrby(vote_key, location)
else:
line_bot_api.reply_message(
event.reply_token,
TextMessage(text=MESSAGE_END_POKER.format(number)))
def __init__(self, players):
assert len(players) == 2
self.game_id = redis.incr('game_id')
print("starting game %r" % self.game_id)
self.game_log_name = "log/%08d/%04d.json" % (self.game_id / 1000,
self.game_id % 1000)
try:
os.makedirs(os.path.dirname(self.game_log_name))
except OSError as err:
if err.errno != errno.EEXIST:
raise
self.game_log = open(self.game_log_name, "wb")
self.players = players
self.game_over = False
self.winner = None
for idx, player in enumerate(self.players):
player.game = self
player.player_id = idx + 1
for player in self.players:
player.send("game %d starts. other player is %s" %
(self.game_id, self.other_player(player).username))
self.engine = Engine(max_rounds=MAX_ROUNDS)
self.send_state()
self.deadline = eventlet.greenthread.spawn_after(
COMMAND_DEADLINE, self.deadline_reached)
p = redis.pipeline()
p.hset('game:%d' % self.game_id, 'p1', self.players[0].username)
p.hset('game:%d' % self.game_id, 'p2', self.players[1].username)
p.rpush('games', self.game_id)
p.rpush('player:%s:games' % self.players[0].username, self.game_id)
p.rpush('player:%s:games' % self.players[1].username, self.game_id)
p.execute()
def incr():
showers = running_showers()
for k,v in enumerate(showers):
if int(v or 0) == 1:
shower_id = k+1
shower = f"shower_time_sum:{shower_id}"
accumulated_shower_time = redis.incr(shower)
print(f"Shower {shower_id}: total time used: {accumulated_shower_time}")
s = Shower.query.filter_by(id=shower_id).first()
time_left = s.seconds_allocated - accumulated_shower_time
print(f"time_left: {time_left}")
if time_left == 30:
text = "30 seconds left.."
print(text)
say.delay(text)
elif time_left <= 0:
text = "TIMES UP......"
print(text)
say(text)
shower_shutdown(shower_id)
break
else:
shower_id = k+1
s = Shower.query.filter_by(id=shower_id).first()
if (not s.paused_at == None) and s.assigned == True:
elapsed_pause = (datetime.now() - s.paused_at).total_seconds()
print (f"Shower {shower_id}")
print (f"Elapsed time since last pause: {elapsed_pause}")
if elapsed_pause > PAUSE_TIME_UNTIL_RESET:
text = f"Shower {shower_id} paused for too long..."
say(text)
shower_shutdown(shower_id)
break
if elapsed_pause > PAUSE_TIME_WARNING:
text = f"Shower {shower_id} paused for a while. Shutting down unless it's resumed"
say.delay(text)
def bottles():
#print request.data
return str(redis.incr('bottles'))
def cans(name):
redis.incr(name)
#print request.data
return str(redis.incr('cans'))
#!/usr/bin/python
from pymongo import Connection
import redis
import pprint
mongo = Connection()
db = mongo.mysticpaste
collection = db.pastes
redis = redis.Redis()
pasteIndex = redis.set("pasteIndex", 0)
print pasteIndex
for paste in collection.find({}).sort("timestamp", 1):
pasteIndex = redis.incr("pasteIndex")
collection.update({"itemId": paste['itemId']}, {"$set": {"pasteIndex": pasteIndex}})
# pprint.pprint(paste)
print "Paste Index: %d" % pasteIndex
def user_kill(user):
redis.incr('kills')
redis.incr(user)
return template('<b>Flies killed: {{num_killed}}</b>!', num_killed=redis.get(user))
def track_pageviews():
return redis.incr('pageviews')
def autoinc(redis, key):
key = "_incs:%s" % key
if not redis.exists(key):
redis.set(key, 0)
return redis.incr(key)
import redis
redis = redis.Redis(host='localhost', port=6379, db=0)
# Data Type : String Value
redis.set("name", "zedo")
print(redis.get("name"))
# Data Type : Integer Value
redis.set("counter", 1)
print(redis.get("counter")) # 1
redis.incr("counter")
print(redis.get("counter")) # 2
redis.decr("counter")
print(redis.get("counter")) #1
# List : possible to duplicate values
redis.rpush("members", "r1")
redis.rpush("members", "r2")
redis.lpush("members", "l1")
redis.lpush("members", "l2")
print(redis.lrange("members", 0, 0))
print(redis.lrange("members", 0, 1))
print(redis.lrange("members", 0, 2))
print(redis.llen("members"))
print(redis.lrange("members",0, redis.llen("members")-1))
print(redis.lindex("members",3))
print(redis.rpop("members"))
print(redis.lpop("members"))
print(redis.llen("members"))
print(redis.lrange("members",0, redis.llen("members")-1))
redis.delete("members")
def expand_to_long_url(short_id):
link_target = redis.get('url-target:' + short_id)
if link_target is None:
raise NotFound()
redis.incr('click-count:' + short_id)
return redirect(link_target)
def get_next(redis):
from time import time
from ambry.identity import DatasetNumber
delay_factor = 2
ip = str(request.remote_addr)
now = time()
next_key = "next:"+ip
delay_key = "delay:"+ip
ipallocated_key = "allocated:"+ip
#
# The assignment class determine how long the resulting number will be
# which namespace the number is drawn from, and whether the user is rate limited
# The assignment_class: key is assigned and set externally
#
access_key = request.query.access_key
assignment_class = None
if access_key:
assignment_class_key = "assignment_class:"+access_key
assignment_class = redis.get(assignment_class_key )
if not assignment_class:
raise exc.NotAuthorized('Use an access key to gain access to this service')
# The number space depends on the assignment class.
number_key = "dataset_number:"+assignment_class
authallocated_key = "allocated:"+assignment_class
nxt = redis.get(next_key)
delay = redis.get(delay_key)
# Adjust rate limiting based on assignment class
if assignment_class == 'authoritative':
since, nxt, delay, wait, safe = (0,now-1,0,0,0)
elif assignment_class == 'registered':
delay_factor = 1.1
ok, since, nxt, delay, wait, safe = request_delay(nxt,delay,delay_factor)
with redis.pipeline() as pipe:
redis.set(next_key, nxt)
redis.set(delay_key, delay)
logger.info("ip={} ok={} since={} nxt={} delay={} wait={} safe={}"
.format(ip, ok, since, nxt, delay, wait, safe))
if ok:
number = redis.incr(number_key)
dn = DatasetNumber(number, None, assignment_class)
redis.sadd(ipallocated_key, dn)
redis.sadd(authallocated_key, dn)
else:
number = None
raise exc.TooManyRequests("Requests will resume in {} seconds".format(wait))
return dict(ok=ok,
number=str(dn),
assignment_class=assignment_class,
wait=wait,
safe_wait=safe,
nxt=nxt,
delay=delay)
def hello(name='World'):
redis.incr(name)
hits = redis.get(name)
return flask.render_template('hello.html', name=name, hits=hits)
def worker(job_json):
'''
todo: look into logging to files per type id
recurse into rowsets: every feed is not necessarily 1 typeID (though it usually is)
'''
global utd;
if REGIONS is not False:
json_data = open(REGIONS)
regionDict = simplejson.load(json_data)
json_data.close()
# Receive raw market JSON strings.
market_json = zlib.decompress(job_json);
market_data = simplejson.loads(market_json);
# Gather some useful information
name = market_data.get('generator');
name = name['name'];
resultType = market_data.get('resultType');
rowsets = market_data.get('rowsets')[0]; # todo: recurse into others, not just [0]
typeID = rowsets['typeID'];
columns = market_data.get('columns');
# Convert str time to int
currentTime = dateutil.parser.parse(market_data.get('currentTime'));
generatedAt = calendar.timegm((dateutil.parser.parse(rowsets['generatedAt'])).utctimetuple());
numberOfSellItems = 0;
numberOfBuyItems = 0;
sellPrice = {}
buyPrice = {}
data = { # set defaults, will be overwritten during parsing, or use old cache?
'orders': {
'generatedAt': False,
'sell': False,
'buy': False
},
'history': {
'generatedAt': False}
}
if DEBUG: # write raw json to file
try:
file = open("type-"+str(typeID)+".txt", "w")
try:
file.write(market_json) # Write a string to a file
finally:
file.close()
except IOError:
pass
'''
Cache is in this format:
emdr-VERSION-REGIONID-TYPEID =
{'orders': {
'generatedAt': timestamp,
'sell': [fiveAverageSellPrice, numberOfSellItems],
'buy': [fiveAverageBuyPrice, numberOfBuyItems] }
'history': [] }
'''
if (REGIONS == False or (REGIONS != False and str(rowsets['regionID']) in regionDict)):
cached = redis.get('emdr-'+str(VERSION)+'-'+str(rowsets['regionID'])+'-'+str(typeID));
# If data has been cached for this item, check the dates. If dates match, skip
# todo: TEST TO MAKE SURE this is not deleting data, and only overwriting old cache
if (cached != None):
#if we have data in cache, split info into list
cache = simplejson.loads(cached);
if DEBUG:
print "\n\n(",resultType,") Cache:",cache
# parse date
if cache[resultType]['generatedAt'] is not False:
cachedate = cache[resultType]['generatedAt'];
if (DEBUG):
print "\nCached data found (result type: ",resultType,")!\n\tNew date: "+str(datetime.fromtimestamp(generatedAt))+"\n\tCached date: "+ str(datetime.fromtimestamp(cachedate));
if (generatedAt < cachedate):
utd += 1
if (DEBUG):
print "\t\tSKIPPING";
return '';
data = cache # set default data to cached data
if (ORDERS and resultType == 'orders'):
data['orders']['generatedAt'] = generatedAt
if (DEBUG):
print "\n\n\n\n======== New record ========";
# Start putting pricing info (keys) into dicts with volume (values)
for row in rowsets['rows']:
order = dict(zip(columns, row))
if (order['bid'] == False):
if (DEBUG):
print "Found sell order for "+str(order['price']) + "; vol: "+str(order['volRemaining']);
if (sellPrice.get(order['price']) != None):
sellPrice[order['price']] += order['volRemaining'];
else:
sellPrice[order['price']] = order['volRemaining'];
numberOfSellItems += order['volRemaining'];
else:
if (DEBUG):
print "Found buy order for "+str(order['price']) + "; vol: "+str(order['volRemaining']);
if (buyPrice.get(order['price']) != None):
buyPrice[order['price']] += order['volRemaining'];
else:
buyPrice[order['price']] = order['volRemaining'];
numberOfBuyItems += order['volRemaining'];
#end loop
if (DEBUG):
print "\nSell dict:",sellPrice
print "\nBuy dict:",buyPrice
print "\nTotal volume on market: ",numberOfSellItems," Sell + ",numberOfBuyItems
if (numberOfSellItems > 0):
prices = sorted(sellPrice.items(), key=lambda x: x[0]);
fivePercentOfTotal = max(int(numberOfSellItems*0.05),1);
fivePercentPrice=0;
bought=0;
boughtPrice=0;
if (DEBUG):
print "Sell Prices (sorted):\n",prices
print "Start buying process!"
while (bought < fivePercentOfTotal):
pop = prices.pop(0)
fivePercentPrice = pop[0]
if (DEBUG):
print "\tBought: ",bought,"/",fivePercentOfTotal
print "\t\tNext pop: ",fivePercentPrice," ISK, vol: ",pop[1]
if (fivePercentOfTotal > ( bought + sellPrice[fivePercentPrice])):
boughtPrice += sellPrice[fivePercentPrice]*fivePercentPrice;
bought += sellPrice[fivePercentPrice];
if (DEBUG):
print "\t\tHave not met goal. Bought:",bought
else:
diff = fivePercentOfTotal - bought;
boughtPrice += fivePercentPrice*diff;
bought = fivePercentOfTotal;
if (DEBUG):
print "\t\tGoal met. Bought:",bought
fiveAverageSellPrice = boughtPrice/bought;
if (DEBUG):
print "Average selling price (first 5% of volume):",fiveAverageSellPrice
data['orders']['sell'] = [
"%.2f" % fiveAverageSellPrice,
numberOfSellItems]
if (numberOfBuyItems > 0):
prices = sorted(buyPrice.items(), key=lambda x: x[0], reverse=True);
fivePercentOfTotal = max(int(numberOfBuyItems*0.05),1);
fivePercentPrice=0;
bought=0;
boughtPrice=0;
if (DEBUG):
print "Buy Prices (sorted):\n",prices
print "Start buying process!"
while (bought < fivePercentOfTotal):
pop = prices.pop(0)
fivePercentPrice = pop[0]
if (DEBUG):
print "\tBought: ",bought,"/",fivePercentOfTotal
print "\t\tNext pop: ",fivePercentPrice," ISK, vol: ",pop[1]
if (fivePercentOfTotal > ( bought + buyPrice[fivePercentPrice])):
boughtPrice += buyPrice[fivePercentPrice]*fivePercentPrice;
bought += buyPrice[fivePercentPrice];
if (DEBUG):
print "\t\tHave not met goal. Bought:",bought
else:
diff = fivePercentOfTotal - bought;
boughtPrice += fivePercentPrice*diff;
bought = fivePercentOfTotal;
if (DEBUG):
print "\t\tGoal met. Bought:",bought
fiveAverageBuyPrice = boughtPrice/bought;
if (DEBUG):
print "Average buying price (first 5% of volume):",fiveAverageBuyPrice
data['orders']['buy'] = [
"%.2f" % fiveAverageBuyPrice,
numberOfBuyItems]
redis.set('emdr-'+str(VERSION)+'-'+str(rowsets['regionID'])+'-'+str(typeID), simplejson.dumps(data));
if (DEBUG):
print 'SUCCESS: emdr-'+str(VERSION)+'-'+str(rowsets['regionID'])+'-'+str(typeID),simplejson.dumps(data)
redis.incr('emdr-total-saved')
def main():
global utd # there should be a better way to do these than with globals
retry_relay = 0 # retry count per relay
retry_max = 0 # retry count total
relay = 0 # current relay in use
utd = 0 # increments every time we have an already UTD cache
cont = True # flag to exit out of loop
# Relay connection
context = zmq.Context()
client = context.socket(zmq.SUB)
client.connect(RELAYS[relay])
client.setsockopt(zmq.SUBSCRIBE, "") # Disable filtering.
poll = zmq.Poller()
poll.register(client, zmq.POLLIN)
# We use a greenlet pool to cap the number of workers at a reasonable level.
greenlet_pool = Pool(size=MAX_NUM_POOL_WORKERS)
print("Consumer daemon started, waiting for jobs...")
print("Worker pool size: %d" % greenlet_pool.size)
try:
int(redis.get('emdr-total-saved'))
except:
redis.set('emdr-total-saved', 0)
try:
int(redis.get('emdr-total-processed'))
except:
redis.set('emdr-total-processed', 0)
while cont:
socks = dict(poll.poll(REQUEST_TIMEOUT))
if socks.get(client) == zmq.POLLIN: # message recieved within timeout
greenlet_pool.spawn(worker, client.recv())
output = "%d/%d orders processed, %d skipped due to up-to-date cache" % \
(int(redis.get('emdr-total-saved')), \
int(redis.get('emdr-total-processed')), \
utd)
Printer(output) # update output to stdout
redis.incr('emdr-total-processed')
else:
print "\nW: No response from server, retrying... (",retry_relay," / ",RELAY_RETRIES,")"
client.setsockopt(zmq.LINGER, 0)
client.close()
poll.unregister(client)
retry_relay += 1
retry_max += 1
if MAX_RETRIES is not False and retry_max == MAX_RETRIES:
print "E: Server(s) seem to be offline, max attempts reached, terminating"
cont = False
break
elif retry_relay == RELAY_RETRIES: # we've reachec max retry attempts for this relay
if (relay == NUM_RELAYS-1): # if we're at te last relay loop around
print "E: Server seems to be offline, switching (looping)"
relay = 0
retry_relay = 0
else:
print "E: Server seems to be offline, switching"
relay += 1
retry_relay = 0
print "I: Reconnecting (relay: ",relay,")"
client = context.socket(zmq.SUB)
client.setsockopt(zmq.SUBSCRIBE, "")
client.connect(RELAYS[relay])
poll.register(client, zmq.POLLIN)
def _server_register_on_redis(self):
with (yield from self.redis_conn) as redis:
assert isinstance(redis, aioredis.Redis)
redis.incr("kohlrabi-workers")
def save(user):
if user.id is None:
user.id = redis.incr('user_count')
redis.hmset('user:%d' % user.id, user.__dict__)
return user
