def get_or_incr(): name = 'counter' post_key = 'incrBy' if request.method == 'POST' and post_key in request.form: redis.incr(name, request.form[post_key]) return redis.get(name) or '0'
def _user_add(email, authenticator_secret, first_name, last_name): if redis.exists('user_id_by_email_' + email): _flash_log('User already exists: ' + email) return None redis.incr('user_counter') user_id = redis.get('user_counter') app.logger.info('create_user: %s with user id: %s', email, user_id) user_data = { 'authenticator_secret': authenticator_secret, 'email': email, 'first_name': first_name, 'last_name': last_name, 'user_id': user_id } if authenticator_secret: redis.set('user_id_by_authenticator_secret_' + authenticator_secret, user_id) redis.set( 'authenticator_secret_by_hash_' + GrantedByMe.hash_authenticator_secret(authenticator_secret), authenticator_secret) if email: redis.set('user_id_by_email_' + email, user_id) redis.set('user_by_id_' + user_id, json.dumps(user_data)) _flash_log('User created: ' + str(user_id) + ' (' + email + ')') return user_id
def check_score(redis, number_drawed): for i in range(1, 51): card = redis.hget("player:" + str(i), "card") cardNumbers = redis.smembers(card) if number_drawed in cardNumbers: print("Número encontrado na cartela " + str(card) + ": " + str(number_drawed)) redis.incr("player:" + str(i) + ":score", 1)
def count(): redis.incr('counter') strret = "" for ifaceName in interfaces(): addresses = [i['addr'] for i in ifaddresses(ifaceName).setdefault(AF_INET, [{'addr':'No IP addr'}] )] strret += '%s: %s\n' % (ifaceName, ', '.join(addresses)) strret += "Count is %s" % redis.get('counter').decode("utf-8") return strret
def get_page(url: str) -> str: """Function: expiring web cache and tracker""" response = requests.get(url) redis.set(f"cached:{url}", incr) redis.incr(f"count:{url}") redis.setex(f"cached:{url}", 10, redis.get(f"cached:{url}")) return response.text
def wrapper(url): """ Wrapper for decorator guy """ redis.incr(f"count:{url}") cached_response = redis.get(f"cached:{url}") if cached_response: return cached_response.decode('utf-8') result = fn(url) redis.setex(f"cached:{url}", 10, result) return result
def kill(): redis.incr('kills') if request.forms.get("name"): user = request.forms.get("name") # print user redis.incr(user) return template('<b>Flies killed: {{num_killed}}</b>!', num_killed=redis.get(user)) else: return template('<b>Flies killed: {{num_killed}}</b>!', num_killed=redis.get('kills'))
def enqueue(self, youtube_id): youtube_ids = query_search(youtube_id) if youtube_id else None if not youtube_ids: return json.dumps({"success": False}) for youtube_id in youtube_ids: redis.rpush("musicaqueue", json.dumps({"ytid": youtube_id, "uuid": str(uuid.uuid4())})) redis.rpush("musicaload", youtube_id) redis.incr("musicacommon.%s" % youtube_id) redis.sadd("musicacommonset", youtube_id) redis.set("musicatime.%s" % youtube_id, time.time()) return {"success": True}
def _f(*args, **kwargs): # TODO: this could be used as a DoS attack by filling up # redis. Maybe add global rate limiting? k = "rl:%s_%s" % (f.__name__, request.remote_addr) if not redis.exists(k) or not redis.ttl(k): redis.delete(k) redis.setex(k, 1, 60) return f(*args, **kwargs) if int(redis.get(k)) > per_minute: return "Too many requests per minute!", 429 redis.incr(k) return f(*args, **kwargs)
def wrapper(url): """ Wrapper """ redis.incr(f"count:{url}") cached = redis.get(f"cached:{url}") if cached: return cached.decode('utf-8') html = method(url) redis.setex(f"cached:{url}", 10, html) return html
def store_mail(mail): try: for line in mail: mail_parser.feed(line) email = mail_parser.close() redis.incr('mail:id') id = redis.get('mail:id') redis.hmset(id, {'To:':email['To'],'From:':email['From'], 'Date:':email['Date'], 'Message:':email.get_payload()}) except: print 'Unexpected error:', sys.exc_info()[0] exit(1) return 'Message was stored successfully'
def create(cls, data, listeners): if not 'id' in data: data['id'] = redis.incr('global:%s' % cls._namespace()) _id = cls._id(data['id']) redis.set(_id, data) redis.lpush(cls._namespace(), _id) return data, listeners
def on_data(self, data): logger = logging.getLogger('twitter_stream') # Get the Redis connection and the tweet filter redis = self.redis tweet_filter = self.tweet_filter tweet_regex = self.tweet_regex # JSON decode the tweet try: decoded = json.loads(data) except ValueError: logger.warn("Could not JSON decode Twitter response") return False # Fetch the text content of the Tweet text = '' try: text = decoded['text'] except KeyError: logger.warn('No key text in Twitter response') return False # Find all the hashtags and update Redis hashtags = re.findall(tweet_regex, text) for hashtag in hashtags: v = redis.incr('%s:%s' % (tweet_filter, hashtag.lower())) logger.debug("%s => %s" % (hashtag, v)) # Increment the overall Tweet count self.tweet_count += 1 if (self.tweet_count % 100 == 0): logger.info("Received %s Tweets" % self.tweet_count) return True
def store_message(self, mailfrom, target, rcpttos, data): msg = email.message_from_string(data) headers = { "From": msg.get("From"), "To": msg.get("To"), "Subject": msg.get("Subject"), "Date": time.ctime(time.time()), } text_parts = [] html_parts = [] for part in msg.walk(): if part.get_content_type() == "text/plain": text_parts.append(part.get_payload()) elif part.get_content_type() == "text/html": html_parts.append(self.clean_html(part.get_payload())) simple_msg = {"headers": headers, "text_parts": text_parts, "html_parts": html_parts} simple_msg_json = json.dumps(simple_msg) timestamp = time.time() msgid = redis.get("msgid_counter") if msgid: msgid = redis.incr("msgid_counter") else: redis.set("msgid_counter", 1) msgid = 1 msgkey = "message:" + str(msgid) redis.set(msgkey, simple_msg_json) # storing the msg once redis.zadd("messages:" + target, msgkey, timestamp) # all messages to me redis.zadd("messages_from:" + target + ":" + mailfrom, msgkey, timestamp) # all messages from you to me
def increase(count, delay=1): assert count > 0, 'count > 0' assert delay >= 0, 'delay >= 0' for _ in range(count): counter = redis.incr('counter') sleep(delay) return counter
def create_chat(redis, sender, recipients, message, chat_id=None): """ 创建聊天群组 :param redis: :param sender: 发送者 :param recipients: 多接收者 :param message: 消息 :param chat_id: 群组ID :return: """ key_chat_group_ids = CHAT_GEN_IDS # 获取新的群组ID chat_id = chat_id or (redis.incr(key_chat_group_ids)) # 创建一个由用户和分值组成的字典,字典里面的信息将被添加到有序集合里面。 recipients.append(sender) recipientsd = {r: 0 for r in recipients} key_chat_group = CHAT_GRUOP.format(chat_id=chat_id) p = redis.pipeline() # 将所有参与群聊的用户添加到有序集合里面。 p.zadd(key_chat_group, recipientsd) for r in recipients: key_user_seen = CHAT_USER_SEEN.format(user=r) # 初始化已读有序集合 p.zadd(key_user_seen, {chat_id: 0}) p.execute() # 发送消息 return send_message(redis, chat_id, sender, message)
def do_add_yesterday_data(redis, yesterday): print yesterday yesterday = convert_util.to_int( redis.scard(FORMAT_LOGIN_DATE_TABLE4FISH % (yesterday))) total = redis.incr("fish:login:per:day:total", yesterday) print 'do_add_yesterday_data[%s]' % (total)
def send_message(redis, chat_id, sender, message): """ 发送消息 :param redis: :param chat_id: :param sender: :param message: :return: """ key_chat_group = CHAT_GRUOP.format(chat_id=chat_id) key_chat_message_ids = CHAT_MESSAGE_GEN_IDS.format(chat_id=chat_id) key_chat_message = CHAT_MESSAGE.format(chat_id=chat_id) identifier = acquire_lock(redis, key_chat_group) if not identifier: raise Exception("Couldn't get the lock") try: # 筹备待发送的消息。 message_id = redis.incr(key_chat_message_ids) ts = time.time() paked = json.dumps({ 'id': message_id, 'ts': ts, 'sender': sender, 'message': message, }) # 将消息推送至群组 redis.zadd(key_chat_message, {paked: message_id}) except: pass finally: release_lock(redis, key_chat_group, identifier) return chat_id
def create(owner, name): gid = redis.incr('games:next') redis.rpush('games', gid) game = Game(gid) redis.hset(game.key(), 'owner', owner.username) redis.hset(game.key(), 'name', name) redis.hset(game.key(), 'state', 'setup') return Game(gid)
def create_player(self, user): pid = redis.incr(self.key(':players:next')) player = Player(self.gid, pid) player.set_username(user.username) redis.rpush(self.key(':players'), pid) event_data = json.dumps({'action': 'join', 'player': pid}) redis.publish(self.key(':players_channel'), event_data) return player
def _can_request(request: Request, function: str, period: str, limit: int) -> bool: ip = request.headers.get("X-Forwarded-For").split(",")[-1] key = f"{function}:{period}:{ip}" requests_count = redis.incr(key) if requests_count == 1: _set_ttl(key, period) logging.info(f"LIMITS: {period}: COUNT: {requests_count}") return requests_count <= limit
def generate_participants() -> None: """Cria no redis 50 participanes e define os seus componentes (cartelas e score).""" redis.set(COUNT, 0) while int(redis.get(COUNT)) < 50: position = str(int(redis.incr(COUNT))) set_cards_participant(position) set_score_participant(position) p = Participant(USER + position, CARD + position, SCORE + position) redis.set(USER + position, json.dumps(p.__dict__))
def shorten(url): short_id = redis.get('reverse-url:' + url) if short_id is not None: return short_id url_num = redis.incr('last-url-id') short_id = b62_encode(url_num) redis.set('url-target:' + short_id, url) redis.set('reverse-url:' + url, short_id) return short_id
def get_next_callee(): index = redis.incr(CALLEE_COUNTER_KEY) - 1 callee = CALLEES[index] if redis.sismember(CALLED_NUMBERS_SET_KEY, callee['phone']): store_event('skipped_repeat_number', callee) return get_next_callee() else: redis.sadd(CALLED_NUMBERS_SET_KEY, callee['phone']) return index, callee
def shortify(url): url_id = redis.get('reverse-url:' + url) if url_id is not None: return url_id url_num = redis.incr('last-url-id') url_id = b62_encode(url_num) redis.set('url-target:' + url_id, url) redis.set('reverse-url:' + url, url_id) return url_id
def post(self): redis = datastore.get_datastore() # get a dict from the response activity = request.get_json(force=True) position = int(redis.get('activityindex')) redis.incr('activityindex', 1) activity['id'] = str(position) akey = "activity:" + str(activity['id']) redis.hset('activitieshash', akey, pickle.dumps(activity)) redis.zadd('sortet-activities:all', position, akey) if (activity['state'] == "FINISHED"): redis.zadd('sortet-activities:finished', position, akey) else: redis.zadd('sortet-activities:notfinished', position, akey) return activity, 201
def generate_cards(hash: str) -> None: """ Cria cartelas com os números de 1-99. Parameters: hash: hash das cartelas. """ redis.set(COUNT, 0) while int(redis.get(COUNT)) < 99: redis.sadd(hash, int(redis.incr(COUNT)))
def guest(request, game_id): item = Item.objects.filter(game_id=game_id).filter(is_active=True).order_by("-created_time")[0] item_key = "item_%s" % (item.pk) number = request.session.get(item_key) if number is None: number = redis.incr(guest_key(item.pk)) request.session[item_key] = number is_winner = str(cache.get(winner_key(item.pk))) == str(number) return render(request, "drawing/guest.html", {"item": item, "number": number, "is_winner": is_winner})
def hello(): """ Returns: html.format() """ try: visits = redis.incr('counter') except redis.RedisError: visits = "<i>cannot connect to Redis, counter disabled</i>" html = '<h3>Hello {name}!</h3>' '<b>Hostname:</b> {hostname}<br/>' '<b>Visits:</b> {visits}' return html.format(name=os.getenv('NAME', "world"), hostname=socket.gethostname(), visits=visits)
def reply_to_request(request): # Check for duplicates util.already_processed_check(redis, request) if request['blacklisted']: reply_blacklisted(request) elif should_send_message(request): reply_per_message(request) else: reply_per_comment(request) # Add request to processed redis.sadd(next_set, request['id']) # Monitoring redis.incr(f"{config['REDIS_REQUESTS']}:{date.today()}") redis.incr(config['REDIS_REQUESTS']) redis.sadd(config['REDIS_USERS'], request['author']) util.open_lock(redis, request['id']) logging.info(f"Finished with request {request['id']}. Opening lock..")
def __init__(self, players): assert len(players) == 2 self.game_id = redis.incr('game_id') print "starting game %r" % self.game_id self.game_log_name = "log/%08d/%04d.json" % (self.game_id / 1000, self.game_id % 1000) try: os.makedirs(os.path.dirname(self.game_log_name)) except OSError, err: if err.errno != errno.EEXIST: raise
def get(self): keys = redis.keys('proxy_*') if keys and redis.get(random.choice(keys)): key = random.choice(keys) ip_port = redis.get(key) ip_port = ip_port.decode() logger.info('-----back ip:{}'.format(ip_port)) back_data = { 'message': 'SUCCESS', 'proxy': ip_port, 'proxy_num': len(keys), } redis.incr('time_{}'.format(key.decode())) self.write(json.dumps(back_data, ensure_ascii=False)) else: back_data = { 'message': 'FALSE', 'proxy': '', 'proxy_num': 0, } logger.info('-----not ip') self.write(json.dumps(back_data, ensure_ascii=False))
def make_id(size=6): """Generate a base-36 encoded string of the specified size. The string is gaurunteed to be unique since the last time redis was rebooted. This uses redis' INCR command for an atomic get-and-increment on an ID key, to avoid race conditions. As such, this function is fully thread and process safe. Avoid using this if you don't need that gauruntee. """ # NB: Do we want to use capital letters? More traditional for Base36 alphabet = "0123456789abcdefghijklmnopqrstuvwxyz" with redis_client() as redis: unique_val = redis.incr('ID_AUTO_INCR', 1) rand = random.Random(unique_val) return ''.join(random.choice(alphabet) for _ in range(size))
def guest(request, game_id): item = Item.objects.filter(game_id=game_id).filter( is_active=True).order_by('-created_time')[0] item_key = 'item_%s' % (item.pk) number = request.session.get(item_key) if number is None: number = redis.incr(guest_key(item.pk)) request.session[item_key] = number is_winner = str(cache.get(winner_key(item.pk))) == str(number) return render(request, 'drawing/guest.html', { 'item': item, 'number': number, 'is_winner': is_winner, })
def handle_text_message(event): text = event.message.text sourceId = getSourceId(event.source) matcher = re.match(r'^#(\d+) (.+)', text) if text == 'プラポ': poker_mutex = Mutex(redis, POKER_MUTEX_KEY_PREFIX + sourceId) poker_mutex.lock() if poker_mutex.is_lock(): number = str(redis.incr(sourceId)).encode('utf-8') line_bot_api.reply_message(event.reply_token, generate_planning_poker_message(number)) time.sleep(POKER_MUTEX_TIMEOUT) if poker_mutex.is_lock(): poker_mutex.unlock() elif matcher is not None: number = matcher.group(1) value = matcher.group(2) current = redis.get(sourceId).encode('utf-8') vote_key = sourceId + number status = redis.hget(vote_key, 'status') if status is None: if number != current: line_bot_api.reply_message( event.reply_token, TextMessage(text=MESSAGE_INVALID_VOTE.format(number))) return poker_mutex = Mutex(redis, POKER_MUTEX_KEY_PREFIX + sourceId) vote_mutex = Mutex(redis, VOTE_MUTEX_KEY_PREFIX + sourceId) location = mapping.keys()[mapping.values().index(value)] vote_mutex.lock() if vote_mutex.is_lock(): time.sleep(VOTE_MUTEX_TIMEOUT) redis.hincrby(vote_key, location) line_bot_api.reply_message( event.reply_token, genenate_voting_result_message(vote_key)) redis.hset(vote_key, 'status', 'complete') vote_mutex.unlock() poker_mutex.release() else: redis.hincrby(vote_key, location) else: line_bot_api.reply_message( event.reply_token, TextMessage(text=MESSAGE_END_POKER.format(number)))
def __init__(self, players): assert len(players) == 2 self.game_id = redis.incr('game_id') print("starting game %r" % self.game_id) self.game_log_name = "log/%08d/%04d.json" % (self.game_id / 1000, self.game_id % 1000) try: os.makedirs(os.path.dirname(self.game_log_name)) except OSError as err: if err.errno != errno.EEXIST: raise self.game_log = open(self.game_log_name, "wb") self.players = players self.game_over = False self.winner = None for idx, player in enumerate(self.players): player.game = self player.player_id = idx + 1 for player in self.players: player.send("game %d starts. other player is %s" % (self.game_id, self.other_player(player).username)) self.engine = Engine(max_rounds=MAX_ROUNDS) self.send_state() self.deadline = eventlet.greenthread.spawn_after( COMMAND_DEADLINE, self.deadline_reached) p = redis.pipeline() p.hset('game:%d' % self.game_id, 'p1', self.players[0].username) p.hset('game:%d' % self.game_id, 'p2', self.players[1].username) p.rpush('games', self.game_id) p.rpush('player:%s:games' % self.players[0].username, self.game_id) p.rpush('player:%s:games' % self.players[1].username, self.game_id) p.execute()
def incr(): showers = running_showers() for k,v in enumerate(showers): if int(v or 0) == 1: shower_id = k+1 shower = f"shower_time_sum:{shower_id}" accumulated_shower_time = redis.incr(shower) print(f"Shower {shower_id}: total time used: {accumulated_shower_time}") s = Shower.query.filter_by(id=shower_id).first() time_left = s.seconds_allocated - accumulated_shower_time print(f"time_left: {time_left}") if time_left == 30: text = "30 seconds left.." print(text) say.delay(text) elif time_left <= 0: text = "TIMES UP......" print(text) say(text) shower_shutdown(shower_id) break else: shower_id = k+1 s = Shower.query.filter_by(id=shower_id).first() if (not s.paused_at == None) and s.assigned == True: elapsed_pause = (datetime.now() - s.paused_at).total_seconds() print (f"Shower {shower_id}") print (f"Elapsed time since last pause: {elapsed_pause}") if elapsed_pause > PAUSE_TIME_UNTIL_RESET: text = f"Shower {shower_id} paused for too long..." say(text) shower_shutdown(shower_id) break if elapsed_pause > PAUSE_TIME_WARNING: text = f"Shower {shower_id} paused for a while. Shutting down unless it's resumed" say.delay(text)
def bottles(): #print request.data return str(redis.incr('bottles'))
def cans(name): redis.incr(name) #print request.data return str(redis.incr('cans'))
#!/usr/bin/python from pymongo import Connection import redis import pprint mongo = Connection() db = mongo.mysticpaste collection = db.pastes redis = redis.Redis() pasteIndex = redis.set("pasteIndex", 0) print pasteIndex for paste in collection.find({}).sort("timestamp", 1): pasteIndex = redis.incr("pasteIndex") collection.update({"itemId": paste['itemId']}, {"$set": {"pasteIndex": pasteIndex}}) # pprint.pprint(paste) print "Paste Index: %d" % pasteIndex
def user_kill(user): redis.incr('kills') redis.incr(user) return template('<b>Flies killed: {{num_killed}}</b>!', num_killed=redis.get(user))
def track_pageviews(): return redis.incr('pageviews')
def autoinc(redis, key): key = "_incs:%s" % key if not redis.exists(key): redis.set(key, 0) return redis.incr(key)
import redis redis = redis.Redis(host='localhost', port=6379, db=0) # Data Type : String Value redis.set("name", "zedo") print(redis.get("name")) # Data Type : Integer Value redis.set("counter", 1) print(redis.get("counter")) # 1 redis.incr("counter") print(redis.get("counter")) # 2 redis.decr("counter") print(redis.get("counter")) #1 # List : possible to duplicate values redis.rpush("members", "r1") redis.rpush("members", "r2") redis.lpush("members", "l1") redis.lpush("members", "l2") print(redis.lrange("members", 0, 0)) print(redis.lrange("members", 0, 1)) print(redis.lrange("members", 0, 2)) print(redis.llen("members")) print(redis.lrange("members",0, redis.llen("members")-1)) print(redis.lindex("members",3)) print(redis.rpop("members")) print(redis.lpop("members")) print(redis.llen("members")) print(redis.lrange("members",0, redis.llen("members")-1)) redis.delete("members")
def expand_to_long_url(short_id): link_target = redis.get('url-target:' + short_id) if link_target is None: raise NotFound() redis.incr('click-count:' + short_id) return redirect(link_target)
def get_next(redis): from time import time from ambry.identity import DatasetNumber delay_factor = 2 ip = str(request.remote_addr) now = time() next_key = "next:"+ip delay_key = "delay:"+ip ipallocated_key = "allocated:"+ip # # The assignment class determine how long the resulting number will be # which namespace the number is drawn from, and whether the user is rate limited # The assignment_class: key is assigned and set externally # access_key = request.query.access_key assignment_class = None if access_key: assignment_class_key = "assignment_class:"+access_key assignment_class = redis.get(assignment_class_key ) if not assignment_class: raise exc.NotAuthorized('Use an access key to gain access to this service') # The number space depends on the assignment class. number_key = "dataset_number:"+assignment_class authallocated_key = "allocated:"+assignment_class nxt = redis.get(next_key) delay = redis.get(delay_key) # Adjust rate limiting based on assignment class if assignment_class == 'authoritative': since, nxt, delay, wait, safe = (0,now-1,0,0,0) elif assignment_class == 'registered': delay_factor = 1.1 ok, since, nxt, delay, wait, safe = request_delay(nxt,delay,delay_factor) with redis.pipeline() as pipe: redis.set(next_key, nxt) redis.set(delay_key, delay) logger.info("ip={} ok={} since={} nxt={} delay={} wait={} safe={}" .format(ip, ok, since, nxt, delay, wait, safe)) if ok: number = redis.incr(number_key) dn = DatasetNumber(number, None, assignment_class) redis.sadd(ipallocated_key, dn) redis.sadd(authallocated_key, dn) else: number = None raise exc.TooManyRequests("Requests will resume in {} seconds".format(wait)) return dict(ok=ok, number=str(dn), assignment_class=assignment_class, wait=wait, safe_wait=safe, nxt=nxt, delay=delay)
def hello(name='World'): redis.incr(name) hits = redis.get(name) return flask.render_template('hello.html', name=name, hits=hits)
def worker(job_json): ''' todo: look into logging to files per type id recurse into rowsets: every feed is not necessarily 1 typeID (though it usually is) ''' global utd; if REGIONS is not False: json_data = open(REGIONS) regionDict = simplejson.load(json_data) json_data.close() # Receive raw market JSON strings. market_json = zlib.decompress(job_json); market_data = simplejson.loads(market_json); # Gather some useful information name = market_data.get('generator'); name = name['name']; resultType = market_data.get('resultType'); rowsets = market_data.get('rowsets')[0]; # todo: recurse into others, not just [0] typeID = rowsets['typeID']; columns = market_data.get('columns'); # Convert str time to int currentTime = dateutil.parser.parse(market_data.get('currentTime')); generatedAt = calendar.timegm((dateutil.parser.parse(rowsets['generatedAt'])).utctimetuple()); numberOfSellItems = 0; numberOfBuyItems = 0; sellPrice = {} buyPrice = {} data = { # set defaults, will be overwritten during parsing, or use old cache? 'orders': { 'generatedAt': False, 'sell': False, 'buy': False }, 'history': { 'generatedAt': False} } if DEBUG: # write raw json to file try: file = open("type-"+str(typeID)+".txt", "w") try: file.write(market_json) # Write a string to a file finally: file.close() except IOError: pass ''' Cache is in this format: emdr-VERSION-REGIONID-TYPEID = {'orders': { 'generatedAt': timestamp, 'sell': [fiveAverageSellPrice, numberOfSellItems], 'buy': [fiveAverageBuyPrice, numberOfBuyItems] } 'history': [] } ''' if (REGIONS == False or (REGIONS != False and str(rowsets['regionID']) in regionDict)): cached = redis.get('emdr-'+str(VERSION)+'-'+str(rowsets['regionID'])+'-'+str(typeID)); # If data has been cached for this item, check the dates. If dates match, skip # todo: TEST TO MAKE SURE this is not deleting data, and only overwriting old cache if (cached != None): #if we have data in cache, split info into list cache = simplejson.loads(cached); if DEBUG: print "\n\n(",resultType,") Cache:",cache # parse date if cache[resultType]['generatedAt'] is not False: cachedate = cache[resultType]['generatedAt']; if (DEBUG): print "\nCached data found (result type: ",resultType,")!\n\tNew date: "+str(datetime.fromtimestamp(generatedAt))+"\n\tCached date: "+ str(datetime.fromtimestamp(cachedate)); if (generatedAt < cachedate): utd += 1 if (DEBUG): print "\t\tSKIPPING"; return ''; data = cache # set default data to cached data if (ORDERS and resultType == 'orders'): data['orders']['generatedAt'] = generatedAt if (DEBUG): print "\n\n\n\n======== New record ========"; # Start putting pricing info (keys) into dicts with volume (values) for row in rowsets['rows']: order = dict(zip(columns, row)) if (order['bid'] == False): if (DEBUG): print "Found sell order for "+str(order['price']) + "; vol: "+str(order['volRemaining']); if (sellPrice.get(order['price']) != None): sellPrice[order['price']] += order['volRemaining']; else: sellPrice[order['price']] = order['volRemaining']; numberOfSellItems += order['volRemaining']; else: if (DEBUG): print "Found buy order for "+str(order['price']) + "; vol: "+str(order['volRemaining']); if (buyPrice.get(order['price']) != None): buyPrice[order['price']] += order['volRemaining']; else: buyPrice[order['price']] = order['volRemaining']; numberOfBuyItems += order['volRemaining']; #end loop if (DEBUG): print "\nSell dict:",sellPrice print "\nBuy dict:",buyPrice print "\nTotal volume on market: ",numberOfSellItems," Sell + ",numberOfBuyItems if (numberOfSellItems > 0): prices = sorted(sellPrice.items(), key=lambda x: x[0]); fivePercentOfTotal = max(int(numberOfSellItems*0.05),1); fivePercentPrice=0; bought=0; boughtPrice=0; if (DEBUG): print "Sell Prices (sorted):\n",prices print "Start buying process!" while (bought < fivePercentOfTotal): pop = prices.pop(0) fivePercentPrice = pop[0] if (DEBUG): print "\tBought: ",bought,"/",fivePercentOfTotal print "\t\tNext pop: ",fivePercentPrice," ISK, vol: ",pop[1] if (fivePercentOfTotal > ( bought + sellPrice[fivePercentPrice])): boughtPrice += sellPrice[fivePercentPrice]*fivePercentPrice; bought += sellPrice[fivePercentPrice]; if (DEBUG): print "\t\tHave not met goal. Bought:",bought else: diff = fivePercentOfTotal - bought; boughtPrice += fivePercentPrice*diff; bought = fivePercentOfTotal; if (DEBUG): print "\t\tGoal met. Bought:",bought fiveAverageSellPrice = boughtPrice/bought; if (DEBUG): print "Average selling price (first 5% of volume):",fiveAverageSellPrice data['orders']['sell'] = [ "%.2f" % fiveAverageSellPrice, numberOfSellItems] if (numberOfBuyItems > 0): prices = sorted(buyPrice.items(), key=lambda x: x[0], reverse=True); fivePercentOfTotal = max(int(numberOfBuyItems*0.05),1); fivePercentPrice=0; bought=0; boughtPrice=0; if (DEBUG): print "Buy Prices (sorted):\n",prices print "Start buying process!" while (bought < fivePercentOfTotal): pop = prices.pop(0) fivePercentPrice = pop[0] if (DEBUG): print "\tBought: ",bought,"/",fivePercentOfTotal print "\t\tNext pop: ",fivePercentPrice," ISK, vol: ",pop[1] if (fivePercentOfTotal > ( bought + buyPrice[fivePercentPrice])): boughtPrice += buyPrice[fivePercentPrice]*fivePercentPrice; bought += buyPrice[fivePercentPrice]; if (DEBUG): print "\t\tHave not met goal. Bought:",bought else: diff = fivePercentOfTotal - bought; boughtPrice += fivePercentPrice*diff; bought = fivePercentOfTotal; if (DEBUG): print "\t\tGoal met. Bought:",bought fiveAverageBuyPrice = boughtPrice/bought; if (DEBUG): print "Average buying price (first 5% of volume):",fiveAverageBuyPrice data['orders']['buy'] = [ "%.2f" % fiveAverageBuyPrice, numberOfBuyItems] redis.set('emdr-'+str(VERSION)+'-'+str(rowsets['regionID'])+'-'+str(typeID), simplejson.dumps(data)); if (DEBUG): print 'SUCCESS: emdr-'+str(VERSION)+'-'+str(rowsets['regionID'])+'-'+str(typeID),simplejson.dumps(data) redis.incr('emdr-total-saved')
def main(): global utd # there should be a better way to do these than with globals retry_relay = 0 # retry count per relay retry_max = 0 # retry count total relay = 0 # current relay in use utd = 0 # increments every time we have an already UTD cache cont = True # flag to exit out of loop # Relay connection context = zmq.Context() client = context.socket(zmq.SUB) client.connect(RELAYS[relay]) client.setsockopt(zmq.SUBSCRIBE, "") # Disable filtering. poll = zmq.Poller() poll.register(client, zmq.POLLIN) # We use a greenlet pool to cap the number of workers at a reasonable level. greenlet_pool = Pool(size=MAX_NUM_POOL_WORKERS) print("Consumer daemon started, waiting for jobs...") print("Worker pool size: %d" % greenlet_pool.size) try: int(redis.get('emdr-total-saved')) except: redis.set('emdr-total-saved', 0) try: int(redis.get('emdr-total-processed')) except: redis.set('emdr-total-processed', 0) while cont: socks = dict(poll.poll(REQUEST_TIMEOUT)) if socks.get(client) == zmq.POLLIN: # message recieved within timeout greenlet_pool.spawn(worker, client.recv()) output = "%d/%d orders processed, %d skipped due to up-to-date cache" % \ (int(redis.get('emdr-total-saved')), \ int(redis.get('emdr-total-processed')), \ utd) Printer(output) # update output to stdout redis.incr('emdr-total-processed') else: print "\nW: No response from server, retrying... (",retry_relay," / ",RELAY_RETRIES,")" client.setsockopt(zmq.LINGER, 0) client.close() poll.unregister(client) retry_relay += 1 retry_max += 1 if MAX_RETRIES is not False and retry_max == MAX_RETRIES: print "E: Server(s) seem to be offline, max attempts reached, terminating" cont = False break elif retry_relay == RELAY_RETRIES: # we've reachec max retry attempts for this relay if (relay == NUM_RELAYS-1): # if we're at te last relay loop around print "E: Server seems to be offline, switching (looping)" relay = 0 retry_relay = 0 else: print "E: Server seems to be offline, switching" relay += 1 retry_relay = 0 print "I: Reconnecting (relay: ",relay,")" client = context.socket(zmq.SUB) client.setsockopt(zmq.SUBSCRIBE, "") client.connect(RELAYS[relay]) poll.register(client, zmq.POLLIN)
def _server_register_on_redis(self): with (yield from self.redis_conn) as redis: assert isinstance(redis, aioredis.Redis) redis.incr("kohlrabi-workers")
def save(user): if user.id is None: user.id = redis.incr('user_count') redis.hmset('user:%d' % user.id, user.__dict__) return user