def _blockplayer(): timeouts = current_app.config["BLOCKEDPLAYER_EXPIRE_TIMEOUTS"] blocked_count_ip_key = f"blocked:{ip}" expire_index = max(0, redis_connection.incr(blocked_count_ip_key) - 1) redis_connection.expire(blocked_count_ip_key, timeouts[-1]) timeout = timeouts[min(expire_index, len(timeouts) - 1)] expires = now + timeout blockedplayers_for_puzzle_key = "blockedplayers:{puzzle}".format( puzzle=puzzle) # Add the player to the blocked players list for the puzzle and # extend the expiration of the key. redis_connection.zadd(blockedplayers_for_puzzle_key, {user: expires}) redis_connection.expire(blockedplayers_for_puzzle_key, timeouts[-1]) err_msg = get_blockedplayers_err_msg(expires, expires - now) sse.publish( "{user}:{piece}:{karma}:{karma_change}".format( user=user, piece=piece, karma=karma + recent_points, karma_change=karma_change, ), type="karma", channel="puzzle:{puzzle_id}".format( puzzle_id=puzzle_data["puzzle_id"]), ) return make_response(json.jsonify(err_msg), 429)
def __call__(self): self.do_task() if self.id == None: # The id may be None if the task is being run manually. Running # a task manually shouldn't change the schedule. return now = int(time()) due = now + self.interval current_app.logger.debug( "{format_due} - task {task_name} {task_id} due date".format( **{ "format_due": ctime(due), "task_name": self.task_name, "task_id": self.id, })) redis_connection.zadd(scheduler_key, {self.id: due})
def all_tasks(): """ Cycle through all tasks in the task registry and run them at their set interval. """ # Reset scheduler to start by removing any previous scheduled tasks redis_connection.delete(scheduler_key) now = int(time()) tasks = {} for index in range(len(task_registry)): # Create each task with an id corresponding to the index tasks[index] = task_registry[index](index) task_ids_scheduled_to_now = dict( zip(tasks.keys(), map(lambda x: now, range(len(tasks))))) # reset all tasks to be scheduled now redis_connection.zadd(scheduler_key, task_ids_scheduled_to_now) def cycle_over_tasks(): "Cycle over each and call the task" for task_id in task_ids: tasks[task_id]() while True: now = int(time()) # Get list of tasks on the schedule that are due. task_ids = list( map(int, redis_connection.zrangebyscore(scheduler_key, 0, now))) # Cycle over each and call the task. Any connection errors will trigger # a longer wait before retrying. try: for task_id in task_ids: tasks[task_id]() except requests.exceptions.ConnectionError as err: current_app.logger.warning( "Connection error. Retrying in {} seconds... \nError: {}". format(SCHEDULER_RETRY_INTERVAL, err)) sleep(SCHEDULER_RETRY_INTERVAL) sleep(SCHEDULER_INTERVAL)
def increase_ban_time(user, seconds): now = int(time.time()) current = int(redis_connection.zscore("bannedusers", user) or now) current = max(current, now) ban_timestamp = min(current + seconds, now + MAX_BAN_TIME) redis_connection.zadd("bannedusers", {user: ban_timestamp}) return { "msg": "Temporarily banned. Ban time has increased by {} seconds".format( seconds), "type": "bannedusers", "user": user, "expires": ban_timestamp, "timeout": ban_timestamp - now, }
def patch(self, puzzle_id): "Pong. Determine the latency for this player." response = {"message": "", "name": "", "data": {"latency": 0}} args = {} xhr_data = request.get_json() if xhr_data: args.update(xhr_data) if request.form: args.update(request.form.to_dict(flat=True)) token = args.get("token") if token is None: response["message"] = "No token" response["name"] = "error" return make_response(json.jsonify(response), 400) user = current_app.secure_cookie.get(u"user") or user_id_from_ip( request.headers.get("X-Real-IP"), skip_generate=True, validate_shared_user=False, ) if user is None: response["message"] = "Player not currently logged in." response["name"] = "error" return make_response(json.jsonify(response), 400) user = int(user) cur = db.cursor() # Validate the puzzle_id result = cur.execute( fetch_query_string("select-id-status-from-puzzle-by-puzzle_id.sql"), {"puzzle_id": puzzle_id}, ).fetchall() if not result: response["message"] = "Puzzle not available" response["name"] = "invalid" cur.close() return make_response(json.jsonify(response), 400) else: (result, col_names) = rowify(result, cur.description) puzzle = result[0].get("id") status = result[0].get("status") if status not in ( ACTIVE, IN_QUEUE, COMPLETED, FROZEN, BUGGY_UNLISTED, NEEDS_MODERATION, REBUILD, IN_RENDER_QUEUE, RENDERING, RENDERING_FAILED, MAINTENANCE, ): response["message"] = "Puzzle no longer valid" response["name"] = "invalid" cur.close() sse.publish( "Puzzle no longer valid", type="invalid", channel="puzzle:{puzzle_id}".format(puzzle_id=puzzle_id), ) return make_response(json.jsonify(response), 200) cur.close() # Determine latency for the player and record timestamp in sorted set. pingtoken_key = get_pingtoken_key(puzzle, user, token) ping_start = redis_connection.get(pingtoken_key) redis_connection.delete(pingtoken_key) ping_end = int(time.time() * 1000) if not ping_start: response["message"] = "Ignoring error when determining latency." response["name"] = "ignored" return make_response(json.jsonify(response), 200) ping_start = int(ping_start) ping_key = get_ping_key(puzzle) redis_connection.zadd(ping_key, {user: ping_end}) redis_connection.expire(ping_key, PING_EXPIRE) latency = ping_end - ping_start # Record the latency for the player redis_connection.lpush( "latency", "{user}:{timestamp}:{latency}".format( user=user, timestamp=ping_end, latency=latency ), ) # Keep only the last 1000 entries to latency redis_connection.ltrim("latency", 0, 999) response["message"] = "Latency" response["data"]["latency"] = latency response["name"] = "success" response = make_response(json.jsonify(response), 200) return response
def publishMessage(msg, karma_change, karma, points=0, complete=False): # print(topic) # print(msg) if current_app.config.get("PUZZLE_PIECES_CACHE_TTL"): stamp = redis_connection.get(f"pzstamp:{puzzle}") if stamp: pcu_key = f"pcu:{stamp}" redis_connection.rpushx(pcu_key, msg) sse.publish( msg, type="move", channel="puzzle:{puzzle_id}".format( puzzle_id=puzzleData["puzzle_id"]), ) if user != ANONYMOUS_USER_ID: points_key = "points:{user}".format(user=user) recent_points = int(redis_connection.get(points_key) or 0) if karma_change < 0 and karma <= 0 and recent_points > 0: redis_connection.decr(points_key) redis_connection.zadd("pcupdates", {puzzle: now}) if user != ANONYMOUS_USER_ID: # bump the m_date for this player on the puzzle and timeline redis_connection.zadd("timeline:{puzzle}".format(puzzle=puzzle), {user: now}) redis_connection.zadd("timeline", {user: now}) # Update player points if points != 0 and user is not None and user != ANONYMOUS_USER_ID: redis_connection.zincrby("score:{puzzle}".format(puzzle=puzzle), amount=1, value=user) redis_connection.sadd("batchuser", user) redis_connection.sadd("batchpuzzle", puzzle) redis_connection.incr("batchscore:{user}".format(user=user), amount=1) redis_connection.incr( "batchpoints:{puzzle}:{user}".format(puzzle=puzzle, user=user), amount=points, ) redis_connection.zincrby("rank", amount=1, value=user) pieces = int(puzzleData["pieces"]) # Skip increasing dots if puzzle is private earns = get_earned_points(pieces, permission=puzzleData.get("permission")) # karma = int(redis_connection.get(karma_key)) ## Max out recent points if (earns != 0 and karma >= current_app.config["MAX_KARMA"] and recent_points < current_app.config["MAX_RECENT_POINTS"]): recent_points = redis_connection.incr(points_key) # Doing small puzzles doesn't increase recent points, just extends points expiration. redis_connection.expire(points_key, current_app.config["RECENT_POINTS_EXPIRE"]) # Extend the karma points expiration since it has increased redis_connection.expire(karma_key, current_app.config["KARMA_POINTS_EXPIRE"]) # Max out karma if karma < current_app.config["MAX_KARMA"]: karma = redis_connection.incr(karma_key) karma_change += 1 redis_connection.incr("batchpoints:{user}".format(user=user), amount=earns) if complete: current_app.logger.info("puzzle {puzzle_id} is complete".format( puzzle_id=puzzleData["puzzle_id"])) sse.publish( "status:{}".format(COMPLETED), channel="puzzle:{puzzle_id}".format( puzzle_id=puzzleData["puzzle_id"]), ) r = requests.patch( "http://{HOSTAPI}:{PORTAPI}/internal/puzzle/{puzzle_id}/details/" .format( HOSTAPI=current_app.config["HOSTAPI"], PORTAPI=current_app.config["PORTAPI"], puzzle_id=puzzleData["puzzle_id"], ), json={ "status": COMPLETED, "m_date": time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()), "queue": QUEUE_END_OF_LINE, }, ) if r.status_code != 200: raise Exception( "Puzzle details api error when updating puzzle to be complete" ) # Delaying helps avoid issues for players that are moving the last # piece of the puzzle as someone else completes it. delay = (current_app.config["MAX_PAUSE_PIECES_TIMEOUT"] + current_app.config["PIECE_MOVE_TIMEOUT"] + 2) current_app.logger.info( f"Delaying puzzle transfer on completed puzzle ({puzzleData['puzzle_id']}) for {delay} seconds" ) job = current_app.cleanupqueue.enqueue_in( timedelta(seconds=delay), "api.jobs.convertPiecesToDB.transfer", puzzle, result_ttl=0, ) purge_route_from_nginx_cache( "/chill/site/front/{puzzle_id}/".format( puzzle_id=puzzleData["puzzle_id"]), current_app.config.get("PURGEURLLIST"), ) if karma_change and user != ANONYMOUS_USER_ID: sse.publish( "{user}:{piece}:{karma}:{karma_change}".format( user=user, piece=piece, karma=karma + recent_points, karma_change=karma_change, ), type="karma", channel="puzzle:{puzzle_id}".format( puzzle_id=puzzleData["puzzle_id"]), ) # end = time.perf_counter() # duration = end - start # redis_connection.rpush("testdata:translate", duration) # return topic and msg mostly for testing return (msg, karma_change)
def do_task(self): super().do_task() made_change = False cur = db.cursor() puzzle = redis_connection.spop("batchpuzzle") while puzzle: last_batch = redis_connection.zrangebyscore( "timeline:{puzzle}".format(puzzle=puzzle), self.last_run, "+inf", withscores=True, ) for (user, update_timestamp) in last_batch: current_app.logger.debug( "user: {user}, {update_timestamp}".format( user=user, update_timestamp=update_timestamp)) user = int(user) points = int( redis_connection.getset( "batchpoints:{puzzle}:{user}".format(puzzle=puzzle, user=user), value=0, ) or "0") redis_connection.expire( "batchpoints:{puzzle}:{user}".format(puzzle=puzzle, user=user), DAY) if points != 0: result = cur.execute( fetch_query_string("select-all-from-puzzle-by-id.sql"), { "puzzle": puzzle }, ).fetchall() if not result: current_app.logger.warn( "no puzzle details found for puzzle {}".format( puzzle)) continue (result, col_names) = rowify(result, cur.description) puzzle_data = result[0] puzzle_id = puzzle_data["puzzle_id"] timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime(update_timestamp)) current_app.logger.debug( "{timestamp} - bumping {points} points on {puzzle} ({puzzle_id}) for player: {player}" .format( puzzle=puzzle, puzzle_id=puzzle_id, player=user, points=points, timestamp=timestamp, )) r = requests.post( "http://{HOSTAPI}:{PORTAPI}/internal/puzzle/{puzzle_id}/timeline/" .format( HOSTAPI=current_app.config["HOSTAPI"], PORTAPI=current_app.config["PORTAPI"], puzzle_id=puzzle_id, ), json={ "player": user, "points": points, "timestamp": timestamp }, ) if r.status_code != 200: current_app.logger.warning( "Puzzle timeline api error. Could not add batchpoints. Skipping {puzzle_id}" .format(puzzle_id=puzzle_id, )) continue made_change = True puzzle = redis_connection.spop("batchpuzzle") if self.first_run: result = cur.execute( read_query_file( "get_list_of_puzzles_in_timeline.sql")).fetchall() if result and len(result): puzzle_list = list(map(lambda x: x[0], result)) for puzzle in puzzle_list: result = cur.execute( read_query_file( "select_user_score_and_timestamp_per_puzzle.sql"), { "puzzle": puzzle }, ).fetchall() if result and len(result): current_app.logger.info( "Set puzzle ({0}) score and puzzle timeline on {1} players" .format(puzzle, len(result))) user_score = dict(map(lambda x: [x[0], x[1]], result)) user_timestamps = dict( map(lambda x: [x[0], int(x[2])], result)) redis_connection.zadd( "timeline:{puzzle}".format(puzzle=puzzle), user_timestamps) redis_connection.zadd( "score:{puzzle}".format(puzzle=puzzle), user_score) made_change = True self.first_run = False self.last_run = int(time()) if made_change: self.log_task() cur.close()
def do_task(self): super().do_task() made_change = False cur = db.cursor() user = redis_connection.spop("batchuser") while user: user = int(user) score = redis_connection.getset( "batchscore:{user}".format(user=user), value=0) redis_connection.expire("batchscore:{user}".format(user=user), DAY) points = redis_connection.getset( "batchpoints:{user}".format(user=user), value=0) redis_connection.expire("batchpoints:{user}".format(user=user), DAY) current_app.logger.debug( "update user {id} with {points} points and score of {score}". format(**{ "id": user, "points": points, "score": score })) r = requests.post( "http://{HOSTAPI}:{PORTAPI}/internal/tasks/{task_name}/start/". format( HOSTAPI=current_app.config["HOSTAPI"], PORTAPI=current_app.config["PORTAPI"], task_name="update_user_points_and_m_date", ), json={ "player": user, "points": points, "score": score, }, ) if r.status_code != 200: current_app.logger.warning( "Internal tasks api error. Could not run task update_user_points_and_m_date for player {}" .format(user)) r = requests.post( "http://{HOSTAPI}:{PORTAPI}/internal/tasks/{task_name}/start/". format( HOSTAPI=current_app.config["HOSTAPI"], PORTAPI=current_app.config["PORTAPI"], task_name="update_bit_icon_expiration", ), json={ "player": user, }, ) if r.status_code != 200: current_app.logger.warning( "Internal tasks api error. Could not run task update_bit_icon_expiration for player {}" .format(user)) user = redis_connection.spop("batchuser") made_change = True if self.first_run: result = cur.execute( read_query_file( "select_user_score_and_timestamp.sql")).fetchall() if result and len(result): current_app.logger.info( "Set rank and timeline on {0} players".format(len(result))) user_scores = dict(map(lambda x: [x[0], x[1]], result)) user_timestamps = dict(map(lambda x: [x[0], int(x[2])], result)) redis_connection.zadd("rank", user_scores) redis_connection.zadd("timeline", user_timestamps) made_change = True self.first_run = False if made_change: self.log_task() cur.close()
def publishMessage(msg, karma_change, points=0, complete=False): # print(topic) # print(msg) sse.publish( msg, type="move", channel="puzzle:{puzzle_id}".format( puzzle_id=puzzleData["puzzle_id"]), ) now = int(time.time()) redis_connection.zadd("pcupdates", {puzzle: now}) # TODO: # return (topic, msg) # bump the m_date for this player on the puzzle and timeline redis_connection.zadd("timeline:{puzzle}".format(puzzle=puzzle), {user: now}) redis_connection.zadd("timeline", {user: now}) # Update player points if points != 0 and user != None: redis_connection.zincrby("score:{puzzle}".format(puzzle=puzzle), amount=1, value=user) redis_connection.sadd("batchuser", user) redis_connection.sadd("batchpuzzle", puzzle) redis_connection.incr("batchscore:{user}".format(user=user), amount=1) redis_connection.incr( "batchpoints:{puzzle}:{user}".format(puzzle=puzzle, user=user), amount=points, ) redis_connection.zincrby("rank", amount=1, value=user) points_key = "points:{user}".format(user=user) pieces = int(puzzleData["pieces"]) # Skip increasing dots if puzzle is private earns = get_earned_points(pieces, permission=puzzleData.get("permission")) karma = int(redis_connection.get(karma_key)) ## Max out recent points # if earns != 0: # recent_points = int(redis_connection.get(points_key) or 0) # if karma + 1 + recent_points + earns < MAX_KARMA: # redis_connection.incr(points_key, amount=earns) # Doing small puzzles doesn't increase recent points, just extends points expiration. redis_connection.expire(points_key, RECENT_POINTS_EXPIRE) karma_change += 1 # Extend the karma points expiration since it has increased redis_connection.expire(karma_key, KARMA_POINTS_EXPIRE) # Max out karma if karma < MAX_KARMA: redis_connection.incr(karma_key) else: # Max out points if earns != 0: recent_points = int(redis_connection.get(points_key) or 0) if recent_points + earns <= MAX_RECENT_POINTS: redis_connection.incr(points_key, amount=earns) redis_connection.incr("batchpoints:{user}".format(user=user), amount=earns) # TODO: Optimize by using redis for puzzle status if complete: current_app.logger.info("puzzle {puzzle_id} is complete".format( puzzle_id=puzzleData["puzzle_id"])) cur = db.cursor() cur.execute( fetch_query_string("update_puzzle_status_for_puzzle.sql"), { "puzzle": puzzle, "status": COMPLETED }, ) cur.execute( fetch_query_string("update_puzzle_m_date_to_now.sql"), { "puzzle": puzzle, "modified": now }, ) cur.execute( fetch_query_string("update_puzzle_queue_for_puzzle.sql"), { "puzzle": puzzle, "queue": QUEUE_END_OF_LINE }, ) db.commit() sse.publish( "status:{}".format(COMPLETED), channel="puzzle:{puzzle_id}".format( puzzle_id=puzzleData["puzzle_id"]), ) job = current_app.cleanupqueue.enqueue_call( func="api.jobs.convertPiecesToDB.transfer", args=(puzzle, ), result_ttl=0) purge_route_from_nginx_cache( "/chill/site/front/{puzzle_id}/".format( puzzle_id=puzzleData["puzzle_id"]), current_app.config.get("PURGEURLLIST"), ) db.commit() cur.close() # return topic and msg mostly for testing return (msg, karma_change)
def patch(self, puzzle_id): "Pong. Determine the latency for this player." response = {"message": "", "name": "", "data": {"latency": 0}} args = {} xhr_data = request.get_json() if xhr_data: args.update(xhr_data) if request.form: args.update(request.form.to_dict(flat=True)) token = args.get("token") if token == None: response["message"] = "No token" response["name"] = "error" return make_response(json.jsonify(response), 400) user = current_app.secure_cookie.get(u"user") or user_id_from_ip( request.headers.get("X-Real-IP"), skip_generate=True) if user == None: response["message"] = "Player not currently logged in." response["name"] = "error" return make_response(json.jsonify(response), 400) user = int(user) cur = db.cursor() # Validate the puzzle_id result = cur.execute( fetch_query_string("select_viewable_puzzle_id.sql"), { "puzzle_id": puzzle_id }, ).fetchall() if not result: response["message"] = "Invalid puzzle id." response["name"] = "error" cur.close() db.commit() return make_response(json.jsonify(response), 400) else: (result, col_names) = rowify(result, cur.description) puzzle = result[0].get("puzzle") status = result[0].get("status") if status != ACTIVE: response["message"] = "Puzzle not active" response["name"] = "invalid" cur.close() db.commit() return make_response(json.jsonify(response), 200) # Determine latency for the player and record timestamp in sorted set. pingtoken_key = get_pingtoken_key(puzzle, user, token) ping_start = redis_connection.get(pingtoken_key) redis_connection.delete(pingtoken_key) ping_end = int(time.time() * 1000) if not ping_start: response["message"] = "Ignoring error when determining latency." response["name"] = "ignored" return make_response(json.jsonify(response), 200) ping_start = int(ping_start) ping_key = get_ping_key(puzzle) redis_connection.zadd(ping_key, {user: ping_end}) redis_connection.expire(ping_key, PING_EXPIRE) latency = ping_end - ping_start # Record the latency for the player redis_connection.lpush( "latency", "{user}:{timestamp}:{latency}".format(user=user, timestamp=ping_end, latency=latency), ) # Keep only the last 1000 entries to latency redis_connection.ltrim("latency", 0, 999) response["message"] = "Latency" response["data"]["latency"] = latency response["name"] = "success" response = make_response(json.jsonify(response), 200) return response