def get_streams(shuffle=True): if lazy_get_redis() is not None: r = lazy_get_redis() if r.get('twitch_streams'): return json.loads(r.get('twitch_streams')) r = requests.get("https://api.twitch.tv/helix/streams", headers=headers, params={ 'user_login': TWITCH_STREAMS }).json() if 'data' not in r or len(r['data']) == 0: r = requests.get("https://api.twitch.tv/helix/streams", headers=headers, params={ 'game_id': '30921' }).json() # RL streams r['data'] = r['data'][:5] stream_data = [] for stream in r['data']: stream['thumbnail_url'] = TwitchStreams.get_size( stream['thumbnail_url'], 800, 450) stream_data.append(stream) if lazy_get_redis() is not None: r = lazy_get_redis() r.set('twitch_streams', json.dumps(stream_data), ex=60 * 2) if shuffle: random.shuffle(stream_data) return stream_data
def calc_item_stats(self, session=None): if session is None: sess = self.session() else: sess = session results = ItemStatsWrapper.create_stats(sess) if lazy_get_redis() is not None: lazy_get_redis().set('item_stats', json.dumps(results))
def calculate_global_stats_by_playlist(session): overall_data = [] numbers = [] game_modes = range(1, 5) for game_mode in game_modes: numbers.append( session.query(func.count(PlayerGame.id)).join( Game, Game.hash == PlayerGame.game).filter( Game.teamsize == (game_mode)).scalar()) for global_stats_metadata in global_stats_metadatas: stats_field = global_stats_metadata.field per_hit_name_suffix = 'ph' if stats_field.endswith(per_hit_name_suffix): _query = session.query( func.round( cast( getattr(PlayerGame, stats_field.replace(per_hit_name_suffix, '')), Numeric) / PlayerGame.total_hits, 2).label('n'), func.count(PlayerGame.id)).filter( PlayerGame.total_hits > 0).group_by('n').order_by('n') else: _query = session.query(getattr( PlayerGame, stats_field), func.count(PlayerGame.id)).group_by( getattr(PlayerGame, stats_field)).order_by( getattr(PlayerGame, stats_field)) datasets = [] if stats_field == 'score': _query = _query.filter(PlayerGame.score % 10 == 0) for i, game_mode in enumerate(game_modes): # print(g) data_query = _query.join(Game, Game.hash == PlayerGame.game).filter( Game.teamsize == game_mode).all() datasets.append({ 'name': f"{game_mode}'s", 'keys': [], 'values': [] }) for k, v in data_query: if k is not None: datasets[-1]['keys'].append(float(k)) datasets[-1]['values'].append(float(v) / float(numbers[i])) overall_data.append( GlobalStatsGraph(name=global_stats_metadata.name, datasets=[ GlobalStatsGraphDataset(**dataset) for dataset in datasets ])) session.close() if lazy_get_redis() is not None: lazy_get_redis().set('global_stats_by_playlist', better_json_dumps(overall_data)) return overall_data
def calc_global_stats(self): sess = self.session() result = player_stat_wrapper.get_global_stats(sess) sess.close() if lazy_get_redis() is not None: lazy_get_redis().set('global_stats', json.dumps(result)) lazy_get_redis().set('global_stats_expire', json.dumps(True)) print('Done') return result
def get_patreon_progress(): if lazy_get_redis() is not None: r = lazy_get_redis() if r.get('patreon_progress'): return tuple(json.loads(r.get('patreon_progress'))) r = requests.get("https://patreon.com/calculated") bs = BeautifulSoup(r.text, "html.parser") progress = bs.find_all(class_=PATREON_PROGRESS_LOOKUP_CLASS)[0].text nums = [int(n[1:]) for n in progress.split(' of ')] if lazy_get_redis() is not None: r = lazy_get_redis() r.set('patreon_progress', json.dumps(nums), ex=60 * 60) return tuple(nums)
def get_redis_result_if_exists(prefix: str, id_): redis_key = prefix + str(id_) r = lazy_get_redis() if r is not None: result = r.get(redis_key) if result is not None: return json.loads(result) return None
def api_get_replay_count(session=None): r = lazy_get_redis() if r is not None and r.get('replay_count') is not None: return jsonify(int(r.get('replay_count'))) count = session.query(Game.hash).count() if r is not None: r.set('replay_count', str(count), ex=60 * 60) return jsonify(count)
def cache_items(self): r = lazy_get_redis() if r is None: raise CalculatedError() items, category_map = self.get_items() for item in items: r.set(f"rlgarage_{item}", json.dumps(items[item])) r.set("rlgarage_category_map", json.dumps(category_map)) r.set("rlgarage_items", json.dumps(items))
def get_recent_replays(session=None): r = lazy_get_redis() if r is not None and r.get('recent_replays') is not None: return json.loads(r.get('recent_replays')) replays = session.query(Game).order_by(desc(Game.upload_date))[:5] replays = [CompactReplay.create_from_game(r).__dict__ for r in replays] if r is not None: r.set('recent_replays', json.dumps(replays), ex=60 * 60) return replays
def get_cached_items(self): r = lazy_get_redis() if r is not None: if r.get("rlgarage_items") is not None: return json.loads(r.get("rlgarage_items")) items = self.get_items() if r is not None: r.set("rlgarage_items", json.dumps(items), ex=60 * 60 * 24) return items
def get_item_list(self, page, limit, override=False): if self.item_map is None: r = lazy_get_redis() self.item_map, self.category_map = json.loads(r.get("rlgarage_items")), \ json.loads(r.get("rlgarage_category_map")) if not override and limit > 500: limit = 500 item_list = list(self.item_map.values()) return { 'items': self.get_item_response(item_list[page * limit: (page + 1) * limit]), 'count': len(item_list) }
def set_redis_result_if_exists(prefix: str, id_: int or str, value, ex=None): redis_key = prefix + str(id_) r = lazy_get_redis() if r is not None: if ex is not None: r.set(redis_key, json.dumps(value, default=date_converter), ex=ex) else: r.set(redis_key, json.dumps(value, default=date_converter))
def get_redis() -> redis.Redis: """ Tries to get redis. Does a fallback if redis is not able to be grabbed from flask. """ try: from flask import current_app return current_app.config['r'] except Exception as e: ErrorLogger.log_error(e) try: from backend.database.startup import lazy_get_redis return lazy_get_redis() except Exception as e: ErrorLogger.log_error(e)
def get_item(self, id_, paint_id=0): if self.item_map is not None and id_ in self.item_map: return self.item_map[id_] r = lazy_get_redis() item = r.get(f'rlgarage_{id_}') if item is None: return None item = json.loads(item) if paint_id > 0 and item['hascoloredicons'] == 1: pic = item['name'].replace(' ', '').replace('\'', '').lower() paint_name = self.paint_map[paint_id] item['image'] = f"https://rocket-league.com/content/media/items/avatar/220px/" \ f"{pic}/{pic}-{paint_name}.png" else: item['image'] = f"https://rocket-league.com/content/media/items/avatar/220px/{item['image']}" return item
def api_get_leaderboards(): if lazy_get_redis() is not None: if lazy_get_redis().get("leaderboards"): resp = Response(response=lazy_get_redis().get("leaderboards"), status=200, mimetype="application/json") return resp leaderboards = Leaderboards.create() if lazy_get_redis() is not None: lazy_get_redis().set("leaderboards", json.dumps([l.__dict__ for l in leaderboards]), ex=24 * 60 * 60) return better_jsonify(leaderboards)
def start_app() -> Tuple[Flask, Dict[str, int]]: # APP SETUP app = Flask(__name__, template_folder=os.path.join('frontend', 'templates'), static_folder=os.path.join('frontend', 'static'), static_url_path='/static2') CalculatedServer.set_up_app_config(app) CORS(app) CalculatedServer.create_needed_folders(app) # Enable GZIP compression Compress(app) session_factory = lazy_startup() with app.app_context(): create_celery_config() CalculatedServer.register_blueprints(app) CalculatedServer.setup_metrics(app) try: import config app.secret_key = config.SECRET_KEY except: # TODO: Specify necessary excepts here. logger.warning('No secret key has been set') app.config['db'] = session_factory app.config['r'] = lazy_get_redis() _session = session_factory() CalculatedServer.add_needed_groups_to_db(_session, SERVER_PERMISSION_GROUPS) ids, ids_to_group = CalculatedServer.get_id_group_dicts( _session, SERVER_PERMISSION_GROUPS) app.config['groups'] = ids_to_group _session.commit() _session.close() return app, ids
def calc_leaderboards(self): leaderboards = Leaderboards.create() if lazy_get_redis() is not None: lazy_get_redis().set("leaderboards", json.dumps([l.__dict__ for l in leaderboards]))
import os import pickle import traceback from concurrent.futures import ThreadPoolExecutor from functools import partial from sqlalchemy.orm import sessionmaker, Session from backend.database.objects import Game from backend.database.startup import lazy_startup, lazy_get_redis from backend.database.utils.utils import convert_pickle_to_db, add_objs_to_db logger = logging.getLogger(__name__) session = lazy_startup() # type: sessionmaker r = lazy_get_redis() pickled_location = os.path.join(os.path.dirname(__file__), '..', 'data', 'parsed') pickles = glob.glob(os.path.join(pickled_location, '*.pkl')) s = session() games = s.query(Game.hash).all() def main(): with ThreadPoolExecutor() as executor: fn = partial(parse_pickle) executor.map(fn, pickles, timeout=120) def parse_pickle(p): s = session() # type: Session
def api_v1_get_itemstats(): r = lazy_get_redis() if r.get('item_stats'): return jsonify(json.loads(r.get('item_stats'))) calc_item_stats.delay() return jsonify({})