def get_streams(shuffle=True):
if lazy_get_redis() is not None:
r = lazy_get_redis()
if r.get('twitch_streams'):
return json.loads(r.get('twitch_streams'))
r = requests.get("https://api.twitch.tv/helix/streams",
headers=headers,
params={
'user_login': TWITCH_STREAMS
}).json()
if 'data' not in r or len(r['data']) == 0:
r = requests.get("https://api.twitch.tv/helix/streams",
headers=headers,
params={
'game_id': '30921'
}).json() # RL streams
r['data'] = r['data'][:5]
stream_data = []
for stream in r['data']:
stream['thumbnail_url'] = TwitchStreams.get_size(
stream['thumbnail_url'], 800, 450)
stream_data.append(stream)
if lazy_get_redis() is not None:
r = lazy_get_redis()
r.set('twitch_streams', json.dumps(stream_data), ex=60 * 2)
if shuffle:
random.shuffle(stream_data)
return stream_data
def calc_item_stats(self, session=None):
if session is None:
sess = self.session()
else:
sess = session
results = ItemStatsWrapper.create_stats(sess)
if lazy_get_redis() is not None:
lazy_get_redis().set('item_stats', json.dumps(results))
def calculate_global_stats_by_playlist(session):
overall_data = []
numbers = []
game_modes = range(1, 5)
for game_mode in game_modes:
numbers.append(
session.query(func.count(PlayerGame.id)).join(
Game, Game.hash == PlayerGame.game).filter(
Game.teamsize == (game_mode)).scalar())
for global_stats_metadata in global_stats_metadatas:
stats_field = global_stats_metadata.field
per_hit_name_suffix = 'ph'
if stats_field.endswith(per_hit_name_suffix):
_query = session.query(
func.round(
cast(
getattr(PlayerGame,
stats_field.replace(per_hit_name_suffix, '')),
Numeric) / PlayerGame.total_hits, 2).label('n'),
func.count(PlayerGame.id)).filter(
PlayerGame.total_hits > 0).group_by('n').order_by('n')
else:
_query = session.query(getattr(
PlayerGame, stats_field), func.count(PlayerGame.id)).group_by(
getattr(PlayerGame, stats_field)).order_by(
getattr(PlayerGame, stats_field))
datasets = []
if stats_field == 'score':
_query = _query.filter(PlayerGame.score % 10 == 0)
for i, game_mode in enumerate(game_modes):
# print(g)
data_query = _query.join(Game,
Game.hash == PlayerGame.game).filter(
Game.teamsize == game_mode).all()
datasets.append({
'name': f"{game_mode}'s",
'keys': [],
'values': []
})
for k, v in data_query:
if k is not None:
datasets[-1]['keys'].append(float(k))
datasets[-1]['values'].append(float(v) / float(numbers[i]))
overall_data.append(
GlobalStatsGraph(name=global_stats_metadata.name,
datasets=[
GlobalStatsGraphDataset(**dataset)
for dataset in datasets
]))
session.close()
if lazy_get_redis() is not None:
lazy_get_redis().set('global_stats_by_playlist',
better_json_dumps(overall_data))
return overall_data
def calc_global_stats(self):
sess = self.session()
result = player_stat_wrapper.get_global_stats(sess)
sess.close()
if lazy_get_redis() is not None:
lazy_get_redis().set('global_stats', json.dumps(result))
lazy_get_redis().set('global_stats_expire', json.dumps(True))
print('Done')
return result
def get_patreon_progress():
if lazy_get_redis() is not None:
r = lazy_get_redis()
if r.get('patreon_progress'):
return tuple(json.loads(r.get('patreon_progress')))
r = requests.get("https://patreon.com/calculated")
bs = BeautifulSoup(r.text, "html.parser")
progress = bs.find_all(class_=PATREON_PROGRESS_LOOKUP_CLASS)[0].text
nums = [int(n[1:]) for n in progress.split(' of ')]
if lazy_get_redis() is not None:
r = lazy_get_redis()
r.set('patreon_progress', json.dumps(nums), ex=60 * 60)
return tuple(nums)
def get_redis_result_if_exists(prefix: str, id_):
redis_key = prefix + str(id_)
r = lazy_get_redis()
if r is not None:
result = r.get(redis_key)
if result is not None:
return json.loads(result)
return None
def api_get_replay_count(session=None):
r = lazy_get_redis()
if r is not None and r.get('replay_count') is not None:
return jsonify(int(r.get('replay_count')))
count = session.query(Game.hash).count()
if r is not None:
r.set('replay_count', str(count), ex=60 * 60)
return jsonify(count)
def cache_items(self):
r = lazy_get_redis()
if r is None:
raise CalculatedError()
items, category_map = self.get_items()
for item in items:
r.set(f"rlgarage_{item}", json.dumps(items[item]))
r.set("rlgarage_category_map", json.dumps(category_map))
r.set("rlgarage_items", json.dumps(items))
def get_recent_replays(session=None):
r = lazy_get_redis()
if r is not None and r.get('recent_replays') is not None:
return json.loads(r.get('recent_replays'))
replays = session.query(Game).order_by(desc(Game.upload_date))[:5]
replays = [CompactReplay.create_from_game(r).__dict__ for r in replays]
if r is not None:
r.set('recent_replays', json.dumps(replays), ex=60 * 60)
return replays
def get_cached_items(self):
r = lazy_get_redis()
if r is not None:
if r.get("rlgarage_items") is not None:
return json.loads(r.get("rlgarage_items"))
items = self.get_items()
if r is not None:
r.set("rlgarage_items", json.dumps(items), ex=60 * 60 * 24)
return items
def get_item_list(self, page, limit, override=False):
if self.item_map is None:
r = lazy_get_redis()
self.item_map, self.category_map = json.loads(r.get("rlgarage_items")), \
json.loads(r.get("rlgarage_category_map"))
if not override and limit > 500:
limit = 500
item_list = list(self.item_map.values())
return {
'items': self.get_item_response(item_list[page * limit: (page + 1) * limit]),
'count': len(item_list)
}
def set_redis_result_if_exists(prefix: str,
id_: int or str,
value,
ex=None):
redis_key = prefix + str(id_)
r = lazy_get_redis()
if r is not None:
if ex is not None:
r.set(redis_key,
json.dumps(value, default=date_converter),
ex=ex)
else:
r.set(redis_key, json.dumps(value, default=date_converter))
def get_redis() -> redis.Redis:
"""
Tries to get redis.
Does a fallback if redis is not able to be grabbed from flask.
"""
try:
from flask import current_app
return current_app.config['r']
except Exception as e:
ErrorLogger.log_error(e)
try:
from backend.database.startup import lazy_get_redis
return lazy_get_redis()
except Exception as e:
ErrorLogger.log_error(e)
def get_item(self, id_, paint_id=0):
if self.item_map is not None and id_ in self.item_map:
return self.item_map[id_]
r = lazy_get_redis()
item = r.get(f'rlgarage_{id_}')
if item is None:
return None
item = json.loads(item)
if paint_id > 0 and item['hascoloredicons'] == 1:
pic = item['name'].replace(' ', '').replace('\'', '').lower()
paint_name = self.paint_map[paint_id]
item['image'] = f"https://rocket-league.com/content/media/items/avatar/220px/" \
f"{pic}/{pic}-{paint_name}.png"
else:
item['image'] = f"https://rocket-league.com/content/media/items/avatar/220px/{item['image']}"
return item
def api_get_leaderboards():
if lazy_get_redis() is not None:
if lazy_get_redis().get("leaderboards"):
resp = Response(response=lazy_get_redis().get("leaderboards"),
status=200,
mimetype="application/json")
return resp
leaderboards = Leaderboards.create()
if lazy_get_redis() is not None:
lazy_get_redis().set("leaderboards", json.dumps([l.__dict__ for l in leaderboards]), ex=24 * 60 * 60)
return better_jsonify(leaderboards)
def start_app() -> Tuple[Flask, Dict[str, int]]:
# APP SETUP
app = Flask(__name__,
template_folder=os.path.join('frontend', 'templates'),
static_folder=os.path.join('frontend', 'static'),
static_url_path='/static2')
CalculatedServer.set_up_app_config(app)
CORS(app)
CalculatedServer.create_needed_folders(app)
# Enable GZIP compression
Compress(app)
session_factory = lazy_startup()
with app.app_context():
create_celery_config()
CalculatedServer.register_blueprints(app)
CalculatedServer.setup_metrics(app)
try:
import config
app.secret_key = config.SECRET_KEY
except: # TODO: Specify necessary excepts here.
logger.warning('No secret key has been set')
app.config['db'] = session_factory
app.config['r'] = lazy_get_redis()
_session = session_factory()
CalculatedServer.add_needed_groups_to_db(_session,
SERVER_PERMISSION_GROUPS)
ids, ids_to_group = CalculatedServer.get_id_group_dicts(
_session, SERVER_PERMISSION_GROUPS)
app.config['groups'] = ids_to_group
_session.commit()
_session.close()
return app, ids
def calc_leaderboards(self):
leaderboards = Leaderboards.create()
if lazy_get_redis() is not None:
lazy_get_redis().set("leaderboards",
json.dumps([l.__dict__ for l in leaderboards]))
import os
import pickle
import traceback
from concurrent.futures import ThreadPoolExecutor
from functools import partial
from sqlalchemy.orm import sessionmaker, Session
from backend.database.objects import Game
from backend.database.startup import lazy_startup, lazy_get_redis
from backend.database.utils.utils import convert_pickle_to_db, add_objs_to_db
logger = logging.getLogger(__name__)
session = lazy_startup() # type: sessionmaker
r = lazy_get_redis()
pickled_location = os.path.join(os.path.dirname(__file__), '..', 'data', 'parsed')
pickles = glob.glob(os.path.join(pickled_location, '*.pkl'))
s = session()
games = s.query(Game.hash).all()
def main():
with ThreadPoolExecutor() as executor:
fn = partial(parse_pickle)
executor.map(fn, pickles, timeout=120)
def parse_pickle(p):
s = session() # type: Session
def api_v1_get_itemstats():
r = lazy_get_redis()
if r.get('item_stats'):
return jsonify(json.loads(r.get('item_stats')))
calc_item_stats.delay()
return jsonify({})
