def __init__(self):
download_global_cache()
log("creating match cache")
global_dict = get_global_ranked_match_cache()
self._global_data = GlobalRankedData(global_dict=global_dict)
self._player_ticks = get_player_ticks()
self._players_to_save = []
def ensure_test_data():
global _test_data_created
if _test_data_created:
return
_test_data_created = True
ensure_test_tables()
log('creating test data')
already_exists = (
Player
.select()
.where(Player.cfn_id == sample_player_id)
.exists()
)
if already_exists:
log('existing test data found')
return
create_test_player(
cfn_id=sample_player_id,
name='TestPlayer',
subscribed=True,
)
Rank.create(
id=1,
created_at=datetime.utcnow(),
player_cfn=sample_player_id,
league_points=100,
placement=1,
favorite_character=1,
)
def __init__(self):
download_global_cache()
log("creating match cache")
global_dict = get_global_ranked_match_cache()
self._global_data = GlobalRankedData(global_dict=global_dict)
self._player_ticks = get_player_ticks()
self._players_to_save = []
def batch_query_match_history(session, pids):
pids_list = list(pids)
total = len(pids_list)
batch = 50
player_matches = {}
any_error = False
log("Begin querying %s players" % total)
for idx in range(0, total, batch):
next_idx = idx + batch
bound = next_idx if next_idx < total else total
sub_pids = pids_list[idx:bound]
log('Attempting to query players %s-%s of %s' % (idx, bound, total))
is_error, matches, player_matches_dict = _bulk_query_match_history(
session, sub_pids
)
any_error = any_error or is_error
save_match_list(matches)
player_matches.update(player_matches_dict)
set_player_updated_at(player_matches.keys())
with DATABASE.atomic():
cache = MatchCache()
for pid, matches in player_matches.items():
cache.process_matches(pid, matches)
cache.save()
return any_error
def ensure_test_data():
global _test_data_created
if _test_data_created:
return
_test_data_created = True
ensure_test_tables()
log('creating test data')
already_exists = (Player.select().where(
Player.cfn_id == sample_player_id).exists())
if already_exists:
log('existing test data found')
return
create_test_player(
cfn_id=sample_player_id,
name='TestPlayer',
subscribed=True,
)
Rank.create(
id=1,
created_at=datetime.utcnow(),
player_cfn=sample_player_id,
league_points=100,
placement=1,
favorite_character=1,
)
def bulk_insert(model, data_source):
total = len(data_source)
batch = 1000
log("Trying to insert %s rows" % total)
with DATABASE.atomic():
for idx in range(0, total, batch):
(model.insert_many(data_source[idx:idx + batch]).execute())
log("Inserted %s" % (idx + batch))
def save(self):
log("saving match cache")
for player in self._players_to_save:
save_player_cache(player)
# this implicitly uploads
save_global_ranked_match_cache(self.global_data)
self._players_to_save = []
log("cache successfully saved")
def save(self):
log("saving match cache")
for player in self._players_to_save:
save_player_cache(player)
# this implicitly uploads
save_global_ranked_match_cache(self.global_data)
self._players_to_save = []
log("cache successfully saved")
def set_player_updated_at(pids):
log("setting players as updated")
if not pids:
log('no players to update!')
return
now_str = _get_now_str()
(Player.update(
updated_at=now_str).where(Player.cfn_id << list(pids)).execute())
def prod_database():
log('connecting to prod db')
return PooledMySQLDatabase(
ENVARS.database_name,
host=ENVARS.host,
user=ENVARS.username,
password=ENVARS.password,
max_connections=32, # 50 max on t2.micro
stale_timeout=300, # 5 minutes.
)
def prod_database():
log('connecting to prod db')
return PooledMySQLDatabase(
ENVARS.database_name,
host=ENVARS.host,
user=ENVARS.username,
password=ENVARS.password,
max_connections=32, # 50 max on t2.micro
stale_timeout=300, # 5 minutes.
)
def set_subscribed(pids):
log("updating affected players to subscribed")
if not pids:
return
(
Player
.update(subscribed=1)
.where(Player.subscribed == 0, Player.cfn_id << list(pids))
.execute()
)
def shut_down(process_id):
if process_id == str(PROCESS_ID):
log('signal received! shutting down...')
with open('temp/server.pid', 'wt') as f:
f.write('')
threading.Timer(1, lambda: _kill_server()).start()
return 'killed: %s' % PROCESS_ID
else:
log('nice try: %r != %r' % (process_id, PROCESS_ID))
abort(404)
def _insert_missing_matches(match_source):
log("determining missing matches")
match_source_ids = [ms['cfn_id'] for ms in match_source]
existing_mids, missing_mids = (_determine_missing_cfn_ids(
Match, match_source_ids))
missing_matches = [
ms for ms in match_source if ms['cfn_id'] in missing_mids
]
log("inserting missing matches")
bulk_insert(Match, missing_matches)
def _insert_missing_players(player_source):
log("determining missing players")
player_source_ids = [ps['cfn_id'] for ps in player_source]
existing_pids, missing_pids = (_determine_missing_cfn_ids(
Player, player_source_ids))
missing_players = [
ps for ps in player_source if ps['cfn_id'] in missing_pids
]
log("inserting missing players")
bulk_insert(Player, missing_players)
return existing_pids
def _query_rival(session, query_name, player_name):
url = RIVAL_URL % ('%s', query_name)
res = query_cfn(session, url)
matching_players = res.json()['response'][0]['searchresult']
for player in matching_players:
cfn_model = CFNPlayerSearchModel(player)
log('logging cfn player')
log(cfn_model.__dict__)
if cfn_model.name.lower() == player_name.lower():
subscribe_to_new_player(cfn_model)
return cfn_model.name
return None
def _query_rival(session, query_name, player_name):
url = RIVAL_URL % ('%s', query_name)
res = query_cfn(session, url)
matching_players = res.json()['response'][0]['searchresult']
for player in matching_players:
cfn_model = CFNPlayerSearchModel(player)
log('logging cfn player')
log(cfn_model.__dict__)
if cfn_model.name.lower() == player_name.lower():
subscribe_to_new_player(cfn_model)
return cfn_model.name
return None
def bulk_insert(model, data_source):
total = len(data_source)
batch = 1000
log("Trying to insert %s rows" % total)
with DATABASE.atomic():
for idx in range(0, total, batch):
(
model
.insert_many(data_source[idx:idx + batch])
.execute()
)
log("Inserted %s" % (idx + batch))
def _insert_missing_matches(match_source):
log("determining missing matches")
match_source_ids = [ms['cfn_id'] for ms in match_source]
existing_mids, missing_mids = (
_determine_missing_cfn_ids(Match, match_source_ids)
)
missing_matches = [
ms for ms in match_source
if ms['cfn_id'] in missing_mids
]
log("inserting missing matches")
bulk_insert(Match, missing_matches)
def set_player_updated_at(pids):
log("setting players as updated")
if not pids:
log('no players to update!')
return
now_str = _get_now_str()
(
Player
.update(updated_at=now_str)
.where(Player.cfn_id << list(pids))
.execute()
)
def _insert_missing_players(player_source):
log("determining missing players")
player_source_ids = [ps['cfn_id'] for ps in player_source]
existing_pids, missing_pids = (
_determine_missing_cfn_ids(Player, player_source_ids)
)
missing_players = [
ps for ps in player_source
if ps['cfn_id'] in missing_pids
]
log("inserting missing players")
bulk_insert(Player, missing_players)
return existing_pids
def _query_player_match_history(session, pid):
url = MATCH_URL % ('%s', pid)
res = query_cfn(session, url)
if res.status_code != 200:
log("got a non-200 response:\n%s" % res.text)
raise MatchQueryException()
try:
match_results = res.json()['response'][0]['matchresults']
except Exception as e:
log_exception(e)
log('failed to extract json, dumping res:\n%s' % res.text)
raise MatchQueryException()
else:
return match_results
def backload_all_players():
log('backloading match_last_updated players')
log('building cache')
cache = MatchCache()
batch_size = 500
backfill_player_ids = load_player_ids_to_backfill(
batch_size=batch_size,
)
log('backloading %s players...' % batch_size)
for player_id in backfill_player_ids:
backload_player(cache, player_id)
log("saving")
cache.save()
log("done")
def backload_volatile_players():
log('backloading volatile players')
log('building cache')
cache = MatchCache()
batch_size = 600
backfill_player_ids = load_volatile_player_ids(
batch_size=batch_size,
)
log('backloading %s players...' % batch_size)
for player_id in backfill_player_ids:
backload_player(cache, player_id)
log("saving")
cache.save()
log("done")
def main():
with create_session() as session:
cd, res = query_cookie_status(session)
log(res.status_code)
log(res.headers)
log(res.text)
log("cookie is good: %s" % (not cd.is_invalid))
def _query_player_match_history(session, pid):
url = MATCH_URL % ('%s', pid)
res = query_cfn(session, url)
if res.status_code != 200:
log("got a non-200 response:\n%s" % res.text)
raise MatchQueryException()
try:
match_results = res.json()['response'][0]['matchresults']
except Exception as e:
log_exception(e)
log('failed to extract json, dumping res:\n%s' % res.text)
raise MatchQueryException()
else:
return match_results
def save_match_list(match_list):
log("arranging fetched data")
matches = dict()
for match in match_list:
mid = int(match['matchid'])
matches[mid] = match
log("processing matches")
match_source = [{
'cfn_id': mid,
'left_player_cfn_id': int(match['leftstartplayer']),
'right_player_cfn_id': int(match['rightstartplayer']),
'data': json.dumps(match),
} for mid, match in matches.items()]
_insert_missing_matches(match_source)
def save_match_list(match_list):
log("arranging fetched data")
matches = dict()
for match in match_list:
mid = int(match['matchid'])
matches[mid] = match
log("processing matches")
match_source = [
{
'cfn_id': mid,
'left_player_cfn_id': int(match['leftstartplayer']),
'right_player_cfn_id': int(match['rightstartplayer']),
'data': json.dumps(match),
} for mid, match in matches.items()
]
_insert_missing_matches(match_source)
def send_error_message(message):
log('sending error email')
text = "server: %s\n\n%s\n\nlogging:\n\n%s" % (
ENVARS.server_name, message, read_logs()
)
return requests.post(
"https://api.mailgun.net/v3/%s/messages" % ENVARS.mailgun_domain_name,
auth=(
"api",
ENVARS.mailgun_api_key,
),
data={
"from": "FGCBOT <robot@%s>" % ENVARS.mailgun_domain_name,
"to": [ENVARS.mailgun_recipient],
"subject": "FGC STATUS %s" % convert_dt_to_nyc(),
"text": text,
},
)
def _bulk_query_match_history(session, pids):
match_list = []
count = 1
total = len(pids)
player_matches = {}
is_error = False
for pid in pids:
log("Fetching match history for %s (%s/%s)" % (pid, count, total))
count += 1
try:
matches = _query_player_match_history(session, pid)
except MatchQueryException:
# pretend we got 0 matches so that player gets marked as "updated"
# this prevents a bunch of bad players from starving the rest
matches = []
is_error = True
match_list.extend(matches)
player_matches[pid] = matches
return is_error, match_list, player_matches
def _bulk_query_match_history(session, pids):
match_list = []
count = 1
total = len(pids)
player_matches = {}
is_error = False
for pid in pids:
log("Fetching match history for %s (%s/%s)" % (pid, count, total))
count += 1
try:
matches = _query_player_match_history(session, pid)
except MatchQueryException:
# pretend we got 0 matches so that player gets marked as "updated"
# this prevents a bunch of bad players from starving the rest
matches = []
is_error = True
match_list.extend(matches)
player_matches[pid] = matches
return is_error, match_list, player_matches
def start_server():
if ENVARS.is_web_testing:
bottle_web.merge(bottle_api)
if ENVARS.is_web_server():
designated_bottle_app = bottle_web
designated_setup = setup_web
elif ENVARS.is_api_server():
designated_bottle_app = bottle_api
designated_setup = setup_api
else:
log('no webserver for instance type: %s' % ENVARS.instance_type)
return
designated_setup()
run(
designated_bottle_app,
server='cherrypy',
host='localhost',
port=5555,
debug=ENVARS.debug,
)
def fix_player_names(session):
players = (
Player
.select()
.where(Player.name == '')
)
player_ids = [p.cfn_id for p in players]
log("found %s players with missing names" % len(player_ids))
if len(player_ids) == 0:
return
for pid in player_ids:
url = LICENSE_URL % ('%s', pid)
res = query_cfn(session, url)
try:
new_name = res.json()['response'][0]['displayid']
except json.decoder.JSONDecodeError as e:
log(res.text)
raise e
log('%s -> %s' % (pid, new_name))
(
Player
.update(name=new_name)
.where(Player.cfn_id == pid)
.execute()
)
def batch_query_match_history(session, pids):
pids_list = list(pids)
total = len(pids_list)
batch = 50
player_matches = {}
any_error = False
log("Begin querying %s players" % total)
for idx in range(0, total, batch):
next_idx = idx + batch
bound = next_idx if next_idx < total else total
sub_pids = pids_list[idx:bound]
log('Attempting to query players %s-%s of %s' % (idx, bound, total))
is_error, matches, player_matches_dict = _bulk_query_match_history(
session, sub_pids)
any_error = any_error or is_error
save_match_list(matches)
player_matches.update(player_matches_dict)
set_player_updated_at(player_matches.keys())
with DATABASE.atomic():
cache = MatchCache()
for pid, matches in player_matches.items():
cache.process_matches(pid, matches)
cache.save()
return any_error
def fix_player_names(session):
players = (Player.select().where(Player.name == ''))
player_ids = [p.cfn_id for p in players]
log("found %s players with missing names" % len(player_ids))
if len(player_ids) == 0:
return
for pid in player_ids:
url = LICENSE_URL % ('%s', pid)
res = query_cfn(session, url)
try:
new_name = res.json()['response'][0]['displayid']
except json.decoder.JSONDecodeError as e:
log(res.text)
raise e
log('%s -> %s' % (pid, new_name))
(Player.update(name=new_name).where(Player.cfn_id == pid).execute())
log_exception,
set_log_file,
)
from py.src.settings import (
DATABASE,
)
from py.src.message import (
send_error_message,
)
from py.src.cfn.api import (
create_session,
test_cookie_status,
record_top_players,
)
if __name__ == "__main__":
set_log_file("task_ranking")
log("task_ranking begin")
with create_session() as session:
try:
test_cookie_status(session)
DATABASE.connect()
record_top_players(session)
except Exception as e:
log_exception(e)
send_error_message("FATAL ERROR when pulling rank data")
if not DATABASE.is_closed():
DATABASE.close()
log("task_ranking complete")
def ensure_test_tables():
ensure_testing()
if not Player.table_exists():
log('creating test tables')
DATABASE.create_tables([Player, Match, Rank])
def load_crontab():
crontab_path = "install/crontab/crontab-%s" % ENVARS.instance_type
process_out = subprocess.check_output(["crontab", crontab_path], )
log("crontab out: %s" % process_out)
)
from py.src.cfn.api import (
create_session,
test_cookie_status,
batch_query_match_history,
fix_player_names,
)
from py.src.store import (
load_subscribed_player_ids,
)
PLAYER_BATCH_SIZE = 300
if __name__ == "__main__":
set_log_file("task_matches")
log("task_matches begin")
with create_session() as session:
try:
test_cookie_status(session)
DATABASE.connect()
pids = load_subscribed_player_ids(batch_size=PLAYER_BATCH_SIZE)
any_error = batch_query_match_history(session, pids)
fix_player_names(session)
except Exception as e:
log_exception(e)
send_error_message("FATAL ERROR when pulling match data")
else:
if any_error:
# suppress non-fatal errors for now
# send_error_message('non-fatal errors when pulling matches')
pass
def test_local_database():
log('connecting to local test db')
return SqliteDatabase('local/test.db')
def _download_file(file_name, file_path):
log("downloading %s to %s" % (file_name, file_path))
_connect_to_bucket().download_file(Key=file_name, Filename=file_path)
def ensure_test_tables():
ensure_testing()
if not Player.table_exists():
log('creating test tables')
DATABASE.create_tables([Player, Match, Rank])
def demo_database():
log('connecting to local demo db')
return SqliteDatabase('local/demo.db')
def load_crontab():
crontab_path = "install/crontab/crontab-%s" % ENVARS.instance_type
process_out = subprocess.check_output(
["crontab", crontab_path],
)
log("crontab out: %s" % process_out)
def test_temp_database():
log('connecting to in-memory test db')
return SqliteDatabase(':memory:')
def set_subscribed(pids):
log("updating affected players to subscribed")
if not pids:
return
(Player.update(subscribed=1).where(Player.subscribed == 0,
Player.cfn_id << list(pids)).execute())
def invalidate_cache():
_refresh_cache()
log('cache refreshed')
return
def test_local_database():
log('connecting to local test db')
return SqliteDatabase('local/test.db')
def test_temp_database():
log('connecting to in-memory test db')
return SqliteDatabase(':memory:')
def _upload_file(file_name, file_path):
log("uploading %s to %s" % (file_path, file_name))
with open(file_path, 'rb') as data:
_connect_to_bucket().put_object(Key=file_name, Body=data)
def main():
log('creating demo db from csv files')
create_demo_db()
def demo_database():
log('connecting to local demo db')
return SqliteDatabase('local/demo.db')
