def import_into_database():
class Cron1Logs2DB(cron.Logs2DB):
def calculate_new_ratings(self, game, game_players):
return
with orm.session_scope() as session:
lookup = orm.Lookup(session)
logs2db = Cron1Logs2DB(session, lookup)
with open('game_import_data.txt', 'r') as f:
logs2db.process_logs(f, 'x')
def calculate_ratings():
with orm.session_scope() as session:
lookup = orm.Lookup(session)
logs2db = cron.Logs2DB(session, lookup)
query = sqlalchemy.sql.text('''
select game.log_time,
game.number,
count(distinct game_player.game_player_id) as num_players
from game
join game_state on game.game_state_id = game_state.game_state_id
join game_player on game.game_id = game_player.game_id
where game_state.name = 'Completed'
group by game.game_id
having num_players > 1
order by game.end_time asc
''')
for row in session.execute(query):
game = lookup.get_game(row.log_time, row.number)
game_players = []
for player_index in range(row.num_players):
game_players.append(lookup.get_game_player(game, player_index))
logs2db.calculate_new_ratings(game, game_players)
def process_logs(write_stats_files):
with orm.session_scope() as session:
lookup = orm.Lookup(session)
logs2db = Logs2DB(session, lookup)
kv_last_log_timestamp = lookup.get_key_value("cron last log timestamp")
last_log_timestamp = (
1408905413
if kv_last_log_timestamp.value is None
else int(kv_last_log_timestamp.value)
)
kv_last_offset = lookup.get_key_value("cron last offset")
last_offset = 0 if kv_last_offset.value is None else int(kv_last_offset.value)
completed_game_users = set()
for log_timestamp, filename in util.get_log_file_filenames(
"py", begin=last_log_timestamp
):
if log_timestamp != last_log_timestamp:
last_offset = 0
with util.open_possibly_gzipped_file(filename) as f:
if last_offset:
f.seek(last_offset)
last_offset, new_completed_game_users = logs2db.process_logs(
f, log_timestamp
)
completed_game_users.update(new_completed_game_users)
last_log_timestamp = log_timestamp
kv_last_log_timestamp.value = last_log_timestamp
kv_last_offset.value = last_offset
session.flush()
if write_stats_files and completed_game_users:
statsgen = StatsGen(session, "stats_temp")
statsgen.output_ratings()
for user in completed_game_users:
record = lookup.get_record(user)
if record:
decoded = ujson.decode(record.encoded)
else:
decoded = get_empty_records()
statsgen.output_user(user.user_id, user.name, decoded)
ratings_filenames = glob.glob("stats_temp/*.json")
users_filenames = glob.glob("stats_temp/users/*.json")
if ratings_filenames:
command = ["zopfli"]
command.extend(ratings_filenames)
command.extend(users_filenames)
subprocess.call(command)
ratings_filenames = ratings_filenames + [
x + ".gz" for x in ratings_filenames
]
users_filenames = users_filenames + [x + ".gz" for x in users_filenames]
command = ["touch", "-r", "stats_temp/ratings.json"]
command.extend(ratings_filenames)
command.extend(users_filenames)
subprocess.call(command)
command = ["mv"]
command.extend(ratings_filenames)
command.append("web/stats/data")
subprocess.call(command)
command = ["mv"]
command.extend(users_filenames)
command.append("web/stats/data/users")
subprocess.call(command)
def main():
user_id_to_name = None
while True:
with orm.session_scope() as session:
lookup = orm.Lookup(session)
logs2db = Logs2DB(session, lookup)
kv_last_log_timestamp = lookup.get_key_value(
'cron last log timestamp')
last_log_timestamp = 1408905413 if kv_last_log_timestamp.value is None else int(
kv_last_log_timestamp.value)
kv_last_offset = lookup.get_key_value('cron last offset')
last_offset = 0 if kv_last_offset.value is None else int(
kv_last_offset.value)
completed_game_users = set()
for log_timestamp, filename in util.get_log_file_filenames(
'py', begin=last_log_timestamp):
if log_timestamp != last_log_timestamp:
last_offset = 0
with util.open_possibly_gzipped_file(filename) as f:
if last_offset:
f.seek(last_offset)
last_offset, new_completed_game_users = logs2db.process_logs(
f, log_timestamp)
completed_game_users.update(new_completed_game_users)
last_log_timestamp = log_timestamp
kv_last_log_timestamp.value = last_log_timestamp
kv_last_offset.value = last_offset
session.flush()
if completed_game_users:
statsgen = StatsGen(session, 'stats_temp')
if not user_id_to_name:
user_id_to_name = statsgen.get_user_id_to_name()
for user in completed_game_users:
user_id_to_name[user.user_id] = user.name
statsgen.output_users(user_id_to_name)
for user in completed_game_users:
statsgen.output_user(user.user_id)
filenames = glob.glob('stats_temp/*.json')
if filenames:
all_filenames = filenames + [x + '.gz' for x in filenames]
command = ['zopfli']
command.extend(filenames)
subprocess.call(command)
command = ['touch', '-r', 'stats_temp/users.json']
command.extend(all_filenames)
subprocess.call(command)
command = ['mv']
command.extend(all_filenames)
command.append('web/stats')
subprocess.call(command)
time.sleep(60)
def main():
user_id_to_name = None
while True:
with orm.session_scope() as session:
lookup = orm.Lookup(session)
logs2db = Logs2DB(session, lookup)
kv_last_filename = lookup.get_key_value('cron last filename')
last_filename = 0 if kv_last_filename.value is None else int(
kv_last_filename.value)
kv_last_offset = lookup.get_key_value('cron last offset')
last_offset = 0 if kv_last_offset.value is None else int(
kv_last_offset.value)
filenames = []
for filename in os.listdir('logs_py'):
try:
filename = int(filename)
except:
continue
if filename >= last_filename:
filenames.append(filename)
filenames.sort()
filename = 0
offset = 0
completed_game_users = set()
for filename in filenames:
offset = last_offset if filename == last_filename else 0
with open('logs_py/' + str(filename), 'r') as f:
if offset:
f.seek(offset)
offset, new_completed_game_users = logs2db.process_logs(
f, filename)
completed_game_users.update(new_completed_game_users)
kv_last_filename.value = filename
kv_last_offset.value = offset
session.flush()
if completed_game_users:
statsgen = StatsGen(session, 'stats_temp')
if not user_id_to_name:
user_id_to_name = statsgen.get_user_id_to_name()
for user in completed_game_users:
user_id_to_name[user.user_id] = user.name
statsgen.output_users(user_id_to_name)
for user in completed_game_users:
statsgen.output_user(user.user_id)
filenames = glob.glob('stats_temp/*.json')
if filenames:
all_filenames = filenames + [x + '.gz' for x in filenames]
command = ['zopfli']
command.extend(filenames)
subprocess.call(command)
command = ['touch', '-r', 'stats_temp/users.json']
command.extend(all_filenames)
subprocess.call(command)
command = ['mv']
command.extend(all_filenames)
command.append('web/stats')
subprocess.call(command)
time.sleep(60)