async def save_replay(uid: int, file_paths: List[str],
start_time: int) -> None:
"""
Saves completed replay.
"""
logger.info("Saving data for uid %s", uid)
logger.debug("Paths %s", str(list(file_paths)))
replay_path = get_replay(file_paths)
output_path = get_replay_path(uid)
os_handler = os.open(output_path,
flags=os.O_CREAT | os.O_WRONLY,
mode=0o664)
with open(os_handler, "wb") as output_file:
with open(replay_path, "rb") as replay_file:
replay_data = replay_file.read()
output_file.write(
json.dumps(await get_replay_info(
uid, replay_data,
start_time)).encode('raw_unicode_escape'))
output_file.write(b'\n')
output_file.write(
base64.b64encode(
zlib.compress(
struct.pack("i", len(replay_data)) + replay_data, 9)))
async def handle_connection(self, reader, writer):
"""
Main method, that handle db_connection
"""
connection = ReplayConnection(reader, writer)
logger.info("<%s> Connection established...", connection)
replay_worker = None
try:
request_type = await connection.determine_type()
uid, replay_name = await connection.get_replay_name()
info = "Saving" if request_type == 0 else "Serving"
logger.info("<%s> %s for %s, '%s'", connection, info, str(uid), replay_name)
replay_worker = WorkerFactory.get_worker(uid, request_type, connection)
WorkerStorage.add_worker(uid, replay_worker)
await replay_worker.run()
except ConnectionError as e:
logger.exception("<%s> Connection problems occurs!", connection)
if replay_worker:
await replay_worker.cleanup()
connection.writer.write(str(e).encode('raw_unicode_escape'))
except Exception as e:
logger.exception("<%s> Something goes terribly wrong!", connection)
if replay_worker:
await replay_worker.cleanup()
connection.writer.write("Wrong request: ".encode('raw_unicode_escape'))
connection.writer.write(str(e).encode('raw_unicode_escape'))
await connection.close()
finally:
try:
await connection.close()
except Exception:
logger.exception("<%s> Something goes terribly wrong during connection close", connection)
def __init__(self, buffer: RawIOBase, *args: List[Any],
**kwargs: Dict[Any, Any]):
super(ReplayWriter, self).__init__(*args, **kwargs)
self.buffer: RawIOBase = buffer
self.position: int = 0
logger.info("<%s> Prepared to save stream %s for", self._connection,
self.get_uid())
async def stop(self):
"""
Stop server and try to cleanup
"""
logger.info("Stopping server on port %s", self._port)
await db.close()
if self._server:
self._server.close()
await self._server.wait_closed()
logger.info("Successfully closed")
async def cleanup(self) -> None:
logger.info("<%s> Closing buffers for %s", self._connection, self.get_uid())
for buffer in self.buffers:
buffer.close()
# remove current worker from storage
WorkerStorage.remove_worker(self.get_uid(), self)
online_workers = WorkerStorage.get_online_workers(self.get_uid())
if len(online_workers) == 0:
ReplayStorage.remove_replay_data(self.get_uid())
logger.info("<%s> Closed buffers for %s", self._connection, self.get_uid())
async def process(self) -> None:
"""
Saves stream into the file
"""
logger.info("<%s> Reading save stream for %s", self._connection,
self.get_uid())
while True:
data = await self._connection.reader.read(WRITE_BUFFER_SIZE)
logger.debug("<%s> Write len data %s on position %s",
self._connection, len(data), self.position)
if not data:
break
self.feed(self.position, data)
self.position += len(data)
logger.info("<%s> Finished save stream for %s with length %s",
self._connection, self.get_uid(), self.position)
async def cleanup(self) -> None:
"""
Closes buffers, removes worker from active workers, saves replay, if there is no writers.
"""
logger.info("<%s> Closing buffer for for %s", self._connection,
self.get_uid())
self.buffer.close()
# remove current worker from storage
WorkerStorage.remove_worker(self.get_uid(), self)
# We will save, if there is no writers
online_workers = WorkerStorage.get_online_workers(self.get_uid())
writers_online = any([
isinstance(online_processor, ReplayWriter)
for online_processor in online_workers
])
if not writers_online and ReplayStorage.has_replays(self.get_uid()):
logger.info("<%s> There is no writers online, saving replay",
self._connection)
await save_replay(
self.get_uid(),
list(ReplayStorage.get_replays(self.get_uid()).keys()),
ReplayStorage.get_replay_start_time(self.get_uid()),
)
if len(online_workers) == 0:
ReplayStorage.remove_replay_data(self.get_uid())
logger.info("<%s> Closed buffer for for %s", self._connection,
self.get_uid())
async def process(self) -> None:
"""
Streams the most common stream of connected players.
Waits, for data, if buffers are "empty".
"""
# send "header" information
logger.info("<%s> Reading header for %s", self._connection, self.get_uid())
data = self.buffers[0].read(self.body_positions[0])
self._connection.writer.write(data)
logger.info("<%s> Header for %s with length %s", self._connection, self.get_uid(), len(data))
# read common stream
await self.process_body()
logger.info("<%s> End reading data for %s. Total length %s",
self._connection, self.get_uid(), self.position)
async def start(self):
"""
Start server on port
"""
logger.info("Starting server on port %s", self._port)
self._server = await asyncio.streams.start_server(self.handle_connection, port=self._port)
async def get_replay_info(game_id: int, replay_data: bytes,
start_time: int) -> Dict:
"""
Returns "header" information for replay.
"""
logger.info("Collecting replay info for uid %s", game_id)
result = {
'uid': game_id,
'launched_at': start_time,
'game_end': time.mktime(datetime.datetime.now().timetuple()),
}
try:
header = parse(replay_data, parse_body=False)['header']
game_version = header.get("version")
result['sim_mods'] = {
mod['uid']: mod['version']
for mod in header.get('mods', []).values()
}
try:
game_stats = await get_game_stats(game_id)
# situation, when we don't wanna loose all information, if mysql is down
if not game_stats:
return result
logger.info("Querying replay info for uid %s from datbase",
game_id)
players = await get_players(game_id)
featured_mods = await get_mod_updates(
game_stats[0].get("game_mod"), game_version)
game_stats_first_row = game_stats[0]
teams: Dict[int, List[str]] = {}
for player in players:
teams.setdefault(player['team'], []).append(player['login'])
featured_mod_versions = {}
for mod in featured_mods:
featured_mod_versions[str(mod['file_id'])] = mod['version']
return dict(
**result, **{
'featured_mod': game_stats_first_row['game_mod'],
'num_players': len(game_stats),
'game_type': int(game_stats_first_row['game_type']),
'recorder': game_stats_first_row['host'],
'host': game_stats_first_row['host'],
'complete': True,
'state': 'PLAYING',
'title': game_stats_first_row['game_name'],
'mapname': game_stats_first_row['map_name'],
'map_file_path': game_stats_first_row['file_name'],
'teams': teams,
'featured_mod_versions': featured_mod_versions,
})
except Exception:
logger.exception("Exception occured during getting replay info %s",
game_id)
raise
except Exception:
logger.error("Exception during getting information about replay %s",
game_id)
raise
import asyncio
from replay_server.constants import PORT
from replay_server.db_conn import db
from replay_server.logger import logger
from replay_server.server import ReplayServer
if __name__ == "__main__":
loop = asyncio.get_event_loop()
logger.info("Checking MySQL DB")
try:
# check, that mysql is available and create connection pool
loop.run_until_complete(asyncio.ensure_future(db.create_pool(loop)))
except Exception as e:
logger.exception("Something goes wrong with MySQL")
logger.info("Started server...")
server = ReplayServer(PORT)
start = asyncio.ensure_future(server.start(), loop=loop)
try:
loop.run_forever()
logger.info("Server started")
except (KeyboardInterrupt, Exception) as e:
if not isinstance(e, KeyboardInterrupt):
logger.exception('Unexpected exception during program run')
logger.info('Stopping server...')
loop.run_until_complete(server.stop())
finally:
loop.close()
logger.info('Server stopped')