def __init__(self):
self.config = Config().config
self.db = Database()
with open(
pkg_resources.resource_filename(
'fcreplay', 'data/supported_games.json')) as f:
self.supported_games = json.load(f)
def __init__(self):
self.config = Config().config
self.db = Database()
self.replay = self.get_replay()
self.description_text = ""
# On replay start create a status file in /tmp
# This is used to determine shutdown status for a replay
with open('/tmp/fcreplay_status', 'w') as f:
f.write(f"{self.replay.id} STARTED")
def __init__(self):
super().__init__()
# Show this as the prompt when asking for input
self.prompt = 'fcreplay> '
# Used as prompt for multiline commands after the first line
self.continuation_prompt = '... '
self.db = Database()
def setUp(self, mock_config, mock_func, mock_create_engine):
db = Database()
mock_create_engine.assert_called(), 'Database should call create_engine'
mock_create_engine.side_effect = Exception
with pytest.raises(Exception) as e:
db = Database()
assert e is Exception, 'Database should raise exception when __init__ fails'
return db
def check_if_finished(challenge_id):
if challenge_exists(challenge_id):
# Checks to see if challenge is already finished
db = Database()
replay = db.get_single_replay(challenge_id=challenge_id)
if replay.status == 'FINISHED':
return ('FINISHED')
else:
return ('NOT_FINISHED')
else:
return ("NO_DATA")
def __init__(self):
self.config = Config().config
self.db = Database()
self.replay = self.get_replay()
self.description_text = ""
self.detected_characters = []
with open(
pkg_resources.resource_filename(
'fcreplay', 'data/supported_games.json')) as f:
self.supported_games = json.load(f)
# On replay start create a status file in /tmp - Legacy?
with open('/tmp/fcreplay_status', 'w') as f:
f.write(f"{self.replay.id} STARTED")
def get_current_job_remaining():
# Returns the time left to complete current job
db = Database()
job = db.get_current_job()
current_time = datetime.datetime.utcnow()
start_time = job.start_time
length = job.length
running_time = int((current_time - start_time).seconds)
time_left = length - running_time
Logging().info(
f"Current job status: running_time: {running_time}, time_left: {time_left}"
)
if time_left <= 0:
# Time left is less than 0, probably uploading or doing something
return 0
else:
return time_left
def get_current_job_id():
db = Database()
job = db.get_current_job()
logging.info(f"Current job ID is: {job.challenge_id}")
return (job.challenge_id)
def destroy_fcreplay(self, failed=False):
"""Destry the current compute engine
Checks for the existance of /tmp/destroying. If it exists then
don't try and destroy fcreplay
Args:
failed (bool, optional): Updates the replay to failed. Defaults to False.
"""
# Create destroying file
try:
Path('/tmp/destroying').touch(0o644, exist_ok=False)
except FileExistsError:
# File already exists, not running
sys.exit(0)
Logging().info("Starting destroy_fcreplay")
RECEIVING_FUNCTION = 'destroy_fcreplay_instance'
HOSTNAME = socket.gethostname()
if 'fcreplay-image-' not in HOSTNAME:
Logging().info(f"Not destroying {HOSTNAME}")
return (False)
# Only retry if failed is false, by default this is false, but sometimes recording
# fails. So we don't want to try and re-record them until we work out why they
# have failed.
if failed is False:
try:
with open('/tmp/fcreplay_status', 'r') as f:
line = f.readline()
local_replay_id = line.split()[0].strip()
local_replay_status = line.split()[1].strip()
if local_replay_status in [
'UPLOADING_TO_IA', 'UPLOADING_TO_YOUTUBE',
'UPLOADED_TO_IA', 'UPLOADED_TO_YOUTUBE'
]:
Logging().error(
f"Not able to safely recover replay {local_replay_id}")
elif local_replay_status not in [
'FINISHED', 'REMOVED_GENERATED_FILES'
]:
# Replay was in the middle of processing, going to set replay to be re-recorded
db = Database()
db.rerecord_replay(challenge_id=local_replay_id)
except FileNotFoundError:
Logging().error('/tmp/fcreplay_status not found')
function_url = f'https://{self.REGION}-{self.PROJECT_ID}.cloudfunctions.net/{RECEIVING_FUNCTION}'
metadata_server_url = \
f"http://metadata/computeMetadata/v1/instance/service-accounts/{self.config['gcloud_compute_service_account']}/identity?audience="
token_full_url = metadata_server_url + function_url
token_headers = {'Metadata-Flavor': 'Google'}
# Fetch the token
token_response = requests.get(token_full_url, headers=token_headers)
jwt = token_response.text
# Provide the token in the request to the receiving function
function_headers = {'Authorization': f'bearer {jwt}'}
function_response = requests.post(function_url,
headers=function_headers,
json={'instance_name': HOSTNAME})
Logging().info(
f"destroy_fcreplay retruned: {function_response.status_code}")
status = function_response.status_code
if self.config['gcloud_shutdown_instance']:
subprocess.run(['shutdown', 'now', '-h'])
return (status)
def __init__(self):
self.started_instances = {}
self.db = Database()
self.max_instances = 1
class Tasker:
def __init__(self):
self.started_instances = {}
self.db = Database()
self.max_instances = 1
def check_for_replay(self):
if self.number_of_instances() >= self.max_instances:
print(
f"Maximum number of instances ({self.max_instances}) reached")
return False
print("Looking for replay")
player_replay = self.db.get_oldest_player_replay()
if player_replay is not None:
print("Found player replay")
self.launch_fcreplay()
return True
replay = self.db.get_oldest_replay()
if replay is not None:
print("Found replay")
self.launch_fcreplay()
return True
print("No replays")
return False
def number_of_instances(self):
d_client = docker.from_env()
containers = d_client.containers.list()
instance_count = 0
for container in containers:
if 'fcreplay-instance-' in container.name:
instance_count += 1
return instance_count
def running_instance(self, instance_hostname):
d_client = docker.from_env()
for i in d_client.containers.list():
if instance_hostname in i.attrs['Config']['Hostname']:
return True
return False
def remove_temp_dirs(self):
remove_instances = []
for docker_hostname in self.started_instances:
if not self.running_instance(docker_hostname):
print(
f"Removing '/avi_storage_temp/{self.started_instances[docker_hostname]}'"
)
shutil.rmtree(
f"/avi_storage_temp/{self.started_instances[docker_hostname]}"
)
remove_instances.append(docker_hostname)
for i in remove_instances:
del self.started_instances[i]
def launch_fcreplay(self):
print("Getting docker env")
d_client = docker.from_env()
instance_uuid = str(uuid.uuid4().hex)
if 'FCREPLAY_NETWORK' not in os.environ:
os.environ['FCREPLAY_NETWORK'] = 'bridge'
# Get fcreplay network list
networks = os.environ['FCREPLAY_NETWORK'].split(',')
print(
f"Starting new instance with temp dir: '{os.environ['AVI_TEMP_DIR']}/{instance_uuid}'"
)
c_instance = d_client.containers.run(
'fcreplay/image:latest',
command='fcrecord',
cpu_count=int(os.environ['CPUS']),
detach=True,
mem_limit=str(os.environ['MEMORY']),
network=networks[0],
remove=True,
name=f"fcreplay-instance-{instance_uuid}",
volumes={
str(os.environ['CLIENT_SECRETS']): {
'bind': '/root/.client_secrets.json',
'mode': 'ro'
},
str(os.environ['CONFIG']): {
'bind': '/root/config.json',
'mode': 'ro'
},
str(os.environ['DESCRIPTION_APPEND']): {
'bind': '/root/description_append.txt',
'mode': 'ro'
},
str(os.environ['IA']): {
'bind': '/root/.ia',
'mode': 'ro'
},
str(os.environ['ROMS']): {
'bind': '/Fightcade/emulator/fbneo/ROMs',
'mode': 'ro'
},
str(os.environ['YOUTUBE_UPLOAD_CREDENTIALS']): {
'bind': '/root/.youtube-upload-credentials.json',
'mode': 'ro'
},
f"{os.environ['AVI_TEMP_DIR']}/{instance_uuid}": {
'bind': '/Fightcade/emulator/fbneo/avi',
'mode': 'rw'
}
})
if len(networks) > 1:
for n in networks[1:]:
print(f"Adding container to network {n}")
d_net = d_client.networks.get(n)
d_net.connect(c_instance)
print("Getting instance uuid")
self.started_instances[c_instance.attrs['Config']
['Hostname']] = instance_uuid
def check_for_docker_network(self):
d_client = docker.from_env()
d_net = d_client.networks.list()
networks = os.environ['FCREPLAY_NETWORK'].split(',')
if set(networks) <= set([i.name for i in d_net]) is False:
print(
f"The folling networks don't exist: {set(networks) - set([i.name for i in d_net])}"
)
return False
return True
def update_video_status(self):
"""Update the status for videos uploaded to archive.org
"""
print("Checking status for completed videos")
# Get all replays that are completed, where video_processed is false
to_check = self.db.get_unprocessed_replays()
for replay in to_check:
# Check if replay has embeded video link. Easy way to do this is to check
# if a thumbnail is created
print(f"Checking: {replay.id}")
r = requests.get(
f"https://archive.org/download/{replay.id.replace('@', '-')}/__ia_thumb.jpg"
)
print(f"ID: {replay.id}, Status: {r.status_code}")
if r.status_code == 200:
self.db.set_replay_processed(challenge_id=replay.id)
def recorder(self, max_instances=1):
if self.check_for_docker_network() is False:
return False
schedule.every(10).to(30).seconds.do(self.remove_temp_dirs)
schedule.every(30).to(60).seconds.do(self.check_for_replay)
self.max_instances = max_instances
if 'MAX_INSTANCES' in os.environ:
self.max_instances = int(os.environ['MAX_INSTANCES'])
while True:
schedule.run_pending()
time.sleep(1)
def check_top_weekly(self):
g = Getreplay()
schedule.every(1).hour.do(g.get_top_weekly)
g.get_top_weekly()
while True:
schedule.run_pending()
time.sleep(1)
def check_video_status(self):
schedule.every(1).hour.do(self.update_video_status)
while True:
schedule.run_pending()
time.sleep(1)
class Getreplay:
def __init__(self):
if 'REMOTE_DEBUG' in os.environ:
import debugpy
debugpy.listen(("0.0.0.0", 5678))
debugpy.wait_for_client()
self.config = Config().config
self.db = Database()
@retry(wait_random_min=5000,
wait_random_max=10000,
stop_max_attempt_number=3)
def get_data(self, query):
r = requests.post("https://www.fightcade.com/api/", json=query)
if r.status_code == 500:
Logging().error("500 Code, trying up to 3 times")
raise IOError("Unable to get data")
else:
return r
def add_replay(self, replay, emulator, game, player_replay=True):
challenge_id = replay['quarkid']
p1_loc = replay['players'][0]['country']
p2_loc = replay['players'][1]['country']
p1 = replay['players'][0]['name']
p2 = replay['players'][1]['name']
date_replay = datetime.datetime.fromtimestamp(replay['date'] // 1000)
length = replay['duration']
created = False
failed = False
status = 'ADDED'
date_added = datetime.datetime.utcnow()
player_requested = player_replay
if 'rank' in replay['players'] or 'rank' in replay['players'][1]:
if replay['players'][0]['rank'] is None:
p1_rank = '0'
else:
p1_rank = replay['players'][0]['rank']
if replay['players'][1]['rank'] is None:
p2_rank = '0'
else:
p2_rank = replay['players'][1]['rank']
else:
p1_rank = '0'
p2_rank = '0'
# Insert into database
Logging().info(f"Looking for {challenge_id}")
# Check if replay exists
data = self.db.get_single_replay(challenge_id=challenge_id)
if data is None:
# Limit the length of videos
if length > int(self.config['min_replay_length']) and length < int(
self.config['max_replay_length']):
Logging().info(f"Adding {challenge_id} to queue")
self.db.add_replay(challenge_id=challenge_id,
p1_loc=p1_loc,
p2_loc=p2_loc,
p1_rank=p1_rank,
p2_rank=p2_rank,
p1=p1,
p2=p2,
date_replay=date_replay,
length=length,
created=created,
failed=failed,
status=status,
date_added=date_added,
player_requested=player_requested,
game=game,
emulator=emulator,
video_processed=False)
return ('ADDED')
else:
Logging().info(f"{challenge_id} is only {length} not adding")
if player_replay:
return ('TOO_SHORT')
else:
Logging().info(f"{challenge_id} already exists")
if player_replay:
# Check if the returned replay is a player replay
if data.player_requested:
return ('ALREADY_EXISTS')
else:
# Update DB to mark returned replay as player replay
self.db.update_player_requested(challenge_id=challenge_id)
return ('MARKED_PLAYER')
return ('ALREADY_EXISTS')
def get_game_replays(self, game):
"""Get game replays
Args:
game (String): Gameid
"""
if game not in self.config['supported_games']:
return ('UNSUPPORTED_GAME')
query = {'req': 'searchquarks', 'gameid': game}
r = self.get_data(query)
for i in r.json()['results']['results']:
if i['emulator'] == 'fbneo' and i['live'] is False:
status = self.add_replay(replay=i,
emaultor=i['emaultor'],
game=game,
player_replay=False)
if status != 'ADDED':
Logging().info(f'Not adding game, Status: {status}')
return ("ADDED")
def get_top_weekly(self):
"""Get the top weekly replays
"""
today = datetime.datetime.today()
start_week = today - timedelta(days=today.weekday())
start_week_ms = int(start_week.timestamp() * 1000)
query = {'req': 'searchquarks', 'best': True, 'since': start_week_ms}
replays = []
for i in range(0, 3):
query['offset'] = i * 15
r = self.get_data(query)
replays += r.json()['results']['results']
for i in replays:
if i['gameid'] not in self.config['supported_games']:
Logging().info(
f"Game {i['gameid']} not supported for replay {i['quarkid']}"
)
continue
status = self.add_replay(replay=i,
emulator=i['emulator'],
game=i['gameid'],
player_replay=False)
if status != 'ADDED':
Logging().info(
f"Not adding replay {i['quarkid']}, Status: {status}")
return ("ADDED")
def get_ranked_replays(self, game, username=None, pages=None):
"""Get ranked replays
Args:
game (String): Gameid
username (String, optional): Player profile name. Defaults to None.
"""
if game not in self.config['supported_games']:
return ('UNSUPPORTED_GAME')
query = {"req": "searchquarks", "best": True, "gameid": game}
if username is not None:
query['username'] = username
replays = []
if pages is None:
query['offset'] = 0
r = self.get_data(query)
replays += r.json()['results']['results']
else:
for page in range(0, pages):
query['offset'] = page
r = self.get_data(query)
replays += r.json()['results']['results']
for i in replays:
if i['emulator'] == 'fbneo' and i['live'] is False:
status = self.add_replay(replay=i,
emulator=i['emulator'],
game=game,
player_replay=False)
if status != 'ADDED':
Logging().info(f'Not adding game, Status: {status}')
return ("ADDED")
def get_replay(self, url, player_requested=False):
"""Get a single replay
Args:
url (String): Link to replay
"""
# Validate url, this could probably be done better
pattern = re.compile(
'^https://replay.fightcade.com/fbneo/.*/[0-9]*-[0-9]*$')
if not pattern.match(url):
return ('INVALID_URL')
# Parse url
emulator = url.split('/')[3]
game = url.split('/')[4]
challenge_id = url.split('/')[5]
Logging().debug(
f"Parsed url: emulator: {emulator}, game: {game}, challenge_id: {challenge_id}"
)
if game not in self.config['supported_games']:
return ('UNSUPPORTED_GAME')
# Get play replays
query = {"req": "searchquarks", "quarkid": challenge_id}
r = self.get_data(query)
# Look for replay in results:
for i in r.json()['results']['results']:
if challenge_id == i['quarkid']:
return self.add_replay(replay=i,
emulator=emulator,
game=game,
player_replay=player_requested)
return False
# This file needs to be in the root of the repository for google cloud
# functions to use the main.py file and be able to import the fcreplay files
import json
import os
import time
import requests
from fcreplay.logging import Logging
from fcreplay.getreplay import Getreplay
from fcreplay.database import Database
from fcreplay.config import Config
config = Config().config
db = Database()
def video_status(request):
Logging().info("Check status for completed videos")
# Get all replays that are completed, where video_processed is false
to_check = db.get_unprocessed_replays()
for replay in to_check:
# Check if replay has embeded video link. Easy way to do this is to check
# if a thumbnail is created
Logging().info(f"Checking: {replay.id}")
r = requests.get(
f"https://archive.org/download/{replay.id.replace('@', '-')}/__ia_thumb.jpg"
)
Logging().info(f"ID: {replay.id}, Status: {r.status_code}")
class Replay:
""" Class for FightCade replays
"""
def __init__(self):
self.config = Config().config
self.db = Database()
self.replay = self.get_replay()
self.description_text = ""
self.detected_characters = []
with open(
pkg_resources.resource_filename(
'fcreplay', 'data/supported_games.json')) as f:
self.supported_games = json.load(f)
# On replay start create a status file in /tmp - Legacy?
with open('/tmp/fcreplay_status', 'w') as f:
f.write(f"{self.replay.id} STARTED")
def handle_fail(func):
"""Handle Failure decorator
"""
def failed(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except Exception:
trace_back = sys.exc_info()[2]
log.error(
f"Excption: {str(traceback.format_tb(trace_back))}, shutting down"
)
log.info(f"Setting {self.replay.id} to failed")
self.db.update_failed_replay(challenge_id=self.replay.id)
self.update_status(status.FAILED)
# Hacky as hell, but ensures everything gets killed
if self.config['kill_all']:
subprocess.run(['pkill', '-9', 'fcadefbneo'])
subprocess.run(['pkill', '-9', 'wine'])
subprocess.run(['pkill', '-9', '-f', 'system32'])
subprocess.run(['/usr/bin/pulseaudio', '-k'])
subprocess.run(['pkill', '-9', 'tail'])
subprocess.run(['killall5'])
subprocess.run(['pkill', '-9', 'sh'])
time.sleep(5)
sys.exit(1)
return failed
@handle_fail
def get_replay(self):
"""Get a replay from the database
"""
log.info('Getting replay from database')
if self.config['player_replay_first']:
replay = self.db.get_oldest_player_replay()
if replay is not None:
log.info('Found player replay to encode')
return replay
else:
log.info('No more player replays')
if self.config['random_replay']:
log.info('Getting random replay')
replay = self.db.get_random_replay()
return replay
else:
log.info('Getting oldest replay')
replay = self.db.get_oldest_replay()
return replay
@handle_fail
def get_characters(self):
"""Get characters (if they exist) from pickle file
"""
c = CharacterDetection()
self.detected_characters = c.get_characters()
for i in self.detected_characters:
self.db.add_detected_characters(challenge_id=self.replay.id,
p1_char=i[0],
p2_char=i[1],
vid_time=i[2],
game=self.replay.game)
@handle_fail
def add_job(self):
"""Update jobs database table with the current replay
"""
start_time = datetime.datetime.utcnow()
self.update_status(status.JOB_ADDED)
self.db.add_job(challenge_id=self.replay.id,
start_time=start_time,
length=self.replay.length)
@handle_fail
def remove_job(self):
"""Remove job from database
"""
self.update_status(status.REMOVED_JOB)
self.db.remove_job(challenge_id=self.replay.id)
@handle_fail
def update_status(self, status):
"""Update the replay status
"""
log.info(f"Set status to {status}")
# This file is legacy?
with open('/tmp/fcreplay_status', 'w') as f:
f.write(f"{self.replay.id} {status}")
self.db.update_status(challenge_id=self.replay.id, status=status)
@handle_fail
def record(self):
"""Start recording a replay
"""
log.info(
f"Starting capture with {self.replay.id} and {self.replay.length}")
time_min = int(self.replay.length / 60)
log.info(f"Capture will take {time_min} minutes")
self.update_status(status.RECORDING)
# Star a recording store recording status
log.debug(f"""Starting record.main with argumens:
fc_challange_id={self.replay.id},
fc_time={self.replay.length},
kill_time={self.config['record_timeout']},
fcadefbneo_path={self.config['fcadefbneo_path']},
game_name={self.replay.game}""")
record_status = Record().main(
fc_challange_id=self.replay.id,
fc_time=self.replay.length,
kill_time=self.config['record_timeout'],
fcadefbneo_path=self.config['fcadefbneo_path'],
game_name=self.replay.game)
# Check recording status
if record_status != "Pass":
log.error(f"Recording failed on {self.replay.id},"
f"Status: {record_status}, exiting.")
if record_status == "FailTimeout":
raise TimeoutError
else:
log.error(f"Unknown error: ${record_status}, exiting")
raise ValueError
log.info("Capture finished")
self.update_status(status.RECORDED)
return True
@handle_fail
def sort_files(self, avi_files_list):
log.info("Sorting files")
if len(avi_files_list) > 1:
avi_dict = {}
for i in avi_files_list:
m = re.search('(.*)_([0-9a-fA-F]+).avi', i)
avi_dict[i] = int(m.group(2), 16)
sorted_avi_files_list = []
for i in sorted(avi_dict.items(), key=lambda x: x[1]):
sorted_avi_files_list.append(i[0])
avi_files = [
f"{self.config['fcadefbneo_path']}/avi/" + i
for i in sorted_avi_files_list
]
else:
avi_files = [
f"{self.config['fcadefbneo_path']}/avi/" + avi_files_list[0]
]
return avi_files
@handle_fail
def encode(self):
log.info("Encoding lossless file")
avi_files_list_glob = glob.glob(
f"{self.config['fcadefbneo_path']}/avi/*.avi")
avi_files_list = []
for f in avi_files_list_glob:
avi_files_list.append(os.path.basename(f))
log.info(f"List of files is: {avi_files_list}")
# Sort files
avi_files = self.sort_files(avi_files_list)
# I can't stress enough how much you should not try and mess with the encoding settings!
# 1. ffmpeg will not handle files generated by fbneo
# 2. The files that fbneo generates need to be transcoded before they are encoded to h264 (h265 doesn't work well with archive.org)
mencoder_options = [
'/opt/mplayer/bin/mencoder', '-oac', 'mp3lame', '-lameopts',
'vbr=3', '-ovc', 'x264', '-x264encopts',
'preset=slow:threads=auto', '-vf',
'flip,scale=960:720,dsize=4/3,expand=1280:720:160:0::', *avi_files,
'-of', 'lavf', '-o',
f"{self.config['fcadefbneo_path']}/avi/{self.replay.id}.mp4"
]
log.info(f"Running mencoder with: {' '.join(mencoder_options)}")
mencoder_rc = subprocess.run(mencoder_options, capture_output=True)
try:
mencoder_rc.check_returncode()
except subprocess.CalledProcessError as e:
log.error(
f"Unable to process avi files. Return code: {e.returncode}, stdout: {mencoder_rc.stdout}, stderr: {mencoder_rc.stderr}"
)
raise e
@handle_fail
def remove_old_avi_files(self):
log.info('Removing old avi files')
old_files = glob.glob(f"{self.config['fcadefbneo_path']}/avi/*.avi")
for f in old_files:
log.info(f"Removing {f}")
os.unlink(f)
@handle_fail
def set_description(self):
"""Set the description of the video
Returns:
Boolean: Success or failure
"""
log.info("Creating description")
if len(self.detected_characters) > 0:
self.description_text = f"({self.replay.p1_loc}) {self.replay.p1} vs "\
f"({self.replay.p2_loc}) {self.replay.p2} - {self.replay.date_replay} "\
f"\nFightcade replay id: {self.replay.id}"
for match in self.detected_characters:
self.description_text += f"\n{self.replay.p1}: {match[0]}, {self.replay.p2}: {match[1]} - {match[2]}" \
f"\n{match[0]} vs {match[1]}"
else:
self.description_text = f"({self.replay.p1_loc}) {self.replay.p1} vs " \
f"({self.replay.p2_loc}) {self.replay.p2} - {self.replay.date_replay}" \
f"\nFightcade replay id: {self.replay.id}"
# Read the append file:
if self.config['description_append_file'][0] is True:
# Check if file exists:
if not os.path.exists(self.config['description_append_file'][1]):
log.error(
f"Description append file {self.config['description_append_file'][1]} doesn't exist"
)
return False
else:
with open(self.config['description_append_file'][1],
'r') as description_append:
self.description_text += "\n" + description_append.read()
self.update_status(status.DESCRIPTION_CREATED)
log.info("Finished creating description")
# Add description to database
log.info('Adding description to database')
self.db.add_description(challenge_id=self.replay.id,
description=self.description_text)
log.debug(
f"Description Text is: {self.description_text.encode('unicode-escape')}"
)
return True
@handle_fail
def create_thumbnail(self):
"""Create thumbnail from video
"""
log.info("Making thumbnail")
self.thumbnail = Thumbnail().get_thumbnail(self.replay)
self.update_status(status.THUMBNAIL_CREATED)
log.info("Finished making thumbnail")
@handle_fail
def update_thumbnail(self):
"""Add text, country and ranks to thumbnail
"""
log.info("Updating thumbnail")
UpdateThumbnail().update_thumbnail(self.replay, self.thumbnail)
@handle_fail
@retry(wait_random_min=30000,
wait_random_max=60000,
stop_max_attempt_number=3)
def upload_to_ia(self):
"""Upload to internet archive
Sometimes it will return a 403, even though the file doesn't already
exist. So we decorate the function with the @retry decorator to try
again in a little bit. Max of 3 tries
"""
self.update_status(status.UPLOADING_TO_IA)
title = f"{self.supported_games[self.replay.game]['game_name']}: ({self.replay.p1_loc}) {self.replay.p1} vs" \
f"({self.replay.p2_loc}) {self.replay.p2} - {self.replay.date_replay}"
filename = f"{self.replay.id}.mp4"
date_short = str(self.replay.date_replay)[10]
# Make identifier for Archive.org
ident = str(self.replay.id).replace("@", "-")
fc_video = get_item(ident)
metadata = {
'title': title,
'mediatype': self.config['ia_settings']['mediatype'],
'collection': self.config['ia_settings']['collection'],
'date': date_short,
'description': self.description_text,
'subject': self.config['ia_settings']['subject'],
'creator': self.config['ia_settings']['creator'],
'language': self.config['ia_settings']['language'],
'licenseurl': self.config['ia_settings']['license_url']
}
log.info("Starting upload to archive.org")
fc_video.upload(f"{self.config['fcadefbneo_path']}/avi/{filename}",
metadata=metadata,
verbose=True)
self.update_status(status.UPLOADED_TO_IA)
log.info("Finished upload to archive.org")
@handle_fail
def upload_to_yt(self):
"""Upload video to youtube
"""
self.update_status(status.UPLOADING_TO_YOUTUBE)
title = f"{self.supported_games[self.replay.game]['game_name']}: ({self.replay.p1_loc}) {self.replay.p1} vs "\
f"({self.replay.p2_loc}) {self.replay.p2} - {self.replay.date_replay}"
filename = f"{self.replay.id}.mp4"
import_format = '%Y-%m-%d %H:%M:%S'
date_raw = datetime.datetime.strptime(str(self.replay.date_replay),
import_format)
if len(title) > 100:
title = title[:99]
log.info(f"Title is: {title}")
# YYYY-MM-DDThh:mm:ss.sZ
youtube_date = date_raw.strftime('%Y-%m-%dT%H:%M:%S.0Z')
# Check if youtube-upload is installed
if shutil.which('youtube-upload') is not None:
# Check if credentials file exists
if not os.path.exists(self.config['youtube_credentials']):
log.error("Youtube credentials don't exist exist")
return False
if not os.path.exists(self.config['youtube_secrets']):
log.error("Youtube secrets don't exist")
return False
# Find number of uploads today
day_log = self.db.get_youtube_day_log()
# Check max uploads
# Get todays date, dd-mm-yyyy
today = datetime.date.today()
# Check the log is for today
if day_log.date.date() == today:
# Check number of uploads
if day_log.count >= int(
self.config['youtube_max_daily_uploads']):
log.info("Maximum uploads reached for today")
return False
else:
# It's a new day, update the counter
log.info("New day for youtube uploads")
self.db.update_youtube_day_log_count(count=1, date=today)
# Create description file
with open(f"{self.config['fcreplay_dir']}/tmp/description.txt",
'w') as description_file:
description_file.write(self.description_text)
# Do upload
log.info("Uploading to youtube")
yt_rc = subprocess.run([
'youtube-upload',
'--credentials-file',
self.config['youtube_credentials'],
'--client-secrets',
self.config['youtube_secrets'],
'-t',
title,
'-c',
'Gaming',
'--description-file',
f"{self.config['fcreplay_dir']}/tmp/description.txt",
'--recording-date',
youtube_date,
'--default-language',
'en',
'--thumbnail',
str(self.thumbnail),
f"{self.config['fcadefbneo_path']}/avi/{filename}",
],
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
youtube_id = yt_rc.stdout.decode().rstrip()
log.info(f"Youtube id: {youtube_id}")
log.info(yt_rc.stderr.decode())
if not self.replay.player_requested:
log.info('Updating day_log')
log.info("Updating counter")
self.db.update_youtube_day_log_count(count=day_log.count + 1,
date=today)
# Remove description file
os.remove(f"{self.config['fcreplay_dir']}/tmp/description.txt")
if len(youtube_id) < 4:
log.info('Unable to upload to youtube')
self.db.set_youtube_uploaded(self.replay.id, False)
else:
self.db.set_youtube_uploaded(self.replay.id, True)
self.db.set_youtube_id(self.replay.id, youtube_id)
self.update_status(status.UPLOADED_TO_YOUTUBE)
log.info('Finished uploading to Youtube')
else:
raise ModuleNotFoundError
@handle_fail
def set_created(self):
self.update_status(status.FINISHED)
self.db.update_created_replay(challenge_id=self.replay.id)
class Replay:
""" Class for FightCade replays
"""
def __init__(self):
self.config = Config().config
self.db = Database()
self.replay = self.get_replay()
self.description_text = ""
# On replay start create a status file in /tmp
# This is used to determine shutdown status for a replay
with open('/tmp/fcreplay_status', 'w') as f:
f.write(f"{self.replay.id} STARTED")
def handle_fail(func):
"""Handle Failure decorator
"""
def failed(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except Exception as e:
trace_back = sys.exc_info()[2]
Logging().error(
f"Excption: {str(traceback.format_tb(trace_back))}, shutting down"
)
Logging().info(f"Setting {self.replay.id} to failed")
self.db.update_failed_replay(challenge_id=self.replay.id)
self.update_status(status.FAILED)
if self.config['gcloud_destroy_on_fail']:
Gcloud().destroy_fcreplay(failed=True)
sys.exit(1)
return failed
@handle_fail
def get_replay(self):
"""Get a replay from the database
"""
Logging().info('Getting replay from database')
if self.config['player_replay_first']:
replay = self.db.get_oldest_player_replay()
if replay is not None:
Logging().info('Found player replay to encode')
return replay
else:
Logging().info('No more player replays')
if self.config['random_replay']:
Logging().info('Getting random replay')
replay = self.db.get_random_replay()
return replay
else:
Logging().info('Getting oldest replay')
replay = self.db.get_oldest_replay()
return replay
@handle_fail
def add_job(self):
"""Update jobs database table with the current replay
"""
start_time = datetime.datetime.utcnow()
self.update_status(status.JOB_ADDED)
self.db.add_job(challenge_id=self.replay.id,
start_time=start_time,
length=self.replay.length)
@handle_fail
def remove_job(self):
"""Remove job from database
"""
self.update_status(status.REMOVED_JOB)
self.db.remove_job(challenge_id=self.replay.id)
@handle_fail
def update_status(self, status):
"""Update the replay status
"""
Logging().info(f"Set status to {status}")
with open('/tmp/fcreplay_status', 'w') as f:
f.write(f"{self.replay.id} {status}")
self.db.update_status(challenge_id=self.replay.id, status=status)
@handle_fail
def record(self):
"""Start recording a replay
"""
Logging().info(
f"Starting capture with {self.replay.id} and {self.replay.length}")
time_min = int(self.replay.length / 60)
Logging().info(f"Capture will take {time_min} minutes")
self.update_status(status.RECORDING)
# Star a recording store recording status
Logging().debug(f"""Starting record.main with argumens:
fc_challange_id={self.replay.id},
fc_time={self.replay.length},
kill_time={self.config['record_timeout']},
fcadefbneo_path={self.config['fcadefbneo_path']},
fcreplay_path={self.config['fcreplay_dir']},
game_name={self.replay.game}""")
record_status = Record().main(
fc_challange_id=self.replay.id,
fc_time=self.replay.length,
kill_time=self.config['record_timeout'],
fcadefbneo_path=self.config['fcadefbneo_path'],
fcreplay_path=self.config['fcreplay_dir'],
game_name=self.replay.game)
# Check recording status
if record_status != "Pass":
Logging().error(f"Recording failed on {self.replay.id},"
"Status: {record_status}, exiting.")
if record_status == "FailTimeout":
raise TimeoutError
else:
Logging().error(f"Unknown error: ${record_status}, exiting")
raise ValueError
Logging().info("Capture finished")
self.update_status(status.RECORDED)
return True
@handle_fail
def move(self):
"""Move files to finished area
"""
avi_files_list = os.listdir(f"{self.config['fcadefbneo_path']}/avi")
for f in avi_files_list:
shutil.move(f"{self.config['fcadefbneo_path']}/avi/{f}",
f"{self.config['fcreplay_dir']}/finished/{f}")
self.update_status(status.MOVED)
@handle_fail
def encode(self):
Logging().info("Encoding file")
avi_files_list = os.listdir(f"{self.config['fcreplay_dir']}/finished")
avi_dict = {
i: int(i.split('_')[1].split('.')[0], 16)
for i in avi_files_list
}
sorted_avi_files_list = []
for i in sorted(avi_dict.items(), key=lambda x: x[1]):
sorted_avi_files_list.append(i[0])
avi_files = [
f"{self.config['fcreplay_dir']}/finished/" + i
for i in sorted_avi_files_list
]
# I can't stress enough how much you should not try and mess with the encoding settings!
# 1. ffmpeg will not handle files generated by fbneo
# 2. x264 for whatever reason inserts audio delay
mencoder_options = [
'mencoder', '-oac', 'mp3lame', '-lameopts', 'vbr=3', '-ovc',
'lavc', '-lavcopts', 'vcodec=mpeg4:vbitrate=4000', '-vf',
'flip,scale=800:600,dsize=4/3', *avi_files, '-of', 'lavf', '-o',
f"{self.config['fcreplay_dir']}/finished/{self.replay.id}.mkv"
]
Logging().info(f"Running mencoder with: {' '.join(mencoder_options)}")
mencoder_rc = subprocess.run(mencoder_options, capture_output=True)
try:
mencoder_rc.check_returncode()
except subprocess.CalledProcessError as e:
Logging().error(
f"Unable to process avi files. Return code: {e.returncode}, stdout: {mencoder_rc.stdout}, stderr: {mencoder_rc.stderr}"
)
raise e
@handle_fail
def set_description(self):
"""Set the description of the video
Returns:
Boolean: Success or failure
"""
Logging().info("Creating description")
self.description_text = f"({self.replay.p1_loc}) {self.replay.p1} vs " \
f"({self.replay.p2_loc}) {self.replay.p2} - {self.replay.date_replay}" \
f"\nFightcade replay id: {self.replay.id}"
# Read the append file:
if self.config['description_append_file'][0] is True:
# Check if file exists:
if not os.path.exists(self.config['description_append_file'][1]):
Logging().error(
f"Description append file {self.config['description_append_file'][1]} doesn't exist"
)
return False
else:
with open(self.config['description_append_file'][1],
'r') as description_append:
self.description_text += "\n" + description_append.read()
self.update_status(status.DESCRIPTION_CREATED)
Logging().info("Finished creating description")
# Add description to database
Logging().info('Adding description to database')
self.db.add_description(challenge_id=self.replay.id,
description=self.description_text)
Logging().debug(
f"Description Text is: {self.description_text.encode('unicode-escape')}"
)
return True
@handle_fail
def create_thumbnail(self):
"""Create thumbnail from video
"""
Logging().info("Making thumbnail")
filename = f"{self.replay.id}.mkv"
subprocess.run([
"ffmpeg", "-ss", "20", "-i",
f"{self.config['fcreplay_dir']}/finished/{filename}", "-vframes:v",
"1", f"{self.config['fcreplay_dir']}/tmp/thumbnail.jpg"
])
self.update_status(status.THUMBNAIL_CREATED)
Logging().info("Finished making thumbnail")
@handle_fail
@retry(wait_random_min=30000,
wait_random_max=60000,
stop_max_attempt_number=3)
def upload_to_ia(self):
"""Upload to internet archive
Sometimes it will return a 403, even though the file doesn't already
exist. So we decorate the function with the @retry decorator to try
again in a little bit. Max of 3 tries
"""
self.update_status(status.UPLOADING_TO_IA)
title = f"{self.config['supported_games'][self.replay.game]['game_name']}: ({self.replay.p1_loc}) {self.replay.p1} vs" \
f"({self.replay.p2_loc}) {self.replay.p2} - {self.replay.date_replay}"
filename = f"{self.replay.id}.mkv"
date_short = str(self.replay.date_replay)[10]
# Make identifier for Archive.org
ident = str(self.replay.id).replace("@", "-")
fc_video = get_item(ident)
metadata = {
'title': title,
'mediatype': self.config['ia_settings']['mediatype'],
'collection': self.config['ia_settings']['collection'],
'date': date_short,
'description': self.description_text,
'subject': self.config['ia_settings']['subject'],
'creator': self.config['ia_settings']['creator'],
'language': self.config['ia_settings']['language'],
'licenseurl': self.config['ia_settings']['license_url']
}
Logging().info("Starting upload to archive.org")
fc_video.upload(f"{self.config['fcreplay_dir']}/finished/{filename}",
metadata=metadata,
verbose=True)
self.update_status(status.UPLOADED_TO_IA)
Logging().info("Finished upload to archive.org")
@handle_fail
def upload_to_yt(self):
"""Upload video to youtube
"""
self.update_status(status.UPLOADING_TO_YOUTUBE)
title = f"{self.config['supported_games'][self.replay.game]['game_name']}: ({self.replay.p1_loc}) {self.replay.p1} vs "\
f"({self.replay.p2_loc}) {self.replay.p2} - {self.replay.date_replay}"
filename = f"{self.replay.id}.mkv"
import_format = '%Y-%m-%d %H:%M:%S'
date_raw = datetime.datetime.strptime(str(self.replay.date_replay),
import_format)
# YYYY-MM-DDThh:mm:ss.sZ
youtube_date = date_raw.strftime('%Y-%m-%dT%H:%M:%S.0Z')
# Check if youtube-upload is installed
if shutil.which('youtube-upload') is not None:
# Check if credentials file exists
if not os.path.exists(self.config['youtube_credentials']):
Logging().error("Youtube credentials don't exist exist")
return False
if not os.path.exists(self.config['youtube_secrets']):
Logging().error("Youtube secrets don't exist")
return False
# Check min and max length:
if (int(self.replay.length) / 60) < int(
self.config['yt_min_length']):
Logging().info("Replay is too short. Not uploading to youtube")
return False
if (int(self.replay.length) / 60) > int(
self.config['yt_max_length']):
Logging().info("Replay is too long. Not uploading to youtube")
return False
# Find number of uploads today
day_log = self.db.get_youtube_day_log()
# Check max uploads
# Get todays date, dd-mm-yyyy
today = datetime.date.today()
# Check the log is for today
if day_log.date.date() == today:
# Check number of uploads
if day_log.count >= int(
self.config['youtube_max_daily_uploads']):
Logging().info("Maximum uploads reached for today")
return False
else:
# It's a new day, update the counter
Logging().info("New day for youtube uploads")
self.db.update_youtube_day_log_count(count=1, date=today)
# Create description file
with open(f"{self.config['fcreplay_dir']}/tmp/description.txt",
'w') as description_file:
description_file.write(self.description_text)
# Do upload
Logging().info("Uploading to youtube")
yt_rc = subprocess.run([
'youtube-upload',
'--credentials-file',
self.config['youtube_credentials'],
'--client-secrets',
self.config['youtube_secrets'],
'-t',
title,
'-c',
'Gaming',
'--description-file',
f"{self.config['fcreplay_dir']}/tmp/description.txt",
'--recording-date',
youtube_date,
'--default-language',
'en',
'--thumbnail',
f"{self.config['fcreplay_dir']}/tmp/thumbnail.jpg",
f"{self.config['fcreplay_dir']}/finished/{filename}",
],
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
Logging().info(yt_rc.stdout.decode())
Logging().info(yt_rc.stderr.decode())
if not self.replay.player_requested:
Logging().info('Updating day_log')
Logging().info("Updating counter")
self.db.update_youtube_day_log_count(count=day_log.count + 1,
date=today)
# Remove description file
os.remove(f"{self.config['fcreplay_dir']}/tmp/description.txt")
self.update_status(status.UPLOADED_TO_YOUTUBE)
Logging().info('Finished uploading to Youtube')
else:
raise ModuleNotFoundError
@handle_fail
def remove_generated_files(self):
"""Remove generated files
Generated files are thumbnail and videofile
"""
Logging().info("Removing old files")
filename = f"{self.replay.id}.mkv"
os.remove(f"{self.config['fcreplay_dir']}/finished/{filename}")
os.remove(f"{self.config['fcreplay_dir']}/tmp/thumbnail.jpg")
self.update_status(status.REMOVED_GENERATED_FILES)
Logging().info("Finished removing files")
@handle_fail
def set_created(self):
self.update_status(status.FINISHED)
self.db.update_created_replay(challenge_id=self.replay.id)
def get_current_job_details():
challenge_id = get_current_job_id()
db = Database()
replay = db.get_single_replay(challenge_id=challenge_id)
logging.info(f"Current job rowdata is: {replay}")
return (replay)
def get_replay_status(challenge_id):
db = Database()
replay = db.get_single_replay(challenge_id=challenge_id)
logging.info(f"Current job STATUS is: {replay.status}")
return (replay.status)
class Cli(cmd2.Cmd):
def __init__(self):
super().__init__()
# Show this as the prompt when asking for input
self.prompt = 'fcreplay> '
# Used as prompt for multiline commands after the first line
self.continuation_prompt = '... '
self.db = Database()
delete_failed_parser = cmd2.Cmd2ArgumentParser(
description='Delete a failed replay')
delete_failed_parser.add_argument('challenge_id',
help='Challenge id of replay')
delete_all_failed_parser = cmd2.Cmd2ArgumentParser(
description='Delete all failed replays')
delete_all_failed_parser.add_argument('-y',
'--yes',
action='store_true',
help='Force yes')
delete_pending_parser = cmd2.Cmd2ArgumentParser(
description='Delete a pending replay')
delete_pending_parser.add_argument('challenge_id',
help='Challenge id of the replay')
delete_all_pending_parser = cmd2.Cmd2ArgumentParser(
description='Delete all pending replays')
delete_all_pending_parser.add_argument('-y',
'--yes',
action='store_true',
help='Force yes')
retry_replay_parser = cmd2.Cmd2ArgumentParser(
description='Mark a replay to be re-encoded')
retry_replay_parser.add_argument('challenge_id',
help='Challenge id of replay')
retry_all_failed_replays_parser = cmd2.Cmd2ArgumentParser(
description='Mark all failed replays to be re-encoded')
retry_all_failed_replays_parser.add_argument('-y',
'--yes',
action='store_true',
help='Force yes')
list_replays_parser = cmd2.Cmd2ArgumentParser(description='List replays')
list_replays_parser.add_argument('type',
type=str,
nargs=1,
choices=['failed', 'finished', 'pending'],
help='Type of replays to return')
list_replays_parser.add_argument('-l',
'--limit',
default=10,
type=int,
help='Limit number of results')
count_parser = cmd2.Cmd2ArgumentParser(description='List replays')
count_parser.add_argument('type',
type=str,
nargs=1,
choices=['failed', 'finished', 'pending', 'all'],
help='Type of replays to count')
def yes_or_no(self, question):
while "the answer is invalid":
reply = str(input(question + ' continue? (y/n): ')).lower().strip()
if reply[:1] == 'y':
return True
if reply[:1] == 'n':
return False
@cmd2.with_argparser(delete_failed_parser)
def do_delete_failed(self, args):
if self.yes_or_no(
f"This will delete failed replay: {args.challenge_id},"):
replay = self.db.get_single_replay(args.challenge_id)
if replay is not None:
if replay.failed is True:
self.db.delete_replay(args.challenge_id)
print(f"Deleated replay {args.challenge_id}")
else:
print(f"Replay {args.challenge_id} isn't a faild replay")
return
else:
print(f"Replay {args.challenge_id} doesn't exist")
return
@cmd2.with_argparser(delete_all_failed_parser)
def do_delete_all_failed(self, args):
if not args.yes:
if not self.yes_or_no("This will delete all failed replays,"):
return
failed_replays = self.db.get_all_failed_replays(limit=9999)
if failed_replays is not None:
for r in failed_replays:
self.db.delete_replay(r.id)
print(f"Removed replay: {r.id}")
else:
print("No failed replays")
return
@cmd2.with_argparser(delete_pending_parser)
def do_delete_pending(self, args):
if self.yes_or_no(
f"This will delete the pending replay: {args.challenge_id},"):
replay = self.db.get_single_replay(args.challenge_id)
if replay is not None:
if replay.failed is not True and replay.finished is not True:
self.db.delete_replay(replay.id)
else:
print("Replay isn't a pending replay")
return
else:
print("No replay found")
return
@cmd2.with_argparser(delete_all_pending_parser)
def do_delete_all_pending(self, args):
if not args.yes:
if not self.yes_or_no("This will delete all pending replays,"):
return
pending_replays = self.db.get_all_queued_replays(limit=9999)
if pending_replays is not None:
for r in pending_replays:
self.db.delete_replay(r.id)
print(f"Removed replay: {r.id}")
else:
print("No pending replays")
return
@cmd2.with_argparser(retry_replay_parser)
def do_retry_replay(self, args):
replay = self.db.get_single_replay(args.challenge_id)
if replay is not None:
self.db.rerecord_replay(args.challenge_id)
print(f"Marked replay {args.challenge_id} to be re-encoded")
else:
print(f"Replay {args.challenge_id} doesn't exist")
@cmd2.with_argparser(retry_all_failed_replays_parser)
def do_retry_all_failed_replays(self, args):
if not args.yes:
if not self.yes_or_no("This will retry all failed replays,"):
return
failed_replays = self.db.get_all_failed_replays()
if failed_replays is None:
print("No failed replays to retry")
else:
for r in failed_replays:
self.db.rerecord_replay(r.id)
print(f"Marked failed replay {r.id} to be re-encoded")
@cmd2.with_argparser(list_replays_parser)
def do_ls(self, args):
replays = None
if 'failed' in args.type:
replays = self.db.get_all_failed_replays(limit=args.limit)
elif 'finished' in args.type:
replays = self.db.get_all_finished_replays(limit=args.limit)
elif 'pending' in args.type:
replays = self.db.get_all_queued_replays(limit=args.limit)
else:
return
if replays is not None:
pp = pprint.PrettyPrinter()
for r in replays:
pp.pprint(r.__dict__)
else:
print(f"No replays found for query: {args}")
@cmd2.with_argparser(count_parser)
def do_count(self, args):
replay_count = None
if 'failed' in args.type:
replay_count = self.db.get_failed_count()
elif 'finished' in args.type:
replay_count = self.db.get_finished_count()
elif 'pending' in args.type:
replay_count = self.db.get_pending_count()
elif 'all' in args.type:
replay_count = self.db.get_all_count()
if replay_count is None:
print("0")
else:
print(replay_count)