def destroy_fcreplay(failed=False):
"""Destry the current compute engine
Checks for the existance of /tmp/destroying. If it exists then
don't try and destroy fcreplay
Args:
failed (bool, optional): Updates the replay to failed. Defaults to False.
"""
# Create destroying file
try:
Path('/tmp/destroying').touch(0o644, exist_ok=False)
except FileExistsError:
# File already exists, not running
sys.exit(0)
logging.info("Starting destroy_fcreplay")
RECEIVING_FUNCTION = 'destroy_fcreplay_instance'
HOSTNAME = socket.gethostname()
if 'fcreplay-image-' not in HOSTNAME:
logging.info(f"Not destroying {HOSTNAME}")
return(False)
# Only retry if failed is false, by default this is false, but sometimes recording
# fails. So we don't want to try and re-record them until we work out why they
# have failed.
if failed is False:
try:
with open('/tmp/fcreplay_status', 'r') as f:
line = f.readline()
local_replay_id = line.split()[0].strip()
local_replay_status = line.split()[1].strip()
if local_replay_status in ['UPLOADING_TO_IA', 'UPLOADING_TO_YOUTUBE', 'UPLOADED_TO_IA', 'UPLOADED_TO_YOUTUBE']:
logging.error(f"Not able to safely recover replay {local_replay_id}")
elif local_replay_status not in ['FINISHED', 'REMOVED_GENERATED_FILES']:
# Replay was in the middle of processing, going to set replay to be re-recorded
db = Database()
db.rerecord_replay(challenge_id=local_replay_id)
except FileNotFoundError:
logging.error('/tmp/fcreplay_status not found')
function_url = f'https://{REGION}-{PROJECT_ID}.cloudfunctions.net/{RECEIVING_FUNCTION}'
metadata_server_url = \
f"http://metadata/computeMetadata/v1/instance/service-accounts/{config['gcloud_compute_service_account']}/identity?audience="
token_full_url = metadata_server_url + function_url
token_headers = {'Metadata-Flavor': 'Google'}
# Fetch the token
token_response = requests.get(token_full_url, headers=token_headers)
jwt = token_response.text
# Provide the token in the request to the receiving function
function_headers = {'Authorization': f'bearer {jwt}'}
function_response = requests.post(function_url, headers=function_headers, json={'instance_name': HOSTNAME})
logging.info(f"destroy_fcreplay retruned: {function_response.status_code}")
status = function_response.status_code
return(status)
class Cli(cmd2.Cmd):
def __init__(self):
super().__init__()
# Show this as the prompt when asking for input
self.prompt = 'fcreplay> '
# Used as prompt for multiline commands after the first line
self.continuation_prompt = '... '
self.db = Database()
delete_failed_parser = cmd2.Cmd2ArgumentParser(
description='Delete a failed replay')
delete_failed_parser.add_argument('challenge_id',
help='Challenge id of replay')
delete_all_failed_parser = cmd2.Cmd2ArgumentParser(
description='Delete all failed replays')
delete_all_failed_parser.add_argument('-y',
'--yes',
action='store_true',
help='Force yes')
delete_pending_parser = cmd2.Cmd2ArgumentParser(
description='Delete a pending replay')
delete_pending_parser.add_argument('challenge_id',
help='Challenge id of the replay')
delete_all_pending_parser = cmd2.Cmd2ArgumentParser(
description='Delete all pending replays')
delete_all_pending_parser.add_argument('-y',
'--yes',
action='store_true',
help='Force yes')
retry_replay_parser = cmd2.Cmd2ArgumentParser(
description='Mark a replay to be re-encoded')
retry_replay_parser.add_argument('challenge_id',
help='Challenge id of replay')
retry_all_failed_replays_parser = cmd2.Cmd2ArgumentParser(
description='Mark all failed replays to be re-encoded')
retry_all_failed_replays_parser.add_argument('-y',
'--yes',
action='store_true',
help='Force yes')
list_replays_parser = cmd2.Cmd2ArgumentParser(description='List replays')
list_replays_parser.add_argument('type',
type=str,
nargs=1,
choices=['failed', 'finished', 'pending'],
help='Type of replays to return')
list_replays_parser.add_argument('-l',
'--limit',
default=10,
type=int,
help='Limit number of results')
count_parser = cmd2.Cmd2ArgumentParser(description='List replays')
count_parser.add_argument('type',
type=str,
nargs=1,
choices=['failed', 'finished', 'pending', 'all'],
help='Type of replays to count')
def yes_or_no(self, question):
while "the answer is invalid":
reply = str(input(question + ' continue? (y/n): ')).lower().strip()
if reply[:1] == 'y':
return True
if reply[:1] == 'n':
return False
@cmd2.with_argparser(delete_failed_parser)
def do_delete_failed(self, args):
if self.yes_or_no(
f"This will delete failed replay: {args.challenge_id},"):
replay = self.db.get_single_replay(args.challenge_id)
if replay is not None:
if replay.failed is True:
self.db.delete_replay(args.challenge_id)
print(f"Deleated replay {args.challenge_id}")
else:
print(f"Replay {args.challenge_id} isn't a faild replay")
return
else:
print(f"Replay {args.challenge_id} doesn't exist")
return
@cmd2.with_argparser(delete_all_failed_parser)
def do_delete_all_failed(self, args):
if not args.yes:
if not self.yes_or_no("This will delete all failed replays,"):
return
failed_replays = self.db.get_all_failed_replays(limit=9999)
if failed_replays is not None:
for r in failed_replays:
self.db.delete_replay(r.id)
print(f"Removed replay: {r.id}")
else:
print("No failed replays")
return
@cmd2.with_argparser(delete_pending_parser)
def do_delete_pending(self, args):
if self.yes_or_no(
f"This will delete the pending replay: {args.challenge_id},"):
replay = self.db.get_single_replay(args.challenge_id)
if replay is not None:
if replay.failed is not True and replay.finished is not True:
self.db.delete_replay(replay.id)
else:
print("Replay isn't a pending replay")
return
else:
print("No replay found")
return
@cmd2.with_argparser(delete_all_pending_parser)
def do_delete_all_pending(self, args):
if not args.yes:
if not self.yes_or_no("This will delete all pending replays,"):
return
pending_replays = self.db.get_all_queued_replays(limit=9999)
if pending_replays is not None:
for r in pending_replays:
self.db.delete_replay(r.id)
print(f"Removed replay: {r.id}")
else:
print("No pending replays")
return
@cmd2.with_argparser(retry_replay_parser)
def do_retry_replay(self, args):
replay = self.db.get_single_replay(args.challenge_id)
if replay is not None:
self.db.rerecord_replay(args.challenge_id)
print(f"Marked replay {args.challenge_id} to be re-encoded")
else:
print(f"Replay {args.challenge_id} doesn't exist")
@cmd2.with_argparser(retry_all_failed_replays_parser)
def do_retry_all_failed_replays(self, args):
if not args.yes:
if not self.yes_or_no("This will retry all failed replays,"):
return
failed_replays = self.db.get_all_failed_replays()
if failed_replays is None:
print("No failed replays to retry")
else:
for r in failed_replays:
self.db.rerecord_replay(r.id)
print(f"Marked failed replay {r.id} to be re-encoded")
@cmd2.with_argparser(list_replays_parser)
def do_ls(self, args):
replays = None
if 'failed' in args.type:
replays = self.db.get_all_failed_replays(limit=args.limit)
elif 'finished' in args.type:
replays = self.db.get_all_finished_replays(limit=args.limit)
elif 'pending' in args.type:
replays = self.db.get_all_queued_replays(limit=args.limit)
else:
return
if replays is not None:
pp = pprint.PrettyPrinter()
for r in replays:
pp.pprint(r.__dict__)
else:
print(f"No replays found for query: {args}")
@cmd2.with_argparser(count_parser)
def do_count(self, args):
replay_count = None
if 'failed' in args.type:
replay_count = self.db.get_failed_count()
elif 'finished' in args.type:
replay_count = self.db.get_finished_count()
elif 'pending' in args.type:
replay_count = self.db.get_pending_count()
elif 'all' in args.type:
replay_count = self.db.get_all_count()
if replay_count is None:
print("0")
else:
print(replay_count)
class Tasker:
def __init__(self):
self.started_instances = {}
self.db = Database()
self.max_instances = 1
self.max_fails = 5
def check_for_replay(self):
if self.number_of_instances() >= self.max_instances:
print(
f"Maximum number of instances ({self.max_instances}) reached")
return False
print("Looking for replay")
player_replay = self.db.get_oldest_player_replay()
if player_replay is not None:
print("Found player replay")
self.launch_fcreplay()
return True
replay = self.db.get_oldest_replay()
if replay is not None:
print("Found replay")
self.launch_fcreplay()
return True
print("No replays")
return False
def number_of_instances(self):
d_client = docker.from_env()
containers = d_client.containers.list()
instance_count = 0
for container in containers:
if 'fcreplay-instance-' in container.name:
instance_count += 1
return instance_count
def running_instance(self, instance_hostname):
d_client = docker.from_env()
for i in d_client.containers.list():
if instance_hostname in i.attrs['Config']['Hostname']:
return True
return False
def remove_temp_dirs(self):
remove_instances = []
for docker_hostname in self.started_instances:
if not self.running_instance(docker_hostname):
print(
f"Removing '/avi_storage_temp/{self.started_instances[docker_hostname]}'"
)
shutil.rmtree(
f"/avi_storage_temp/{self.started_instances[docker_hostname]}"
)
remove_instances.append(docker_hostname)
for i in remove_instances:
del self.started_instances[i]
def retry_failed_videos(self):
print('Setting failed videos to retry')
failed_replays = self.db.get_all_failed_replays(limit=1000)
for r in failed_replays:
if r.fail_count < self.max_fails:
self.db.rerecord_replay(r.id)
print(f"Marked failed replay {r.id} to be re-encoded")
def delete_failed_videos(self):
"""Delete replays that have failed 5 times to record
"""
failed_replays = self.db.get_all_failed_replays(limit=1000)
for r in failed_replays:
if r.fail_count >= self.max_fails:
self.db.delete_replay(r.id)
def launch_fcreplay(self):
print("Getting docker env")
d_client = docker.from_env()
instance_uuid = str(uuid.uuid4().hex)
if 'FCREPLAY_NETWORK' not in os.environ:
os.environ['FCREPLAY_NETWORK'] = 'bridge'
# Get fcreplay network list
networks = os.environ['FCREPLAY_NETWORK'].split(',')
print(
f"Starting new instance with temp dir: '{os.environ['AVI_TEMP_DIR']}/{instance_uuid}'"
)
c_instance = d_client.containers.run(
'fcreplay/image:latest',
command='fcrecord',
cpu_count=int(os.environ['CPUS']),
detach=True,
mem_limit=str(os.environ['MEMORY']),
network=networks[0],
remove=True,
name=f"fcreplay-instance-{instance_uuid}",
volumes={
str(os.environ['CLIENT_SECRETS']): {
'bind': '/root/.client_secrets.json',
'mode': 'ro'
},
str(os.environ['CONFIG']): {
'bind': '/root/config.json',
'mode': 'ro'
},
str(os.environ['DESCRIPTION_APPEND']): {
'bind': '/root/description_append.txt',
'mode': 'ro'
},
str(os.environ['IA']): {
'bind': '/root/.ia',
'mode': 'ro'
},
str(os.environ['ROMS']): {
'bind': '/Fightcade/emulator/fbneo/ROMs',
'mode': 'ro'
},
str(os.environ['YOUTUBE_UPLOAD_CREDENTIALS']): {
'bind': '/root/.youtube-upload-credentials.json',
'mode': 'ro'
},
f"{os.environ['AVI_TEMP_DIR']}/{instance_uuid}": {
'bind': '/Fightcade/emulator/fbneo/avi',
'mode': 'rw'
}
})
if len(networks) > 1:
for n in networks[1:]:
print(f"Adding container to network {n}")
d_net = d_client.networks.get(n)
d_net.connect(c_instance)
print("Getting instance uuid")
self.started_instances[c_instance.attrs['Config']
['Hostname']] = instance_uuid
def check_for_docker_network(self):
d_client = docker.from_env()
d_net = d_client.networks.list()
networks = os.environ['FCREPLAY_NETWORK'].split(',')
if set(networks) <= set([i.name for i in d_net]) is False:
print(
f"The folling networks don't exist: {set(networks) - set([i.name for i in d_net])}"
)
return False
return True
def update_video_status(self):
"""Update the status for videos uploaded to archive.org
"""
print("Checking status for completed videos")
# Get all replays that are completed, where video_processed is false
to_check = self.db.get_unprocessed_replays()
for replay in to_check:
# Check if replay has embeded video link. Easy way to do this is to check
# if a thumbnail is created
print(f"Checking: {replay.id}")
if replay.video_youtube_uploaded:
print(
f"Checking url: http://img.youtube.com/vi/{replay.video_youtube_id}/0.jpg"
)
try:
r = requests.get(
f"http://img.youtube.com/vi/{replay.video_youtube_id}/0.jpg"
)
except Exception as e:
print(f"Caught exception: {e}, when checking {replay.id}")
continue
else:
print(
f"Checking url: https://archive.org/download/{replay.id.replace('@', '-')}/__ia_thumb.jpg"
)
try:
r = requests.get(
f"https://archive.org/download/{replay.id.replace('@', '-')}/__ia_thumb.jpg"
)
except Exception as e:
print(f"Caught exception: {e}, when checking {replay.id}")
continue
print(f"ID: {replay.id}, Status: {r.status_code}")
if r.status_code == 200:
self.db.set_replay_processed(challenge_id=replay.id)
def recorder(self, max_instances=1):
if self.check_for_docker_network() is False:
return False
schedule.every(10).to(30).seconds.do(self.remove_temp_dirs)
schedule.every(30).to(60).seconds.do(self.check_for_replay)
self.max_instances = max_instances
if 'MAX_INSTANCES' in os.environ:
self.max_instances = int(os.environ['MAX_INSTANCES'])
self.check_for_replay()
while True:
schedule.run_pending()
time.sleep(1)
def check_top_weekly(self):
g = Getreplay()
schedule.every(1).hour.do(g.get_top_weekly)
g.get_top_weekly()
while True:
schedule.run_pending()
time.sleep(1)
def check_video_status(self):
self.update_video_status()
schedule.every(1).hour.do(self.update_video_status)
while True:
schedule.run_pending()
time.sleep(1)
def schedule_retry_failed_replays(self):
self.retry_failed_videos()
schedule.every(1).hour.do(self.retry_failed_videos)
while True:
schedule.run_pending()
time.sleep(1)
def schedule_delete_failed_replays(self):
self.delete_failed_videos()
schedule.every(1).hour.do(self.delete_failed_videos)
while True:
schedule.run_pending()
time.sleep(1)