def main():
# Prepare GitHub repo
clone_status, clone_log = run_shell(
['git', 'clone', os.environ['PUBLISHER_REMOTE'], 'data/publish_repo'])
if clone_status != 0:
logger.warn(clone_log)
if not os.path.exists('data/publish_repo/.git'):
raise Exception('Failed to clone repo')
run_shell(['git', 'config', '--global', 'user.name', 'Publisher Bot'])
run_shell(['git', 'config', '--global', 'user.email', 'publisher_bot'])
PublisherController().run()
def run_duel(image_1, image_2, environment, output_file, log_1_file,
log_2_file):
"""
:param image_1: str
:param image_2: str
:param environment: str
:param output_file: str
:param log_1_file: str
:param log_2_file: str
:returns: int, str
"""
# Grab (or wait for) a namespace
namespace = available_namespaces.get()
logger.info(f'Run duel in namespace {namespace}')
# Run
try:
status, duel_logs = run_shell([
'./run_duel.sh', namespace, image_1, image_2, environment,
output_file, log_1_file, log_2_file
])
finally:
available_namespaces.put_nowait(namespace)
return status, duel_logs
def main():
# Get my private IP
global private_ip
private_ip = requests.get('http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address').text
logger.info(f'My private ip is {private_ip}')
# Get my commit
global commit
status, commit = run_shell(['git', 'rev-parse', 'HEAD'])
assert status == 0, 'Failed to get commit'
commit = commit.strip()
logger.info(f'My commit is {commit}')
while True:
logger.info('Start cycle')
timeout_operations()
desired = count_desired()
current = TaskWorker.objects.filter(
state__in=[TaskWorker.CREATING, TaskWorker.READY]
).count()
logger.info(f'Current = {current}, desired = {desired}')
if current > desired:
stop_n(current - desired)
elif current < desired:
create_n(desired - current)
destroy_stopped()
time.sleep(sleep_time.total_seconds())
def execute_task(self, task):
# Create temp folder
with tempfile.TemporaryDirectory() as tmpdir:
try:
# Load ZIP file
with task.zip_file.open('rb') as fp:
submission = ZipFile(fp)
# Unzip
submission.extractall(tmpdir)
except:
return self.TaskResult.error('Failed to unzip provided file',
traceback.format_exc())
# Check zip files
for expected_file in ['environment.yml', 'player.py']:
if not os.path.exists(os.path.join(tmpdir, expected_file)):
return self.TaskResult.error(
f'The expected file {expected_file} was not found in the provided ZIP'
)
# Copy extra image files
for image_file in os.scandir('builder/image'):
shutil.copy2(image_file.path, tmpdir)
# Build image
image_tag = str(uuid.uuid4())
image_name = f'{image_base_name}:{image_tag}'
image_result, image_log = run_shell(
['docker', 'build', '--tag', image_name, tmpdir])
if image_result != 0:
return self.TaskResult.error('Image failed to be built',
image_log)
if push_image:
# Push image
push_result, push_log = run_shell(
['docker', 'push', image_name])
if push_result != 0:
image_log += '\n---\n' + push_log
return self.TaskResult.error('Image failed to be pushed',
image_log)
task.image_name = image_name
return self.TaskResult.success(image_log)
def upload_player_logs(service, log_file):
logger.info(f'Read {service} logs')
log_status, log = run_shell([
'docker-compose',
'--project-name', namespace,
'--file', 'run_duel/docker-compose.yml',
'logs',
service,
])
if log_status != 0:
logger.warning('Failed')
log = 'Failed to read container logs'
logger.info(f'Upload {service} logs to {log_file}')
bucket.blob(log_file).upload_from_string(log)
def __init__(self, image_name, namespace, cpu_limit, memory_limit,
service):
"""
:param image_name: str
:param namespace: str
:param cpu_limit: float
:param memory_limit: str
:param service: str
"""
self.service = service
# Start docker
status, docker_log = run_shell(
[
'docker-compose',
'--project-name',
namespace,
'--file',
'run_duel/docker-compose.yml',
'up',
'--detach',
service,
], {
'PLAYER_IMAGE': image_name,
'PLAYER_CPU_LIMIT': str(cpu_limit),
'PLAYER_MEMORY_LIMIT': memory_limit,
})
assert status == 0, 'Docker image failed to start:\n' + docker_log
# Health check
max_health_time = time.time() + MAX_HEALTH_CHECK_SECONDS
is_healthy = False
while time.time() < max_health_time:
try:
response = requests.get(
f'http://{self.service}:8000/health-check')
if response.status_code == 200:
is_healthy = True
break
except:
pass
time.sleep(HEALTH_CHECK_INTERVAL_SECONDS)
assert is_healthy, 'Docker image failed initial health check'
def execute_task(self, task):
# Note that this task is not thread safe: at most one can be executed at a time
# Create temp folder
with tempfile.TemporaryDirectory() as tmpdir:
# Unzip
try:
with task.zip_file.open('rb') as fp:
submission = ZipFile(fp)
submission.extractall(tmpdir)
except:
return self.TaskResult.error('Failed to unzip provided file',
traceback.format_exc())
# Rewrite big files with a warning (GitHub accepts up to 100MiB, but we will use a much lower limit of 1MiB
# to make sure the full repo won't too big either)
for root, dirs, files in os.walk(tmpdir):
for name in files:
file_path = os.path.join(root, name)
size_MiB = os.path.getsize(file_path) / 1024 / 1024
if size_MiB > 1:
logger.warning(
f'File {file_path} is too big ({size_MiB:.1f}MiB)')
with open(file_path, 'w') as fp:
fp.write(
f'!!! The original file was too large !!!\nPlease find the complete ZIP on the platform'
)
# Prepare commands
competitor = task.competitor
environment = competitor.environment
submitter = competitor.submitter
destination = f'data/publish_repo/{environment.slug}/{competitor.name}'
commit_message = f'Publish {competitor.name} v{task.version_number} for {environment.name} by {submitter.username}'
cmds = [
['git', '-C', 'data/publish_repo', 'fetch'],
[
'git', '-C', 'data/publish_repo', 'reset', '--hard',
'origin/master'
],
['rm', '-rf', destination],
['mkdir', '-p', destination],
['cp', '-r', tmpdir + '/.', destination],
['git', '-C', 'data/publish_repo', 'add', '-A'],
[
'git', '-C', 'data/publish_repo', 'commit',
'--allow-empty', '-m', commit_message
],
['git', '-C', 'data/publish_repo', 'push'],
]
# Run them sequentially
logs = ''
for cmd in cmds:
status, extra_logs = run_shell(cmd)
logs += f'$ {cmd}\n{extra_logs}'
if status != 0:
return self.TaskResult.error(
f'Operation failed with status {status}', logs)
_, commit = run_shell(
['git', '-C', 'data/publish_repo', 'rev-parse', 'HEAD'])
task.publish_url = f'{PUBLISHER_WEB_URL}/blob/{commit.strip()}/{environment.slug}/{competitor.name}/player.py'
return self.TaskResult.success(logs)