def main(): # Prepare GitHub repo clone_status, clone_log = run_shell( ['git', 'clone', os.environ['PUBLISHER_REMOTE'], 'data/publish_repo']) if clone_status != 0: logger.warn(clone_log) if not os.path.exists('data/publish_repo/.git'): raise Exception('Failed to clone repo') run_shell(['git', 'config', '--global', 'user.name', 'Publisher Bot']) run_shell(['git', 'config', '--global', 'user.email', 'publisher_bot']) PublisherController().run()
def run_duel(image_1, image_2, environment, output_file, log_1_file, log_2_file): """ :param image_1: str :param image_2: str :param environment: str :param output_file: str :param log_1_file: str :param log_2_file: str :returns: int, str """ # Grab (or wait for) a namespace namespace = available_namespaces.get() logger.info(f'Run duel in namespace {namespace}') # Run try: status, duel_logs = run_shell([ './run_duel.sh', namespace, image_1, image_2, environment, output_file, log_1_file, log_2_file ]) finally: available_namespaces.put_nowait(namespace) return status, duel_logs
def main(): # Get my private IP global private_ip private_ip = requests.get('http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address').text logger.info(f'My private ip is {private_ip}') # Get my commit global commit status, commit = run_shell(['git', 'rev-parse', 'HEAD']) assert status == 0, 'Failed to get commit' commit = commit.strip() logger.info(f'My commit is {commit}') while True: logger.info('Start cycle') timeout_operations() desired = count_desired() current = TaskWorker.objects.filter( state__in=[TaskWorker.CREATING, TaskWorker.READY] ).count() logger.info(f'Current = {current}, desired = {desired}') if current > desired: stop_n(current - desired) elif current < desired: create_n(desired - current) destroy_stopped() time.sleep(sleep_time.total_seconds())
def execute_task(self, task): # Create temp folder with tempfile.TemporaryDirectory() as tmpdir: try: # Load ZIP file with task.zip_file.open('rb') as fp: submission = ZipFile(fp) # Unzip submission.extractall(tmpdir) except: return self.TaskResult.error('Failed to unzip provided file', traceback.format_exc()) # Check zip files for expected_file in ['environment.yml', 'player.py']: if not os.path.exists(os.path.join(tmpdir, expected_file)): return self.TaskResult.error( f'The expected file {expected_file} was not found in the provided ZIP' ) # Copy extra image files for image_file in os.scandir('builder/image'): shutil.copy2(image_file.path, tmpdir) # Build image image_tag = str(uuid.uuid4()) image_name = f'{image_base_name}:{image_tag}' image_result, image_log = run_shell( ['docker', 'build', '--tag', image_name, tmpdir]) if image_result != 0: return self.TaskResult.error('Image failed to be built', image_log) if push_image: # Push image push_result, push_log = run_shell( ['docker', 'push', image_name]) if push_result != 0: image_log += '\n---\n' + push_log return self.TaskResult.error('Image failed to be pushed', image_log) task.image_name = image_name return self.TaskResult.success(image_log)
def upload_player_logs(service, log_file): logger.info(f'Read {service} logs') log_status, log = run_shell([ 'docker-compose', '--project-name', namespace, '--file', 'run_duel/docker-compose.yml', 'logs', service, ]) if log_status != 0: logger.warning('Failed') log = 'Failed to read container logs' logger.info(f'Upload {service} logs to {log_file}') bucket.blob(log_file).upload_from_string(log)
def __init__(self, image_name, namespace, cpu_limit, memory_limit, service): """ :param image_name: str :param namespace: str :param cpu_limit: float :param memory_limit: str :param service: str """ self.service = service # Start docker status, docker_log = run_shell( [ 'docker-compose', '--project-name', namespace, '--file', 'run_duel/docker-compose.yml', 'up', '--detach', service, ], { 'PLAYER_IMAGE': image_name, 'PLAYER_CPU_LIMIT': str(cpu_limit), 'PLAYER_MEMORY_LIMIT': memory_limit, }) assert status == 0, 'Docker image failed to start:\n' + docker_log # Health check max_health_time = time.time() + MAX_HEALTH_CHECK_SECONDS is_healthy = False while time.time() < max_health_time: try: response = requests.get( f'http://{self.service}:8000/health-check') if response.status_code == 200: is_healthy = True break except: pass time.sleep(HEALTH_CHECK_INTERVAL_SECONDS) assert is_healthy, 'Docker image failed initial health check'
def execute_task(self, task): # Note that this task is not thread safe: at most one can be executed at a time # Create temp folder with tempfile.TemporaryDirectory() as tmpdir: # Unzip try: with task.zip_file.open('rb') as fp: submission = ZipFile(fp) submission.extractall(tmpdir) except: return self.TaskResult.error('Failed to unzip provided file', traceback.format_exc()) # Rewrite big files with a warning (GitHub accepts up to 100MiB, but we will use a much lower limit of 1MiB # to make sure the full repo won't too big either) for root, dirs, files in os.walk(tmpdir): for name in files: file_path = os.path.join(root, name) size_MiB = os.path.getsize(file_path) / 1024 / 1024 if size_MiB > 1: logger.warning( f'File {file_path} is too big ({size_MiB:.1f}MiB)') with open(file_path, 'w') as fp: fp.write( f'!!! The original file was too large !!!\nPlease find the complete ZIP on the platform' ) # Prepare commands competitor = task.competitor environment = competitor.environment submitter = competitor.submitter destination = f'data/publish_repo/{environment.slug}/{competitor.name}' commit_message = f'Publish {competitor.name} v{task.version_number} for {environment.name} by {submitter.username}' cmds = [ ['git', '-C', 'data/publish_repo', 'fetch'], [ 'git', '-C', 'data/publish_repo', 'reset', '--hard', 'origin/master' ], ['rm', '-rf', destination], ['mkdir', '-p', destination], ['cp', '-r', tmpdir + '/.', destination], ['git', '-C', 'data/publish_repo', 'add', '-A'], [ 'git', '-C', 'data/publish_repo', 'commit', '--allow-empty', '-m', commit_message ], ['git', '-C', 'data/publish_repo', 'push'], ] # Run them sequentially logs = '' for cmd in cmds: status, extra_logs = run_shell(cmd) logs += f'$ {cmd}\n{extra_logs}' if status != 0: return self.TaskResult.error( f'Operation failed with status {status}', logs) _, commit = run_shell( ['git', '-C', 'data/publish_repo', 'rev-parse', 'HEAD']) task.publish_url = f'{PUBLISHER_WEB_URL}/blob/{commit.strip()}/{environment.slug}/{competitor.name}/player.py' return self.TaskResult.success(logs)