def init_mission(args): from checkio_client.actions.repo import link_folder_to_repo, clone_repo_to_folder domain_data = conf.default_domain_data mission = args.mission[0] folder = Folder(mission) if folder.exists(): print('Folder exists already') return if args.template: template = args.template else: domain_data = conf.default_domain_data template = domain_data['repo_template'] clone_repo_to_folder(template, folder.mission_folder()) if args.repository: print('Send to git...') link_folder_to_repo(folder.mission_folder(), args.repository) recompile_mission(mission) if not args.without_container: rebuild_mission(mission) init_home_file(mission) print('Done')
def start_server(slug, interface_folder, action, path_to_code, python3, tmp_file_name=None): env_name = conf.default_domain_data['interpreter'] docker_filename = '/root/' + os.path.basename(path_to_code) client = docker.from_env() folder = Folder(slug) logging.info('Interface Folder {}'.format(interface_folder)) return client.containers.run(folder.image_name_cli(), ' '.join(('python -u', '/root/interface/src/main.py', slug, action, env_name, docker_filename, str(conf.console_server_port), str(conf.log_level), tmp_file_name or '-')), volumes={ tmp_folder(interface_folder): { 'bind': '/root/interface', 'mode': 'rw' }, path_to_code: { 'bind': docker_filename, 'mode': 'rw' } }, ports={ (str(conf.console_server_port) + '/tcp'): conf.console_server_port }, name=NAME_CLI_INTERFACE, network_mode=NAME_CLI_NETWORK, stdout=True, detach=True, tty=True )
def init_home_file(slug, force=False): initial_file = conf.default_domain_data['interpreter'] folder = Folder(slug) solution = folder.solution_path() if os.path.exists(solution) and not force: return write_solution(slug, initial_file, solution)
def recompile_mission(slug): folder = Folder(slug) compiled_path = folder.compiled_folder_path() logging.info("Relink folder to %s", compiled_path) if os.path.exists(compiled_path): shutil.rmtree(compiled_path) mission_source = MissionFilesCompiler(compiled_path) mission_source.compile(source_path=folder.mission_folder(), use_link=True)
def can_check_mission(slug): folder = Folder(slug) docker = DockerClient() try: docker.images.get(folder.image_name()) docker.images.get(folder.image_name_cli()) except ImageNotFound: return False else: return True
def write_solution(slug, initial_file, solution_path): domain = conf.default_domain_data logging.info("Write a solution into %s for %s %s", solution_path, slug, initial_file) folder = Folder(slug) dirname = os.path.dirname(solution_path) os.makedirs(dirname, exist_ok=True) with open(solution_path, 'w', encoding='utf8') as fh: fh.write(code_for_file(slug, folder.initial_code(initial_file))) st = os.stat(solution_path) os.chmod(solution_path, st.st_mode | stat.S_IEXEC)
def start_docker(slug, ref_extra_volume=None): command = "{} {} 1 2 {}".format(NAME_CLI_INTERFACE, conf.console_server_port, str(conf.log_level)) client = DockerClient() folder = Folder(slug) logging.info('Verification: {}'.format(folder.verification_folder_path())) copy_tree(folder.verification_folder_path(), folder.container_verification_folder_path()) for root, dirs, files in os.walk(folder.container_verification_folder_path()): for momo in dirs: os.chmod(os.path.join(root, momo), 0o777) for momo in files: os.chmod(os.path.join(root, momo), 0o777) volumes={ tmp_folder(folder.compiled_referee_folder_path()): { 'bind': '/opt/mission/src', 'mode': 'rw' }, tmp_folder(folder.compiled_envs_folder_path()): { 'bind': '/opt/mission/envs', 'mode': 'rw' } } if ref_extra_volume is not None: volumes.update(ref_extra_volume) return client.run(slug, command, volumes=volumes, network_mode=NAME_CLI_NETWORK, name='mission__' + slug, detach=True)
def mission_git_getter(url, slug): # TODO: checkout into mission solder # compile it # build docker # prepare cli interface folder = Folder(slug) destination_path = folder.mission_folder() logging.info('Getting a new mission through the git...') logging.info('from %s to %s', url, destination_path) if os.path.exists(destination_path): answer = input('Folder {} exists already.' ' Do you want to overwite it? [y]/n :'.format(destination_path)) if answer is '' or answer.lower().startswith('y'): if os.path.islink(destination_path): os.remove(destination_path) else: shutil.rmtree(destination_path) else: return if os.path.exists(url): logging.info('Linking folder %s -> %s', url, destination_path) os.symlink(os.path.abspath(url), destination_path) return re_ret = re.search(RE_REPO_BRANCH, url) if re_ret: checkout_url, branch = re_ret.groups() else: checkout_url = url branch = 'master' logging.debug('URL info: checkioout url:%s branch:%s', url, branch) try: git.Repo.clone_from(checkout_url, destination_path, branch=branch) except git.GitCommandError as e: raise Exception(u"{}, {}".format(e or '', e.stderr)) folder.mission_config_write({ 'type': 'git', 'url': url }) logging.info('Prepare mission {} from {}'.format(slug, url))
def rebuild_cli_interface(slug): folder = Folder(slug) build_folder = folder.interface_cli_folder_path() img_name = folder.image_name_cli() logging.info("[interface] Build docker image %s from %s", img_name, build_folder) docker = DockerClient() try: docker.build(name_image=img_name, path=tmp_folder(build_folder)) except BuildError as e: logging.warning('Build Error:') for item in e.build_log: logging.warning(item.get('stream')) logging.warning(list(e.build_log)) logging.warning(e.msg)
def rebuild_native(slug): folder = Folder(slug) logging.info('Building virtualenv in %s', folder.native_env_folder_path()) if os.path.exists(folder.native_env_folder_path()): shutil.rmtree(folder.native_env_folder_path()) logging_sys("virtualenv --system-site-packages -p python3 " + folder.native_env_folder_path()) logging_sys("{pip3} install -r {requirements}".format( pip3=folder.native_env_bin('pip3'), requirements=folder.referee_requirements())) logging_sys("{pip3} install -r {requirements}".format( pip3=folder.native_env_bin('pip3'), requirements=folder.interface_cli_requirements()))
def execute_referee(command, slug, solution, without_container=False, interface_child=False, referee_only=False, interface_only=False, ref_extra_volume=None): def start_interface(tmp_file_name=None): return start_server(slug, folder.interface_cli_folder_path(), command, solution, folder.native_env_bin('python3'), tmp_file_name) def start_container(): return start_docker(slug, ref_extra_volume) client = docker.from_env() folder = Folder(slug) prepare_docker() if interface_only: return start_interface() cleanup_containers() cli_logs = start_interface() ref_logs = start_container() cli_read_size = 0 ref_read_size = 0 while True: #print('...', client.containers.get(cli_logs.id).status) cli_data = cli_logs.logs()[cli_read_size:] cli_read_size += len(cli_data) ref_data = ref_logs.logs()[ref_read_size:] ref_read_size += len(ref_data) if cli_data: print(cli_data.decode('utf-8')) if ref_data: print(ref_data.decode('utf-8')) if (client.containers.get(cli_logs.id).status, client.containers.get(ref_logs.id).status) == ('exited', 'exited'): break time.sleep(0.01)
def rebuild_mission(slug): cleanup_containers() folder = Folder(slug) docker = DockerClient() verification_folder_path = folder.container_verification_folder_path() logging.info("[mission] Build docker image %s from %s", folder.image_name(), verification_folder_path) if os.path.exists(verification_folder_path): shutil.rmtree(verification_folder_path) copy_tree(folder.verification_folder_path(), verification_folder_path) try: docker.build( name_image=folder.image_name(), path=verification_folder_path) except BuildError as e: logging.warning('Build Error:') for item in e.build_log: logging.warning(item.get('stream')) logging.warning(list(e.build_log)) logging.warning(e.msg) return rebuild_cli_interface(slug)
def execute_referee(command, slug, solution, without_container=False, interface_child=False, referee_only=False, interface_only=False, ref_extra_volume=None): def start_interface(tmp_file_name=None): return start_server(slug, folder.interface_cli_folder_path(), command, solution, folder.native_env_bin('python3'), tmp_file_name) def start_container(): return start_docker(slug, ref_extra_volume) client = docker.from_env() folder = Folder(slug) prepare_docker() if interface_only: return start_interface() cleanup_containers() cli_logs = start_interface() ref_logs = start_container() cli_read_size = 0 ref_read_size = 0 system_data = '' is_system_collecting = False SYSTEM_START = '---SYSTEM---' SYSTEM_END = '---END-SYSTEM---' while True: #print('...', client.containers.get(cli_logs.id).status) cli_data = cli_logs.logs()[cli_read_size:] cli_read_size += len(cli_data) ref_data = ref_logs.logs()[ref_read_size:] ref_read_size += len(ref_data) if cli_data: cli_data = cli_data.decode('utf-8') was_splited = False if not is_system_collecting and SYSTEM_START in cli_data: cli_data, new_system_data = cli_data.split(SYSTEM_START) system_data += new_system_data is_system_collecting = True was_splited = True if is_system_collecting and SYSTEM_END in cli_data: new_system_data, cli_data = cli_data.split(SYSTEM_END) system_data += new_system_data is_system_collecting = False was_splited = True if is_system_collecting and not was_splited: system_data += cli_data cli_data = '' #print ('-'* 5 , 'CLI') print(cli_data) #print('-'*5, 'SYS') if ref_data: print(ref_data.decode('utf-8')) if (client.containers.get(cli_logs.id).status, client.containers.get(ref_logs.id).status) == ('exited', 'exited'): break time.sleep(0.01) return system_data