def init_mission(args):
from checkio_client.actions.repo import link_folder_to_repo, clone_repo_to_folder
domain_data = conf.default_domain_data
mission = args.mission[0]
folder = Folder(mission)
if folder.exists():
print('Folder exists already')
return
if args.template:
template = args.template
else:
domain_data = conf.default_domain_data
template = domain_data['repo_template']
clone_repo_to_folder(template, folder.mission_folder())
if args.repository:
print('Send to git...')
link_folder_to_repo(folder.mission_folder(), args.repository)
recompile_mission(mission)
if not args.without_container:
rebuild_mission(mission)
init_home_file(mission)
print('Done')
def start_server(slug, interface_folder, action, path_to_code, python3,
tmp_file_name=None):
env_name = conf.default_domain_data['interpreter']
docker_filename = '/root/' + os.path.basename(path_to_code)
client = docker.from_env()
folder = Folder(slug)
logging.info('Interface Folder {}'.format(interface_folder))
return client.containers.run(folder.image_name_cli(),
' '.join(('python -u', '/root/interface/src/main.py', slug, action, env_name, docker_filename,
str(conf.console_server_port), str(conf.log_level), tmp_file_name or '-')),
volumes={
tmp_folder(interface_folder): {
'bind': '/root/interface',
'mode': 'rw'
},
path_to_code: {
'bind': docker_filename,
'mode': 'rw'
}
},
ports={
(str(conf.console_server_port) + '/tcp'): conf.console_server_port
},
name=NAME_CLI_INTERFACE,
network_mode=NAME_CLI_NETWORK,
stdout=True,
detach=True,
tty=True
)
def init_home_file(slug, force=False):
initial_file = conf.default_domain_data['interpreter']
folder = Folder(slug)
solution = folder.solution_path()
if os.path.exists(solution) and not force:
return
write_solution(slug, initial_file, solution)
def recompile_mission(slug):
folder = Folder(slug)
compiled_path = folder.compiled_folder_path()
logging.info("Relink folder to %s", compiled_path)
if os.path.exists(compiled_path):
shutil.rmtree(compiled_path)
mission_source = MissionFilesCompiler(compiled_path)
mission_source.compile(source_path=folder.mission_folder(), use_link=True)
def can_check_mission(slug):
folder = Folder(slug)
docker = DockerClient()
try:
docker.images.get(folder.image_name())
docker.images.get(folder.image_name_cli())
except ImageNotFound:
return False
else:
return True
def write_solution(slug, initial_file, solution_path):
domain = conf.default_domain_data
logging.info("Write a solution into %s for %s %s", solution_path, slug,
initial_file)
folder = Folder(slug)
dirname = os.path.dirname(solution_path)
os.makedirs(dirname, exist_ok=True)
with open(solution_path, 'w', encoding='utf8') as fh:
fh.write(code_for_file(slug, folder.initial_code(initial_file)))
st = os.stat(solution_path)
os.chmod(solution_path, st.st_mode | stat.S_IEXEC)
def start_docker(slug, ref_extra_volume=None):
command = "{} {} 1 2 {}".format(NAME_CLI_INTERFACE, conf.console_server_port, str(conf.log_level))
client = DockerClient()
folder = Folder(slug)
logging.info('Verification: {}'.format(folder.verification_folder_path()))
copy_tree(folder.verification_folder_path(), folder.container_verification_folder_path())
for root, dirs, files in os.walk(folder.container_verification_folder_path()):
for momo in dirs:
os.chmod(os.path.join(root, momo), 0o777)
for momo in files:
os.chmod(os.path.join(root, momo), 0o777)
volumes={
tmp_folder(folder.compiled_referee_folder_path()): {
'bind': '/opt/mission/src',
'mode': 'rw'
},
tmp_folder(folder.compiled_envs_folder_path()): {
'bind': '/opt/mission/envs',
'mode': 'rw'
}
}
if ref_extra_volume is not None:
volumes.update(ref_extra_volume)
return client.run(slug, command,
volumes=volumes,
network_mode=NAME_CLI_NETWORK,
name='mission__' + slug,
detach=True)
def mission_git_getter(url, slug):
# TODO: checkout into mission solder
# compile it
# build docker
# prepare cli interface
folder = Folder(slug)
destination_path = folder.mission_folder()
logging.info('Getting a new mission through the git...')
logging.info('from %s to %s', url, destination_path)
if os.path.exists(destination_path):
answer = input('Folder {} exists already.'
' Do you want to overwite it? [y]/n :'.format(destination_path))
if answer is '' or answer.lower().startswith('y'):
if os.path.islink(destination_path):
os.remove(destination_path)
else:
shutil.rmtree(destination_path)
else:
return
if os.path.exists(url):
logging.info('Linking folder %s -> %s', url, destination_path)
os.symlink(os.path.abspath(url), destination_path)
return
re_ret = re.search(RE_REPO_BRANCH, url)
if re_ret:
checkout_url, branch = re_ret.groups()
else:
checkout_url = url
branch = 'master'
logging.debug('URL info: checkioout url:%s branch:%s', url, branch)
try:
git.Repo.clone_from(checkout_url, destination_path, branch=branch)
except git.GitCommandError as e:
raise Exception(u"{}, {}".format(e or '', e.stderr))
folder.mission_config_write({
'type': 'git',
'url': url
})
logging.info('Prepare mission {} from {}'.format(slug, url))
def rebuild_cli_interface(slug):
folder = Folder(slug)
build_folder = folder.interface_cli_folder_path()
img_name = folder.image_name_cli()
logging.info("[interface] Build docker image %s from %s", img_name,
build_folder)
docker = DockerClient()
try:
docker.build(name_image=img_name, path=tmp_folder(build_folder))
except BuildError as e:
logging.warning('Build Error:')
for item in e.build_log:
logging.warning(item.get('stream'))
logging.warning(list(e.build_log))
logging.warning(e.msg)
def rebuild_native(slug):
folder = Folder(slug)
logging.info('Building virtualenv in %s', folder.native_env_folder_path())
if os.path.exists(folder.native_env_folder_path()):
shutil.rmtree(folder.native_env_folder_path())
logging_sys("virtualenv --system-site-packages -p python3 " +
folder.native_env_folder_path())
logging_sys("{pip3} install -r {requirements}".format(
pip3=folder.native_env_bin('pip3'),
requirements=folder.referee_requirements()))
logging_sys("{pip3} install -r {requirements}".format(
pip3=folder.native_env_bin('pip3'),
requirements=folder.interface_cli_requirements()))
def execute_referee(command, slug, solution, without_container=False, interface_child=False,
referee_only=False, interface_only=False, ref_extra_volume=None):
def start_interface(tmp_file_name=None):
return start_server(slug, folder.interface_cli_folder_path(), command, solution,
folder.native_env_bin('python3'), tmp_file_name)
def start_container():
return start_docker(slug, ref_extra_volume)
client = docker.from_env()
folder = Folder(slug)
prepare_docker()
if interface_only:
return start_interface()
cleanup_containers()
cli_logs = start_interface()
ref_logs = start_container()
cli_read_size = 0
ref_read_size = 0
while True:
#print('...', client.containers.get(cli_logs.id).status)
cli_data = cli_logs.logs()[cli_read_size:]
cli_read_size += len(cli_data)
ref_data = ref_logs.logs()[ref_read_size:]
ref_read_size += len(ref_data)
if cli_data:
print(cli_data.decode('utf-8'))
if ref_data:
print(ref_data.decode('utf-8'))
if (client.containers.get(cli_logs.id).status,
client.containers.get(ref_logs.id).status) == ('exited', 'exited'):
break
time.sleep(0.01)
def rebuild_mission(slug):
cleanup_containers()
folder = Folder(slug)
docker = DockerClient()
verification_folder_path = folder.container_verification_folder_path()
logging.info("[mission] Build docker image %s from %s", folder.image_name(), verification_folder_path)
if os.path.exists(verification_folder_path):
shutil.rmtree(verification_folder_path)
copy_tree(folder.verification_folder_path(), verification_folder_path)
try:
docker.build(
name_image=folder.image_name(),
path=verification_folder_path)
except BuildError as e:
logging.warning('Build Error:')
for item in e.build_log:
logging.warning(item.get('stream'))
logging.warning(list(e.build_log))
logging.warning(e.msg)
return
rebuild_cli_interface(slug)
def execute_referee(command,
slug,
solution,
without_container=False,
interface_child=False,
referee_only=False,
interface_only=False,
ref_extra_volume=None):
def start_interface(tmp_file_name=None):
return start_server(slug, folder.interface_cli_folder_path(), command,
solution, folder.native_env_bin('python3'),
tmp_file_name)
def start_container():
return start_docker(slug, ref_extra_volume)
client = docker.from_env()
folder = Folder(slug)
prepare_docker()
if interface_only:
return start_interface()
cleanup_containers()
cli_logs = start_interface()
ref_logs = start_container()
cli_read_size = 0
ref_read_size = 0
system_data = ''
is_system_collecting = False
SYSTEM_START = '---SYSTEM---'
SYSTEM_END = '---END-SYSTEM---'
while True:
#print('...', client.containers.get(cli_logs.id).status)
cli_data = cli_logs.logs()[cli_read_size:]
cli_read_size += len(cli_data)
ref_data = ref_logs.logs()[ref_read_size:]
ref_read_size += len(ref_data)
if cli_data:
cli_data = cli_data.decode('utf-8')
was_splited = False
if not is_system_collecting and SYSTEM_START in cli_data:
cli_data, new_system_data = cli_data.split(SYSTEM_START)
system_data += new_system_data
is_system_collecting = True
was_splited = True
if is_system_collecting and SYSTEM_END in cli_data:
new_system_data, cli_data = cli_data.split(SYSTEM_END)
system_data += new_system_data
is_system_collecting = False
was_splited = True
if is_system_collecting and not was_splited:
system_data += cli_data
cli_data = ''
#print ('-'* 5 , 'CLI')
print(cli_data)
#print('-'*5, 'SYS')
if ref_data:
print(ref_data.decode('utf-8'))
if (client.containers.get(cli_logs.id).status,
client.containers.get(ref_logs.id).status) == ('exited',
'exited'):
break
time.sleep(0.01)
return system_data