def check_metadata():
# Check if metadata files exist
try:
with open('metadata/description.md', 'r',
encoding='utf-8') as description_file:
_check_description(description_file)
with open('metadata/summary.md', 'r',
encoding='utf-8') as summary_file:
s = summary_file.read()
if not SUMMARY_RANGE['min'] <= len(s) <= SUMMARY_RANGE['max']:
counted_error(
'Summary should be minimally %d, maximally %d characters long.',
SUMMARY_RANGE['min'], SUMMARY_RANGE['max'])
if os.path.exists('metadata/writeup.md'):
with open('metadata/writeup.md', 'r',
encoding='utf-8') as writeup_file:
_check_writeup(writeup_file)
except FileNotFoundError as e:
counted_error('Missing %s from metadata.' % e.filename)
except UnicodeDecodeError as e:
counted_error('Could decode markdown text: \n\n%s \n\tDetails: %s' %
(e.object, e))
except Exception as e:
counted_error('Could not read file in metadata. \n\tDetails: %s' % e)
def _check_description(description_file):
description = description_file.read()
if not DESCRIPTION_RANGE['min'] <= len(
description) <= DESCRIPTION_RANGE['max']:
counted_error(
'Description should be minimum %d and maximum %d characters long.',
DESCRIPTION_RANGE['min'], DESCRIPTION_RANGE['max'])
section_too_long_pattern = re.compile(r'^ {0,3}#{1,6} +.{151,}$',
re.MULTILINE)
section_too_short_pattern = re.compile(r'^ {0,3}#{1,6} +.{0,4}$',
re.MULTILINE)
section_long = re.findall(section_too_long_pattern, description)
section_short = re.findall(section_too_short_pattern, description)
if section_short:
counted_error('Sections must be between 5 and 150 characters.\n'
'\tDetails: %s' % section_short)
if section_long:
counted_error('Sections must be between 5 and 150 characters.\n'
'\tDetails: %s' % section_long)
tick = zip(*[re.finditer("```", description)] * 2)
ignore = [((p[0].span()[1], p[1].span()[0])) for p in tick]
section_not_capitalized_pattern = re.compile(r'^ {0,3}#{1,6} +[^A-Z].+.*$',
re.MULTILINE)
section_not_capitalized = filter_source(ignore,
section_not_capitalized_pattern,
description)
if section_not_capitalized:
counted_error('Sections must start with a capitalized letter.\n'
'\tDetails: %s' % section_not_capitalized)
# Description can contain only contain H4- or H5-style sections.
maxh3_pattern = re.compile(r'^ {0,3}#{1,3} +.*$', re.MULTILINE)
maxh3 = filter_source(ignore, maxh3_pattern, description)
if maxh3:
counted_error(
'A section name font size is too large in description.md. '
'Please, use H4 (####) or H5 (#####) section names.\n'
'\tDetails: %s' % maxh3)
def check_controller():
check_dockerfile('controller/Dockerfile')
static_flag = None
try:
static_flag = read_config()['flag']
except KeyError:
logging.info(
'[This is not an error] The flag is missing from the config file.\n\tYou should implement '
'a dynamic solution checker (e.g., random flags, unit tests) in the controller.'
)
except FileNotFoundError as e:
counted_error('Could not open %s' % e.filename)
try:
with open('controller/opt/server.py') as server_file:
server = server_file.read()
except FileNotFoundError as e:
counted_error('Could not open %s' % e.filename)
except Exception as e:
counted_error(
'An error occurred while checking controller/opt/server.py. \n\tDetails: %s'
% e)
else:
# Invoke test endpoint if not static
_http_request('http://%s:%d/secret/test' %
(FORWARD_ADDR, CONTROLLER_PORT))
# Check controller's solution_check endpoint.
solution_check_pattern = 'def solution_check\(\):'
solution_check = re.findall(solution_check_pattern, server)
if not (solution_check or static_flag):
counted_error(
'Function "solution_check()" is missing from controller/opt/server.py. '
'\n\tPlease implement it and check user solutions dynamically (e.g., random flag '
'checking)\n\tor insert a static flag into config.yml.')
def check_yml(filename, is_static: bool = False):
try:
config = read_config(filename)
check_config(config, is_static)
except FileNotFoundError as e:
counted_error('Could not open %s' % e.filename)
except KeyError as e:
counted_error('Key "%s" is missing from %s' % (e.args[0], filename))
except Exception as e:
counted_error('An error occurred while loading %s. \n\tDetails: %s' %
(filename, e))
def check_dockerfile(filename):
repo_pattern = 'FROM ((docker\.io\/)?avatao|eu\.gcr\.io\/avatao-challengestore)\/'
try:
with open(filename, 'r') as f:
d = f.read()
if re.search(repo_pattern, d) is None:
counted_error(
'Please use avatao base images for your challenges. Our base images'
' are available at https://hub.docker.com/u/avatao/')
except FileNotFoundError as e:
counted_error('Could not open %s' % e.filename)
except Exception as e:
counted_error('An error occurred while loading %s. \n\tDetails: %s' %
(filename, e))
def check_config(config: dict, is_static):
invalid_keys = set(config.keys()) - set(CONFIG_KEYS)
if len(invalid_keys) > 0:
counted_error('Invalid key(s) found in config.yml: %s' % invalid_keys)
if config['version'][:1] != 'v':
counted_error(
'Invalid version. The version number must start with the letter v')
elif config['version'] == 'v1':
counted_error('This version is deprecated, please use v2.0.0')
elif config['version'] != 'v2.0.0':
counted_error(
'Invalid version. The supplied config version is not supported')
# Difficulty
try:
assert DIFFICULTY_RANGE['min'] <= int(
config['difficulty']) <= DIFFICULTY_RANGE['max']
except Exception:
counted_error('Invalid difficulty in config.yml. '
'Valid values: %d - %d' %
(DIFFICULTY_RANGE['min'], DIFFICULTY_RANGE['max']))
# Name
try:
assert NAME_RANGE['min'] <= len(config['name']) <= NAME_RANGE['max']
except Exception:
counted_error('Invalid challenge name in config.yml. '
'Name should be a string between %d - %d characters.' %
(NAME_RANGE['min'], NAME_RANGE['max']))
for template_config in glob(
os.path.join(TOOLBOX_PATH, 'templates', '*', 'config.yml')):
if config['name'] == read_config(template_config).get('name'):
counted_error('Please, set the challenge name in the config file.')
# Skills
if not isinstance(config['skills'], list):
counted_error(
'Invalid skills in config.yml. Skills should be placed into a list.\n'
'\tValid skills are listed here: \n'
'\thttps://platform.avatao.com/api-explorer/#/api/core/skills/')
# Recommendations
if not isinstance(config['recommendations'], dict):
counted_error(
'Invalid recommendations in config.yml. Recommendations should be added in the following '
'format:\n\n'
'recommendations:\n'
'\twww.example.com: \'Example webpage\'\n'
'\thttp://www.example2.com: \'Example2 webpage\''
'\thttp://example3.com: \'Example3 webpage\'')
url_re = re.compile(
r'(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)'
r'(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]'
r'+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’]))')
for item in config['recommendations'].items():
if url_re.fullmatch(item[0]) is None:
counted_error('Invalid recommended URL (%s) found in config.yml' %
item[0])
if not isinstance(item[1], str):
counted_error(
'The name of recommended url (%s) should be a string in config.yml'
% item[1])
# Owners
if not isinstance(config['owners'], list):
counted_error(
'Challenge owners (%s) should be placed into a list in config.yml'
% config['owners'])
email_re = re.compile(
r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
for owner in config.get('owners', []):
if email_re.fullmatch(owner) is None:
counted_error(
'Invalid owner email (%s) found in config.yml. '
'Make sure you list the email addresses of the owners.' %
owner)
if not is_static:
controller_found = False
for item in config['crp_config'].values():
if not isinstance(item, dict):
counted_error('Items of crp_config must be dictionaries.')
if 'image' in item and item['image'].find('/') < 0:
counted_error(
'If the image is explicitly defined, it must be relative '
'to the registry - e.g. challenge:solvable.')
if 'capabilities' in item:
invalid_caps = set(item['capabilities']) - CAPABILITIES
if len(invalid_caps) > 0:
counted_error('Invalid capabilities: %s. Valid values: %s',
invalid_caps, CAPABILITIES)
if 'mem_limit' in item:
if not isinstance(item['mem_limit'], str):
counted_error(
'Invalid mem_limit value: %s, The mem_limit should be a string like: 100M'
)
if item['mem_limit'][-1] not in "M":
counted_error(
'Invalid mem_limit value: %s, The mem_limit should be a string ending with '
'M (megabytes). No other unit is allowed.')
try:
mem_limit_number_part = int(item['mem_limit'][:-1])
if mem_limit_number_part > 999:
counted_error(
'Invalid mem_limit value: %s, The mem_limit can not be greater than 999M.'
)
except Exception:
counted_error(
'Invalid mem_limit value: %s, mem_limit must start with a number and end with '
'M (megabytes). No other unit is allowed.')
for port in item.get('ports', []):
try:
port, protocol = port.split('/', 1)
try:
port = int(port)
except Exception:
counted_error(
'Invalid port. The port should be a number between 1 and 65535.'
)
if PORT_RANGE['min'] > port or PORT_RANGE['max'] < port:
counted_error(
'Invalid port. The port should be a number between 1 and 65535'
)
if protocol not in PROTOCOLS:
counted_error(
'Invalid protocol in config.yml (crp_config): %s. Valid values: %s',
protocol, PROTOCOLS)
elif protocol == CONTROLLER_PROTOCOL:
controller_found = True
except Exception:
counted_error('Invalid port format. [port/protocol]')
if not config.get('flag') and not controller_found:
counted_error(
'Missing controller port [5555/%s] for a dynamic challenge.' %
CONTROLLER_PROTOCOL)
if str(config.get('enable_flag_input')).lower() not in ('true', 'false',
'1', '0'):
counted_error('Invalid enable_flag_input. Should be a boolean.')
if is_static:
try:
assert isinstance(config['flag'], str)
except AssertionError:
counted_error('Invalid flag. Should be a string.')
except KeyError:
counted_error(
'Missing flag. Static challenges must have a static flag set.')
def _http_request(url):
# Check controller's test endpoint.
curl_cmd = ['curl', '-s']
if url[-4:] == 'test':
method = 'GET'
func = 'Test'
else:
method = 'POST'
func = 'Solution checker'
curl_cmd += ['-X POST']
curl_cmd += [url]
try:
output = subprocess.check_output(curl_cmd).decode('utf-8')
logging.info('%s output: \n\n%s' % (func, output))
# Check if solution checker returns a well-formatted JSON response
solved = False
if func == 'Solution checker':
response = json.loads(output)
solved = bool(response['solved'])
assert isinstance(response['message'], str)
except (OSError, subprocess.CalledProcessError) as e:
# Do not abort if the challenge is not running. TODO: Improve this check!
logging.warning('Curl returned with error. \n\tDetails: %s' % e)
except ValueError as e:
counted_error(
'Could not parse the response of solution checker. Bad JSON format. \n\tDetails: %s'
% e)
except NameError as e:
counted_error(
'JSON attribute "solved" must be a boolean value. \n\tDetails: %s'
% e)
except KeyError as e:
if e.args[0] == 'solved':
counted_error(
'JSON key "%s" is missing from solution checker response. ' %
e.args[0])
elif e.args[0] == 'message':
logging.warning(
'JSON key "%s" is missing from solution checker response. \n\t In certain cases '
'(e.g., programming challenges) messages help users to complete the challenge.'
% e.args[0])
except AssertionError:
counted_error('JSON attribute "message" must be a string value.')
else:
if (re.search('OK', output) and func == 'Test') or solved:
logging.info('%s endpoint passed!' % func)
elif re.search('404', output) and func == 'Test':
logging.warning(
'Test endpoint is not found. '
'\n\t Please implement it so as to make sure your challenge is working correctly.'
)
elif re.search('405', output):
counted_error('%s endpoint should implement HTTP %s.' %
(func, method))
elif re.search('500', output) or not solved:
counted_error('%s endpoint failed!' % func)
def _check_writeup(writeup_file):
writeup = writeup_file.read()
# Check if writeup.md starts with the challenge name in H1 style.
config = read_config('config.yml')
config_challenge_name = config['name']
h1_pattern = '^%s\n={1,}\n\n' % re.escape(config_challenge_name)
try:
writeup_challenge_name = re.search('^.*\n', writeup).group()[:-1]
assert writeup_challenge_name == config_challenge_name
re.search(h1_pattern, writeup).group()
except AssertionError:
logging.warning(
'The challenge name in writeup.md (%s) and config.yml (%s) should be the same.'
% (writeup_challenge_name, config_challenge_name))
except Exception:
logging.warning(
'The challenge names in writeup.md should be in H1 style. For example:\n\n'
'Challenge name\n==============\n')
# Check if costs and H2 sections are correct in writeup.md
cost_pattern = '\nCost: [0-9]{1,2}%\n'
costs = [
int(re.search(r'[0-9]{1,2}', cost).group())
for cost in re.findall(cost_pattern, writeup)
]
if sum(costs) != COST_SUM:
counted_error(
'The sum of costs in writeup.md should be %d%%.\n\tPlease make sure you have the following '
'format (take care of the white spaces, starting and ending newlines):\n%s'
% (COST_SUM, cost_pattern))
h2_pattern = r'\n## [A-Z].{5,150}\n'
h2 = re.findall(h2_pattern, writeup)
if len(h2) < MIN_WRITEUP_SECTIONS:
counted_error('There should be at least %d sections in writeup.md' %
MIN_WRITEUP_SECTIONS)
h2_last = h2.pop()
if h2_last != '\n## Complete solution\n':
counted_error(
'The last section should be called "Complete solution" in writeup.md'
)
h2costs = [
re.search(h2_pattern, h2cost).group()
for h2cost in re.findall(h2_pattern + cost_pattern, writeup)
]
missing_costs = set(h2) - set(h2costs)
if len(missing_costs) > 0:
counted_error(
'No cost is defined in writeup.md for section(s): %s\n\tPlease make sure you have the following '
'format (take care of the starting and ending newlines):\n%s' %
(missing_costs, cost_pattern))
reference_style_links_pattern = r'(\[.*?\]\[.*?\])'
if re.search(reference_style_links_pattern, writeup) is not None:
logging.warning(
'The writeup contains reference style links like [this one][0], which might render incorrectly.\n\t'
'Please use inline style ones like [this](http://example.net/)')
def check_config(config: dict, is_static):
invalid_keys = set(config.keys()) - set(CONFIG_KEYS)
if len(invalid_keys) > 0:
counted_error('Invalid key(s) found in config.yml: %s' % invalid_keys)
if config['version'][:1] != 'v':
counted_error(
'Invalid version. The version number must start with the letter v')
elif config['version'] == 'v1':
counted_error('This version is deprecated, please use v2.0.0')
elif config['version'] != 'v2.0.0':
counted_error(
'Invalid version. The supplied config version is not supported')
# Difficulty
try:
assert DIFFICULTY_RANGE['min'] <= int(
config['difficulty']) <= DIFFICULTY_RANGE['max']
except Exception:
counted_error('Invalid difficulty in config.yml. '
'Valid values: %d - %d' %
(DIFFICULTY_RANGE['min'], DIFFICULTY_RANGE['max']))
# Name
try:
assert NAME_RANGE['min'] <= len(config['name']) <= NAME_RANGE['max']
except Exception:
counted_error('Invalid challenge name in config.yml. '
'Name should be a string between %d - %d characters.' %
(NAME_RANGE['min'], NAME_RANGE['max']))
for template_config in glob(
os.path.join(TOOLBOX_PATH, 'templates', '*', 'config.yml')):
if config['name'] == read_config(template_config).get('name'):
counted_error('Please, set the challenge name in the config file.')
# Skills
if not isinstance(config['skills'], list):
counted_error(
'Invalid skills in config.yml. Skills should be placed into a list.\n'
'\tValid skills are listed here: \n'
'\thttps://platform.avatao.com/api-explorer/#/api/core/skills/')
# Recommendations
if not isinstance(config['recommendations'], dict):
counted_error(
'Invalid recommendations in config.yml. Recommendations should be added in the following '
'format:\n\n'
'recommendations:\n'
'\twww.example.com: \'Example webpage\'\n'
'\thttp://www.example2.com: \'Example2 webpage\''
'\thttp://example3.com: \'Example3 webpage\'')
# https://mathiasbynens.be/demo/url-regex - https://gist.github.com/dperini/729294
url_strip_chr = r'!"#$%&\'()*+,-./@:;<=>[\\]^_`{|}~'
url_re = re.compile(
r'^(?:(?:https?|ftp)://)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})'
r'(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})'
r'(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}'
r'(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|'
r'(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*'
r'(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?$')
for item in config['recommendations'].items():
if url_re.fullmatch(item[0].strip(url_strip_chr)) is None:
counted_error('Invalid recommended URL (%s) found in config.yml' %
item[0])
if not isinstance(item[1], str):
counted_error(
'The name of recommended url (%s) should be a string in config.yml'
% item[1])
# Owners
if not isinstance(config['owners'], list):
counted_error(
'Challenge owners (%s) should be placed into a list in config.yml'
% config['owners'])
email_re = re.compile(
r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
for owner in config.get('owners', []):
if email_re.fullmatch(owner) is None:
counted_error(
'Invalid owner email (%s) found in config.yml. '
'Make sure you list the email addresses of the owners.' %
owner)
if not is_static:
controller_found = False
for item in config['crp_config'].values():
if not isinstance(item, dict):
counted_error('Items of crp_config must be dictionaries.')
if 'image' in item and item['image'].find('/') < 0:
counted_error(
'If the image is explicitly defined, it must be relative '
'to the registry - e.g. challenge:solvable.')
if 'capabilities' in item:
invalid_caps = set(item['capabilities']) - CAPABILITIES
if len(invalid_caps) > 0:
counted_error(
'Forbidden capabilities: %s\n\tAllowed capabilities: %s',
invalid_caps, CAPABILITIES)
if 'kernel_params' in item:
invalid_parameters = set(
item['kernel_params']) - KERNEL_PARAMETERS
if len(invalid_parameters) > 0:
counted_error(
'Forbidden kernel parameters: %s\n\tAllowed parameters: %s',
invalid_parameters, KERNEL_PARAMETERS)
if 'mem_limit' in item:
if not isinstance(item['mem_limit'], str):
counted_error(
'Invalid mem_limit value: %s, The mem_limit should be a string like: 100M'
)
if item['mem_limit'][-1] not in "M":
counted_error(
'Invalid mem_limit value: %s, The mem_limit should be a string ending with '
'M (megabytes). No other unit is allowed.')
try:
mem_limit_number_part = int(item['mem_limit'][:-1])
if mem_limit_number_part > 999:
counted_error(
'Invalid mem_limit value: %s, The mem_limit can not be greater than 999M.'
)
except Exception:
counted_error(
'Invalid mem_limit value: %s, mem_limit must start with a number and end with '
'M (megabytes). No other unit is allowed.')
for port in item.get('ports', []):
try:
port, protocol = port.split('/', 1)
try:
port = int(port)
except Exception:
counted_error(
'Invalid port. The port should be a number between 1 and 65535.'
)
if PORT_RANGE['min'] > port or PORT_RANGE['max'] < port:
counted_error(
'Invalid port. The port should be a number between 1 and 65535'
)
if protocol not in PROTOCOLS:
counted_error(
'Invalid protocol in config.yml (crp_config): %s. Valid values: %s',
protocol, PROTOCOLS)
elif protocol == CONTROLLER_PROTOCOL:
controller_found = True
except Exception:
counted_error('Invalid port format. [port/protocol]')
if not config.get('flag') and not controller_found:
counted_error(
'Missing controller port [5555/%s] for a dynamic challenge.' %
CONTROLLER_PROTOCOL)
if str(config.get('enable_flag_input')).lower() not in ('true', 'false',
'1', '0'):
counted_error('Invalid enable_flag_input. Should be a boolean.')
if is_static:
try:
assert isinstance(config['flag'], str)
except AssertionError:
counted_error('Invalid flag. Should be a string.')
except KeyError:
counted_error(
'Missing flag. Static challenges must have a static flag set.')
if 'flag' in config and config['flag'][0:6] == 'regex:':
try:
re.compile(config['flag'][6:])
except Exception:
counted_error('Failed to compile regex flag.')
