class DirectoryFlagStorage(AbstractFlagStorage):
def __init__(self, directory=defaults.FLAG_STORAGE_DIRECTORY):
self._logger = Logger(self)
self.directory = directory
self._current_flags = collections.deque()
self._all_flags = set()
async def add_flag(self, flag: models.Flag):
if flag.flag not in self._all_flags:
self._current_flags.append(flag)
self._logger.info(
f'Added flag [{flag}] to the storage. Current size of storage is {len(self._current_flags)}'
)
self._all_flags.add(flag.flag)
else:
self._logger.debug(
f'Flag [{flag}] is duplicating, don\'t add it to the storage')
async def get_not_sent_flags(self, limit=30):
not_sent_flags = []
while self._current_flags and len(not_sent_flags) < limit:
flag = self._current_flags.popleft()
if flag.status == models.FlagStatus.DONT_SEND:
not_sent_flags.append(flag)
# Continue monitoring not sent flags, but add them to the end of queue
self._current_flags.extend(not_sent_flags)
# TODO (andgein): Write not fetched flags to file
return not_sent_flags
def read_teams_from_file(file_name: str):
"""
Reads teams from CSV file. File should be in UTF-8 and has two columns: team name and vulnbox address.
Common usage is `TEAMS = read_teams_from_file('teams.csv')` in `settings.py`
:param file_name: name of CSV file
:return: list of pairs
"""
from farm.logging import Logger
logger = Logger('Configurator')
logger.info('Reading teams list from %s' % file_name)
result = []
with open(file_name, 'r', newline='', encoding='utf-8') as f:
csv_reader = csv.reader(f)
for line_index, row in enumerate(csv_reader):
if len(row) != 2:
raise ValueError(
'Invalid teams files %s: row #%d "%s" contains more or less than 2 items'
% (file_name, line_index, ','.join(row)))
result.append(tuple(row))
logger.info('Read %d teams from %s' % (len(result), file_name))
return result
class Farm:
def __init__(self,
teams: [models.Team],
exploit_storage: AbstractExploitStorage,
flag_submitter: AbstractFlagSubmitter,
flag_storage: AbstractFlagStorage,
flag_format: str,
round_timeout=None,
submitter_sleep=None):
self.teams = teams
self.exploit_storage = exploit_storage
self.flag_submitter = flag_submitter
self.flag_storage = flag_storage
self.flag_format = flag_format
self._flag_re_str = re.compile(flag_format)
self._flag_re_bytes = re.compile(flag_format.encode())
self.round_timeout = defaults.ROUND_TIMEOUT if round_timeout is None else round_timeout
self.submitter_sleep = defaults.SUBMITTER_SLEEP if submitter_sleep is None else submitter_sleep
self.logger = Logger(self)
@classmethod
def create_from_configurator(cls, configurator: Configurator):
return cls(
configurator.get_teams(),
configurator.get_exploit_storage(),
configurator.get_flag_submitter(),
configurator.get_flag_storage(),
configurator.get_flag_format(),
configurator.get_round_timeout(),
configurator.get_submitter_sleep(),
)
@classmethod
def debug_exploit(cls, exploit, vulnbox, loop=None):
if loop is None:
loop = asyncio.get_event_loop()
configurator = Configurator()
farm = cls(
[models.Team('Target', vulnbox)],
OneExploitStorage(exploit),
None,
None,
configurator.get_flag_format(),
)
task = asyncio.wait([farm._run_one_round()], loop=loop)
loop.run_until_complete(task)
def run(self, loop=None):
if loop is None:
loop = asyncio.get_event_loop()
task = asyncio.wait(
[self._run_exploits(), self._run_submitter()], loop=loop)
loop.run_until_complete(task)
async def _run_exploits(self):
round = 0
while True:
round += 1
try:
self.logger.info('EXPLOITS ROUND %d' % round)
await self._run_one_round()
except asyncio.TimeoutError as e:
# It's ok: just some exploit didn't finish him work
pass
except Exception as e:
self.logger.error(
'Error occurred while running exploits: %s' % e, e)
async def _run_one_round(self):
try:
exploits = await self.exploit_storage.get_exploits()
except Exception as e:
self.logger.error(
f'Can\'t get exploits from exploits storage: {e}', e)
return
if not exploits:
# It no exploits loaded sleep self.round_timeout
await asyncio.sleep(self.round_timeout)
return
with async_timeout.timeout(self.round_timeout):
tasks = []
for team in self.teams:
for exploit in exploits:
tasks.append(self._run_exploit(team, exploit))
while tasks:
done, tasks = await asyncio.wait(
tasks, return_when=asyncio.FIRST_COMPLETED)
for task in done:
exploit_result, exploit_name, team = await task
# TODO (andgein): dump exploit output somewhere
flags = self._extract_flags(exploit_result)
if len(flags) > 0:
self.logger.info(
'Found flags in exploit output: [%s]' %
", ".join(map(str, flags)))
if self.flag_storage is not None:
await self.flag_storage.add_flags(
flags, exploit_name, team.name)
async def _run_exploit(self, team, exploit):
exploit_name = str(exploit)
try:
self.logger.info(f'Start exploit [{exploit}] on team [{team}]')
run_attack = exploit.attack(team.vulnbox)
# In case of async generators (https://www.python.org/dev/peps/pep-0525/) we can't await it, we should
# iterate over it via `async for` operator
if isinstance(run_attack, types.AsyncGeneratorType):
return [data async for data in run_attack], exploit_name, team
# Otherwise just await it because `exploit.attack()` is asynchronous function
return await run_attack, exploit_name, team
except Exception as e:
message = f'Exception on exploit [{exploit}] on team [{team}]: {e}'
self.logger.warning(message)
# Duplicate message in exploit's log with stacktrace details and ERROR level
exploit.logger.error(message + "\n" + traceback.format_exc())
return '', exploit_name, team
def _extract_flags(self, exploit_result):
if type(exploit_result) in [str, bytes]:
exploit_result = [exploit_result]
# TODO (andgein): log which exploit it was
if not isinstance(exploit_result, (tuple, list, types.GeneratorType)):
error_message = 'Exploit returned invalid result: %s, should be tuple, list, generator, str or bytes, not %s' % (
exploit_result, type(exploit_result))
self.logger.error(error_message)
return []
flags = []
for line in exploit_result:
if type(line) is bytes:
line_flags = re.findall(self._flag_re_bytes, line)
elif type(line) is str:
line_flags = re.findall(self._flag_re_str, line)
else:
self.logger.error(
'Exploit returned invalid result in list: %s, should be str or bytes, not %s'
% (line, type(line)))
line_flags = []
flags.extend(line_flags)
return flags
async def _run_submitter(self):
while True:
try:
flags = await self.flag_storage.get_not_sent_flags()
except Exception as e:
self.logger.error(
f'Can\'t get not sent flags from storage: {e}', e)
continue
if len(flags) > 0:
self.logger.info(
f'Got {len(flags)} not sent flags from flag storage and '
f'sending them to checksystem: [{", ".join(f.flag for f in flags)}]'
)
try:
await self.flag_submitter.send_flags(flags)
except Exception as e:
self.logger.error(f'Can\'t send flags to checksystem: {e}',
e)
await asyncio.sleep(self.submitter_sleep)
class Configurator:
DEFAULT_SETTINGS_MODULE_NAME = 'settings'
def __init__(self, settings_module_name=None):
self._logger = Logger(self)
self.settings_module_name = self.DEFAULT_SETTINGS_MODULE_NAME if settings_module_name is None else settings_module_name
self.settings_module = importlib.import_module(
self.settings_module_name)
def _get_setting_variable(self,
setting_name,
default_value=None,
may_missing=False):
if hasattr(self.settings_module, setting_name):
return getattr(self.settings_module, setting_name)
if may_missing:
return default_value
raise ValueError('Specify variable %s in settings file (%s)' %
(setting_name, self.settings_module_name))
def _get_setting_object(self, setting_name, may_missing=False):
object_spec = self._get_setting_variable(setting_name,
may_missing=may_missing)
if object_spec is None:
return None
# If you don't need args and kwargs,
# you can use just 'module.name.class_name' instead of { 'type': 'module.name.class_name' }
if type(object_spec) is str:
object_spec = {'type': object_spec}
if type(object_spec) is not dict:
raise ValueError(
'Variable %s in settings file (%s) should be dict or str, not %s'
% (setting_name, self.settings_module_name, type(object_spec)))
object_type_name = self._get_dict_value(object_spec, 'type',
setting_name)
object_args = object_spec.get('args', ())
object_kwargs = object_spec.get('kwargs', {})
try:
object_type = utils.import_type(object_type_name)
except ValueError as e:
raise ValueError('Can not find type %s for initializing %s: %s' %
(object_type_name, setting_name, e))
self._logger.info('Creating %s with arguments %s and kwarguments %s' %
(object_type.__name__, object_args, object_kwargs))
return object_type(*object_args, **object_kwargs)
def _get_dict_value(self, dict_object, param, setting_name):
try:
return dict_object[param]
except KeyError:
raise ValueError(
'Variable %s in settings file (%s) should has key "%s"' %
(setting_name, self.settings_module_name, param))
def get_flag_format(self):
return self._get_setting_variable('FLAG_FORMAT')
def get_flag_submitter(self):
return self._get_setting_object('FLAG_SUBMITTER')
def get_teams(self):
teams = self._get_setting_variable('TEAMS')
return list(Team(*team) for team in teams)
def get_exploit_storage(self):
return self._get_setting_object('EXPLOIT_STORAGE')
def get_flag_storage(self):
return self._get_setting_object('FLAG_STORAGE')
def get_round_timeout(self):
return self._get_setting_variable('ROUND_TIMEOUT',
defaults.ROUND_TIMEOUT,
may_missing=True)
def get_submitter_sleep(self):
return self._get_setting_variable('SUBMITTER_SLEEP',
defaults.SUBMITTER_SLEEP,
may_missing=True)