async def load_plugins(self):
"""
Store all plugins in the data store
:return:
"""
for plug in os.listdir('plugins'):
if not os.path.isdir('plugins/%s' % plug) or not os.path.isfile(
'plugins/%s/hook.py' % plug):
self.log.error(
'Problem validating the "%s" plugin. Ensure CALDERA was cloned recursively.'
% plug)
exit(0)
self.log.debug('Loading plugin: %s' % plug)
plugin = Plugin(name=plug)
await self.get_service('data_svc').store(plugin)
if plugin.name in self.config['enabled_plugins']:
plugin.enabled = True
for plug in await self._services.get('data_svc').locate('plugins'):
if plug.name in self.config['enabled_plugins'] or plug.enabled:
await plug.enable(self.get_services())
templates = [
'plugins/%s/templates' % p.name.lower()
for p in await self.get_service('data_svc').locate('plugins')
]
aiohttp_jinja2.setup(self.application,
loader=jinja2.FileSystemLoader(templates))
async def load_plugins(self):
"""
Store all plugins in the data store
:return:
"""
for plug in os.listdir('plugins'):
if plug.startswith('.'):
continue
if not os.path.isdir('plugins/%s' % plug) or not os.path.isfile(
'plugins/%s/hook.py' % plug):
self.log.error(
'Problem locating the "%s" plugin. Ensure CALDERA was cloned recursively.'
% plug)
exit(0)
plugin = Plugin(name=plug)
if await plugin.load():
await self.get_service('data_svc').store(plugin)
if plugin.name in self.config['plugins']:
plugin.enabled = True
for plugin in self.config['plugins']:
plug = await self._services.get('data_svc').locate(
'plugins', match=dict(name=plugin))
[await p.enable(self.get_services()) for p in plug]
self.log.debug('Enabling %s plugin' % plugin)
templates = [
'plugins/%s/templates' % p.name.lower()
for p in await self.get_service('data_svc').locate('plugins')
]
templates.append('templates')
aiohttp_jinja2.setup(self.application,
loader=jinja2.FileSystemLoader(templates))
async def load(p):
plugin = Plugin(name=p)
if plugin.load_plugin():
await self.get_service('data_svc').store(plugin)
if plugin.name in self.get_config('plugins'):
await plugin.enable(self.get_services())
self.log.info('Enabled plugin: %s' % plugin.name)
async def load_plugins(self, enabled):
"""
Store all plugins in the data store
:param enabled: a list of all plugins to enable right away
:return:
"""
for plug in os.listdir('plugins'):
if not os.path.isdir('plugins/%s' % plug) or not os.path.isfile(
'plugins/%s/hook.py' % plug):
self.log.error(
'Problem validating the "%s" plugin. Ensure CALDERA was cloned recursively.'
% plug)
exit(0)
self.log.debug('Loading plugin: %s' % plug)
if os.path.isfile('plugins/%s/requirements.txt' % plug):
self.log.warning(
'Ensure you have installed the PIP requirements for plugin=%s'
% plug)
plugin = Plugin(name=plug)
await self.get_service('data_svc').store(plugin)
if plugin.name in enabled:
plugin.enabled = True
for plug in await self._services.get('data_svc').locate(
'plugins', match=dict(enabled=True)):
await plug.enable(self.application, self.get_services())
templates = [
'plugins/%s/templates' % p.name.lower()
for p in await self.get_service('data_svc').locate('plugins')
]
aiohttp_jinja2.setup(self.application,
loader=jinja2.FileSystemLoader(templates))
async def load(p):
plugin = Plugin(name=p)
if plugin.load_plugin():
await self.get_service('data_svc').store(plugin)
if plugin.name in self.get_config('plugins'):
await plugin.enable(self.get_services())
self.log.debug('Enabled plugin: %s' % plugin.name)
if not plugin.version:
self._errors.append(Error(plugin.name, 'plugin code is not a release version'))
async def _load(self, plugins=()):
try:
async_tasks = []
if not plugins:
plugins = [
p for p in await self.locate('plugins')
if p.data_dir and p.enabled
]
if not [plugin for plugin in plugins if plugin.data_dir == 'data']:
plugins.append(Plugin(data_dir='data'))
for plug in plugins:
await self._load_payloads(plug)
await self._load_abilities(plug, async_tasks)
await self._load_objectives(plug)
await self._load_adversaries(plug)
await self._load_planners(plug)
await self._load_sources(plug)
await self._load_packers(plug)
for task in async_tasks:
await task
await self._load_extensions()
await self._load_data_encoders(plugins)
await self._verify_data_sets()
except Exception as e:
self.log.debug(repr(e), exc_info=True)
async def load_plugins(self, plugins):
for plug in plugins:
if plug.startswith('.'):
continue
if not os.path.isdir('plugins/%s' % plug) or not os.path.isfile(
'plugins/%s/hook.py' % plug):
self.log.error(
'Problem locating the "%s" plugin. Ensure code base was cloned recursively.'
% plug)
exit(0)
plugin = Plugin(name=plug)
if await plugin.load():
await self.get_service('data_svc').store(plugin)
if plugin.name in self.get_config('plugins'):
await plugin.enable(self.get_services())
self.log.debug('Enabled plugin: %s' % plugin.name)
if not plugin.version:
self._errors.append(
Error(plugin.name,
'plugin code is not a release version'))
templates = [
'plugins/%s/templates' % p.lower()
for p in self.get_config('plugins')
]
templates.append('templates')
aiohttp_jinja2.setup(self.application,
loader=jinja2.FileSystemLoader(templates))
async def persist_adversary(self, data):
"""
Save a new adversary from either the GUI or REST API. This writes a new YML file into the core data/ directory.
:param data:
:return: the ID of the created adversary
"""
i = data.pop('i')
if not i:
i = str(uuid.uuid4())
_, file_path = await self.get_service('file_svc').find_file_path(
'%s.yml' % i, location='data')
if not file_path:
file_path = 'data/adversaries/%s.yml' % i
with open(file_path, 'w+') as f:
f.seek(0)
p = defaultdict(list)
for ability in data.pop('phases'):
p[int(ability['phase'])].append(ability['id'])
f.write(
yaml.dump(
dict(id=i,
name=data.pop('name'),
description=data.pop('description'),
phases=dict(p))))
f.truncate()
await self._services.get('data_svc').reload_data(
[Plugin(data_dir='data')])
return [
a.display for a in await self._services.get('data_svc').locate(
'adversaries', dict(adversary_id=i))
]
async def load_plugins(self):
"""
Store all plugins in the data store
:return:
"""
for plug in os.listdir('plugins'):
if not os.path.isdir('plugins/%s' % plug) or not os.path.isfile(
'plugins/%s/hook.py' % plug):
self.log.error(
'Problem locating the "%s" plugin. Ensure code base was cloned recursively.'
% plug)
exit(0)
plugin = Plugin(name=plug)
if await plugin.load():
await self.get_service('data_svc').store(plugin)
if plugin.name in self.get_config('plugins'):
await plugin.enable(self.get_services())
self.log.debug('Enabled plugin: %s' % plugin.name)
templates = [
'plugins/%s/templates' % p.lower()
for p in self.get_config('plugins')
]
templates.append('templates')
aiohttp_jinja2.setup(self.application,
loader=jinja2.FileSystemLoader(templates))
async def add_app_plugin(self):
await self._services.get('data_svc').store(Plugin(
name='app',
description='A plugin designed to hold global application data',
data_dir='data',
access=self.Access.APP)
)
def _generate_plugin(enabled=False, gui=False, data_dir=None, access=None):
name = ''.join(
random.choice(string.ascii_lowercase) for x in range(10))
description = 'this is a good description'
address = '/plugin/%s/gui' % name if gui else None
return Plugin(name=name,
description=description,
address=address,
enabled=enabled,
data_dir=data_dir,
access=access)
async def watch_ability_files(self):
await asyncio.sleep(int(self.get_config('ability_refresh')))
plugins = [p for p in await self.get_service('data_svc').locate('plugins', dict(enabled=True)) if p.data_dir]
plugins.append(Plugin(data_dir='data'))
while True:
for p in plugins:
files = (os.path.join(rt, fle) for rt, _, f in os.walk(p.data_dir+'/abilities') for fle in f if
time.time() - os.stat(os.path.join(rt, fle)).st_mtime < int(self.get_config('ability_refresh')))
for f in files:
self.log.debug('[%s] Reloading %s' % (p.name, f))
await self.get_service('data_svc').load_ability_file(filename=f, access=p.access)
await asyncio.sleep(int(self.get_config('ability_refresh')))
async def persist_source(self, data):
_, file_path = await self.get_service('file_svc').find_file_path(
'%s.yml' % data.get('id'), location='data')
if not file_path:
file_path = 'data/sources/%s.yml' % data.get('id')
with open(file_path, 'w+') as f:
f.seek(0)
f.write(yaml.dump(data))
await self._services.get('data_svc').reload_data(
[Plugin(data_dir='data')])
return [
s.display for s in await self._services.get('data_svc').locate(
'sources', dict(id=data.get('id')))
]
async def _load(self, plugins=()):
try:
if not plugins:
plugins = [p for p in await self.locate('plugins') if p.data_dir]
plugins.append(Plugin(data_dir='data'))
for plug in plugins:
await self._load_payloads(plug)
await self._load_abilities(plug)
await self._verify_ability_set()
for plug in plugins:
await self._load_adversaries(plug)
await self._load_sources(plug)
await self._load_planners(plug)
except Exception as e:
self.log.debug(repr(e))
async def persist_ability(self, data):
_, file_path = await self.get_service('file_svc').find_file_path(
'%s.yml' % data.get('id'), location='data')
if not file_path:
d = 'data/abilities/%s' % data.get('tactic')
if not os.path.exists(d):
os.makedirs(d)
file_path = '%s/%s.yml' % (d, data.get('id'))
with open(file_path, 'w+') as f:
f.seek(0)
f.write(yaml.dump([data]))
await self._services.get('data_svc').reload_data(
[Plugin(data_dir='data')])
return [
a.display for a in await self._services.get('data_svc').locate(
'abilities', dict(ability_id=data.get('id')))
]
async def _load(self, plugins=()):
try:
if not plugins:
plugins = [p for p in await self.locate('plugins') if p.data_dir and p.enabled]
plugins.append(Plugin(data_dir='data'))
for plug in plugins:
await self._load_payloads(plug)
await self._load_abilities(plug)
await self._load_objectives(plug)
await self._verify_ability_set()
for plug in plugins:
await self._load_adversaries(plug)
await self._load_sources(plug)
await self._load_planners(plug)
await self._load_extensions()
await self._verify_data_sets()
except Exception as e:
self.log.debug(repr(e), exc_info=True)
def run_tasks(services):
loop = asyncio.get_event_loop()
loop.create_task(build_docs())
loop.run_until_complete(data_svc.restore_state())
loop.run_until_complete(RestApi(services).enable())
loop.run_until_complete(app_svc.register_contacts())
loop.run_until_complete(app_svc.load_plugins())
loop.run_until_complete(data_svc.load_data([Plugin(data_dir='data')]))
loop.run_until_complete(data_svc.load_data())
loop.create_task(app_svc.start_sniffer_untrusted_agents())
loop.create_task(app_svc.resume_operations())
loop.create_task(app_svc.run_scheduler())
loop.run_until_complete(start_server())
try:
logging.info('All systems ready.')
loop.run_forever()
except KeyboardInterrupt:
loop.run_until_complete(services.get('app_svc').teardown())
async def add_app_plugin(self):
await self._services.get('data_svc').store(
Plugin(name='app', data_dir='data', access=self.Access.APP))
def test_plugin(loop, api_v2_client):
plugin = Plugin(name="test_plugin", enabled=True, description="a test plugin", address="test_address")
loop.run_until_complete(BaseService.get_service('data_svc').store(plugin))
return plugin
