def update_modules(self, repository):
base_dir = path.dirname(path.dirname(path.abspath(__file__)))
modules_dir = path.join(base_dir, 'modules')
self._remove_compiled_files(modules_dir)
installed_modules = self.list_installed_modules_for(repository)
updated_named_configs = []
for name, obj in self.walk_modules(modules_dir, repository=repository):
updated_named_configs += self.update(obj, name, repository)
if obj.name:
installed_modules.discard(obj.name)
# Delete all modules that are no longer in the repository
for missing_module in installed_modules:
print "Deleting '{}'".format(missing_module)
ModuleInfo.get_collection().remove({'name': missing_module})
# Disable all modules that have incomplete named configs
for updated_named_config in unique_for_key(updated_named_configs,
'name'):
if incomplete_config(updated_named_config['config']):
for name, obj in self.walk_modules(modules_dir):
if obj.name and updated_named_config[
'name'] in obj.named_configs:
info = ModuleInfo.get(name=obj.name)
if info['enabled']:
print "Disabling {} for incomplete named config {}".format(
obj.name, updated_named_config['name'])
info.update_value('enabled', False)
def load_all_modules(self):
for module in ModuleInfo.get_collection().find({'enabled': True}):
if module['type'] == "Processing":
self.add_processing_module(module)
elif module['type'] == "Virtualization":
self.add_virtualization_module(module)
else:
self.add_module(module)
def list_installed_modules_for(self, repository):
results = set()
for module in ModuleInfo.get_collection().find():
if module['path'].startswith('fame.modules.{}.'.format(
repository['name'])):
results.add(module['name'])
return results
def get(self, id):
"""Get the analysis with `id`.
.. :quickref: Analysis; Get an analysis
Resulting object is in the ``analysis`` field.
:param id: id of the analysis.
:>json dict _id: ObjectId dict.
:>json dict analyst: analyst's ObjectId.
:>json dict date: date dict.
:>json list executed_modules: list of executed modules.
:>json list pending_modules: list of pending modules.
:>json list waiting_modules: list of waiting modules.
:>json list canceled_modules: list of canceled modules.
:>json list executed_modules: list of executed modules.
:>json string module: the name of the target module.
:>json string status: status of the analysis (one of `pending`, `running`, `finished` or `error`).
:>json list tags: the list of tags.
:>json list probable_names: the list of probable names.
:>json list iocs: list of dict describing observables.
:>json dict results: detailed results for each module, with the module name being the key.
:>json dict generated_files: a dict of generated files, the key being the file type.
:>json list extracted_files: a list of extracted files.
:>json dict support_files: a dict of support files, the key being the module name.
"""
analysis = {
'analysis':
clean_analyses(get_or_404(current_user.analyses, _id=id))
}
file = current_user.files.find_one(
{'_id': analysis['analysis']['file']})
analysis['analysis']['file'] = clean_files(file)
ti_modules = [
m.name for m in dispatcher.get_threat_intelligence_modules()
]
av_modules = [m.name for m in dispatcher.get_antivirus_modules()]
if 'extracted_files' in analysis['analysis']:
files = []
for id in analysis['analysis']['extracted_files']:
files.append(current_user.files.find_one({'_id': id}))
analysis['analysis']['extracted_files'] = clean_files(files)
modules = dict()
for module in ModuleInfo.get_collection().find():
modules[module['name']] = ModuleInfo(module)
return render(analysis,
'analyses/show.html',
ctx={
'analysis': analysis,
'modules': modules,
'av_modules': av_modules,
'ti_modules': ti_modules
})
def list(self):
"""List enabled Processing modules
.. :quickref: Module; List enabled Processing modules
:>json list modules: list of enabled modules.
"""
modules = ModuleInfo.get_collection().find({
'enabled': True,
'type': 'Processing'
})
return render({'modules': clean_modules(list(modules))})
def list(self):
"""List enabled Processing and Preloading modules
.. :quickref: Module; List enabled Processing and Preloading modules
:>json list modules: list of enabled modules.
"""
modules = ModuleInfo.get_collection().find({
'enabled': True,
'type': {
'$in': ['Processing', 'Preloading']
}
})
return render(clean_modules(list(modules)))
def enable(self, id):
"""Enable a module
.. :quickref: Module; Enable a module
Requires the `manage_modules` permission.
If successful, will return the module in ``module``.
Otherwise, errors will be available in ``errors``.
:param id: id of the module to enable.
"""
module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id))
if 'error' in module:
flash(
"Cannot enable '{}' because of errors installing dependencies."
.format(module['name']), 'danger')
return validation_error(url_for('ModulesView:index'))
# See if module is properly configured
module_class = get_class(module['path'], module['class'])
module_class.info = module
try:
module_class()
except MissingConfiguration as e:
if e.name:
flash(
"You must configure '{}' before trying to enable '{}'".
format(e.name, module['name']), 'warning')
return validation_error(
url_for('ModulesView:configuration', id=e.id))
else:
flash(
"You must configure '{}' before trying to enable it.".
format(module['name']), 'warning')
return validation_error(
url_for('ModulesView:configure', id=module['_id']))
module.update_value('enabled', True)
dispatcher.reload()
readme = module.get_readme()
if readme:
flash(readme, 'persistent')
return redirect({'module': clean_modules(module)},
url_for('ModulesView:index'))
def disable(self, id):
"""Disable a module
.. :quickref: Module; Disable a module
Requires the `manage_modules` permission.
:param id: id of the module to disable.
:>json Module module: resulting module.
"""
module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id))
module.update_value('enabled', False)
dispatcher.reload()
return redirect({'module': clean_modules(module)},
url_for('ModulesView:index'))
def update_module_requirements(self):
for module in ModuleInfo.get_collection().find():
module = ModuleInfo(module)
if 'error' in module:
del(module['error'])
if module['type'] == "Processing":
should_update = (module['queue'] in self.queues)
elif module['type'] in ["Threat Intelligence", "Reporting", "Filetype"]:
should_update = True
else:
should_update = (not fame_config.remote)
if should_update:
self.update_python_requirements(module)
self.launch_install_scripts(module)
module.save()
def configure(self, id):
"""Configure a module.
.. :quickref: Module; Configure a module
Requires the `manage_modules` permission.
For each configuration available, you should set the value in a form
parameter named ``config_NAME``. For boolean values, any value not ``0``
or ``False`` is considered to be ``True``.
If the setting should be an option (be available per analysis), you have
to set ``config_NAME_option`` to any value but ``0`` or ``False``.
If successful, will return the module in ``module``.
Otherwise, errors will be available in ``errors``.
:param id: id of the named configuration.
:form acts_on: comma-delimited list of FAME types this module can act on
(only for Processing modules).
:form triggered_by: comma-delimited list of triggers (only for Processing
modules).
:form queue: name of the queue to use for this module (for Processing and
Preloading modules).
"""
module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id))
module['readme'] = module.get_readme()
if request.method == "POST":
if module['type'] == 'Filetype':
if 'acts_on' in request.form:
module.update_setting_value(
'acts_on', request.form.get('acts_on', ''))
elif module['type'] == 'Processing':
if 'acts_on' in request.form:
module.update_setting_value(
'acts_on', request.form.get('acts_on', ''))
if 'triggered_by' in request.form:
module.update_setting_value(
'triggered_by', request.form.get('triggered_by', ''))
if 'queue' in request.form:
update_queue(module, request.form.get('queue', ''))
elif module['type'] == "Preloading":
if 'queue' in request.form:
update_queue(module, request.form.get('queue', ''))
if 'priority' in request.form:
update_priority(module, request.form.get('priority', ''))
errors = update_config(module['config'],
options=(module['type']
in ['Preloading', 'Processing']))
if errors is not None:
return errors
module.save()
dispatcher.reload()
return redirect({'module': clean_modules(module)},
url_for('ModulesView:index'))
else:
return render({'module': clean_modules(module)},
'modules/module_configuration.html')
def index(self):
"""Get the list of modules.
.. :quickref: Module; Get the list of modules
Requires the `manage_modules` permission.
The response is a dict with several elements:
* ``modules``, which is a list of modules, sorted by type::
"modules": {
"Antivirus": [
...
],
"Preloading": [
...
],
"Processing": [
{
"_id": {
"$oid": "MODULE_ID"
},
"acts_on": [
ACTS_ON_FAME_TYPES
],
"class": "CLASS_NAME",
"config": [ CONFIG_OPTIONS ],
"description": "DESCRIPTION",
"enabled": false,
"generates": [GENERATES],
"name": "NAME",
"path": "MODULE_PATH",
"queue": "QUEUE",
"triggered_by": [
TRIGGERS
],
"type": "Processing"
},
...
],
"Reporting": [
...
],
"Threat Intelligence": [
...
],
"Filetype": [
...
]
}
* ``repositories``: list of configured repositories::
"repositories": [
{
"_id": {
"$oid": "ID"
},
"address": "[email protected]:certsocietegenerale/fame_modules.git",
"name": "community",
"private": false,
"status": "active"
},
...
]
* ``configs``: list of named configurations::
"configs": [
{
"_id": {
"$oid": "ID"
},
"config": [
{
"description": "List of patterns (strings) to look for in malware configurations. There should be one pattern per line.",
"name": "monitor",
"type": "text",
"value": null
}
],
"description": "Needed in order to be able to track malware targets",
"name": "malware_config"
},
...
]
"""
types = {
'Preloading': [],
'Processing': [],
'Reporting': [],
'Threat Intelligence': [],
'Antivirus': [],
'Virtualization': [],
'Filetype': []
}
for module in ModuleInfo.get_collection().find():
types[module['type']].append(clean_modules(module))
for type in types:
types[type] = sorted(types[type], key=get_name)
configs = Config.get_collection().find()
repositories = clean_repositories(
list(Repository.get_collection().find()))
return render(
{
'modules': types,
'configs': configs,
'repositories': repositories
}, 'modules/index.html')
def configure(self, id):
"""Configure a module.
.. :quickref: Module; Configure a module
Requires the `manage_modules` permission.
For each configuration available, you should set the value in a form
parameter named ``config_NAME``. For boolean values, any value not ``0``
or ``False`` is considered to be ``True``.
If the setting should be an option (be available per analysis), you have
to set ``config_NAME_option`` to any value but ``0`` or ``False``.
If successful, will return the module in ``module``.
Otherwise, errors will be available in ``errors``.
:param id: id of the named configuration.
:form acts_on: comma-delimited list of FAME types this module can act on
(only for Processing modules).
:form triggered_by: comma-delimited list of triggers (only for Processing
modules).
:form queue: name of the queue to use for this module (only for Processing
modules).
"""
module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id))
if request.method == "POST":
if module['type'] == 'Processing':
if 'acts_on' in request.form:
module.update_setting_value(
'acts_on', request.form.get('acts_on', ''))
if 'triggered_by' in request.form:
module.update_setting_value(
'triggered_by', request.form.get('triggered_by', ''))
if 'queue' in request.form:
new_queue = request.form.get('queue')
if module['queue'] == '':
flash('queue cannot be empty', 'danger')
return validation_error()
else:
if module['queue'] != new_queue:
module.update_setting_value('queue', new_queue)
updates = Internals(
get_or_404(Internals.get_collection(),
name="updates"))
updates.update_value("last_update", time())
flash(
'Workers will reload once they are done with their current tasks',
'success')
errors = update_config(module['config'],
options=(module['type'] == 'Processing'))
if errors is not None:
return errors
module.save()
dispatcher.reload()
return redirect({'module': clean_modules(module)},
url_for('ModulesView:index'))
else:
return render({'module': clean_modules(module)},
'modules/module_configuration.html')
