def get(self, id):
"""Get a user.
.. :quickref: User; Get a user
The user is returned in the ``user`` field.
:param id: user id
:>json ObjectId _id: user's ObjectId.
:>json string name: full name.
:>json string: email address.
:>json boolean enabled: ``True`` if the user is enabled.
:>json list groups: list of groups the user belongs to.
:>json list default_sharing: list of groups used by the user as default sharing preferences.
:>json list permissions: list of user's permissions
"""
self.ensure_permission(id)
user = User(get_or_404(User.get_collection(), _id=id))
return render(
{
'user': clean_users(user),
'permissions': dispatcher.permissions
}, 'users/profile.html')
def index(self):
"""Get the list of analyses.
.. :quickref: Analysis; Get the list of analyses
Response is paginated and will only contain 25 results. The most recent
analyses appear first.
:query page: page number.
:type page: int
:>json list analyses: list of analyses (see :http:get:`/analyses/(id)` for details on the format of an analysis).
"""
page = int(request.args.get('page', 1))
analyses = current_user.analyses.find().sort('_id', DESCENDING).limit(PER_PAGE).skip((page - 1) * PER_PAGE)
pagination = Pagination(page=page, per_page=PER_PAGE, total=analyses.count(), css_framework='bootstrap3')
analyses = {'analyses': clean_analyses(list(analyses))}
for analysis in analyses['analyses']:
file = current_user.files.find_one({'_id': analysis['file']})
analysis['file'] = clean_files(file)
if 'analyst' in analysis:
analyst = store.users.find_one({'_id': analysis['analyst']})
analysis['analyst'] = clean_users(analyst)
return render(analyses, 'analyses/index.html', ctx={'data': analyses, 'pagination': pagination})
def configuration(self, id):
"""Configure a named configuration.
.. :quickref: Module; Configure a named configuration
Requires the `manage_modules` permission.
For each configuration available, you should set the value in a form
parameter named ``config_NAME``. For boolean values, any value not ``0``
or ``False`` is considered to be ``True``.
If successful, will return the named configuration in ``config``.
Otherwise, errors will be available in ``errors``.
:param id: id of the named configuration.
"""
config = Config(get_or_404(Config.get_collection(), _id=id))
if request.method == "POST":
errors = update_config(config['config'])
if errors is not None:
return errors
config.save()
dispatcher.reload()
return redirect({'config': config}, url_for('ModulesView:index'))
else:
return render({'config': config}, 'modules/configuration.html')
def index(self):
"""Get the list of objects.
.. :quickref: File; Get the list of objects
Response is paginated and will only contain 25 results. The most recent
objects appear first.
:query page: page number.
:type page: int
:>json list files: list of files (see :http:get:`/files/(id)` for details on the format of a file).
"""
page = int(request.args.get('page', 1))
files = current_user.files.find().sort(
'_id', DESCENDING).limit(PER_PAGE).skip((page - 1) * PER_PAGE)
pagination = Pagination(page=page,
per_page=PER_PAGE,
total=files.count(),
css_framework='bootstrap3')
files = {'files': clean_files(list(files))}
return render(files,
'files/index.html',
ctx={
'data': files,
'pagination': pagination
})
def get(self, id):
"""Get the analysis with `id`.
.. :quickref: Analysis; Get an analysis
Resulting object is in the ``analysis`` field.
:param id: id of the analysis.
:>json dict _id: ObjectId dict.
:>json dict analyst: analyst's ObjectId.
:>json dict date: date dict.
:>json list executed_modules: list of executed modules.
:>json list pending_modules: list of pending modules.
:>json list waiting_modules: list of waiting modules.
:>json list canceled_modules: list of canceled modules.
:>json list executed_modules: list of executed modules.
:>json string module: the name of the target module.
:>json string status: status of the analysis (one of `pending`, `running`, `finished` or `error`).
:>json list tags: the list of tags.
:>json list probable_names: the list of probable names.
:>json list iocs: list of dict describing observables.
:>json dict results: detailed results for each module, with the module name being the key.
:>json dict generated_files: a dict of generated files, the key being the file type.
:>json list extracted_files: a list of extracted files.
:>json dict support_files: a dict of support files, the key being the module name.
"""
analysis = {
'analysis':
clean_analyses(get_or_404(current_user.analyses, _id=id))
}
file = current_user.files.find_one(
{'_id': analysis['analysis']['file']})
analysis['analysis']['file'] = clean_files(file)
ti_modules = [
m.name for m in dispatcher.get_threat_intelligence_modules()
]
av_modules = [m.name for m in dispatcher.get_antivirus_modules()]
if 'extracted_files' in analysis['analysis']:
files = []
for id in analysis['analysis']['extracted_files']:
files.append(current_user.files.find_one({'_id': id}))
analysis['analysis']['extracted_files'] = clean_files(files)
modules = dict()
for module in ModuleInfo.get_collection().find():
modules[module['name']] = ModuleInfo(module)
return render(analysis,
'analyses/show.html',
ctx={
'analysis': analysis,
'modules': modules,
'av_modules': av_modules,
'ti_modules': ti_modules
})
def repository_new(self):
"""Add a repository
.. :quickref: Module; Add repository
Requires the `manage_modules` permission.
If successful, will return the repository in ``repository``.
Otherwise, errors will be available in ``errors``.
:form name: name of the repository (should be a valid package name).
:form address: HTTPs or SSH address of the repository.
:form private: boolean specifying if the repository is private. See
Administration Guide for more details on private repositories.
"""
deploy_key = get_deploy_key()
repository = Repository()
if request.method == 'POST':
for field in ['name', 'address']:
repository[field] = request.form.get(field)
if repository[field] is None or repository[field] == "":
flash("{} is required.".format(field), 'danger')
return validation_error()
existing_repository = Repository.get(
**{field: repository[field]})
if existing_repository:
flash(
"There is already a repository with this {}.".format(
field), 'danger')
return validation_error()
value = request.form.get('private')
repository['private'] = (value is not None) and (value not in [
'0', 'False'
])
if repository['private'] and deploy_key is None:
flash(
"Private repositories are disabled because of a problem with your installation (you do not have a deploy key in 'conf/id_rsa.pub')",
'danger')
return validation_error()
repository['status'] = 'cloning'
repository.save()
repository.clone()
return redirect({'repository': clean_repositories(repository)},
url_for('ModulesView:index'))
return render({
'repository': repository,
'deploy_key': deploy_key
}, 'modules/repository_new.html')
def list(self):
"""List enabled Processing modules
.. :quickref: Module; List enabled Processing modules
:>json list modules: list of enabled modules.
"""
modules = ModuleInfo.get_collection().find({
'enabled': True,
'type': 'Processing'
})
return render({'modules': clean_modules(list(modules))})
def index(self):
"""Get the index of malware configurations.
.. :quickref: Malware Configurations; Get the index
Requires the `config` permission.
The response is a dict with the following format::
{
"MONITOR1": {
"active_botnets": [
"FAMILY:BOTNETID"
],
"botnets": [
"FAMILY:BOTNETID"
],
"count": 1,
"targets": {
"TARGET1": {
"active_botnets": [
"FAMILY:BOTNETID"
],
"botnets": [
"FAMILY:BOTNETID"
],
"count": 1
},
"TARGET2": {
...
}
}
},
"MONITOR2": {
...
}
}
"""
monitors = {}
for record in targets_aggregation():
clean_record(record)
monitor = get_monitor(monitors, record)
target = get_target(monitor, record)
increment([monitor, target], 'count', record['count'])
add([monitor, target], 'botnets', record['botnet'])
if record['last_action'] != ACTION_REMOVED:
add([monitor, target], 'active_botnets', record['botnet'])
return render(monitors, 'configs/index.html')
def return_file(file):
analyses = list(current_user.analyses.find({'_id': {'$in': file['file']['analysis']}}))
file['av_modules'] = [m.name for m in dispatcher.get_antivirus_modules()]
for analysis in analyses:
if 'analyst' in analysis:
analyst = store.users.find_one({'_id': analysis['analyst']})
analysis['analyst'] = clean_users(analyst)
file['file']['analysis'] = clean_analyses(analyses)
return render(file, 'files/show.html', ctx={
'data': file,
'options': dispatcher.options,
'comments_enabled': comments_enabled()})
def list(self):
"""List enabled Processing and Preloading modules
.. :quickref: Module; List enabled Processing and Preloading modules
:>json list modules: list of enabled modules.
"""
modules = ModuleInfo.get_collection().find({
'enabled': True,
'type': {
'$in': ['Processing', 'Preloading']
}
})
return render(clean_modules(list(modules)))
def get(self, id):
"""Get the object with `id`.
.. :quickref: File; Get an object
Resulting object is in the ``file`` field.
:param id: id of the object.
:>json dict _id: ObjectId dict.
:>json string md5: MD5 hash.
:>json string sha1: SHA1 hash.
:>json string sha256: SHA256 hash.
:>json string type: FAME type.
:>json string mime: mime type.
:>json string detailed_type: detailed type.
:>json list groups: list of groups (as strings) that have access to this file.
:>json list owners: list of groups (as strings) that submitted this file.
:>json list probable_names: list of probable names (as strings).
:>json list analysis: list of analyses' ObjectIds.
:>json list parent_analyses: list of analyses (as ObjectIds) that extracted this object.
:>json dict antivirus: dict with antivirus names as keys.
"""
file = {'file': clean_files(get_or_404(current_user.files, _id=id))}
analyses = list(
current_user.analyses.find(
{'_id': {
'$in': file['file']['analysis']
}}))
file['av_modules'] = [
m.name for m in dispatcher.get_antivirus_modules()
]
for analysis in analyses:
if 'analyst' in analysis:
analyst = store.users.find_one({'_id': analysis['analyst']})
analysis['analyst'] = clean_users(analyst)
file['file']['analysis'] = clean_analyses(analyses)
return render(file,
'files/show.html',
ctx={
'data': file,
'options': dispatcher.options
})
def index(self):
"""Get all users.
.. :quickref: User; Get the list of users
Requires the `manage_users` permission.
The result is in the ``users`` field.
:>jsonarr ObjectId _id: user's ObjectId.
:>jsonarr string name: full name.
:>jsonarr string: email address.
:>jsonarr boolean enabled: ``True`` if the user is enabled.
:>jsonarr list groups: list of groups the user belongs to.
:>jsonarr list default_sharing: list of groups used by the user as default sharing preferences.
:>jsonarr list permissions: list of user's permissions
"""
users = {"users": clean_users(list(User.find()))}
return render(users, 'users/index.html')
def return_file(file):
analyses = list(
current_user.analyses.find({"_id": {
"$in": file["file"]["analysis"]
}}))
file["av_modules"] = [m.name for m in dispatcher.get_antivirus_modules()]
for analysis in analyses:
if "analyst" in analysis:
analyst = store.users.find_one({"_id": analysis["analyst"]})
analysis["analyst"] = clean_users(analyst)
file["file"]["analysis"] = clean_analyses(analyses)
return render(
file,
"files/show.html",
ctx={
"data": file,
"options": dispatcher.options,
"comments_enabled": comments_enabled()
},
)
def post(self):
query = request.form['query']
files = []
for file in current_user.files.find({'$text': {'$search': query}}):
files.append(file)
analyses = []
for analysis in current_user.analyses.find(
{'$text': {
'$search': query
}}):
file = current_user.files.find_one({'_id': analysis['file']})
analysis['file'] = clean_files(file)
analyses.append(analysis)
results = {
'files': clean_files(files),
'analyses': clean_analyses(analyses)
}
return render(results, 'search.html')
def index(self):
pending_analyses = []
stale_analyses = []
for analysis in store.analysis.find({'status': 'pending'}):
file = store.files.find_one({'_id': analysis['file']})
analysis['file'] = file
pending_analyses.append(analysis)
for analysis in store.analysis.find({
'status': 'running',
'waiting_modules': {
'$ne': []
}
}):
file = store.files.find_one({'_id': analysis['file']})
analysis['file'] = file
stale_analyses.append(analysis)
return render(
{
'pending_analyses': pending_analyses,
'stale_analyses': stale_analyses
}, "system/index.html")
def post(self):
file = request.files['file']
f = File(filename=secure_filename(file.filename), stream=file.stream)
return render({'file': f})
def configure(self, id):
"""Configure a module.
.. :quickref: Module; Configure a module
Requires the `manage_modules` permission.
For each configuration available, you should set the value in a form
parameter named ``config_NAME``. For boolean values, any value not ``0``
or ``False`` is considered to be ``True``.
If the setting should be an option (be available per analysis), you have
to set ``config_NAME_option`` to any value but ``0`` or ``False``.
If successful, will return the module in ``module``.
Otherwise, errors will be available in ``errors``.
:param id: id of the named configuration.
:form acts_on: comma-delimited list of FAME types this module can act on
(only for Processing modules).
:form triggered_by: comma-delimited list of triggers (only for Processing
modules).
:form queue: name of the queue to use for this module (for Processing and
Preloading modules).
"""
module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id))
module['readme'] = module.get_readme()
if request.method == "POST":
if module['type'] == 'Filetype':
if 'acts_on' in request.form:
module.update_setting_value(
'acts_on', request.form.get('acts_on', ''))
elif module['type'] == 'Processing':
if 'acts_on' in request.form:
module.update_setting_value(
'acts_on', request.form.get('acts_on', ''))
if 'triggered_by' in request.form:
module.update_setting_value(
'triggered_by', request.form.get('triggered_by', ''))
if 'queue' in request.form:
update_queue(module, request.form.get('queue', ''))
elif module['type'] == "Preloading":
if 'queue' in request.form:
update_queue(module, request.form.get('queue', ''))
if 'priority' in request.form:
update_priority(module, request.form.get('priority', ''))
errors = update_config(module['config'],
options=(module['type']
in ['Preloading', 'Processing']))
if errors is not None:
return errors
module.save()
dispatcher.reload()
return redirect({'module': clean_modules(module)},
url_for('ModulesView:index'))
else:
return render({'module': clean_modules(module)},
'modules/module_configuration.html')
def index(self):
"""Get the list of modules.
.. :quickref: Module; Get the list of modules
Requires the `manage_modules` permission.
The response is a dict with several elements:
* ``modules``, which is a list of modules, sorted by type::
"modules": {
"Antivirus": [
...
],
"Preloading": [
...
],
"Processing": [
{
"_id": {
"$oid": "MODULE_ID"
},
"acts_on": [
ACTS_ON_FAME_TYPES
],
"class": "CLASS_NAME",
"config": [ CONFIG_OPTIONS ],
"description": "DESCRIPTION",
"enabled": false,
"generates": [GENERATES],
"name": "NAME",
"path": "MODULE_PATH",
"queue": "QUEUE",
"triggered_by": [
TRIGGERS
],
"type": "Processing"
},
...
],
"Reporting": [
...
],
"Threat Intelligence": [
...
],
"Filetype": [
...
]
}
* ``repositories``: list of configured repositories::
"repositories": [
{
"_id": {
"$oid": "ID"
},
"address": "[email protected]:certsocietegenerale/fame_modules.git",
"name": "community",
"private": false,
"status": "active"
},
...
]
* ``configs``: list of named configurations::
"configs": [
{
"_id": {
"$oid": "ID"
},
"config": [
{
"description": "List of patterns (strings) to look for in malware configurations. There should be one pattern per line.",
"name": "monitor",
"type": "text",
"value": null
}
],
"description": "Needed in order to be able to track malware targets",
"name": "malware_config"
},
...
]
"""
types = {
'Preloading': [],
'Processing': [],
'Reporting': [],
'Threat Intelligence': [],
'Antivirus': [],
'Virtualization': [],
'Filetype': []
}
for module in ModuleInfo.get_collection().find():
types[module['type']].append(clean_modules(module))
for type in types:
types[type] = sorted(types[type], key=get_name)
configs = Config.get_collection().find()
repositories = clean_repositories(
list(Repository.get_collection().find()))
return render(
{
'modules': types,
'configs': configs,
'repositories': repositories
}, 'modules/index.html')
def new(self):
context = {'user': {}, 'permissions': dispatcher.permissions}
return render(context, 'users/new.html')
def configure(self, id):
"""Configure a module.
.. :quickref: Module; Configure a module
Requires the `manage_modules` permission.
For each configuration available, you should set the value in a form
parameter named ``config_NAME``. For boolean values, any value not ``0``
or ``False`` is considered to be ``True``.
If the setting should be an option (be available per analysis), you have
to set ``config_NAME_option`` to any value but ``0`` or ``False``.
If successful, will return the module in ``module``.
Otherwise, errors will be available in ``errors``.
:param id: id of the named configuration.
:form acts_on: comma-delimited list of FAME types this module can act on
(only for Processing modules).
:form triggered_by: comma-delimited list of triggers (only for Processing
modules).
:form queue: name of the queue to use for this module (only for Processing
modules).
"""
module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id))
if request.method == "POST":
if module['type'] == 'Processing':
if 'acts_on' in request.form:
module.update_setting_value(
'acts_on', request.form.get('acts_on', ''))
if 'triggered_by' in request.form:
module.update_setting_value(
'triggered_by', request.form.get('triggered_by', ''))
if 'queue' in request.form:
new_queue = request.form.get('queue')
if module['queue'] == '':
flash('queue cannot be empty', 'danger')
return validation_error()
else:
if module['queue'] != new_queue:
module.update_setting_value('queue', new_queue)
updates = Internals(
get_or_404(Internals.get_collection(),
name="updates"))
updates.update_value("last_update", time())
flash(
'Workers will reload once they are done with their current tasks',
'success')
errors = update_config(module['config'],
options=(module['type'] == 'Processing'))
if errors is not None:
return errors
module.save()
dispatcher.reload()
return redirect({'module': clean_modules(module)},
url_for('ModulesView:index'))
else:
return render({'module': clean_modules(module)},
'modules/module_configuration.html')
def show(self):
"""Get a malware configuration timeline.
.. :quickref: Malware Configurations; Get a timeline
Requires the `config` permission.
You can get the timeline of your choice by conbining several filters.
:query monitor: (optional) filter by monitor.
:query target: (optional) filter by target.
:query botnet: (optional) filter by botnet.
:query type: (optional) filter by type.
:>json list botnets: the list of available botnets.
:>json list monitors: the list of available monitors.
:>json list targets: the list of available targets.
:>json list types: the list of available configuration block types.
:>json list config_blocks: a sorted list of configuration blocks matching
this query. Each configuration block has the following format::
{
"_id": {
"$oid": "CONFIG_BLOCK_ID"
},
"action": "CONFIG_BLOCK_ACTION", # new, update, removed or added
"additional": null,
"analyses": [
{
"$oid": "ANALYSIS_ID"
}
],
"botnet": "FAMILY:BOTNETID",
"content": "CONFIG_BLOCK_CONTENT",
"created": {
"$date": CREATION_DATE
},
"monitor": "MATCHING_MONITOR",
"target": "TARGET",
"type": "CONFIG_BLOCK_TYPE",
"updated": {
"$date": MODIFICATION_DATE
}
}
"""
query = build_query(['monitor', 'target', 'botnet', 'type'])
history = {}
monitors = set()
targets = set()
types = set()
botnets = set()
config_blocks = []
for block in store.config_blocks.find(query).sort('updated'):
config_blocks.append(block)
monitors.add(block['monitor'])
targets.add(block['target'])
types.add(block['type'])
botnets.add(block['botnet'])
label = "{}:{}:{}".format(block['target'], block['type'],
block['botnet'])
if block['action'] == ACTION_UPDATE:
block['diff'] = ''.join(
ndiff(history[label].splitlines(1),
block['content'].splitlines(1)))
if block['action'] != ACTION_REMOVED:
history[label] = block['content']
result = {
'config_blocks': config_blocks,
'monitors': monitors,
'targets': targets,
'types': types,
'botnets': botnets
}
return render(result, 'configs/show.html')
