def create_comment_configuration():
comments = Config.get(name='comments')
if comments is None:
comments = Config({
'name': 'comments',
'description': 'Analysis comments configuration.',
'config': [
{
'name': 'enable',
'description': 'Let users add comments to an analysis.',
'type': 'bool',
'default': True,
'value': True
},
{
'name': 'minimum_length',
'description': 'Define a minimal character count to be enforced when submitting an analysis',
'type': 'integer',
'default': 0,
'value': None
}
]
})
comments.save()
def configuration(self, id):
"""Configure a named configuration.
.. :quickref: Module; Configure a named configuration
Requires the `manage_modules` permission.
For each configuration available, you should set the value in a form
parameter named ``config_NAME``. For boolean values, any value not ``0``
or ``False`` is considered to be ``True``.
If successful, will return the named configuration in ``config``.
Otherwise, errors will be available in ``errors``.
:param id: id of the named configuration.
"""
config = Config(get_or_404(Config.get_collection(), _id=id))
if request.method == "POST":
errors = update_config(config['config'])
if errors is not None:
return errors
config.save()
dispatcher.reload()
return redirect({'config': config}, url_for('ModulesView:index'))
else:
return render({'config': config}, 'modules/configuration.html')
def create_virustotal_configuration():
vt = Config.get(name='virustotal')
if vt is None:
vt = Config({
'name': 'virustotal',
'description': 'VirusTotal API configuration, in order to be able to submit hashes.',
'config': [
{
'name': 'api_key',
'description': 'VirusTotal Intelligence API key.',
'type': 'str',
'value': None
}
]})
vt.save()
def __init__(self, template_path=None):
self.template_path = template_path
self.env = None
if self.template_path:
self.env = Environment(loader=FileSystemLoader(template_path))
try:
self.config = Config.get(name="email").get_values()
self.is_configured = True
try:
self.smtp = smtplib.SMTP(self.config.host, self.config.port)
if self.config.tls:
self.smtp.ehlo()
self.smtp.starttls()
if self.config.username and self.config.password:
self.smtp.login(self.config.username, self.config.password)
self.is_connected = True
else:
status = self.smtp.noop()[0]
self.is_connected = (status == 250)
except Exception as e:
print(e)
self.is_connected = False
except MissingConfiguration:
self.is_configured = False
self.is_connected = False
def notify_new_comment(self, analysis_id, commentator_id, comment):
commentator = store.users.find_one({'_id': commentator_id})
analysis = store.analysis.find_one({'_id': ObjectId(analysis_id)})
analyst_id = analysis['analyst']
recipients = set()
# First let's add submiter analyst and check if he is not commentator
if commentator_id != analyst_id:
analyst = store.users.find_one({'_id': analysis['analyst']})
recipients.add(analyst['email'])
# iter on commentators and add them as recipient
for comment in self['comments']:
if comment['analyst'] not in [analyst_id, commentator_id]:
recipient = store.users.find_one({'_id': comment['analyst']})
recipients.add(recipient['email'])
if len(recipients):
config = Config.get(name="email").get_values()
analysis_url = "{0}/analyses/{1}".format(fame_config.fame_url,
analysis_id)
body = notification_body_tpl.format(commentator['name'],
analysis_url,
comment['comment'])
email_server = EmailServer()
if email_server.is_connected:
msg = email_server.new_message(
"[FAME] New comment on analysis", body)
msg.send(list(recipients))
def add_extracted_file(self, filepath, automatic_analysis=True):
self.log('debug', u"Adding extracted file '{}'".format(filepath))
fd = open(filepath, 'rb')
filename = os.path.basename(filepath)
f = File(filename=filename, stream=fd, create=False)
if not f.existing:
if fame_config.remote:
response = send_file_to_remote(filepath, '/files/')
f = File(response.json()['file'])
else:
f = File(filename=os.path.basename(filepath), stream=fd)
# Automatically analyze extracted file if magic is enabled and module did not disable it
if self.magic_enabled() and automatic_analysis:
modules = None
config = Config.get(name="extracted").get_values()
if config is not None and "modules" in config:
modules = config["modules"].split()
f.analyze(self['groups'], self['analyst'], modules,
self['options'])
fd.close()
self.append_to('extracted_files', f['_id'])
f.add_parent_analysis(self)
def create_reverseit_configuration():
reverseit = Config.get(name='reverseit')
if reverseit is None:
reverseit = Config({
'name':
'reverseit',
'description':
'Reverseit API configuration, in order to be able to submit hashes.',
'config': [{
'name': 'api_key',
'description': 'Reverseit API key.',
'type': 'str',
'value': None
}]
})
reverseit.save()
def comments_enabled():
# Determine if comments are enabled
config = Config.get(name="comments")
comments_enabled = False
if config:
comments_enabled = config.get_values()['enable']
return comments_enabled
def create_extracted_schedule():
extracted = Config.get(name="extracted")
if extracted is None:
extracted = Config({
"name": "extracted",
"description": "Define which modules are scheduled by default on extracted files",
"config": [
{
"name": "modules",
"type": "text",
"value": """peepdf
document_preview
exiftool
office_macros
virustotal_public
"""
}
]
})
extracted.save()
def new(self):
config = Config.get(name="virustotal")
hash_capable = False
if config:
try:
config.get_values()
hash_capable = True
except:
hash_capable = False
return render_template('analyses/new.html', hash_capable=hash_capable, options=dispatcher.options)
def _validate_comment(self, comment):
config = Config.get(name="comments")
if config:
config = config.get_values()
if config['enable'] and config['minimum_length'] > len(comment):
flash(
'Comment has to contain at least {} characters'.format(
config['minimum_length']), 'danger')
return False
return True
def init_config(self):
for named_config in self.named_configs:
config = Config.get(name=named_config)
if config is None:
raise MissingConfiguration("Missing '{}' configuration".format(named_config))
setattr(self, named_config, config.get_values())
for config in self.info['config']:
if (config['value'] is None) and ('default' not in config):
raise MissingConfiguration("Missing configuration value: {}".format(config['name']))
setattr(self, config['name'], config['value'])
if config['value'] is None:
setattr(self, config['name'], config['default'])
def new(self):
# See if Hash submission is available
config = Config.get(name="virustotal")
hash_capable = False
if config:
try:
config.get_values()
hash_capable = True
except Exception:
hash_capable = False
return render_template('analyses/new.html',
hash_capable=hash_capable,
comments_enabled=comments_enabled(),
options=dispatcher.options)
def configuration():
print("########## Configuration ##########\n")
for config in Config.find():
print(("{}: {}".format(config['name'], not incomplete_config(config['config']))))
print("\nModules:\n")
for module in ModuleInfo.find():
state = "Disabled"
configured = "Configured"
if module['enabled']:
state = "Enabled"
if incomplete_config(module['config']):
configured = "Not Configured"
print(("{: <25} {: <20} {: <10} {: <15}".format(module['name'], module['type'], state, configured)))
def _set_type(self, hash_only=False):
if hash_only:
# cannot say anything about the file if we only know the hash
self['type'] = "hash"
return
config = Config.get(name="types").get_values()
config = ConfigObject(from_string=config.mappings)
detailed_types = config.get('details')
extensions = config.get('extensions')
self['type'] = self['mime']
# First, look at extensions
for ext in extensions:
if self['filepath'].split('.')[-1].lower() == ext:
self['type'] = extensions.get(ext)
break
# Otherwise, look in 'detailed_types'
else:
for t in detailed_types:
if self['detailed_type'].lower().startswith(t.lower()):
self['type'] = detailed_types.get(t)
break
# Or mime types
else:
types = config.get("types")
if types.get(self['mime']) is not None:
self['type'] = types.get(self['mime'])
# Run Filetype modules, starting with the most specific ones
filetype_modules = dispatcher.get_filetype_modules_for(self['type'])
filetype_modules += dispatcher.get_filetype_modules_for('*')
for module in filetype_modules:
try:
known_type = module.recognize(self['filepath'], self['type'])
if known_type:
self['type'] = known_type
break
except:
pass
def _get_object_to_analyze(self):
file = request.files.get('file') or None
url = request.form.get('url') or None
hash = request.form.get('hash') or None
f = None
if file:
f = File(filename=file.filename, stream=file.stream)
elif url:
stream = StringIO(url)
f = File(filename='url', stream=stream)
if not f.existing:
f.update_value('type', 'url')
f.update_value('names', [url])
elif hash:
config = Config.get(name="virustotal")
if config:
try:
config = config.get_values()
params = {'apikey': config.api_key, 'hash': hash}
response = requests.get(
'https://www.virustotal.com/vtapi/v2/file/download',
params=params)
if response.status_code == 403:
flash('This requires a valid API key.', 'danger')
elif response.status_code == 404:
flash('No file found with this hash.', 'danger')
elif response.status_code == 200:
f = File(filename='{}.bin'.format(hash),
stream=StringIO(response.content))
except MissingConfiguration:
flash("VirusTotal is not properly configured.", 'danger')
else:
flash(
"There seems to be a problem with your installation (no 'virustotal' configuration)",
'danger')
else:
flash('You have to submit a file, a URL or a hash', 'danger')
return f
def update(self, module, path, repository):
named_configs = []
if module.name:
module_info = ModuleInfo.get(name=module.name)
# Ignore duplicates
if module_info and not module_info['path'].startswith(
'fame.modules.{}.'.format(repository['name'])):
print "Duplicate name '{}', ignoring module.".format(
module.name)
return None
# Handle named configs
for named_config in module.named_configs:
config = Config.get(name=named_config)
# Creation
if config is None:
config = Config(module.named_config(named_config))
config.save()
# Update
else:
config.update_config(module.named_config(named_config))
named_configs.append(config)
# Handle module info
if module_info is None:
module_info = module.static_info()
module_info['enabled'] = False
else:
module_info.update_config(module.static_info())
module_info['class'] = module.__name__
module_info['path'] = path
module_info.save()
return named_configs
def _set_type(self):
config = Config.get(name="types").get_values()
config = ConfigObject(from_string=config.mappings)
detailed_types = config.get('details')
extensions = config.get('extensions')
self['type'] = self['mime']
# First, look at extensions
for ext in extensions:
if self['filepath'].split('.')[-1].lower() == ext:
self['type'] = extensions.get(ext)
break
# Otherwise, look in 'detailed_types'
else:
for t in detailed_types:
if self['detailed_type'].lower().startswith(t.lower()):
self['type'] = detailed_types.get(t)
break
# Finally, look at mime types
else:
types = config.get("types")
if types.get(self['mime']) is not None:
self['type'] = types.get(self['mime'])
def index(self):
"""Get the list of modules.
.. :quickref: Module; Get the list of modules
Requires the `manage_modules` permission.
The response is a dict with several elements:
* ``modules``, which is a list of modules, sorted by type::
"modules": {
"Antivirus": [
...
],
"Preloading": [
...
],
"Processing": [
{
"_id": {
"$oid": "MODULE_ID"
},
"acts_on": [
ACTS_ON_FAME_TYPES
],
"class": "CLASS_NAME",
"config": [ CONFIG_OPTIONS ],
"description": "DESCRIPTION",
"enabled": false,
"generates": [GENERATES],
"name": "NAME",
"path": "MODULE_PATH",
"queue": "QUEUE",
"triggered_by": [
TRIGGERS
],
"type": "Processing"
},
...
],
"Reporting": [
...
],
"Threat Intelligence": [
...
],
"Filetype": [
...
]
}
* ``repositories``: list of configured repositories::
"repositories": [
{
"_id": {
"$oid": "ID"
},
"address": "[email protected]:certsocietegenerale/fame_modules.git",
"name": "community",
"private": false,
"status": "active"
},
...
]
* ``configs``: list of named configurations::
"configs": [
{
"_id": {
"$oid": "ID"
},
"config": [
{
"description": "List of patterns (strings) to look for in malware configurations. There should be one pattern per line.",
"name": "monitor",
"type": "text",
"value": null
}
],
"description": "Needed in order to be able to track malware targets",
"name": "malware_config"
},
...
]
"""
types = {
'Preloading': [],
'Processing': [],
'Reporting': [],
'Threat Intelligence': [],
'Antivirus': [],
'Virtualization': [],
'Filetype': []
}
for module in ModuleInfo.get_collection().find():
types[module['type']].append(clean_modules(module))
for type in types:
types[type] = sorted(types[type], key=get_name)
configs = Config.get_collection().find()
repositories = clean_repositories(
list(Repository.get_collection().find()))
return render(
{
'modules': types,
'configs': configs,
'repositories': repositories
}, 'modules/index.html')
def create_types():
types = Config.get(name='types')
if types is None:
types = Config({
'name': 'types',
'description': 'Mappings for file type determination.',
'config': [
{
'name': 'mappings',
'type': 'text',
'value': """[types]
application/x-dosexec = executable
application/vnd.openxmlformats-officedocument.wordprocessingml.document = word
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet = excel
application/msword = word
application/vnd.ms-excel = excel
application/vnd.ms-powerpoint = powerpoint
text/html = html
text/rtf = rtf
application/x-coredump = memory_dump
application/pdf = pdf
application/zip = zip
text/x-mail = eml
message/rfc822 = eml
application/CDFV2-unknown = msg
application/java-archive = jar
application/x-7z-compressed = 7z
application/x-rar = rar
application/x-iso9660-image = iso
[details]
MIME entity, ISO-8859 text, with CRLF line terminators = word
MIME entity, ISO-8859 text, with very long lines, with CRLF line terminators = word
Dalvik dex file = dex
[extensions]
exe = executable
scr = executable
doc = word
docx = word
docm = word
xls = excel
xlsx = excel
xslm = excel
ppt = powerpoint
pptx = powerpoint
rtf = rtf
html = html
js = javascript
pdf = pdf
apk = apk
jar = jar
zip = zip
msg = msg
eml = eml
iso = iso
msi = executable
7z = 7z
rar = rar""",
'description': "In order to determine the file type, FAME will use the `python-magic` library. It will then try to find a match in 'mappings' for either the extension, the detailed type or the mime type (in this order of priority). If no matching type was found, the mime type will be used."
}
]
})
types.save()
