def get(self, id): """Get a user. .. :quickref: User; Get a user The user is returned in the ``user`` field. :param id: user id :>json ObjectId _id: user's ObjectId. :>json string name: full name. :>json string: email address. :>json boolean enabled: ``True`` if the user is enabled. :>json list groups: list of groups the user belongs to. :>json list default_sharing: list of groups used by the user as default sharing preferences. :>json list permissions: list of user's permissions """ self.ensure_permission(id) user = User(get_or_404(User.get_collection(), _id=id)) return render( { 'user': clean_users(user), 'permissions': dispatcher.permissions }, 'users/profile.html')
def index(self): """Get the list of analyses. .. :quickref: Analysis; Get the list of analyses Response is paginated and will only contain 25 results. The most recent analyses appear first. :query page: page number. :type page: int :>json list analyses: list of analyses (see :http:get:`/analyses/(id)` for details on the format of an analysis). """ page = int(request.args.get('page', 1)) analyses = current_user.analyses.find().sort('_id', DESCENDING).limit(PER_PAGE).skip((page - 1) * PER_PAGE) pagination = Pagination(page=page, per_page=PER_PAGE, total=analyses.count(), css_framework='bootstrap3') analyses = {'analyses': clean_analyses(list(analyses))} for analysis in analyses['analyses']: file = current_user.files.find_one({'_id': analysis['file']}) analysis['file'] = clean_files(file) if 'analyst' in analysis: analyst = store.users.find_one({'_id': analysis['analyst']}) analysis['analyst'] = clean_users(analyst) return render(analyses, 'analyses/index.html', ctx={'data': analyses, 'pagination': pagination})
def configuration(self, id): """Configure a named configuration. .. :quickref: Module; Configure a named configuration Requires the `manage_modules` permission. For each configuration available, you should set the value in a form parameter named ``config_NAME``. For boolean values, any value not ``0`` or ``False`` is considered to be ``True``. If successful, will return the named configuration in ``config``. Otherwise, errors will be available in ``errors``. :param id: id of the named configuration. """ config = Config(get_or_404(Config.get_collection(), _id=id)) if request.method == "POST": errors = update_config(config['config']) if errors is not None: return errors config.save() dispatcher.reload() return redirect({'config': config}, url_for('ModulesView:index')) else: return render({'config': config}, 'modules/configuration.html')
def index(self): """Get the list of objects. .. :quickref: File; Get the list of objects Response is paginated and will only contain 25 results. The most recent objects appear first. :query page: page number. :type page: int :>json list files: list of files (see :http:get:`/files/(id)` for details on the format of a file). """ page = int(request.args.get('page', 1)) files = current_user.files.find().sort( '_id', DESCENDING).limit(PER_PAGE).skip((page - 1) * PER_PAGE) pagination = Pagination(page=page, per_page=PER_PAGE, total=files.count(), css_framework='bootstrap3') files = {'files': clean_files(list(files))} return render(files, 'files/index.html', ctx={ 'data': files, 'pagination': pagination })
def get(self, id): """Get the analysis with `id`. .. :quickref: Analysis; Get an analysis Resulting object is in the ``analysis`` field. :param id: id of the analysis. :>json dict _id: ObjectId dict. :>json dict analyst: analyst's ObjectId. :>json dict date: date dict. :>json list executed_modules: list of executed modules. :>json list pending_modules: list of pending modules. :>json list waiting_modules: list of waiting modules. :>json list canceled_modules: list of canceled modules. :>json list executed_modules: list of executed modules. :>json string module: the name of the target module. :>json string status: status of the analysis (one of `pending`, `running`, `finished` or `error`). :>json list tags: the list of tags. :>json list probable_names: the list of probable names. :>json list iocs: list of dict describing observables. :>json dict results: detailed results for each module, with the module name being the key. :>json dict generated_files: a dict of generated files, the key being the file type. :>json list extracted_files: a list of extracted files. :>json dict support_files: a dict of support files, the key being the module name. """ analysis = { 'analysis': clean_analyses(get_or_404(current_user.analyses, _id=id)) } file = current_user.files.find_one( {'_id': analysis['analysis']['file']}) analysis['analysis']['file'] = clean_files(file) ti_modules = [ m.name for m in dispatcher.get_threat_intelligence_modules() ] av_modules = [m.name for m in dispatcher.get_antivirus_modules()] if 'extracted_files' in analysis['analysis']: files = [] for id in analysis['analysis']['extracted_files']: files.append(current_user.files.find_one({'_id': id})) analysis['analysis']['extracted_files'] = clean_files(files) modules = dict() for module in ModuleInfo.get_collection().find(): modules[module['name']] = ModuleInfo(module) return render(analysis, 'analyses/show.html', ctx={ 'analysis': analysis, 'modules': modules, 'av_modules': av_modules, 'ti_modules': ti_modules })
def repository_new(self): """Add a repository .. :quickref: Module; Add repository Requires the `manage_modules` permission. If successful, will return the repository in ``repository``. Otherwise, errors will be available in ``errors``. :form name: name of the repository (should be a valid package name). :form address: HTTPs or SSH address of the repository. :form private: boolean specifying if the repository is private. See Administration Guide for more details on private repositories. """ deploy_key = get_deploy_key() repository = Repository() if request.method == 'POST': for field in ['name', 'address']: repository[field] = request.form.get(field) if repository[field] is None or repository[field] == "": flash("{} is required.".format(field), 'danger') return validation_error() existing_repository = Repository.get( **{field: repository[field]}) if existing_repository: flash( "There is already a repository with this {}.".format( field), 'danger') return validation_error() value = request.form.get('private') repository['private'] = (value is not None) and (value not in [ '0', 'False' ]) if repository['private'] and deploy_key is None: flash( "Private repositories are disabled because of a problem with your installation (you do not have a deploy key in 'conf/id_rsa.pub')", 'danger') return validation_error() repository['status'] = 'cloning' repository.save() repository.clone() return redirect({'repository': clean_repositories(repository)}, url_for('ModulesView:index')) return render({ 'repository': repository, 'deploy_key': deploy_key }, 'modules/repository_new.html')
def list(self): """List enabled Processing modules .. :quickref: Module; List enabled Processing modules :>json list modules: list of enabled modules. """ modules = ModuleInfo.get_collection().find({ 'enabled': True, 'type': 'Processing' }) return render({'modules': clean_modules(list(modules))})
def index(self): """Get the index of malware configurations. .. :quickref: Malware Configurations; Get the index Requires the `config` permission. The response is a dict with the following format:: { "MONITOR1": { "active_botnets": [ "FAMILY:BOTNETID" ], "botnets": [ "FAMILY:BOTNETID" ], "count": 1, "targets": { "TARGET1": { "active_botnets": [ "FAMILY:BOTNETID" ], "botnets": [ "FAMILY:BOTNETID" ], "count": 1 }, "TARGET2": { ... } } }, "MONITOR2": { ... } } """ monitors = {} for record in targets_aggregation(): clean_record(record) monitor = get_monitor(monitors, record) target = get_target(monitor, record) increment([monitor, target], 'count', record['count']) add([monitor, target], 'botnets', record['botnet']) if record['last_action'] != ACTION_REMOVED: add([monitor, target], 'active_botnets', record['botnet']) return render(monitors, 'configs/index.html')
def return_file(file): analyses = list(current_user.analyses.find({'_id': {'$in': file['file']['analysis']}})) file['av_modules'] = [m.name for m in dispatcher.get_antivirus_modules()] for analysis in analyses: if 'analyst' in analysis: analyst = store.users.find_one({'_id': analysis['analyst']}) analysis['analyst'] = clean_users(analyst) file['file']['analysis'] = clean_analyses(analyses) return render(file, 'files/show.html', ctx={ 'data': file, 'options': dispatcher.options, 'comments_enabled': comments_enabled()})
def list(self): """List enabled Processing and Preloading modules .. :quickref: Module; List enabled Processing and Preloading modules :>json list modules: list of enabled modules. """ modules = ModuleInfo.get_collection().find({ 'enabled': True, 'type': { '$in': ['Processing', 'Preloading'] } }) return render(clean_modules(list(modules)))
def get(self, id): """Get the object with `id`. .. :quickref: File; Get an object Resulting object is in the ``file`` field. :param id: id of the object. :>json dict _id: ObjectId dict. :>json string md5: MD5 hash. :>json string sha1: SHA1 hash. :>json string sha256: SHA256 hash. :>json string type: FAME type. :>json string mime: mime type. :>json string detailed_type: detailed type. :>json list groups: list of groups (as strings) that have access to this file. :>json list owners: list of groups (as strings) that submitted this file. :>json list probable_names: list of probable names (as strings). :>json list analysis: list of analyses' ObjectIds. :>json list parent_analyses: list of analyses (as ObjectIds) that extracted this object. :>json dict antivirus: dict with antivirus names as keys. """ file = {'file': clean_files(get_or_404(current_user.files, _id=id))} analyses = list( current_user.analyses.find( {'_id': { '$in': file['file']['analysis'] }})) file['av_modules'] = [ m.name for m in dispatcher.get_antivirus_modules() ] for analysis in analyses: if 'analyst' in analysis: analyst = store.users.find_one({'_id': analysis['analyst']}) analysis['analyst'] = clean_users(analyst) file['file']['analysis'] = clean_analyses(analyses) return render(file, 'files/show.html', ctx={ 'data': file, 'options': dispatcher.options })
def index(self): """Get all users. .. :quickref: User; Get the list of users Requires the `manage_users` permission. The result is in the ``users`` field. :>jsonarr ObjectId _id: user's ObjectId. :>jsonarr string name: full name. :>jsonarr string: email address. :>jsonarr boolean enabled: ``True`` if the user is enabled. :>jsonarr list groups: list of groups the user belongs to. :>jsonarr list default_sharing: list of groups used by the user as default sharing preferences. :>jsonarr list permissions: list of user's permissions """ users = {"users": clean_users(list(User.find()))} return render(users, 'users/index.html')
def return_file(file): analyses = list( current_user.analyses.find({"_id": { "$in": file["file"]["analysis"] }})) file["av_modules"] = [m.name for m in dispatcher.get_antivirus_modules()] for analysis in analyses: if "analyst" in analysis: analyst = store.users.find_one({"_id": analysis["analyst"]}) analysis["analyst"] = clean_users(analyst) file["file"]["analysis"] = clean_analyses(analyses) return render( file, "files/show.html", ctx={ "data": file, "options": dispatcher.options, "comments_enabled": comments_enabled() }, )
def post(self): query = request.form['query'] files = [] for file in current_user.files.find({'$text': {'$search': query}}): files.append(file) analyses = [] for analysis in current_user.analyses.find( {'$text': { '$search': query }}): file = current_user.files.find_one({'_id': analysis['file']}) analysis['file'] = clean_files(file) analyses.append(analysis) results = { 'files': clean_files(files), 'analyses': clean_analyses(analyses) } return render(results, 'search.html')
def index(self): pending_analyses = [] stale_analyses = [] for analysis in store.analysis.find({'status': 'pending'}): file = store.files.find_one({'_id': analysis['file']}) analysis['file'] = file pending_analyses.append(analysis) for analysis in store.analysis.find({ 'status': 'running', 'waiting_modules': { '$ne': [] } }): file = store.files.find_one({'_id': analysis['file']}) analysis['file'] = file stale_analyses.append(analysis) return render( { 'pending_analyses': pending_analyses, 'stale_analyses': stale_analyses }, "system/index.html")
def post(self): file = request.files['file'] f = File(filename=secure_filename(file.filename), stream=file.stream) return render({'file': f})
def configure(self, id): """Configure a module. .. :quickref: Module; Configure a module Requires the `manage_modules` permission. For each configuration available, you should set the value in a form parameter named ``config_NAME``. For boolean values, any value not ``0`` or ``False`` is considered to be ``True``. If the setting should be an option (be available per analysis), you have to set ``config_NAME_option`` to any value but ``0`` or ``False``. If successful, will return the module in ``module``. Otherwise, errors will be available in ``errors``. :param id: id of the named configuration. :form acts_on: comma-delimited list of FAME types this module can act on (only for Processing modules). :form triggered_by: comma-delimited list of triggers (only for Processing modules). :form queue: name of the queue to use for this module (for Processing and Preloading modules). """ module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id)) module['readme'] = module.get_readme() if request.method == "POST": if module['type'] == 'Filetype': if 'acts_on' in request.form: module.update_setting_value( 'acts_on', request.form.get('acts_on', '')) elif module['type'] == 'Processing': if 'acts_on' in request.form: module.update_setting_value( 'acts_on', request.form.get('acts_on', '')) if 'triggered_by' in request.form: module.update_setting_value( 'triggered_by', request.form.get('triggered_by', '')) if 'queue' in request.form: update_queue(module, request.form.get('queue', '')) elif module['type'] == "Preloading": if 'queue' in request.form: update_queue(module, request.form.get('queue', '')) if 'priority' in request.form: update_priority(module, request.form.get('priority', '')) errors = update_config(module['config'], options=(module['type'] in ['Preloading', 'Processing'])) if errors is not None: return errors module.save() dispatcher.reload() return redirect({'module': clean_modules(module)}, url_for('ModulesView:index')) else: return render({'module': clean_modules(module)}, 'modules/module_configuration.html')
def index(self): """Get the list of modules. .. :quickref: Module; Get the list of modules Requires the `manage_modules` permission. The response is a dict with several elements: * ``modules``, which is a list of modules, sorted by type:: "modules": { "Antivirus": [ ... ], "Preloading": [ ... ], "Processing": [ { "_id": { "$oid": "MODULE_ID" }, "acts_on": [ ACTS_ON_FAME_TYPES ], "class": "CLASS_NAME", "config": [ CONFIG_OPTIONS ], "description": "DESCRIPTION", "enabled": false, "generates": [GENERATES], "name": "NAME", "path": "MODULE_PATH", "queue": "QUEUE", "triggered_by": [ TRIGGERS ], "type": "Processing" }, ... ], "Reporting": [ ... ], "Threat Intelligence": [ ... ], "Filetype": [ ... ] } * ``repositories``: list of configured repositories:: "repositories": [ { "_id": { "$oid": "ID" }, "address": "[email protected]:certsocietegenerale/fame_modules.git", "name": "community", "private": false, "status": "active" }, ... ] * ``configs``: list of named configurations:: "configs": [ { "_id": { "$oid": "ID" }, "config": [ { "description": "List of patterns (strings) to look for in malware configurations. There should be one pattern per line.", "name": "monitor", "type": "text", "value": null } ], "description": "Needed in order to be able to track malware targets", "name": "malware_config" }, ... ] """ types = { 'Preloading': [], 'Processing': [], 'Reporting': [], 'Threat Intelligence': [], 'Antivirus': [], 'Virtualization': [], 'Filetype': [] } for module in ModuleInfo.get_collection().find(): types[module['type']].append(clean_modules(module)) for type in types: types[type] = sorted(types[type], key=get_name) configs = Config.get_collection().find() repositories = clean_repositories( list(Repository.get_collection().find())) return render( { 'modules': types, 'configs': configs, 'repositories': repositories }, 'modules/index.html')
def new(self): context = {'user': {}, 'permissions': dispatcher.permissions} return render(context, 'users/new.html')
def configure(self, id): """Configure a module. .. :quickref: Module; Configure a module Requires the `manage_modules` permission. For each configuration available, you should set the value in a form parameter named ``config_NAME``. For boolean values, any value not ``0`` or ``False`` is considered to be ``True``. If the setting should be an option (be available per analysis), you have to set ``config_NAME_option`` to any value but ``0`` or ``False``. If successful, will return the module in ``module``. Otherwise, errors will be available in ``errors``. :param id: id of the named configuration. :form acts_on: comma-delimited list of FAME types this module can act on (only for Processing modules). :form triggered_by: comma-delimited list of triggers (only for Processing modules). :form queue: name of the queue to use for this module (only for Processing modules). """ module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id)) if request.method == "POST": if module['type'] == 'Processing': if 'acts_on' in request.form: module.update_setting_value( 'acts_on', request.form.get('acts_on', '')) if 'triggered_by' in request.form: module.update_setting_value( 'triggered_by', request.form.get('triggered_by', '')) if 'queue' in request.form: new_queue = request.form.get('queue') if module['queue'] == '': flash('queue cannot be empty', 'danger') return validation_error() else: if module['queue'] != new_queue: module.update_setting_value('queue', new_queue) updates = Internals( get_or_404(Internals.get_collection(), name="updates")) updates.update_value("last_update", time()) flash( 'Workers will reload once they are done with their current tasks', 'success') errors = update_config(module['config'], options=(module['type'] == 'Processing')) if errors is not None: return errors module.save() dispatcher.reload() return redirect({'module': clean_modules(module)}, url_for('ModulesView:index')) else: return render({'module': clean_modules(module)}, 'modules/module_configuration.html')
def show(self): """Get a malware configuration timeline. .. :quickref: Malware Configurations; Get a timeline Requires the `config` permission. You can get the timeline of your choice by conbining several filters. :query monitor: (optional) filter by monitor. :query target: (optional) filter by target. :query botnet: (optional) filter by botnet. :query type: (optional) filter by type. :>json list botnets: the list of available botnets. :>json list monitors: the list of available monitors. :>json list targets: the list of available targets. :>json list types: the list of available configuration block types. :>json list config_blocks: a sorted list of configuration blocks matching this query. Each configuration block has the following format:: { "_id": { "$oid": "CONFIG_BLOCK_ID" }, "action": "CONFIG_BLOCK_ACTION", # new, update, removed or added "additional": null, "analyses": [ { "$oid": "ANALYSIS_ID" } ], "botnet": "FAMILY:BOTNETID", "content": "CONFIG_BLOCK_CONTENT", "created": { "$date": CREATION_DATE }, "monitor": "MATCHING_MONITOR", "target": "TARGET", "type": "CONFIG_BLOCK_TYPE", "updated": { "$date": MODIFICATION_DATE } } """ query = build_query(['monitor', 'target', 'botnet', 'type']) history = {} monitors = set() targets = set() types = set() botnets = set() config_blocks = [] for block in store.config_blocks.find(query).sort('updated'): config_blocks.append(block) monitors.add(block['monitor']) targets.add(block['target']) types.add(block['type']) botnets.add(block['botnet']) label = "{}:{}:{}".format(block['target'], block['type'], block['botnet']) if block['action'] == ACTION_UPDATE: block['diff'] = ''.join( ndiff(history[label].splitlines(1), block['content'].splitlines(1))) if block['action'] != ACTION_REMOVED: history[label] = block['content'] result = { 'config_blocks': config_blocks, 'monitors': monitors, 'targets': targets, 'types': types, 'botnets': botnets } return render(result, 'configs/show.html')