def update_modules(self, repository): base_dir = path.dirname(path.dirname(path.abspath(__file__))) modules_dir = path.join(base_dir, 'modules') self._remove_compiled_files(modules_dir) installed_modules = self.list_installed_modules_for(repository) updated_named_configs = [] for name, obj in self.walk_modules(modules_dir, repository=repository): updated_named_configs += self.update(obj, name, repository) if obj.name: installed_modules.discard(obj.name) # Delete all modules that are no longer in the repository for missing_module in installed_modules: print "Deleting '{}'".format(missing_module) ModuleInfo.get_collection().remove({'name': missing_module}) # Disable all modules that have incomplete named configs for updated_named_config in unique_for_key(updated_named_configs, 'name'): if incomplete_config(updated_named_config['config']): for name, obj in self.walk_modules(modules_dir): if obj.name and updated_named_config[ 'name'] in obj.named_configs: info = ModuleInfo.get(name=obj.name) if info['enabled']: print "Disabling {} for incomplete named config {}".format( obj.name, updated_named_config['name']) info.update_value('enabled', False)
def get(self, id): """Get the analysis with `id`. .. :quickref: Analysis; Get an analysis Resulting object is in the ``analysis`` field. :param id: id of the analysis. :>json dict _id: ObjectId dict. :>json dict analyst: analyst's ObjectId. :>json dict date: date dict. :>json list executed_modules: list of executed modules. :>json list pending_modules: list of pending modules. :>json list waiting_modules: list of waiting modules. :>json list canceled_modules: list of canceled modules. :>json list executed_modules: list of executed modules. :>json string module: the name of the target module. :>json string status: status of the analysis (one of `pending`, `running`, `finished` or `error`). :>json list tags: the list of tags. :>json list probable_names: the list of probable names. :>json list iocs: list of dict describing observables. :>json dict results: detailed results for each module, with the module name being the key. :>json dict generated_files: a dict of generated files, the key being the file type. :>json list extracted_files: a list of extracted files. :>json dict support_files: a dict of support files, the key being the module name. """ analysis = { 'analysis': clean_analyses(get_or_404(current_user.analyses, _id=id)) } file = current_user.files.find_one( {'_id': analysis['analysis']['file']}) analysis['analysis']['file'] = clean_files(file) ti_modules = [ m.name for m in dispatcher.get_threat_intelligence_modules() ] av_modules = [m.name for m in dispatcher.get_antivirus_modules()] if 'extracted_files' in analysis['analysis']: files = [] for id in analysis['analysis']['extracted_files']: files.append(current_user.files.find_one({'_id': id})) analysis['analysis']['extracted_files'] = clean_files(files) modules = dict() for module in ModuleInfo.get_collection().find(): modules[module['name']] = ModuleInfo(module) return render(analysis, 'analyses/show.html', ctx={ 'analysis': analysis, 'modules': modules, 'av_modules': av_modules, 'ti_modules': ti_modules })
def disable(self, id): """Disable a module .. :quickref: Module; Disable a module Requires the `manage_modules` permission. :param id: id of the module to disable. :>json Module module: resulting module. """ module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id)) module.update_value('enabled', False) dispatcher.reload() return redirect({'module': clean_modules(module)}, url_for('ModulesView:index'))
def load_all_modules(self): for module in ModuleInfo.get_collection().find({'enabled': True}): if module['type'] == "Processing": self.add_processing_module(module) elif module['type'] == "Virtualization": self.add_virtualization_module(module) else: self.add_module(module)
def list_installed_modules_for(self, repository): results = set() for module in ModuleInfo.get_collection().find(): if module['path'].startswith('fame.modules.{}.'.format( repository['name'])): results.add(module['name']) return results
def update_module_requirements(self): for module in ModuleInfo.get_collection().find(): module = ModuleInfo(module) if 'error' in module: del(module['error']) if module['type'] == "Processing": should_update = (module['queue'] in self.queues) elif module['type'] in ["Threat Intelligence", "Reporting", "Filetype"]: should_update = True else: should_update = (not fame_config.remote) if should_update: self.update_python_requirements(module) self.launch_install_scripts(module) module.save()
def list(self): """List enabled Processing modules .. :quickref: Module; List enabled Processing modules :>json list modules: list of enabled modules. """ modules = ModuleInfo.get_collection().find({ 'enabled': True, 'type': 'Processing' }) return render({'modules': clean_modules(list(modules))})
def enable(self, id): """Enable a module .. :quickref: Module; Enable a module Requires the `manage_modules` permission. If successful, will return the module in ``module``. Otherwise, errors will be available in ``errors``. :param id: id of the module to enable. """ module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id)) if 'error' in module: flash( "Cannot enable '{}' because of errors installing dependencies." .format(module['name']), 'danger') return validation_error(url_for('ModulesView:index')) # See if module is properly configured module_class = get_class(module['path'], module['class']) module_class.info = module try: module_class() except MissingConfiguration, e: if e.name: flash( "You must configure '{}' before trying to enable '{}'". format(e.name, module['name']), 'warning') return validation_error( url_for('ModulesView:configuration', id=e.id)) else: flash( "You must configure '{}' before trying to enable it.". format(module['name']), 'warning') return validation_error( url_for('ModulesView:configure', id=module['_id']))
def delete(self): # First, remove modules from database for module in ModuleInfo.find(): if module['path'].startswith('fame.modules.{}.'.format(self['name'])): module.delete() # Then, delete the files try: rmtree(self.path()) except: pass # Finally, delete record of repository MongoDict.delete(self)
def _validate_form(self, groups, module): for group in groups: if group in current_user['groups']: break else: flash('You have to at least share with one of your groups.', 'danger') return False if module: if not ModuleInfo.get(name=module): flash('"{}" is not a valid module'.format(module), 'danger') return False return True
def list(self): """List enabled Processing and Preloading modules .. :quickref: Module; List enabled Processing and Preloading modules :>json list modules: list of enabled modules. """ modules = ModuleInfo.get_collection().find({ 'enabled': True, 'type': { '$in': ['Processing', 'Preloading'] } }) return render(clean_modules(list(modules)))
def configuration(): print("########## Configuration ##########\n") for config in Config.find(): print(("{}: {}".format(config['name'], not incomplete_config(config['config'])))) print("\nModules:\n") for module in ModuleInfo.find(): state = "Disabled" configured = "Configured" if module['enabled']: state = "Enabled" if incomplete_config(module['config']): configured = "Not Configured" print(("{: <25} {: <20} {: <10} {: <15}".format(module['name'], module['type'], state, configured)))
def _validate_form(self, groups, modules, options): for group in groups: if group in current_user['groups']: break else: flash('You have to at least share with one of your groups.', 'danger') return False if modules: for module in modules: if not ModuleInfo.get(name=module): flash('"{}" is not a valid module'.format(module), 'danger') return False else: if not options['magic_enabled']: flash('You have to select at least one module to execute when magic is disabled', 'danger') return False return True
def next_preloading_module(self, selected_modules=[], excluded_modules=[]): candidate_module = None smallest_priority = None for module in self.get_preloading_modules(): if ((not selected_modules or module.info['name'] in selected_modules) and module.info['name'] not in excluded_modules): module_info = ModuleInfo.get(name=module.info['name']) if smallest_priority is None or module_info[ 'priority'] < smallest_priority: candidate_module = module_info['name'] smallest_priority = module_info['priority'] if candidate_module: return candidate_module else: raise DispatchingException("No more preloading module available")
def update(self, module, path, repository): named_configs = [] if module.name: module_info = ModuleInfo.get(name=module.name) # Ignore duplicates if module_info and not module_info['path'].startswith( 'fame.modules.{}.'.format(repository['name'])): print "Duplicate name '{}', ignoring module.".format( module.name) return None # Handle named configs for named_config in module.named_configs: config = Config.get(name=named_config) # Creation if config is None: config = Config(module.named_config(named_config)) config.save() # Update else: config.update_config(module.named_config(named_config)) named_configs.append(config) # Handle module info if module_info is None: module_info = module.static_info() module_info['enabled'] = False else: module_info.update_config(module.static_info()) module_info['class'] = module.__name__ module_info['path'] = path module_info.save() return named_configs
def index(self): """Get the list of modules. .. :quickref: Module; Get the list of modules Requires the `manage_modules` permission. The response is a dict with several elements: * ``modules``, which is a list of modules, sorted by type:: "modules": { "Antivirus": [ ... ], "Preloading": [ ... ], "Processing": [ { "_id": { "$oid": "MODULE_ID" }, "acts_on": [ ACTS_ON_FAME_TYPES ], "class": "CLASS_NAME", "config": [ CONFIG_OPTIONS ], "description": "DESCRIPTION", "enabled": false, "generates": [GENERATES], "name": "NAME", "path": "MODULE_PATH", "queue": "QUEUE", "triggered_by": [ TRIGGERS ], "type": "Processing" }, ... ], "Reporting": [ ... ], "Threat Intelligence": [ ... ], "Filetype": [ ... ] } * ``repositories``: list of configured repositories:: "repositories": [ { "_id": { "$oid": "ID" }, "address": "[email protected]:certsocietegenerale/fame_modules.git", "name": "community", "private": false, "status": "active" }, ... ] * ``configs``: list of named configurations:: "configs": [ { "_id": { "$oid": "ID" }, "config": [ { "description": "List of patterns (strings) to look for in malware configurations. There should be one pattern per line.", "name": "monitor", "type": "text", "value": null } ], "description": "Needed in order to be able to track malware targets", "name": "malware_config" }, ... ] """ types = { 'Preloading': [], 'Processing': [], 'Reporting': [], 'Threat Intelligence': [], 'Antivirus': [], 'Virtualization': [], 'Filetype': [] } for module in ModuleInfo.get_collection().find(): types[module['type']].append(clean_modules(module)) for type in types: types[type] = sorted(types[type], key=get_name) configs = Config.get_collection().find() repositories = clean_repositories( list(Repository.get_collection().find())) return render( { 'modules': types, 'configs': configs, 'repositories': repositories }, 'modules/index.html')
def configure(self, id): """Configure a module. .. :quickref: Module; Configure a module Requires the `manage_modules` permission. For each configuration available, you should set the value in a form parameter named ``config_NAME``. For boolean values, any value not ``0`` or ``False`` is considered to be ``True``. If the setting should be an option (be available per analysis), you have to set ``config_NAME_option`` to any value but ``0`` or ``False``. If successful, will return the module in ``module``. Otherwise, errors will be available in ``errors``. :param id: id of the named configuration. :form acts_on: comma-delimited list of FAME types this module can act on (only for Processing modules). :form triggered_by: comma-delimited list of triggers (only for Processing modules). :form queue: name of the queue to use for this module (for Processing and Preloading modules). """ module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id)) module['readme'] = module.get_readme() if request.method == "POST": if module['type'] == 'Filetype': if 'acts_on' in request.form: module.update_setting_value( 'acts_on', request.form.get('acts_on', '')) elif module['type'] == 'Processing': if 'acts_on' in request.form: module.update_setting_value( 'acts_on', request.form.get('acts_on', '')) if 'triggered_by' in request.form: module.update_setting_value( 'triggered_by', request.form.get('triggered_by', '')) if 'queue' in request.form: update_queue(module, request.form.get('queue', '')) elif module['type'] == "Preloading": if 'queue' in request.form: update_queue(module, request.form.get('queue', '')) if 'priority' in request.form: update_priority(module, request.form.get('priority', '')) errors = update_config(module['config'], options=(module['type'] in ['Preloading', 'Processing'])) if errors is not None: return errors module.save() dispatcher.reload() return redirect({'module': clean_modules(module)}, url_for('ModulesView:index')) else: return render({'module': clean_modules(module)}, 'modules/module_configuration.html')
def configure(self, id): """Configure a module. .. :quickref: Module; Configure a module Requires the `manage_modules` permission. For each configuration available, you should set the value in a form parameter named ``config_NAME``. For boolean values, any value not ``0`` or ``False`` is considered to be ``True``. If the setting should be an option (be available per analysis), you have to set ``config_NAME_option`` to any value but ``0`` or ``False``. If successful, will return the module in ``module``. Otherwise, errors will be available in ``errors``. :param id: id of the named configuration. :form acts_on: comma-delimited list of FAME types this module can act on (only for Processing modules). :form triggered_by: comma-delimited list of triggers (only for Processing modules). :form queue: name of the queue to use for this module (only for Processing modules). """ module = ModuleInfo(get_or_404(ModuleInfo.get_collection(), _id=id)) if request.method == "POST": if module['type'] == 'Processing': if 'acts_on' in request.form: module.update_setting_value( 'acts_on', request.form.get('acts_on', '')) if 'triggered_by' in request.form: module.update_setting_value( 'triggered_by', request.form.get('triggered_by', '')) if 'queue' in request.form: new_queue = request.form.get('queue') if module['queue'] == '': flash('queue cannot be empty', 'danger') return validation_error() else: if module['queue'] != new_queue: module.update_setting_value('queue', new_queue) updates = Internals( get_or_404(Internals.get_collection(), name="updates")) updates.update_value("last_update", time()) flash( 'Workers will reload once they are done with their current tasks', 'success') errors = update_config(module['config'], options=(module['type'] == 'Processing')) if errors is not None: return errors module.save() dispatcher.reload() return redirect({'module': clean_modules(module)}, url_for('ModulesView:index')) else: return render({'module': clean_modules(module)}, 'modules/module_configuration.html')