def taxonomies(): active_taxonomies = r_serv_tags.smembers('active_taxonomies') taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) id = [] name = [] description = [] version = [] enabled = [] n_tags = [] for taxonomie in list_taxonomies: id.append(taxonomie) name.append(taxonomies.get(taxonomie).name) description.append(taxonomies.get(taxonomie).description) version.append(taxonomies.get(taxonomie).version) if taxonomie in active_taxonomies: enabled.append(True) else: enabled.append(False) n = str(r_serv_tags.scard('active_tag_' + taxonomie)) n_tags.append(n + '/' + str(len(taxonomies.get(taxonomie).machinetags())) ) return render_template("taxonomies.html", id=id, all_name = name, description = description, version = version, enabled = enabled, n_tags=n_tags)
def taxonomies(): active_taxonomies = r_serv_tags.smembers('active_taxonomies') taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) id = [] name = [] description = [] version = [] enabled = [] n_tags = [] for taxonomie in list_taxonomies: id.append(taxonomie) name.append(taxonomies.get(taxonomie).name) description.append(taxonomies.get(taxonomie).description) version.append(taxonomies.get(taxonomie).version) if taxonomie in active_taxonomies: enabled.append(True) else: enabled.append(False) n = str(r_serv_tags.scard('active_tag_' + taxonomie)) n_tags.append(n + '/' + str(len(taxonomies.get(taxonomie).machinetags()))) return render_template("taxonomies.html", id=id, all_name=name, description=description, version=version, enabled=enabled, n_tags=n_tags)
def build_unsafe_tags(): unsafe_tags = set() ## CE content unsafe_tags.add('dark-web:topic="pornography-child-exploitation"') # add copine-scale tags taxonomies = Taxonomies() copine_scale = taxonomies.get('copine-scale') if copine_scale: for tag in copine_scale.machinetags(): unsafe_tags.add(tag) return unsafe_tags
def disable_taxonomie(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) id = request.args.get('taxonomie') if id in list_taxonomies: r_serv_tags.srem('active_taxonomies', id) for tag in taxonomies.get(id).machinetags(): r_serv_tags.srem('active_tag_' + id, tag) return redirect(url_for('Tags.taxonomies')) else: return "INCORRECT INPUT"
def edit_taxonomie_tag(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) arg1 = request.args.getlist('tag_enabled') arg2 = request.args.getlist('tag_disabled') id = request.args.get('taxonomie') #verify input if id in list_taxonomies: list_tag = taxonomies.get(id).machinetags() #check tags validity if (all(elem in list_tag for elem in arg1) or (len(arg1) == 0)) and (all(elem in list_tag for elem in arg2) or (len(arg2) == 0)): active_tag = r_serv_tags.smembers('active_tag_' + id) diff = list(set(arg1) ^ set(list_tag)) #remove tags for tag in diff: r_serv_tags.srem('active_tag_' + id, tag) #all tags unchecked if len(arg1) == 0 and len(arg2) == 0: r_serv_tags.srem('active_taxonomies', id) #add new tags for tag in arg2: r_serv_tags.sadd('active_taxonomies', id) r_serv_tags.sadd('active_tag_' + id, tag) return redirect(url_for('Tags.taxonomies')) else: return "INCORRECT INPUT" else: return "INCORRECT INPUT"
def enable_taxonomy(taxonomie, enable_tags=True): ''' Enable a taxonomy. (UI) :param taxonomie: MISP taxonomy :type taxonomie: str :param enable_tags: crawled domain :type enable_tags: boolean ''' taxonomies = Taxonomies() if enable_tags: taxonomie_info = taxonomies.get(taxonomie) if taxonomie_info: # activate taxonomie r_serv_tags.sadd('active_taxonomies', taxonomie) # activate taxonomie tags for tag in taxonomie_info.machinetags(): r_serv_tags.sadd('active_tag_{}'.format(taxonomie), tag) else: print('Error: {}, please update pytaxonomies'.format(taxonomie))
def edit_taxonomie_tag(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) arg1 = request.args.getlist('tag_enabled') arg2 = request.args.getlist('tag_disabled') id = request.args.get('taxonomie') #verify input if id in list_taxonomies: list_tag = taxonomies.get(id).machinetags() #check tags validity if ( all(elem in list_tag for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag for elem in arg2) or (len(arg2) == 0) ): active_tag = r_serv_tags.smembers('active_tag_' + id) diff = list(set(arg1) ^ set(list_tag)) #remove tags for tag in diff: r_serv_tags.srem('active_tag_' + id, tag) #all tags unchecked if len(arg1) == 0 and len(arg2) == 0: r_serv_tags.srem('active_taxonomies', id) #add new tags for tag in arg2: r_serv_tags.sadd('active_taxonomies', id) r_serv_tags.sadd('active_tag_' + id, tag) return redirect(url_for('Tags.taxonomies')) else: return "INCORRECT INPUT" else: return "INCORRECT INPUT"
def edit_taxonomie(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) id = request.args.get('taxonomie') #verify input if id in list(taxonomies.keys()): active_tag = r_serv_tags.smembers('active_tag_' + id) list_tag = taxonomies.get(id).machinetags() list_tag_desc = taxonomies.get(id).machinetags_expanded() active_taxonomies = r_serv_tags.smembers('active_taxonomies') if id in active_taxonomies: active = True else: active = False n = str(r_serv_tags.scard('active_tag_' + id)) badge = n + '/' + str(len(taxonomies.get(id).machinetags())) name = taxonomies.get(id).name description = taxonomies.get(id).description version = taxonomies.get(id).version status = [] for tag in list_tag: if tag in active_tag: status.append(True) else: status.append(False) return render_template("edit_taxonomie.html", id=id, name=name, badge = badge, description = description, version = version, active=active, all_tags = list_tag, list_tag_desc=list_tag_desc, status = status) else: return 'INVALID TAXONOMIE'
def edit_taxonomie(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) id = request.args.get('taxonomie') #verify input if id in list(taxonomies.keys()): active_tag = r_serv_tags.smembers('active_tag_' + id) list_tag = taxonomies.get(id).machinetags() list_tag_desc = taxonomies.get(id).machinetags_expanded() active_taxonomies = r_serv_tags.smembers('active_taxonomies') if id in active_taxonomies: active = True else: active = False n = str(r_serv_tags.scard('active_tag_' + id)) badge = n + '/' + str(len(taxonomies.get(id).machinetags())) name = taxonomies.get(id).name description = taxonomies.get(id).description version = taxonomies.get(id).version status = [] for tag in list_tag: if tag in active_tag: status.append(True) else: status.append(False) return render_template("edit_taxonomie.html", id=id, name=name, badge=badge, description=description, version=version, active=active, all_tags=list_tag, list_tag_desc=list_tag_desc, status=status) else: return 'INVALID TAXONOMIE'
return page_not_found(e) @login_required def page_not_found(e): # avoid endpoint enumeration return render_template('error/404.html'), 404 # ========== INITIAL taxonomies ============ default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"] # enable default taxonomies for taxo in default_taxonomies: Tag.enable_taxonomy(taxo) # ========== INITIAL tags auto export ============ taxonomies = Taxonomies() infoleak_tags = taxonomies.get('infoleak').machinetags() infoleak_automatic_tags = [] for tag in taxonomies.get('infoleak').machinetags(): if tag.split('=')[0][:] == 'infoleak:automatic-detection': r_serv_db.sadd('list_export_tags', tag) r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"') # ============ MAIN ============ if __name__ == "__main__": app.run(host=host, port=FLASK_PORT, threaded=True, ssl_context=ssl_context)
def tag(self): if not HAVE_PYTAX: self.log( 'error', "Missing dependency, install PyTaxonomies (`pip install git+https://github.com/MISP/PyTaxonomies.git`)" ) return taxonomies = Taxonomies() if self.args.list: self.log( 'table', dict(header=['Name', 'Description'], rows=[(title, tax.description) for title, tax in taxonomies.items()])) elif self.args.search: matches = taxonomies.search(self.args.search) if not matches: self.log('error', 'No tags matching "{}".'.format(self.args.search)) return self.log('success', 'Tags matching "{}":'.format(self.args.search)) for t in taxonomies.search(self.args.search): self.log('item', t) elif self.args.details: taxonomy = taxonomies.get(self.args.details) if not taxonomy: self.log('error', 'No taxonomy called "{}".'.format(self.args.details)) return if taxonomy.description: self.log('info', taxonomy.description) elif taxonomy.expanded: self.log('info', taxonomy.expanded) if taxonomy.refs: self.log('info', 'References:') for r in taxonomy.refs: self.log('item', r) if not taxonomy.has_entries(): header = ['Description', 'Predicate', 'Machinetag'] rows = [] for p in taxonomy.predicates.values(): rows.append( [p.description, p.predicate, taxonomy.make_machinetag(p)]) self.log('table', dict(header=header, rows=rows)) else: for p in taxonomy.predicates.values(): if p.description: self.log('info', p.description) elif p.expanded: self.log('info', p.expanded) else: self.log('info', p.predicate) if not p.entries: self.log('item', taxonomy.make_machinetag(p)) else: header = ['Description', 'Predicate', 'Machinetag'] rows = [] for e in p.entries.values(): if e.description: descr = e.description else: descr = e.expanded rows.append( [descr, e.value, taxonomy.make_machinetag(p, e)]) self.log('table', dict(header=header, rows=rows)) elif self.args.event: if not __sessions__.is_attached_misp(): return try: taxonomies.revert_machinetag(self.args.event) except Exception: self.log( 'error', 'Not a valid machine tag available in misp-taxonomies: "{}".'. format(self.args.event)) return __sessions__.current.misp_event.event.add_tag(self.args.event) self._change_event() elif self.args.attribute: if not __sessions__.is_attached_misp(): return identifier, tag = self.args.attribute try: taxonomies.revert_machinetag(tag) except Exception: self.log( 'error', 'Not a valid machine tag available in misp-taxonomies: "{}".'. format(tag)) return __sessions__.current.misp_event.event.add_attribute_tag( tag, identifier) self._change_event()
@login_required def page_not_found(e): # avoid endpoint enumeration return render_template('error/404.html'), 404 # ========== INITIAL taxonomies ============ # add default ail taxonomies r_serv_tags.sadd('active_taxonomies', 'infoleak') r_serv_tags.sadd('active_taxonomies', 'gdpr') r_serv_tags.sadd('active_taxonomies', 'fpf') # add default tags taxonomies = Taxonomies() for tag in taxonomies.get('infoleak').machinetags(): r_serv_tags.sadd('active_tag_infoleak', tag) for tag in taxonomies.get('gdpr').machinetags(): r_serv_tags.sadd('active_tag_gdpr', tag) for tag in taxonomies.get('fpf').machinetags(): r_serv_tags.sadd('active_tag_fpf', tag) # ========== INITIAL tags auto export ============ infoleak_tags = taxonomies.get('infoleak').machinetags() infoleak_automatic_tags = [] for tag in taxonomies.get('infoleak').machinetags(): if tag.split('=')[0][:] == 'infoleak:automatic-detection': r_serv_db.sadd('list_export_tags', tag) r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"') # ============ MAIN ============
def tag(self): if not HAVE_PYTAX: self.log('error', "Missing dependency, install PyTaxonomies (`pip install git+https://github.com/MISP/PyTaxonomies.git`)") return try: taxonomies = Taxonomies(manifest_path=os.path.join(self.local_dir_taxonomies, 'MANIFEST.json')) except Exception as e: self.log('error', 'Unable to open the taxonomies, please fix the config file ([misp] - misp_taxonomies_directory): {}'.format(e)) return if self.args.list: self.log('table', dict(header=['Name', 'Description'], rows=[(title, tax.description) for title, tax in taxonomies.items()])) elif self.args.search: matches = taxonomies.search(self.args.search) if not matches: self.log('error', 'No tags matching "{}".'.format(self.args.search)) return self.log('success', 'Tags matching "{}":'.format(self.args.search)) for t in taxonomies.search(self.args.search): self.log('item', t) elif self.args.details: taxonomy = taxonomies.get(self.args.details) if not taxonomy: self.log('error', 'No taxonomy called "{}".'.format(self.args.details)) return if taxonomy.description: self.log('info', taxonomy.description) elif taxonomy.expanded: self.log('info', taxonomy.expanded) if taxonomy.refs: self.log('info', 'References:') for r in taxonomy.refs: self.log('item', r) if not taxonomy.has_entries(): header = ['Description', 'Predicate', 'Machinetag'] rows = [] for p in taxonomy.predicates.values(): rows.append([p.description, p.predicate, taxonomy.make_machinetag(p)]) self.log('table', dict(header=header, rows=rows)) else: for p in taxonomy.predicates.values(): if p.description: self.log('info', p.description) elif p.expanded: self.log('info', p.expanded) else: self.log('info', p.predicate) if not p.entries: self.log('item', taxonomy.make_machinetag(p)) else: header = ['Description', 'Predicate', 'Machinetag'] rows = [] for e in p.entries.values(): if e.description: descr = e.description else: descr = e.expanded rows.append([descr, e.value, taxonomy.make_machinetag(p, e)]) self.log('table', dict(header=header, rows=rows)) elif self.args.event: if not __sessions__.is_attached_misp(): return try: taxonomies.revert_machinetag(self.args.event) except: self.log('error', 'Not a valid machine tag available in misp-taxonomies: "{}".'.format(self.args.event)) return __sessions__.current.misp_event.event.add_tag(self.args.event) self._change_event() elif self.args.attribute: if not __sessions__.is_attached_misp(): return identifier, tag = self.args.attribute try: taxonomies.revert_machinetag(tag) except: self.log('error', 'Not a valid machine tag available in misp-taxonomies: "{}".'.format(tag)) return __sessions__.current.misp_event.event.add_attribute_tag(tag, identifier) self._change_event()
return render_template("searchbox.html") # ========== INITIAL taxonomies ============ r_serv_tags = redis.StrictRedis( host=cfg.get("ARDB_Tags", "host"), port=cfg.getint("ARDB_Tags", "port"), db=cfg.getint("ARDB_Tags", "db"), decode_responses=True) # add default ail taxonomies r_serv_tags.sadd('active_taxonomies', 'infoleak') r_serv_tags.sadd('active_taxonomies', 'gdpr') r_serv_tags.sadd('active_taxonomies', 'fpf') # add default tags taxonomies = Taxonomies() for tag in taxonomies.get('infoleak').machinetags(): r_serv_tags.sadd('active_tag_infoleak', tag) for tag in taxonomies.get('gdpr').machinetags(): r_serv_tags.sadd('active_tag_gdpr', tag) for tag in taxonomies.get('fpf').machinetags(): r_serv_tags.sadd('active_tag_fpf', tag) # ========== INITIAL tags auto export ============ r_serv_db = redis.StrictRedis( host=cfg.get("ARDB_DB", "host"), port=cfg.getint("ARDB_DB", "port"), db=cfg.getint("ARDB_DB", "db"), decode_responses=True) infoleak_tags = taxonomies.get('infoleak').machinetags() infoleak_automatic_tags = [] for tag in taxonomies.get('infoleak').machinetags():