def setUp(self): self.maxDiff = None self.taxonomies_offline = Taxonomies() self.loaded_tax = {} for t in self.taxonomies_offline.manifest['taxonomies']: with open( '{}/{}/{}'.format(self.taxonomies_offline.url, t['name'], 'machinetag.json'), 'r') as f: self.loaded_tax[t['name']] = json.load(f)
def build_unsafe_tags(): unsafe_tags = set() ## CE content unsafe_tags.add('dark-web:topic="pornography-child-exploitation"') # add copine-scale tags taxonomies = Taxonomies() copine_scale = taxonomies.get('copine-scale') if copine_scale: for tag in copine_scale.machinetags(): unsafe_tags.add(tag) return unsafe_tags
def explore_pytaxonomies(): """ the tool should go in an endless loop asking for user input with auto complete until the right taxonomy is choosen example: PAP<enter> PAP:AMBER PAP:WHITE PAP:GREEN PAP:RED PAP:RED<enter> --> finished https://github.com/MISP/PyTaxonomies """ from pytaxonomies import Taxonomies taxonomies = Taxonomies() again_user_input = input("do you want to search within the pyTaxonomies? (y/n) ") if again_user_input.lower() == 'y': again = True else: again = False while again: try: char = input("Term you want to search for e.g. PAP, TLP, ...") # read 1 character from stdin # try autocomplete print("Suggestions") if (char is "") or (len(char) == 0): search_results = taxonomies.keys() else: #search_results = taxonomies.get(char).machinetags_expanded() all = taxonomies.all_machinetags() resultarray = [] for iterm in all: for item2 in iterm: if char in item2.lower(): # lower to be able to find PAP if you look for p resultarray.append(item2) for resultitem in resultarray: print(resultitem) # print(search_results) again_user_input = input("again?") if again_user_input.lower() == 'n': again = False else: again = True except AttributeError as e: print("Seems we did not find the value "+str(e))
def get_all_tags_taxonomies(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) active_taxonomie = r_serv_tags.smembers('active_taxonomies') list_tags = [] for taxonomie in active_taxonomie: #l_tags = taxonomies.get(taxonomie).machinetags() l_tags = r_serv_tags.smembers('active_tag_' + taxonomie) for tag in l_tags: list_tags.append( tag ) return jsonify(list_tags)
def get_all_tags_taxonomies(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) active_taxonomie = r_serv_tags.smembers('active_taxonomies') list_tags = [] for taxonomie in active_taxonomie: #l_tags = taxonomies.get(taxonomie).machinetags() l_tags = r_serv_tags.smembers('active_tag_' + taxonomie) for tag in l_tags: list_tags.append(tag) return jsonify(list_tags)
def disable_taxonomie(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) id = request.args.get('taxonomie') if id in list_taxonomies: r_serv_tags.srem('active_taxonomies', id) for tag in taxonomies.get(id).machinetags(): r_serv_tags.srem('active_tag_' + id, tag) return redirect(url_for('Tags.taxonomies')) else: return "INCORRECT INPUT"
def main() -> None: argParser = argparse.ArgumentParser(description='Use MISP taxonomies') argParser.add_argument('-a', '--all', action='store_true', help='Print all taxonomies as machine tags') argParser.add_argument('-l', '--local', default=None, help='Use local manifest file.') args = argParser.parse_args() if args.local: t = Taxonomies(manifest_path=args.local) else: t = Taxonomies() if args.all: print(t)
def edit_taxonomie(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) id = request.args.get('taxonomie') #verify input if id in list(taxonomies.keys()): active_tag = r_serv_tags.smembers('active_tag_' + id) list_tag = taxonomies.get(id).machinetags() list_tag_desc = taxonomies.get(id).machinetags_expanded() active_taxonomies = r_serv_tags.smembers('active_taxonomies') if id in active_taxonomies: active = True else: active = False n = str(r_serv_tags.scard('active_tag_' + id)) badge = n + '/' + str(len(taxonomies.get(id).machinetags())) name = taxonomies.get(id).name description = taxonomies.get(id).description version = taxonomies.get(id).version status = [] for tag in list_tag: if tag in active_tag: status.append(True) else: status.append(False) return render_template("edit_taxonomie.html", id=id, name=name, badge=badge, description=description, version=version, active=active, all_tags=list_tag, list_tag_desc=list_tag_desc, status=status) else: return 'INVALID TAXONOMIE'
def taxonomies(): active_taxonomies = r_serv_tags.smembers('active_taxonomies') taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) id = [] name = [] description = [] version = [] enabled = [] n_tags = [] for taxonomie in list_taxonomies: id.append(taxonomie) name.append(taxonomies.get(taxonomie).name) description.append(taxonomies.get(taxonomie).description) version.append(taxonomies.get(taxonomie).version) if taxonomie in active_taxonomies: enabled.append(True) else: enabled.append(False) n = str(r_serv_tags.scard('active_tag_' + taxonomie)) n_tags.append(n + '/' + str(len(taxonomies.get(taxonomie).machinetags()))) return render_template("taxonomies.html", id=id, all_name=name, description=description, version=version, enabled=enabled, n_tags=n_tags)
def addTags(): tags = request.args.get('tags') tagsgalaxies = request.args.get('tagsgalaxies') path = request.args.get('path') list_tag = tags.split(',') list_tag_galaxies = tagsgalaxies.split(',') taxonomies = Taxonomies() active_taxonomies = r_serv_tags.smembers('active_taxonomies') active_galaxies = r_serv_tags.smembers('active_galaxies') if not path: return 'INCORRECT INPUT0' if list_tag != ['']: for tag in list_tag: # verify input tax = tag.split(':')[0] if tax in active_taxonomies: if tag in r_serv_tags.smembers('active_tag_' + tax): #add tag r_serv_metadata.sadd('tag:' + path, tag) r_serv_tags.sadd(tag, path) #add new tag in list of all used tags r_serv_tags.sadd('list_tags', tag) else: return 'INCORRECT INPUT1' else: return 'INCORRECT INPUT2' if list_tag_galaxies != ['']: for tag in list_tag_galaxies: # verify input gal = tag.split(':')[1] gal = gal.split('=')[0] if gal in active_galaxies: if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal): #add tag r_serv_metadata.sadd('tag:' + path, tag) r_serv_tags.sadd(tag, path) #add new tag in list of all used tags r_serv_tags.sadd('list_tags', tag) else: return 'INCORRECT INPUT3' else: return 'INCORRECT INPUT4' return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
def edit_taxonomie_tag(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) arg1 = request.args.getlist('tag_enabled') arg2 = request.args.getlist('tag_disabled') id = request.args.get('taxonomie') #verify input if id in list_taxonomies: list_tag = taxonomies.get(id).machinetags() #check tags validity if (all(elem in list_tag for elem in arg1) or (len(arg1) == 0)) and (all(elem in list_tag for elem in arg2) or (len(arg2) == 0)): active_tag = r_serv_tags.smembers('active_tag_' + id) diff = list(set(arg1) ^ set(list_tag)) #remove tags for tag in diff: r_serv_tags.srem('active_tag_' + id, tag) #all tags unchecked if len(arg1) == 0 and len(arg2) == 0: r_serv_tags.srem('active_taxonomies', id) #add new tags for tag in arg2: r_serv_tags.sadd('active_taxonomies', id) r_serv_tags.sadd('active_tag_' + id, tag) return redirect(url_for('Tags.taxonomies')) else: return "INCORRECT INPUT" else: return "INCORRECT INPUT"
def enable_taxonomy(taxonomie, enable_tags=True): ''' Enable a taxonomy. (UI) :param taxonomie: MISP taxonomy :type taxonomie: str :param enable_tags: crawled domain :type enable_tags: boolean ''' taxonomies = Taxonomies() if enable_tags: taxonomie_info = taxonomies.get(taxonomie) if taxonomie_info: # activate taxonomie r_serv_tags.sadd('active_taxonomies', taxonomie) # activate taxonomie tags for tag in taxonomie_info.machinetags(): r_serv_tags.sadd('active_tag_{}'.format(taxonomie), tag) else: print('Error: {}, please update pytaxonomies'.format(taxonomie))
def edit_taxonomie(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) id = request.args.get('taxonomie') #verify input if id in list(taxonomies.keys()): active_tag = r_serv_tags.smembers('active_tag_' + id) list_tag = taxonomies.get(id).machinetags() list_tag_desc = taxonomies.get(id).machinetags_expanded() active_taxonomies = r_serv_tags.smembers('active_taxonomies') if id in active_taxonomies: active = True else: active = False n = str(r_serv_tags.scard('active_tag_' + id)) badge = n + '/' + str(len(taxonomies.get(id).machinetags())) name = taxonomies.get(id).name description = taxonomies.get(id).description version = taxonomies.get(id).version status = [] for tag in list_tag: if tag in active_tag: status.append(True) else: status.append(False) return render_template("edit_taxonomie.html", id=id, name=name, badge = badge, description = description, version = version, active=active, all_tags = list_tag, list_tag_desc=list_tag_desc, status = status) else: return 'INVALID TAXONOMIE'
def getTaxonomies(dataframe): taxonomies = Taxonomies() taxonomies = list(taxonomies.keys()) notInTaxo = [] count = 0 for taxonomy in taxonomies: empty = True for it in dataframe.iterrows(): if it[0].startswith(taxonomy): empty = False dataframe = dataframe.drop([it[0]]) count = count + 1 if empty is True: notInTaxo.append(taxonomy) if dataframe.empty: emptyOther = True else: emptyOther = False for taxonomy in notInTaxo: taxonomies.remove(taxonomy) return taxonomies, emptyOther
def edit_taxonomie_tag(): taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) arg1 = request.args.getlist('tag_enabled') arg2 = request.args.getlist('tag_disabled') id = request.args.get('taxonomie') #verify input if id in list_taxonomies: list_tag = taxonomies.get(id).machinetags() #check tags validity if ( all(elem in list_tag for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag for elem in arg2) or (len(arg2) == 0) ): active_tag = r_serv_tags.smembers('active_tag_' + id) diff = list(set(arg1) ^ set(list_tag)) #remove tags for tag in diff: r_serv_tags.srem('active_tag_' + id, tag) #all tags unchecked if len(arg1) == 0 and len(arg2) == 0: r_serv_tags.srem('active_taxonomies', id) #add new tags for tag in arg2: r_serv_tags.sadd('active_taxonomies', id) r_serv_tags.sadd('active_tag_' + id, tag) return redirect(url_for('Tags.taxonomies')) else: return "INCORRECT INPUT" else: return "INCORRECT INPUT"
def taxonomies(): active_taxonomies = r_serv_tags.smembers('active_taxonomies') taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) id = [] name = [] description = [] version = [] enabled = [] n_tags = [] for taxonomie in list_taxonomies: id.append(taxonomie) name.append(taxonomies.get(taxonomie).name) description.append(taxonomies.get(taxonomie).description) version.append(taxonomies.get(taxonomie).version) if taxonomie in active_taxonomies: enabled.append(True) else: enabled.append(False) n = str(r_serv_tags.scard('active_tag_' + taxonomie)) n_tags.append(n + '/' + str(len(taxonomies.get(taxonomie).machinetags())) ) return render_template("taxonomies.html", id=id, all_name = name, description = description, version = version, enabled = enabled, n_tags=n_tags)
def delete_push_tag(): tag = request.args.get('tag') infoleak_tags = Taxonomies().get('infoleak').machinetags() if tag not in infoleak_tags and r_serv_db.sismember('list_export_tags', tag): r_serv_db.srem('list_export_tags', tag) r_serv_db.srem('whitelist_misp', tag) r_serv_db.srem('whitelist_hive', tag) to_return = {} to_return["tag"] = tag return jsonify(to_return) else: return 'this tag can\'t be removed', 400
def get_tags_taxonomie(): taxonomie = request.args.get('taxonomie') taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) active_taxonomie = r_serv_tags.smembers('active_taxonomies') #verify input if taxonomie in list_taxonomies: if taxonomie in active_taxonomie: list_tags = [] l_tags = r_serv_tags.smembers('active_tag_' + taxonomie) for tag in l_tags: list_tags.append(tag) return jsonify(list_tags) else: return 'this taxonomie is disable' else: return 'INCORRECT INPUT'
def get_tags_taxonomie(): taxonomie = request.args.get('taxonomie') taxonomies = Taxonomies() list_taxonomies = list(taxonomies.keys()) active_taxonomie = r_serv_tags.smembers('active_taxonomies') #verify input if taxonomie in list_taxonomies: if taxonomie in active_taxonomie: list_tags = [] l_tags = r_serv_tags.smembers('active_tag_' + taxonomie) for tag in l_tags: list_tags.append( tag ) return jsonify(list_tags) else: return 'this taxonomie is disable' else: return 'INCORRECT INPUT'
def addTagsVerification(tags, tagsgalaxies): list_tag = tags.split(',') list_tag_galaxies = tagsgalaxies.split(',') taxonomies = Taxonomies() active_taxonomies = r_serv_tags.smembers('active_taxonomies') active_galaxies = r_serv_tags.smembers('active_galaxies') if list_tag != ['']: for tag in list_tag: # verify input tax = tag.split(':')[0] if tax in active_taxonomies: if tag in r_serv_tags.smembers('active_tag_' + tax): pass else: return False else: return False if list_tag_galaxies != ['']: for tag in list_tag_galaxies: # verify input gal = tag.split(':')[1] gal = gal.split('=')[0] if gal in active_galaxies: if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal): pass else: return False else: return False return True
def revert_tag_from_taxonomies(tag): taxonomies = Taxonomies() try: return taxonomies.revert_machinetag(tag) except Exception: return []
def searchbox(): return render_template("searchbox.html") # ========== INITIAL taxonomies ============ r_serv_tags = redis.StrictRedis( host=cfg.get("ARDB_Tags", "host"), port=cfg.getint("ARDB_Tags", "port"), db=cfg.getint("ARDB_Tags", "db"), decode_responses=True) # add default ail taxonomies r_serv_tags.sadd('active_taxonomies', 'infoleak') r_serv_tags.sadd('active_taxonomies', 'gdpr') r_serv_tags.sadd('active_taxonomies', 'fpf') # add default tags taxonomies = Taxonomies() for tag in taxonomies.get('infoleak').machinetags(): r_serv_tags.sadd('active_tag_infoleak', tag) for tag in taxonomies.get('gdpr').machinetags(): r_serv_tags.sadd('active_tag_gdpr', tag) for tag in taxonomies.get('fpf').machinetags(): r_serv_tags.sadd('active_tag_fpf', tag) # ========== INITIAL tags auto export ============ r_serv_db = redis.StrictRedis( host=cfg.get("ARDB_DB", "host"), port=cfg.getint("ARDB_DB", "port"), db=cfg.getint("ARDB_DB", "db"), decode_responses=True) infoleak_tags = taxonomies.get('infoleak').machinetags() infoleak_automatic_tags = []
def search_taxonomies(query): taxonomies = Taxonomies() found = taxonomies.search(query) if not found: found = taxonomies.search(query, expanded=True) return found
return page_not_found(e) @login_required def page_not_found(e): # avoid endpoint enumeration return render_template('error/404.html'), 404 # ========== INITIAL taxonomies ============ # add default ail taxonomies r_serv_tags.sadd('active_taxonomies', 'infoleak') r_serv_tags.sadd('active_taxonomies', 'gdpr') r_serv_tags.sadd('active_taxonomies', 'fpf') # add default tags taxonomies = Taxonomies() for tag in taxonomies.get('infoleak').machinetags(): r_serv_tags.sadd('active_tag_infoleak', tag) for tag in taxonomies.get('gdpr').machinetags(): r_serv_tags.sadd('active_tag_gdpr', tag) for tag in taxonomies.get('fpf').machinetags(): r_serv_tags.sadd('active_tag_fpf', tag) # ========== INITIAL tags auto export ============ infoleak_tags = taxonomies.get('infoleak').machinetags() infoleak_automatic_tags = [] for tag in taxonomies.get('infoleak').machinetags(): if tag.split('=')[0][:] == 'infoleak:automatic-detection': r_serv_db.sadd('list_export_tags', tag) r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"')
class TestPyTaxonomies(unittest.TestCase): def setUp(self): self.taxonomies_offline = Taxonomies() self.loaded_tax = {} for t in self.taxonomies_offline.manifest['taxonomies']: with open('{}/{}/{}'.format(self.taxonomies_offline.url, t['name'], 'machinetag.json'), 'r') as f: self.loaded_tax[t['name']] = json.load(f) def test_compareOnlineOffilne(self): taxonomies_online = Taxonomies(manifest_path=None) for t_online, t_offline in zip(taxonomies_online.values(), self.taxonomies_offline.values()): self.assertEqual(str(t_online), str(t_offline)) self.assertEqual(str(taxonomies_online), str(self.taxonomies_offline)) def test_expanded_machinetags(self): self.taxonomies_offline.all_machinetags(expanded=True) def test_machinetags(self): self.taxonomies_offline.all_machinetags() def test_dict(self): len(self.taxonomies_offline) for n, t in self.taxonomies_offline.items(): len(t) for p, value in t.items(): continue def test_search(self): self.taxonomies_offline.search('phish') def test_search_expanded(self): self.taxonomies_offline.search('phish', expanded=True) def test_print_classes(self): for taxonomy in self.taxonomies_offline.values(): print(taxonomy) for predicate in taxonomy.values(): print(predicate) for entry in predicate.values(): print(entry) def test_amountEntries(self): for tax in self.taxonomies_offline.values(): tax.amount_entries() def test_missingDependency(self): pytaxonomies.api.HAS_REQUESTS = False with self.assertRaises(Exception): Taxonomies(manifest_path=None) Taxonomies() pytaxonomies.api.HAS_REQUESTS = True def test_revert_machinetags(self): for tax in self.taxonomies_offline.values(): for p in tax.values(): if tax.has_entries(): for e in p.values(): mt = tax.make_machinetag(p, e) self.taxonomies_offline.revert_machinetag(mt) else: mt = tax.make_machinetag(p) self.taxonomies_offline.revert_machinetag(mt) def test_json(self): for key, t in self.taxonomies_offline.items(): json.dumps(t, cls=EncodeTaxonomies) def test_recreate_dump(self): self.maxDiff = None for key, t in self.taxonomies_offline.items(): out = t.to_dict() self.assertDictEqual(out, self.loaded_tax[t.name]) def test_validate_schema(self): self.taxonomies_offline.validate_with_schema()
return Navbar( 'MISP taxonomies viewer and editor', View('Taxonomies', 'taxonomies'), View('Search', 'search'), ) app = Flask(__name__) app.secret_key = '<changeme>' Bootstrap(app) app.config['BOOTSTRAP_SERVE_LOCAL'] = True app.debug = True nav.init_app(app) #t = Taxonomies(manifest_path="../../misp-taxonomies/MANIFEST.json") t = Taxonomies() class SearchForm(FlaskForm): query = StringField('Query', validators=[DataRequired()]) submit = SubmitField('Search') @app.route('/', methods=['GET']) def index(): return taxonomies() @app.route('/taxonomies/', defaults={'name': None}) @app.route('/taxonomies/<name>', methods=['GET']) def taxonomies(name=None):
def edit_tag_export(): misp_auto_events = r_serv_db.get('misp:auto-events') hive_auto_alerts = r_serv_db.get('hive:auto-alerts') whitelist_misp = r_serv_db.scard('whitelist_misp') whitelist_hive = r_serv_db.scard('whitelist_hive') list_export_tags = list(r_serv_db.smembers('list_export_tags')) status_misp = [] status_hive = [] infoleak_tags = Taxonomies().get('infoleak').machinetags() is_infoleak_tag = [] for tag in list_export_tags: if r_serv_db.sismember('whitelist_misp', tag): status_misp.append(True) else: status_misp.append(False) for tag in list_export_tags: if r_serv_db.sismember('whitelist_hive', tag): status_hive.append(True) else: status_hive.append(False) if tag in infoleak_tags: is_infoleak_tag.append(True) else: is_infoleak_tag.append(False) if misp_auto_events is not None: if int(misp_auto_events) == 1: misp_active = True else: misp_active = False else: misp_active = False if hive_auto_alerts is not None: if int(hive_auto_alerts) == 1: hive_active = True else: hive_active = False else: hive_active = False nb_tags = str(r_serv_db.scard('list_export_tags')) nb_tags_whitelist_misp = str( r_serv_db.scard('whitelist_misp')) + ' / ' + nb_tags nb_tags_whitelist_hive = str( r_serv_db.scard('whitelist_hive')) + ' / ' + nb_tags return render_template("edit_tag_export.html", misp_active=misp_active, hive_active=hive_active, list_export_tags=list_export_tags, is_infoleak_tag=is_infoleak_tag, status_misp=status_misp, status_hive=status_hive, nb_tags_whitelist_misp=nb_tags_whitelist_misp, nb_tags_whitelist_hive=nb_tags_whitelist_hive, flag_misp=flag_misp, flag_hive=flag_hive)
def setUp(self): self.taxonomies = Taxonomies() self.manifest_path = "./misp-taxonomies/MANIFEST.json" self.taxonomies_offline = Taxonomies(manifest_path=self.manifest_path) self.json_load_taxonomies()
class TestPyTaxonomies(unittest.TestCase): def setUp(self): self.taxonomies = Taxonomies() self.manifest_path = "./misp-taxonomies/MANIFEST.json" self.taxonomies_offline = Taxonomies(manifest_path=self.manifest_path) self.json_load_taxonomies() def __load_path(self, path): with open(path, 'r') as f: return json.load(f) def json_load_taxonomies(self): self.manifest = self.__load_path(self.manifest_path) self.loaded_tax = {} for t in self.manifest['taxonomies']: path = '{}/{}/{}'.format( os.path.dirname(os.path.realpath(self.manifest_path)), t['name'], self.manifest['path']) self.loaded_tax[t['name']] = self.__load_path(path) def test_compareOnlineOffilne(self): self.assertEqual(str(self.taxonomies), str(self.taxonomies_offline)) def test_expanded_machinetags(self): self.taxonomies.all_machinetags(expanded=True) def test_machinetags(self): self.taxonomies.all_machinetags() def test_dict(self): len(self.taxonomies) for n, t in self.taxonomies.items(): len(t) for p, value in t.items(): continue def test_search(self): self.taxonomies.search('phish') def test_search_expanded(self): self.taxonomies.search('phish', expanded=True) def test_print_classes(self): for taxonomy in self.taxonomies.values(): print(taxonomy) for predicate in taxonomy.values(): print(predicate) for entry in predicate.values(): print(entry) def test_amountEntries(self): for tax in self.taxonomies.values(): tax.amount_entries() def test_missingDependency(self): pytaxonomies.api.HAS_REQUESTS = False with self.assertRaises(Exception): Taxonomies() Taxonomies(manifest_path="./misp-taxonomies/MANIFEST.json") pytaxonomies.api.HAS_REQUESTS = True def test_revert_machinetags(self): for tax in self.taxonomies.values(): for p in tax.values(): if tax.has_entries(): for e in p.values(): mt = tax.make_machinetag(p, e) self.taxonomies.revert_machinetag(mt) else: mt = tax.make_machinetag(p) self.taxonomies.revert_machinetag(mt) def test_json(self): for key, t in self.taxonomies.items(): json.dumps(t, cls=EncodeTaxonomies) def test_recreate_dump(self): self.maxDiff = None for key, t in self.taxonomies.items(): out = t._json() print(t.name) self.assertCountEqual(out, self.loaded_tax[t.name])
class ReportGenerator(): def __init__(self, profile="daily_report"): self.taxonomies = Taxonomies() self.report = '' profile_name = "profiles.{}".format(profile) self.template = importlib.import_module(name=profile_name) def from_remote(self, event_id): from pymisp import PyMISP from keys import misp_url, misp_key, misp_verifycert misp = PyMISP(misp_url, misp_key, misp_verifycert) result = misp.get(event_id) self.misp_event = MISPEvent() self.misp_event.load(result) def from_file(self, path): self.misp_event = MISPEvent() self.misp_event.load_file(path) def attributes(self): if not self.misp_event.attributes: return '' list_attributes = [] for attribute in self.misp_event.attributes: if attribute.type in self.template.types_to_attach: list_attributes.append("* {}".format(defang(attribute.value))) for obj in self.misp_event.Object: if obj.name in self.template.objects_to_attach: for attribute in obj.Attribute: if attribute.type in self.template.types_to_attach: list_attributes.append("* {}".format( defang(attribute.value))) return self.template.attributes.format( list_attributes="\n".join(list_attributes)) def _get_tag_info(self, machinetag): return self.taxonomies.revert_machinetag(machinetag) def report_headers(self): content = {'org_name': 'name', 'date': date.today().isoformat()} self.report += self.template.headers.format(**content) def event_level_tags(self): if not self.misp_event.Tag: return '' for tag in self.misp_event.Tag: # Only look for TLP for now if tag['name'].startswith('tlp'): tax, predicate = self._get_tag_info(tag['name']) return self.template.event_level_tags.format( value=predicate.predicate.upper(), expanded=predicate.expanded) def title(self): internal_id = '' summary = '' # Get internal refs for report for obj in self.misp_event.Object: if obj.name != 'report': continue for a in obj.Attribute: if a.object_relation == 'case-number': internal_id = a.value if a.object_relation == 'summary': summary = a.value return self.template.title.format(internal_id=internal_id, title=self.misp_event.info, summary=summary) def asciidoc(self, lang='en'): self.report += self.title() self.report += self.event_level_tags() self.report += self.attributes()
def test_missingDependency(self): pytaxonomies.api.HAS_REQUESTS = False with self.assertRaises(Exception): Taxonomies() Taxonomies(manifest_path="./misp-taxonomies/MANIFEST.json") pytaxonomies.api.HAS_REQUESTS = True
def __init__(self, profile="daily_report"): self.taxonomies = Taxonomies() self.report = '' profile_name = "profiles.{}".format(profile) self.template = importlib.import_module(name=profile_name)
def get_taxonomies(): return Taxonomies()
def tag(self): if not HAVE_PYTAX: self.log( 'error', "Missing dependency, install PyTaxonomies (`pip install git+https://github.com/MISP/PyTaxonomies.git`)" ) return taxonomies = Taxonomies() if self.args.list: self.log( 'table', dict(header=['Name', 'Description'], rows=[(title, tax.description) for title, tax in taxonomies.items()])) elif self.args.search: matches = taxonomies.search(self.args.search) if not matches: self.log('error', 'No tags matching "{}".'.format(self.args.search)) return self.log('success', 'Tags matching "{}":'.format(self.args.search)) for t in taxonomies.search(self.args.search): self.log('item', t) elif self.args.details: taxonomy = taxonomies.get(self.args.details) if not taxonomy: self.log('error', 'No taxonomy called "{}".'.format(self.args.details)) return if taxonomy.description: self.log('info', taxonomy.description) elif taxonomy.expanded: self.log('info', taxonomy.expanded) if taxonomy.refs: self.log('info', 'References:') for r in taxonomy.refs: self.log('item', r) if not taxonomy.has_entries(): header = ['Description', 'Predicate', 'Machinetag'] rows = [] for p in taxonomy.predicates.values(): rows.append( [p.description, p.predicate, taxonomy.make_machinetag(p)]) self.log('table', dict(header=header, rows=rows)) else: for p in taxonomy.predicates.values(): if p.description: self.log('info', p.description) elif p.expanded: self.log('info', p.expanded) else: self.log('info', p.predicate) if not p.entries: self.log('item', taxonomy.make_machinetag(p)) else: header = ['Description', 'Predicate', 'Machinetag'] rows = [] for e in p.entries.values(): if e.description: descr = e.description else: descr = e.expanded rows.append( [descr, e.value, taxonomy.make_machinetag(p, e)]) self.log('table', dict(header=header, rows=rows)) elif self.args.event: if not __sessions__.is_attached_misp(): return try: taxonomies.revert_machinetag(self.args.event) except Exception: self.log( 'error', 'Not a valid machine tag available in misp-taxonomies: "{}".'. format(self.args.event)) return __sessions__.current.misp_event.event.add_tag(self.args.event) self._change_event() elif self.args.attribute: if not __sessions__.is_attached_misp(): return identifier, tag = self.args.attribute try: taxonomies.revert_machinetag(tag) except Exception: self.log( 'error', 'Not a valid machine tag available in misp-taxonomies: "{}".'. format(tag)) return __sessions__.current.misp_event.event.add_attribute_tag( tag, identifier) self._change_event()
def tag(self): if not HAVE_PYTAX: self.log('error', "Missing dependency, install PyTaxonomies (`pip install git+https://github.com/MISP/PyTaxonomies.git`)") return try: taxonomies = Taxonomies(manifest_path=os.path.join(self.local_dir_taxonomies, 'MANIFEST.json')) except Exception as e: self.log('error', 'Unable to open the taxonomies, please fix the config file ([misp] - misp_taxonomies_directory): {}'.format(e)) return if self.args.list: self.log('table', dict(header=['Name', 'Description'], rows=[(title, tax.description) for title, tax in taxonomies.items()])) elif self.args.search: matches = taxonomies.search(self.args.search) if not matches: self.log('error', 'No tags matching "{}".'.format(self.args.search)) return self.log('success', 'Tags matching "{}":'.format(self.args.search)) for t in taxonomies.search(self.args.search): self.log('item', t) elif self.args.details: taxonomy = taxonomies.get(self.args.details) if not taxonomy: self.log('error', 'No taxonomy called "{}".'.format(self.args.details)) return if taxonomy.description: self.log('info', taxonomy.description) elif taxonomy.expanded: self.log('info', taxonomy.expanded) if taxonomy.refs: self.log('info', 'References:') for r in taxonomy.refs: self.log('item', r) if not taxonomy.has_entries(): header = ['Description', 'Predicate', 'Machinetag'] rows = [] for p in taxonomy.predicates.values(): rows.append([p.description, p.predicate, taxonomy.make_machinetag(p)]) self.log('table', dict(header=header, rows=rows)) else: for p in taxonomy.predicates.values(): if p.description: self.log('info', p.description) elif p.expanded: self.log('info', p.expanded) else: self.log('info', p.predicate) if not p.entries: self.log('item', taxonomy.make_machinetag(p)) else: header = ['Description', 'Predicate', 'Machinetag'] rows = [] for e in p.entries.values(): if e.description: descr = e.description else: descr = e.expanded rows.append([descr, e.value, taxonomy.make_machinetag(p, e)]) self.log('table', dict(header=header, rows=rows)) elif self.args.event: if not __sessions__.is_attached_misp(): return try: taxonomies.revert_machinetag(self.args.event) except: self.log('error', 'Not a valid machine tag available in misp-taxonomies: "{}".'.format(self.args.event)) return __sessions__.current.misp_event.event.add_tag(self.args.event) self._change_event() elif self.args.attribute: if not __sessions__.is_attached_misp(): return identifier, tag = self.args.attribute try: taxonomies.revert_machinetag(tag) except: self.log('error', 'Not a valid machine tag available in misp-taxonomies: "{}".'.format(tag)) return __sessions__.current.misp_event.event.add_attribute_tag(tag, identifier) self._change_event()
class ReportGenerator(): def __init__(self, profile="daily_report"): self.taxonomies = Taxonomies() self.report = '' profile_name = "profiles.{}".format(profile) self.template = importlib.import_module(name=profile_name) def from_remote(self, event_id): from pymisp import PyMISP from keys import misp_url, misp_key, misp_verifycert misp = PyMISP(misp_url, misp_key, misp_verifycert) result = misp.get(event_id) self.misp_event = MISPEvent() self.misp_event.load(result) def from_file(self, path): self.misp_event = MISPEvent() self.misp_event.load_file(path) def attributes(self): if not self.misp_event.attributes: return '' list_attributes = [] for attribute in self.misp_event.attributes: if attribute.type in self.template.types_to_attach: list_attributes.append("* {}".format(defang(attribute.value))) for obj in self.misp_event.Object: if obj.name in self.template.objects_to_attach: for attribute in obj.Attribute: if attribute.type in self.template.types_to_attach: list_attributes.append("* {}".format(defang(attribute.value))) return self.template.attributes.format(list_attributes="\n".join(list_attributes)) def _get_tag_info(self, machinetag): return self.taxonomies.revert_machinetag(machinetag) def report_headers(self): content = {'org_name': 'name', 'date': date.today().isoformat()} self.report += self.template.headers.format(**content) def event_level_tags(self): if not self.misp_event.Tag: return '' for tag in self.misp_event.Tag: # Only look for TLP for now if tag['name'].startswith('tlp'): tax, predicate = self._get_tag_info(tag['name']) return self.template.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) def title(self): internal_id = '' summary = '' # Get internal refs for report for obj in self.misp_event.Object: if obj.name != 'report': continue for a in obj.Attribute: if a.object_relation == 'case-number': internal_id = a.value if a.object_relation == 'summary': summary = a.value return self.template.title.format(internal_id=internal_id, title=self.misp_event.info, summary=summary) def asciidoc(self, lang='en'): self.report += self.title() self.report += self.event_level_tags() self.report += self.attributes()
def setUp(self): self.taxonomies_offline = Taxonomies() self.loaded_tax = {} for t in self.taxonomies_offline.manifest['taxonomies']: with open('{}/{}/{}'.format(self.taxonomies_offline.url, t['name'], 'machinetag.json'), 'r') as f: self.loaded_tax[t['name']] = json.load(f)
class TestPyTaxonomies(unittest.TestCase): def setUp(self): self.maxDiff = None self.taxonomies_offline = Taxonomies() self.loaded_tax = {} for t in self.taxonomies_offline.manifest['taxonomies']: with open( '{}/{}/{}'.format(self.taxonomies_offline.url, t['name'], 'machinetag.json'), 'r') as f: self.loaded_tax[t['name']] = json.load(f) def test_compareOnlineOffilne(self): taxonomies_online = Taxonomies(manifest_path=None) for t_online, t_offline in zip(taxonomies_online.values(), self.taxonomies_offline.values()): self.assertEqual(str(t_online), str(t_offline)) self.assertEqual(str(taxonomies_online), str(self.taxonomies_offline)) def test_expanded_machinetags(self): self.taxonomies_offline.all_machinetags(expanded=True) def test_machinetags(self): self.taxonomies_offline.all_machinetags() def test_dict(self): len(self.taxonomies_offline) for n, t in self.taxonomies_offline.items(): len(t) for p, value in t.items(): continue def test_search(self): self.taxonomies_offline.search('phish') def test_search_expanded(self): self.taxonomies_offline.search('phish', expanded=True) def test_print_classes(self): for taxonomy in self.taxonomies_offline.values(): print(taxonomy) for predicate in taxonomy.values(): print(predicate) for entry in predicate.values(): print(entry) def test_amountEntries(self): for tax in self.taxonomies_offline.values(): tax.amount_entries() def test_missingDependency(self): pytaxonomies.api.HAS_REQUESTS = False with self.assertRaises(Exception): Taxonomies(manifest_path=None) Taxonomies() pytaxonomies.api.HAS_REQUESTS = True def test_revert_machinetags(self): for tax in self.taxonomies_offline.values(): for p in tax.values(): if tax.has_entries(): for e in p.values(): mt = tax.make_machinetag(p, e) self.taxonomies_offline.revert_machinetag(mt) else: mt = tax.make_machinetag(p) self.taxonomies_offline.revert_machinetag(mt) def test_json(self): for key, t in self.taxonomies_offline.items(): t.to_json() def test_recreate_dump(self): self.maxDiff = None for key, t in self.taxonomies_offline.items(): out = t.to_dict() self.assertDictEqual(out, self.loaded_tax[t.name]) def test_validate_schema(self): self.taxonomies_offline.validate_with_schema()
def test_compareOnlineOffilne(self): taxonomies_online = Taxonomies(manifest_path=None) for t_online, t_offline in zip(taxonomies_online.values(), self.taxonomies_offline.values()): self.assertEqual(str(t_online), str(t_offline)) self.assertEqual(str(taxonomies_online), str(self.taxonomies_offline))
return page_not_found(e) @login_required def page_not_found(e): # avoid endpoint enumeration return render_template('error/404.html'), 404 # ========== INITIAL taxonomies ============ default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"] # enable default taxonomies for taxo in default_taxonomies: Tag.enable_taxonomy(taxo) # ========== INITIAL tags auto export ============ taxonomies = Taxonomies() infoleak_tags = taxonomies.get('infoleak').machinetags() infoleak_automatic_tags = [] for tag in taxonomies.get('infoleak').machinetags(): if tag.split('=')[0][:] == 'infoleak:automatic-detection': r_serv_db.sadd('list_export_tags', tag) r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"') # ============ MAIN ============ if __name__ == "__main__": app.run(host=host, port=FLASK_PORT, threaded=True, ssl_context=ssl_context)
def test_missingDependency(self): pytaxonomies.api.HAS_REQUESTS = False with self.assertRaises(Exception): Taxonomies(manifest_path=None) Taxonomies() pytaxonomies.api.HAS_REQUESTS = True