def setUp(self):
self.maxDiff = None
self.taxonomies_offline = Taxonomies()
self.loaded_tax = {}
for t in self.taxonomies_offline.manifest['taxonomies']:
with open(
'{}/{}/{}'.format(self.taxonomies_offline.url, t['name'],
'machinetag.json'), 'r') as f:
self.loaded_tax[t['name']] = json.load(f)
def build_unsafe_tags():
unsafe_tags = set()
## CE content
unsafe_tags.add('dark-web:topic="pornography-child-exploitation"')
# add copine-scale tags
taxonomies = Taxonomies()
copine_scale = taxonomies.get('copine-scale')
if copine_scale:
for tag in copine_scale.machinetags():
unsafe_tags.add(tag)
return unsafe_tags
def explore_pytaxonomies():
"""
the tool should go in an endless loop asking for user input with auto complete until the right taxonomy is choosen
example:
PAP<enter>
PAP:AMBER
PAP:WHITE
PAP:GREEN
PAP:RED
PAP:RED<enter> --> finished
https://github.com/MISP/PyTaxonomies
"""
from pytaxonomies import Taxonomies
taxonomies = Taxonomies()
again_user_input = input("do you want to search within the pyTaxonomies? (y/n) ")
if again_user_input.lower() == 'y':
again = True
else:
again = False
while again:
try:
char = input("Term you want to search for e.g. PAP, TLP, ...") # read 1 character from stdin
# try autocomplete
print("Suggestions")
if (char is "") or (len(char) == 0):
search_results = taxonomies.keys()
else:
#search_results = taxonomies.get(char).machinetags_expanded()
all = taxonomies.all_machinetags()
resultarray = []
for iterm in all:
for item2 in iterm:
if char in item2.lower(): # lower to be able to find PAP if you look for p
resultarray.append(item2)
for resultitem in resultarray:
print(resultitem)
# print(search_results)
again_user_input = input("again?")
if again_user_input.lower() == 'n':
again = False
else:
again = True
except AttributeError as e:
print("Seems we did not find the value "+str(e))
def get_all_tags_taxonomies():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
active_taxonomie = r_serv_tags.smembers('active_taxonomies')
list_tags = []
for taxonomie in active_taxonomie:
#l_tags = taxonomies.get(taxonomie).machinetags()
l_tags = r_serv_tags.smembers('active_tag_' + taxonomie)
for tag in l_tags:
list_tags.append( tag )
return jsonify(list_tags)
def get_all_tags_taxonomies():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
active_taxonomie = r_serv_tags.smembers('active_taxonomies')
list_tags = []
for taxonomie in active_taxonomie:
#l_tags = taxonomies.get(taxonomie).machinetags()
l_tags = r_serv_tags.smembers('active_tag_' + taxonomie)
for tag in l_tags:
list_tags.append(tag)
return jsonify(list_tags)
def disable_taxonomie():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = request.args.get('taxonomie')
if id in list_taxonomies:
r_serv_tags.srem('active_taxonomies', id)
for tag in taxonomies.get(id).machinetags():
r_serv_tags.srem('active_tag_' + id, tag)
return redirect(url_for('Tags.taxonomies'))
else:
return "INCORRECT INPUT"
def disable_taxonomie():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = request.args.get('taxonomie')
if id in list_taxonomies:
r_serv_tags.srem('active_taxonomies', id)
for tag in taxonomies.get(id).machinetags():
r_serv_tags.srem('active_tag_' + id, tag)
return redirect(url_for('Tags.taxonomies'))
else:
return "INCORRECT INPUT"
def main() -> None:
argParser = argparse.ArgumentParser(description='Use MISP taxonomies')
argParser.add_argument('-a',
'--all',
action='store_true',
help='Print all taxonomies as machine tags')
argParser.add_argument('-l',
'--local',
default=None,
help='Use local manifest file.')
args = argParser.parse_args()
if args.local:
t = Taxonomies(manifest_path=args.local)
else:
t = Taxonomies()
if args.all:
print(t)
def edit_taxonomie():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = request.args.get('taxonomie')
#verify input
if id in list(taxonomies.keys()):
active_tag = r_serv_tags.smembers('active_tag_' + id)
list_tag = taxonomies.get(id).machinetags()
list_tag_desc = taxonomies.get(id).machinetags_expanded()
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
if id in active_taxonomies:
active = True
else:
active = False
n = str(r_serv_tags.scard('active_tag_' + id))
badge = n + '/' + str(len(taxonomies.get(id).machinetags()))
name = taxonomies.get(id).name
description = taxonomies.get(id).description
version = taxonomies.get(id).version
status = []
for tag in list_tag:
if tag in active_tag:
status.append(True)
else:
status.append(False)
return render_template("edit_taxonomie.html",
id=id,
name=name,
badge=badge,
description=description,
version=version,
active=active,
all_tags=list_tag,
list_tag_desc=list_tag_desc,
status=status)
else:
return 'INVALID TAXONOMIE'
def taxonomies():
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = []
name = []
description = []
version = []
enabled = []
n_tags = []
for taxonomie in list_taxonomies:
id.append(taxonomie)
name.append(taxonomies.get(taxonomie).name)
description.append(taxonomies.get(taxonomie).description)
version.append(taxonomies.get(taxonomie).version)
if taxonomie in active_taxonomies:
enabled.append(True)
else:
enabled.append(False)
n = str(r_serv_tags.scard('active_tag_' + taxonomie))
n_tags.append(n + '/' +
str(len(taxonomies.get(taxonomie).machinetags())))
return render_template("taxonomies.html",
id=id,
all_name=name,
description=description,
version=version,
enabled=enabled,
n_tags=n_tags)
def addTags():
tags = request.args.get('tags')
tagsgalaxies = request.args.get('tagsgalaxies')
path = request.args.get('path')
list_tag = tags.split(',')
list_tag_galaxies = tagsgalaxies.split(',')
taxonomies = Taxonomies()
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
active_galaxies = r_serv_tags.smembers('active_galaxies')
if not path:
return 'INCORRECT INPUT0'
if list_tag != ['']:
for tag in list_tag:
# verify input
tax = tag.split(':')[0]
if tax in active_taxonomies:
if tag in r_serv_tags.smembers('active_tag_' + tax):
#add tag
r_serv_metadata.sadd('tag:' + path, tag)
r_serv_tags.sadd(tag, path)
#add new tag in list of all used tags
r_serv_tags.sadd('list_tags', tag)
else:
return 'INCORRECT INPUT1'
else:
return 'INCORRECT INPUT2'
if list_tag_galaxies != ['']:
for tag in list_tag_galaxies:
# verify input
gal = tag.split(':')[1]
gal = gal.split('=')[0]
if gal in active_galaxies:
if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal):
#add tag
r_serv_metadata.sadd('tag:' + path, tag)
r_serv_tags.sadd(tag, path)
#add new tag in list of all used tags
r_serv_tags.sadd('list_tags', tag)
else:
return 'INCORRECT INPUT3'
else:
return 'INCORRECT INPUT4'
return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
def edit_taxonomie_tag():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
arg1 = request.args.getlist('tag_enabled')
arg2 = request.args.getlist('tag_disabled')
id = request.args.get('taxonomie')
#verify input
if id in list_taxonomies:
list_tag = taxonomies.get(id).machinetags()
#check tags validity
if (all(elem in list_tag for elem in arg1) or
(len(arg1) == 0)) and (all(elem in list_tag
for elem in arg2) or (len(arg2) == 0)):
active_tag = r_serv_tags.smembers('active_tag_' + id)
diff = list(set(arg1) ^ set(list_tag))
#remove tags
for tag in diff:
r_serv_tags.srem('active_tag_' + id, tag)
#all tags unchecked
if len(arg1) == 0 and len(arg2) == 0:
r_serv_tags.srem('active_taxonomies', id)
#add new tags
for tag in arg2:
r_serv_tags.sadd('active_taxonomies', id)
r_serv_tags.sadd('active_tag_' + id, tag)
return redirect(url_for('Tags.taxonomies'))
else:
return "INCORRECT INPUT"
else:
return "INCORRECT INPUT"
def enable_taxonomy(taxonomie, enable_tags=True):
'''
Enable a taxonomy. (UI)
:param taxonomie: MISP taxonomy
:type taxonomie: str
:param enable_tags: crawled domain
:type enable_tags: boolean
'''
taxonomies = Taxonomies()
if enable_tags:
taxonomie_info = taxonomies.get(taxonomie)
if taxonomie_info:
# activate taxonomie
r_serv_tags.sadd('active_taxonomies', taxonomie)
# activate taxonomie tags
for tag in taxonomie_info.machinetags():
r_serv_tags.sadd('active_tag_{}'.format(taxonomie), tag)
else:
print('Error: {}, please update pytaxonomies'.format(taxonomie))
def edit_taxonomie():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = request.args.get('taxonomie')
#verify input
if id in list(taxonomies.keys()):
active_tag = r_serv_tags.smembers('active_tag_' + id)
list_tag = taxonomies.get(id).machinetags()
list_tag_desc = taxonomies.get(id).machinetags_expanded()
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
if id in active_taxonomies:
active = True
else:
active = False
n = str(r_serv_tags.scard('active_tag_' + id))
badge = n + '/' + str(len(taxonomies.get(id).machinetags()))
name = taxonomies.get(id).name
description = taxonomies.get(id).description
version = taxonomies.get(id).version
status = []
for tag in list_tag:
if tag in active_tag:
status.append(True)
else:
status.append(False)
return render_template("edit_taxonomie.html",
id=id,
name=name,
badge = badge,
description = description,
version = version,
active=active,
all_tags = list_tag,
list_tag_desc=list_tag_desc,
status = status)
else:
return 'INVALID TAXONOMIE'
def getTaxonomies(dataframe):
taxonomies = Taxonomies()
taxonomies = list(taxonomies.keys())
notInTaxo = []
count = 0
for taxonomy in taxonomies:
empty = True
for it in dataframe.iterrows():
if it[0].startswith(taxonomy):
empty = False
dataframe = dataframe.drop([it[0]])
count = count + 1
if empty is True:
notInTaxo.append(taxonomy)
if dataframe.empty:
emptyOther = True
else:
emptyOther = False
for taxonomy in notInTaxo:
taxonomies.remove(taxonomy)
return taxonomies, emptyOther
def getTaxonomies(dataframe):
taxonomies = Taxonomies()
taxonomies = list(taxonomies.keys())
notInTaxo = []
count = 0
for taxonomy in taxonomies:
empty = True
for it in dataframe.iterrows():
if it[0].startswith(taxonomy):
empty = False
dataframe = dataframe.drop([it[0]])
count = count + 1
if empty is True:
notInTaxo.append(taxonomy)
if dataframe.empty:
emptyOther = True
else:
emptyOther = False
for taxonomy in notInTaxo:
taxonomies.remove(taxonomy)
return taxonomies, emptyOther
def edit_taxonomie_tag():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
arg1 = request.args.getlist('tag_enabled')
arg2 = request.args.getlist('tag_disabled')
id = request.args.get('taxonomie')
#verify input
if id in list_taxonomies:
list_tag = taxonomies.get(id).machinetags()
#check tags validity
if ( all(elem in list_tag for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag for elem in arg2) or (len(arg2) == 0) ):
active_tag = r_serv_tags.smembers('active_tag_' + id)
diff = list(set(arg1) ^ set(list_tag))
#remove tags
for tag in diff:
r_serv_tags.srem('active_tag_' + id, tag)
#all tags unchecked
if len(arg1) == 0 and len(arg2) == 0:
r_serv_tags.srem('active_taxonomies', id)
#add new tags
for tag in arg2:
r_serv_tags.sadd('active_taxonomies', id)
r_serv_tags.sadd('active_tag_' + id, tag)
return redirect(url_for('Tags.taxonomies'))
else:
return "INCORRECT INPUT"
else:
return "INCORRECT INPUT"
def taxonomies():
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = []
name = []
description = []
version = []
enabled = []
n_tags = []
for taxonomie in list_taxonomies:
id.append(taxonomie)
name.append(taxonomies.get(taxonomie).name)
description.append(taxonomies.get(taxonomie).description)
version.append(taxonomies.get(taxonomie).version)
if taxonomie in active_taxonomies:
enabled.append(True)
else:
enabled.append(False)
n = str(r_serv_tags.scard('active_tag_' + taxonomie))
n_tags.append(n + '/' + str(len(taxonomies.get(taxonomie).machinetags())) )
return render_template("taxonomies.html",
id=id,
all_name = name,
description = description,
version = version,
enabled = enabled,
n_tags=n_tags)
def delete_push_tag():
tag = request.args.get('tag')
infoleak_tags = Taxonomies().get('infoleak').machinetags()
if tag not in infoleak_tags and r_serv_db.sismember('list_export_tags', tag):
r_serv_db.srem('list_export_tags', tag)
r_serv_db.srem('whitelist_misp', tag)
r_serv_db.srem('whitelist_hive', tag)
to_return = {}
to_return["tag"] = tag
return jsonify(to_return)
else:
return 'this tag can\'t be removed', 400
def get_tags_taxonomie():
taxonomie = request.args.get('taxonomie')
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
active_taxonomie = r_serv_tags.smembers('active_taxonomies')
#verify input
if taxonomie in list_taxonomies:
if taxonomie in active_taxonomie:
list_tags = []
l_tags = r_serv_tags.smembers('active_tag_' + taxonomie)
for tag in l_tags:
list_tags.append(tag)
return jsonify(list_tags)
else:
return 'this taxonomie is disable'
else:
return 'INCORRECT INPUT'
def get_tags_taxonomie():
taxonomie = request.args.get('taxonomie')
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
active_taxonomie = r_serv_tags.smembers('active_taxonomies')
#verify input
if taxonomie in list_taxonomies:
if taxonomie in active_taxonomie:
list_tags = []
l_tags = r_serv_tags.smembers('active_tag_' + taxonomie)
for tag in l_tags:
list_tags.append( tag )
return jsonify(list_tags)
else:
return 'this taxonomie is disable'
else:
return 'INCORRECT INPUT'
def addTagsVerification(tags, tagsgalaxies):
list_tag = tags.split(',')
list_tag_galaxies = tagsgalaxies.split(',')
taxonomies = Taxonomies()
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
active_galaxies = r_serv_tags.smembers('active_galaxies')
if list_tag != ['']:
for tag in list_tag:
# verify input
tax = tag.split(':')[0]
if tax in active_taxonomies:
if tag in r_serv_tags.smembers('active_tag_' + tax):
pass
else:
return False
else:
return False
if list_tag_galaxies != ['']:
for tag in list_tag_galaxies:
# verify input
gal = tag.split(':')[1]
gal = gal.split('=')[0]
if gal in active_galaxies:
if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal):
pass
else:
return False
else:
return False
return True
def revert_tag_from_taxonomies(tag):
taxonomies = Taxonomies()
try:
return taxonomies.revert_machinetag(tag)
except Exception:
return []
def searchbox():
return render_template("searchbox.html")
# ========== INITIAL taxonomies ============
r_serv_tags = redis.StrictRedis(
host=cfg.get("ARDB_Tags", "host"),
port=cfg.getint("ARDB_Tags", "port"),
db=cfg.getint("ARDB_Tags", "db"),
decode_responses=True)
# add default ail taxonomies
r_serv_tags.sadd('active_taxonomies', 'infoleak')
r_serv_tags.sadd('active_taxonomies', 'gdpr')
r_serv_tags.sadd('active_taxonomies', 'fpf')
# add default tags
taxonomies = Taxonomies()
for tag in taxonomies.get('infoleak').machinetags():
r_serv_tags.sadd('active_tag_infoleak', tag)
for tag in taxonomies.get('gdpr').machinetags():
r_serv_tags.sadd('active_tag_gdpr', tag)
for tag in taxonomies.get('fpf').machinetags():
r_serv_tags.sadd('active_tag_fpf', tag)
# ========== INITIAL tags auto export ============
r_serv_db = redis.StrictRedis(
host=cfg.get("ARDB_DB", "host"),
port=cfg.getint("ARDB_DB", "port"),
db=cfg.getint("ARDB_DB", "db"),
decode_responses=True)
infoleak_tags = taxonomies.get('infoleak').machinetags()
infoleak_automatic_tags = []
def search_taxonomies(query):
taxonomies = Taxonomies()
found = taxonomies.search(query)
if not found:
found = taxonomies.search(query, expanded=True)
return found
return page_not_found(e)
@login_required
def page_not_found(e):
# avoid endpoint enumeration
return render_template('error/404.html'), 404
# ========== INITIAL taxonomies ============
# add default ail taxonomies
r_serv_tags.sadd('active_taxonomies', 'infoleak')
r_serv_tags.sadd('active_taxonomies', 'gdpr')
r_serv_tags.sadd('active_taxonomies', 'fpf')
# add default tags
taxonomies = Taxonomies()
for tag in taxonomies.get('infoleak').machinetags():
r_serv_tags.sadd('active_tag_infoleak', tag)
for tag in taxonomies.get('gdpr').machinetags():
r_serv_tags.sadd('active_tag_gdpr', tag)
for tag in taxonomies.get('fpf').machinetags():
r_serv_tags.sadd('active_tag_fpf', tag)
# ========== INITIAL tags auto export ============
infoleak_tags = taxonomies.get('infoleak').machinetags()
infoleak_automatic_tags = []
for tag in taxonomies.get('infoleak').machinetags():
if tag.split('=')[0][:] == 'infoleak:automatic-detection':
r_serv_db.sadd('list_export_tags', tag)
r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"')
class TestPyTaxonomies(unittest.TestCase):
def setUp(self):
self.taxonomies_offline = Taxonomies()
self.loaded_tax = {}
for t in self.taxonomies_offline.manifest['taxonomies']:
with open('{}/{}/{}'.format(self.taxonomies_offline.url, t['name'], 'machinetag.json'), 'r') as f:
self.loaded_tax[t['name']] = json.load(f)
def test_compareOnlineOffilne(self):
taxonomies_online = Taxonomies(manifest_path=None)
for t_online, t_offline in zip(taxonomies_online.values(), self.taxonomies_offline.values()):
self.assertEqual(str(t_online), str(t_offline))
self.assertEqual(str(taxonomies_online), str(self.taxonomies_offline))
def test_expanded_machinetags(self):
self.taxonomies_offline.all_machinetags(expanded=True)
def test_machinetags(self):
self.taxonomies_offline.all_machinetags()
def test_dict(self):
len(self.taxonomies_offline)
for n, t in self.taxonomies_offline.items():
len(t)
for p, value in t.items():
continue
def test_search(self):
self.taxonomies_offline.search('phish')
def test_search_expanded(self):
self.taxonomies_offline.search('phish', expanded=True)
def test_print_classes(self):
for taxonomy in self.taxonomies_offline.values():
print(taxonomy)
for predicate in taxonomy.values():
print(predicate)
for entry in predicate.values():
print(entry)
def test_amountEntries(self):
for tax in self.taxonomies_offline.values():
tax.amount_entries()
def test_missingDependency(self):
pytaxonomies.api.HAS_REQUESTS = False
with self.assertRaises(Exception):
Taxonomies(manifest_path=None)
Taxonomies()
pytaxonomies.api.HAS_REQUESTS = True
def test_revert_machinetags(self):
for tax in self.taxonomies_offline.values():
for p in tax.values():
if tax.has_entries():
for e in p.values():
mt = tax.make_machinetag(p, e)
self.taxonomies_offline.revert_machinetag(mt)
else:
mt = tax.make_machinetag(p)
self.taxonomies_offline.revert_machinetag(mt)
def test_json(self):
for key, t in self.taxonomies_offline.items():
json.dumps(t, cls=EncodeTaxonomies)
def test_recreate_dump(self):
self.maxDiff = None
for key, t in self.taxonomies_offline.items():
out = t.to_dict()
self.assertDictEqual(out, self.loaded_tax[t.name])
def test_validate_schema(self):
self.taxonomies_offline.validate_with_schema()
return Navbar(
'MISP taxonomies viewer and editor',
View('Taxonomies', 'taxonomies'),
View('Search', 'search'),
)
app = Flask(__name__)
app.secret_key = '<changeme>'
Bootstrap(app)
app.config['BOOTSTRAP_SERVE_LOCAL'] = True
app.debug = True
nav.init_app(app)
#t = Taxonomies(manifest_path="../../misp-taxonomies/MANIFEST.json")
t = Taxonomies()
class SearchForm(FlaskForm):
query = StringField('Query', validators=[DataRequired()])
submit = SubmitField('Search')
@app.route('/', methods=['GET'])
def index():
return taxonomies()
@app.route('/taxonomies/', defaults={'name': None})
@app.route('/taxonomies/<name>', methods=['GET'])
def taxonomies(name=None):
def edit_tag_export():
misp_auto_events = r_serv_db.get('misp:auto-events')
hive_auto_alerts = r_serv_db.get('hive:auto-alerts')
whitelist_misp = r_serv_db.scard('whitelist_misp')
whitelist_hive = r_serv_db.scard('whitelist_hive')
list_export_tags = list(r_serv_db.smembers('list_export_tags'))
status_misp = []
status_hive = []
infoleak_tags = Taxonomies().get('infoleak').machinetags()
is_infoleak_tag = []
for tag in list_export_tags:
if r_serv_db.sismember('whitelist_misp', tag):
status_misp.append(True)
else:
status_misp.append(False)
for tag in list_export_tags:
if r_serv_db.sismember('whitelist_hive', tag):
status_hive.append(True)
else:
status_hive.append(False)
if tag in infoleak_tags:
is_infoleak_tag.append(True)
else:
is_infoleak_tag.append(False)
if misp_auto_events is not None:
if int(misp_auto_events) == 1:
misp_active = True
else:
misp_active = False
else:
misp_active = False
if hive_auto_alerts is not None:
if int(hive_auto_alerts) == 1:
hive_active = True
else:
hive_active = False
else:
hive_active = False
nb_tags = str(r_serv_db.scard('list_export_tags'))
nb_tags_whitelist_misp = str(
r_serv_db.scard('whitelist_misp')) + ' / ' + nb_tags
nb_tags_whitelist_hive = str(
r_serv_db.scard('whitelist_hive')) + ' / ' + nb_tags
return render_template("edit_tag_export.html",
misp_active=misp_active,
hive_active=hive_active,
list_export_tags=list_export_tags,
is_infoleak_tag=is_infoleak_tag,
status_misp=status_misp,
status_hive=status_hive,
nb_tags_whitelist_misp=nb_tags_whitelist_misp,
nb_tags_whitelist_hive=nb_tags_whitelist_hive,
flag_misp=flag_misp,
flag_hive=flag_hive)
def setUp(self):
self.taxonomies = Taxonomies()
self.manifest_path = "./misp-taxonomies/MANIFEST.json"
self.taxonomies_offline = Taxonomies(manifest_path=self.manifest_path)
self.json_load_taxonomies()
class TestPyTaxonomies(unittest.TestCase):
def setUp(self):
self.taxonomies = Taxonomies()
self.manifest_path = "./misp-taxonomies/MANIFEST.json"
self.taxonomies_offline = Taxonomies(manifest_path=self.manifest_path)
self.json_load_taxonomies()
def __load_path(self, path):
with open(path, 'r') as f:
return json.load(f)
def json_load_taxonomies(self):
self.manifest = self.__load_path(self.manifest_path)
self.loaded_tax = {}
for t in self.manifest['taxonomies']:
path = '{}/{}/{}'.format(
os.path.dirname(os.path.realpath(self.manifest_path)),
t['name'], self.manifest['path'])
self.loaded_tax[t['name']] = self.__load_path(path)
def test_compareOnlineOffilne(self):
self.assertEqual(str(self.taxonomies), str(self.taxonomies_offline))
def test_expanded_machinetags(self):
self.taxonomies.all_machinetags(expanded=True)
def test_machinetags(self):
self.taxonomies.all_machinetags()
def test_dict(self):
len(self.taxonomies)
for n, t in self.taxonomies.items():
len(t)
for p, value in t.items():
continue
def test_search(self):
self.taxonomies.search('phish')
def test_search_expanded(self):
self.taxonomies.search('phish', expanded=True)
def test_print_classes(self):
for taxonomy in self.taxonomies.values():
print(taxonomy)
for predicate in taxonomy.values():
print(predicate)
for entry in predicate.values():
print(entry)
def test_amountEntries(self):
for tax in self.taxonomies.values():
tax.amount_entries()
def test_missingDependency(self):
pytaxonomies.api.HAS_REQUESTS = False
with self.assertRaises(Exception):
Taxonomies()
Taxonomies(manifest_path="./misp-taxonomies/MANIFEST.json")
pytaxonomies.api.HAS_REQUESTS = True
def test_revert_machinetags(self):
for tax in self.taxonomies.values():
for p in tax.values():
if tax.has_entries():
for e in p.values():
mt = tax.make_machinetag(p, e)
self.taxonomies.revert_machinetag(mt)
else:
mt = tax.make_machinetag(p)
self.taxonomies.revert_machinetag(mt)
def test_json(self):
for key, t in self.taxonomies.items():
json.dumps(t, cls=EncodeTaxonomies)
def test_recreate_dump(self):
self.maxDiff = None
for key, t in self.taxonomies.items():
out = t._json()
print(t.name)
self.assertCountEqual(out, self.loaded_tax[t.name])
class ReportGenerator():
def __init__(self, profile="daily_report"):
self.taxonomies = Taxonomies()
self.report = ''
profile_name = "profiles.{}".format(profile)
self.template = importlib.import_module(name=profile_name)
def from_remote(self, event_id):
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
misp = PyMISP(misp_url, misp_key, misp_verifycert)
result = misp.get(event_id)
self.misp_event = MISPEvent()
self.misp_event.load(result)
def from_file(self, path):
self.misp_event = MISPEvent()
self.misp_event.load_file(path)
def attributes(self):
if not self.misp_event.attributes:
return ''
list_attributes = []
for attribute in self.misp_event.attributes:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(defang(attribute.value)))
for obj in self.misp_event.Object:
if obj.name in self.template.objects_to_attach:
for attribute in obj.Attribute:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(
defang(attribute.value)))
return self.template.attributes.format(
list_attributes="\n".join(list_attributes))
def _get_tag_info(self, machinetag):
return self.taxonomies.revert_machinetag(machinetag)
def report_headers(self):
content = {'org_name': 'name', 'date': date.today().isoformat()}
self.report += self.template.headers.format(**content)
def event_level_tags(self):
if not self.misp_event.Tag:
return ''
for tag in self.misp_event.Tag:
# Only look for TLP for now
if tag['name'].startswith('tlp'):
tax, predicate = self._get_tag_info(tag['name'])
return self.template.event_level_tags.format(
value=predicate.predicate.upper(),
expanded=predicate.expanded)
def title(self):
internal_id = ''
summary = ''
# Get internal refs for report
for obj in self.misp_event.Object:
if obj.name != 'report':
continue
for a in obj.Attribute:
if a.object_relation == 'case-number':
internal_id = a.value
if a.object_relation == 'summary':
summary = a.value
return self.template.title.format(internal_id=internal_id,
title=self.misp_event.info,
summary=summary)
def asciidoc(self, lang='en'):
self.report += self.title()
self.report += self.event_level_tags()
self.report += self.attributes()
def test_missingDependency(self):
pytaxonomies.api.HAS_REQUESTS = False
with self.assertRaises(Exception):
Taxonomies()
Taxonomies(manifest_path="./misp-taxonomies/MANIFEST.json")
pytaxonomies.api.HAS_REQUESTS = True
def __init__(self, profile="daily_report"):
self.taxonomies = Taxonomies()
self.report = ''
profile_name = "profiles.{}".format(profile)
self.template = importlib.import_module(name=profile_name)
def get_taxonomies():
return Taxonomies()
def tag(self):
if not HAVE_PYTAX:
self.log(
'error',
"Missing dependency, install PyTaxonomies (`pip install git+https://github.com/MISP/PyTaxonomies.git`)"
)
return
taxonomies = Taxonomies()
if self.args.list:
self.log(
'table',
dict(header=['Name', 'Description'],
rows=[(title, tax.description)
for title, tax in taxonomies.items()]))
elif self.args.search:
matches = taxonomies.search(self.args.search)
if not matches:
self.log('error',
'No tags matching "{}".'.format(self.args.search))
return
self.log('success', 'Tags matching "{}":'.format(self.args.search))
for t in taxonomies.search(self.args.search):
self.log('item', t)
elif self.args.details:
taxonomy = taxonomies.get(self.args.details)
if not taxonomy:
self.log('error',
'No taxonomy called "{}".'.format(self.args.details))
return
if taxonomy.description:
self.log('info', taxonomy.description)
elif taxonomy.expanded:
self.log('info', taxonomy.expanded)
if taxonomy.refs:
self.log('info', 'References:')
for r in taxonomy.refs:
self.log('item', r)
if not taxonomy.has_entries():
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for p in taxonomy.predicates.values():
rows.append(
[p.description, p.predicate,
taxonomy.make_machinetag(p)])
self.log('table', dict(header=header, rows=rows))
else:
for p in taxonomy.predicates.values():
if p.description:
self.log('info', p.description)
elif p.expanded:
self.log('info', p.expanded)
else:
self.log('info', p.predicate)
if not p.entries:
self.log('item', taxonomy.make_machinetag(p))
else:
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for e in p.entries.values():
if e.description:
descr = e.description
else:
descr = e.expanded
rows.append(
[descr, e.value,
taxonomy.make_machinetag(p, e)])
self.log('table', dict(header=header, rows=rows))
elif self.args.event:
if not __sessions__.is_attached_misp():
return
try:
taxonomies.revert_machinetag(self.args.event)
except Exception:
self.log(
'error',
'Not a valid machine tag available in misp-taxonomies: "{}".'.
format(self.args.event))
return
__sessions__.current.misp_event.event.add_tag(self.args.event)
self._change_event()
elif self.args.attribute:
if not __sessions__.is_attached_misp():
return
identifier, tag = self.args.attribute
try:
taxonomies.revert_machinetag(tag)
except Exception:
self.log(
'error',
'Not a valid machine tag available in misp-taxonomies: "{}".'.
format(tag))
return
__sessions__.current.misp_event.event.add_attribute_tag(
tag, identifier)
self._change_event()
def __init__(self, profile="daily_report"):
self.taxonomies = Taxonomies()
self.report = ''
profile_name = "profiles.{}".format(profile)
self.template = importlib.import_module(name=profile_name)
def tag(self):
if not HAVE_PYTAX:
self.log('error', "Missing dependency, install PyTaxonomies (`pip install git+https://github.com/MISP/PyTaxonomies.git`)")
return
try:
taxonomies = Taxonomies(manifest_path=os.path.join(self.local_dir_taxonomies, 'MANIFEST.json'))
except Exception as e:
self.log('error', 'Unable to open the taxonomies, please fix the config file ([misp] - misp_taxonomies_directory): {}'.format(e))
return
if self.args.list:
self.log('table', dict(header=['Name', 'Description'], rows=[(title, tax.description)
for title, tax in taxonomies.items()]))
elif self.args.search:
matches = taxonomies.search(self.args.search)
if not matches:
self.log('error', 'No tags matching "{}".'.format(self.args.search))
return
self.log('success', 'Tags matching "{}":'.format(self.args.search))
for t in taxonomies.search(self.args.search):
self.log('item', t)
elif self.args.details:
taxonomy = taxonomies.get(self.args.details)
if not taxonomy:
self.log('error', 'No taxonomy called "{}".'.format(self.args.details))
return
if taxonomy.description:
self.log('info', taxonomy.description)
elif taxonomy.expanded:
self.log('info', taxonomy.expanded)
if taxonomy.refs:
self.log('info', 'References:')
for r in taxonomy.refs:
self.log('item', r)
if not taxonomy.has_entries():
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for p in taxonomy.predicates.values():
rows.append([p.description, p.predicate, taxonomy.make_machinetag(p)])
self.log('table', dict(header=header, rows=rows))
else:
for p in taxonomy.predicates.values():
if p.description:
self.log('info', p.description)
elif p.expanded:
self.log('info', p.expanded)
else:
self.log('info', p.predicate)
if not p.entries:
self.log('item', taxonomy.make_machinetag(p))
else:
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for e in p.entries.values():
if e.description:
descr = e.description
else:
descr = e.expanded
rows.append([descr, e.value, taxonomy.make_machinetag(p, e)])
self.log('table', dict(header=header, rows=rows))
elif self.args.event:
if not __sessions__.is_attached_misp():
return
try:
taxonomies.revert_machinetag(self.args.event)
except:
self.log('error', 'Not a valid machine tag available in misp-taxonomies: "{}".'.format(self.args.event))
return
__sessions__.current.misp_event.event.add_tag(self.args.event)
self._change_event()
elif self.args.attribute:
if not __sessions__.is_attached_misp():
return
identifier, tag = self.args.attribute
try:
taxonomies.revert_machinetag(tag)
except:
self.log('error', 'Not a valid machine tag available in misp-taxonomies: "{}".'.format(tag))
return
__sessions__.current.misp_event.event.add_attribute_tag(tag, identifier)
self._change_event()
class ReportGenerator():
def __init__(self, profile="daily_report"):
self.taxonomies = Taxonomies()
self.report = ''
profile_name = "profiles.{}".format(profile)
self.template = importlib.import_module(name=profile_name)
def from_remote(self, event_id):
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
misp = PyMISP(misp_url, misp_key, misp_verifycert)
result = misp.get(event_id)
self.misp_event = MISPEvent()
self.misp_event.load(result)
def from_file(self, path):
self.misp_event = MISPEvent()
self.misp_event.load_file(path)
def attributes(self):
if not self.misp_event.attributes:
return ''
list_attributes = []
for attribute in self.misp_event.attributes:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(defang(attribute.value)))
for obj in self.misp_event.Object:
if obj.name in self.template.objects_to_attach:
for attribute in obj.Attribute:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(defang(attribute.value)))
return self.template.attributes.format(list_attributes="\n".join(list_attributes))
def _get_tag_info(self, machinetag):
return self.taxonomies.revert_machinetag(machinetag)
def report_headers(self):
content = {'org_name': 'name',
'date': date.today().isoformat()}
self.report += self.template.headers.format(**content)
def event_level_tags(self):
if not self.misp_event.Tag:
return ''
for tag in self.misp_event.Tag:
# Only look for TLP for now
if tag['name'].startswith('tlp'):
tax, predicate = self._get_tag_info(tag['name'])
return self.template.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded)
def title(self):
internal_id = ''
summary = ''
# Get internal refs for report
for obj in self.misp_event.Object:
if obj.name != 'report':
continue
for a in obj.Attribute:
if a.object_relation == 'case-number':
internal_id = a.value
if a.object_relation == 'summary':
summary = a.value
return self.template.title.format(internal_id=internal_id, title=self.misp_event.info,
summary=summary)
def asciidoc(self, lang='en'):
self.report += self.title()
self.report += self.event_level_tags()
self.report += self.attributes()
def setUp(self):
self.taxonomies_offline = Taxonomies()
self.loaded_tax = {}
for t in self.taxonomies_offline.manifest['taxonomies']:
with open('{}/{}/{}'.format(self.taxonomies_offline.url, t['name'], 'machinetag.json'), 'r') as f:
self.loaded_tax[t['name']] = json.load(f)
class TestPyTaxonomies(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.taxonomies_offline = Taxonomies()
self.loaded_tax = {}
for t in self.taxonomies_offline.manifest['taxonomies']:
with open(
'{}/{}/{}'.format(self.taxonomies_offline.url, t['name'],
'machinetag.json'), 'r') as f:
self.loaded_tax[t['name']] = json.load(f)
def test_compareOnlineOffilne(self):
taxonomies_online = Taxonomies(manifest_path=None)
for t_online, t_offline in zip(taxonomies_online.values(),
self.taxonomies_offline.values()):
self.assertEqual(str(t_online), str(t_offline))
self.assertEqual(str(taxonomies_online), str(self.taxonomies_offline))
def test_expanded_machinetags(self):
self.taxonomies_offline.all_machinetags(expanded=True)
def test_machinetags(self):
self.taxonomies_offline.all_machinetags()
def test_dict(self):
len(self.taxonomies_offline)
for n, t in self.taxonomies_offline.items():
len(t)
for p, value in t.items():
continue
def test_search(self):
self.taxonomies_offline.search('phish')
def test_search_expanded(self):
self.taxonomies_offline.search('phish', expanded=True)
def test_print_classes(self):
for taxonomy in self.taxonomies_offline.values():
print(taxonomy)
for predicate in taxonomy.values():
print(predicate)
for entry in predicate.values():
print(entry)
def test_amountEntries(self):
for tax in self.taxonomies_offline.values():
tax.amount_entries()
def test_missingDependency(self):
pytaxonomies.api.HAS_REQUESTS = False
with self.assertRaises(Exception):
Taxonomies(manifest_path=None)
Taxonomies()
pytaxonomies.api.HAS_REQUESTS = True
def test_revert_machinetags(self):
for tax in self.taxonomies_offline.values():
for p in tax.values():
if tax.has_entries():
for e in p.values():
mt = tax.make_machinetag(p, e)
self.taxonomies_offline.revert_machinetag(mt)
else:
mt = tax.make_machinetag(p)
self.taxonomies_offline.revert_machinetag(mt)
def test_json(self):
for key, t in self.taxonomies_offline.items():
t.to_json()
def test_recreate_dump(self):
self.maxDiff = None
for key, t in self.taxonomies_offline.items():
out = t.to_dict()
self.assertDictEqual(out, self.loaded_tax[t.name])
def test_validate_schema(self):
self.taxonomies_offline.validate_with_schema()
def test_compareOnlineOffilne(self):
taxonomies_online = Taxonomies(manifest_path=None)
for t_online, t_offline in zip(taxonomies_online.values(), self.taxonomies_offline.values()):
self.assertEqual(str(t_online), str(t_offline))
self.assertEqual(str(taxonomies_online), str(self.taxonomies_offline))
return page_not_found(e)
@login_required
def page_not_found(e):
# avoid endpoint enumeration
return render_template('error/404.html'), 404
# ========== INITIAL taxonomies ============
default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"]
# enable default taxonomies
for taxo in default_taxonomies:
Tag.enable_taxonomy(taxo)
# ========== INITIAL tags auto export ============
taxonomies = Taxonomies()
infoleak_tags = taxonomies.get('infoleak').machinetags()
infoleak_automatic_tags = []
for tag in taxonomies.get('infoleak').machinetags():
if tag.split('=')[0][:] == 'infoleak:automatic-detection':
r_serv_db.sadd('list_export_tags', tag)
r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"')
# ============ MAIN ============
if __name__ == "__main__":
app.run(host=host, port=FLASK_PORT, threaded=True, ssl_context=ssl_context)
def test_missingDependency(self):
pytaxonomies.api.HAS_REQUESTS = False
with self.assertRaises(Exception):
Taxonomies(manifest_path=None)
Taxonomies()
pytaxonomies.api.HAS_REQUESTS = True
