def taxonomies():
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = []
name = []
description = []
version = []
enabled = []
n_tags = []
for taxonomie in list_taxonomies:
id.append(taxonomie)
name.append(taxonomies.get(taxonomie).name)
description.append(taxonomies.get(taxonomie).description)
version.append(taxonomies.get(taxonomie).version)
if taxonomie in active_taxonomies:
enabled.append(True)
else:
enabled.append(False)
n = str(r_serv_tags.scard('active_tag_' + taxonomie))
n_tags.append(n + '/' + str(len(taxonomies.get(taxonomie).machinetags())) )
return render_template("taxonomies.html",
id=id,
all_name = name,
description = description,
version = version,
enabled = enabled,
n_tags=n_tags)
def taxonomies():
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = []
name = []
description = []
version = []
enabled = []
n_tags = []
for taxonomie in list_taxonomies:
id.append(taxonomie)
name.append(taxonomies.get(taxonomie).name)
description.append(taxonomies.get(taxonomie).description)
version.append(taxonomies.get(taxonomie).version)
if taxonomie in active_taxonomies:
enabled.append(True)
else:
enabled.append(False)
n = str(r_serv_tags.scard('active_tag_' + taxonomie))
n_tags.append(n + '/' +
str(len(taxonomies.get(taxonomie).machinetags())))
return render_template("taxonomies.html",
id=id,
all_name=name,
description=description,
version=version,
enabled=enabled,
n_tags=n_tags)
def build_unsafe_tags():
unsafe_tags = set()
## CE content
unsafe_tags.add('dark-web:topic="pornography-child-exploitation"')
# add copine-scale tags
taxonomies = Taxonomies()
copine_scale = taxonomies.get('copine-scale')
if copine_scale:
for tag in copine_scale.machinetags():
unsafe_tags.add(tag)
return unsafe_tags
def disable_taxonomie():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = request.args.get('taxonomie')
if id in list_taxonomies:
r_serv_tags.srem('active_taxonomies', id)
for tag in taxonomies.get(id).machinetags():
r_serv_tags.srem('active_tag_' + id, tag)
return redirect(url_for('Tags.taxonomies'))
else:
return "INCORRECT INPUT"
def disable_taxonomie():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = request.args.get('taxonomie')
if id in list_taxonomies:
r_serv_tags.srem('active_taxonomies', id)
for tag in taxonomies.get(id).machinetags():
r_serv_tags.srem('active_tag_' + id, tag)
return redirect(url_for('Tags.taxonomies'))
else:
return "INCORRECT INPUT"
def edit_taxonomie_tag():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
arg1 = request.args.getlist('tag_enabled')
arg2 = request.args.getlist('tag_disabled')
id = request.args.get('taxonomie')
#verify input
if id in list_taxonomies:
list_tag = taxonomies.get(id).machinetags()
#check tags validity
if (all(elem in list_tag for elem in arg1) or
(len(arg1) == 0)) and (all(elem in list_tag
for elem in arg2) or (len(arg2) == 0)):
active_tag = r_serv_tags.smembers('active_tag_' + id)
diff = list(set(arg1) ^ set(list_tag))
#remove tags
for tag in diff:
r_serv_tags.srem('active_tag_' + id, tag)
#all tags unchecked
if len(arg1) == 0 and len(arg2) == 0:
r_serv_tags.srem('active_taxonomies', id)
#add new tags
for tag in arg2:
r_serv_tags.sadd('active_taxonomies', id)
r_serv_tags.sadd('active_tag_' + id, tag)
return redirect(url_for('Tags.taxonomies'))
else:
return "INCORRECT INPUT"
else:
return "INCORRECT INPUT"
def enable_taxonomy(taxonomie, enable_tags=True):
'''
Enable a taxonomy. (UI)
:param taxonomie: MISP taxonomy
:type taxonomie: str
:param enable_tags: crawled domain
:type enable_tags: boolean
'''
taxonomies = Taxonomies()
if enable_tags:
taxonomie_info = taxonomies.get(taxonomie)
if taxonomie_info:
# activate taxonomie
r_serv_tags.sadd('active_taxonomies', taxonomie)
# activate taxonomie tags
for tag in taxonomie_info.machinetags():
r_serv_tags.sadd('active_tag_{}'.format(taxonomie), tag)
else:
print('Error: {}, please update pytaxonomies'.format(taxonomie))
def edit_taxonomie_tag():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
arg1 = request.args.getlist('tag_enabled')
arg2 = request.args.getlist('tag_disabled')
id = request.args.get('taxonomie')
#verify input
if id in list_taxonomies:
list_tag = taxonomies.get(id).machinetags()
#check tags validity
if ( all(elem in list_tag for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag for elem in arg2) or (len(arg2) == 0) ):
active_tag = r_serv_tags.smembers('active_tag_' + id)
diff = list(set(arg1) ^ set(list_tag))
#remove tags
for tag in diff:
r_serv_tags.srem('active_tag_' + id, tag)
#all tags unchecked
if len(arg1) == 0 and len(arg2) == 0:
r_serv_tags.srem('active_taxonomies', id)
#add new tags
for tag in arg2:
r_serv_tags.sadd('active_taxonomies', id)
r_serv_tags.sadd('active_tag_' + id, tag)
return redirect(url_for('Tags.taxonomies'))
else:
return "INCORRECT INPUT"
else:
return "INCORRECT INPUT"
def edit_taxonomie():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = request.args.get('taxonomie')
#verify input
if id in list(taxonomies.keys()):
active_tag = r_serv_tags.smembers('active_tag_' + id)
list_tag = taxonomies.get(id).machinetags()
list_tag_desc = taxonomies.get(id).machinetags_expanded()
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
if id in active_taxonomies:
active = True
else:
active = False
n = str(r_serv_tags.scard('active_tag_' + id))
badge = n + '/' + str(len(taxonomies.get(id).machinetags()))
name = taxonomies.get(id).name
description = taxonomies.get(id).description
version = taxonomies.get(id).version
status = []
for tag in list_tag:
if tag in active_tag:
status.append(True)
else:
status.append(False)
return render_template("edit_taxonomie.html",
id=id,
name=name,
badge = badge,
description = description,
version = version,
active=active,
all_tags = list_tag,
list_tag_desc=list_tag_desc,
status = status)
else:
return 'INVALID TAXONOMIE'
def edit_taxonomie():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = request.args.get('taxonomie')
#verify input
if id in list(taxonomies.keys()):
active_tag = r_serv_tags.smembers('active_tag_' + id)
list_tag = taxonomies.get(id).machinetags()
list_tag_desc = taxonomies.get(id).machinetags_expanded()
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
if id in active_taxonomies:
active = True
else:
active = False
n = str(r_serv_tags.scard('active_tag_' + id))
badge = n + '/' + str(len(taxonomies.get(id).machinetags()))
name = taxonomies.get(id).name
description = taxonomies.get(id).description
version = taxonomies.get(id).version
status = []
for tag in list_tag:
if tag in active_tag:
status.append(True)
else:
status.append(False)
return render_template("edit_taxonomie.html",
id=id,
name=name,
badge=badge,
description=description,
version=version,
active=active,
all_tags=list_tag,
list_tag_desc=list_tag_desc,
status=status)
else:
return 'INVALID TAXONOMIE'
return page_not_found(e)
@login_required
def page_not_found(e):
# avoid endpoint enumeration
return render_template('error/404.html'), 404
# ========== INITIAL taxonomies ============
default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"]
# enable default taxonomies
for taxo in default_taxonomies:
Tag.enable_taxonomy(taxo)
# ========== INITIAL tags auto export ============
taxonomies = Taxonomies()
infoleak_tags = taxonomies.get('infoleak').machinetags()
infoleak_automatic_tags = []
for tag in taxonomies.get('infoleak').machinetags():
if tag.split('=')[0][:] == 'infoleak:automatic-detection':
r_serv_db.sadd('list_export_tags', tag)
r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"')
# ============ MAIN ============
if __name__ == "__main__":
app.run(host=host, port=FLASK_PORT, threaded=True, ssl_context=ssl_context)
def tag(self):
if not HAVE_PYTAX:
self.log(
'error',
"Missing dependency, install PyTaxonomies (`pip install git+https://github.com/MISP/PyTaxonomies.git`)"
)
return
taxonomies = Taxonomies()
if self.args.list:
self.log(
'table',
dict(header=['Name', 'Description'],
rows=[(title, tax.description)
for title, tax in taxonomies.items()]))
elif self.args.search:
matches = taxonomies.search(self.args.search)
if not matches:
self.log('error',
'No tags matching "{}".'.format(self.args.search))
return
self.log('success', 'Tags matching "{}":'.format(self.args.search))
for t in taxonomies.search(self.args.search):
self.log('item', t)
elif self.args.details:
taxonomy = taxonomies.get(self.args.details)
if not taxonomy:
self.log('error',
'No taxonomy called "{}".'.format(self.args.details))
return
if taxonomy.description:
self.log('info', taxonomy.description)
elif taxonomy.expanded:
self.log('info', taxonomy.expanded)
if taxonomy.refs:
self.log('info', 'References:')
for r in taxonomy.refs:
self.log('item', r)
if not taxonomy.has_entries():
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for p in taxonomy.predicates.values():
rows.append(
[p.description, p.predicate,
taxonomy.make_machinetag(p)])
self.log('table', dict(header=header, rows=rows))
else:
for p in taxonomy.predicates.values():
if p.description:
self.log('info', p.description)
elif p.expanded:
self.log('info', p.expanded)
else:
self.log('info', p.predicate)
if not p.entries:
self.log('item', taxonomy.make_machinetag(p))
else:
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for e in p.entries.values():
if e.description:
descr = e.description
else:
descr = e.expanded
rows.append(
[descr, e.value,
taxonomy.make_machinetag(p, e)])
self.log('table', dict(header=header, rows=rows))
elif self.args.event:
if not __sessions__.is_attached_misp():
return
try:
taxonomies.revert_machinetag(self.args.event)
except Exception:
self.log(
'error',
'Not a valid machine tag available in misp-taxonomies: "{}".'.
format(self.args.event))
return
__sessions__.current.misp_event.event.add_tag(self.args.event)
self._change_event()
elif self.args.attribute:
if not __sessions__.is_attached_misp():
return
identifier, tag = self.args.attribute
try:
taxonomies.revert_machinetag(tag)
except Exception:
self.log(
'error',
'Not a valid machine tag available in misp-taxonomies: "{}".'.
format(tag))
return
__sessions__.current.misp_event.event.add_attribute_tag(
tag, identifier)
self._change_event()
@login_required
def page_not_found(e):
# avoid endpoint enumeration
return render_template('error/404.html'), 404
# ========== INITIAL taxonomies ============
# add default ail taxonomies
r_serv_tags.sadd('active_taxonomies', 'infoleak')
r_serv_tags.sadd('active_taxonomies', 'gdpr')
r_serv_tags.sadd('active_taxonomies', 'fpf')
# add default tags
taxonomies = Taxonomies()
for tag in taxonomies.get('infoleak').machinetags():
r_serv_tags.sadd('active_tag_infoleak', tag)
for tag in taxonomies.get('gdpr').machinetags():
r_serv_tags.sadd('active_tag_gdpr', tag)
for tag in taxonomies.get('fpf').machinetags():
r_serv_tags.sadd('active_tag_fpf', tag)
# ========== INITIAL tags auto export ============
infoleak_tags = taxonomies.get('infoleak').machinetags()
infoleak_automatic_tags = []
for tag in taxonomies.get('infoleak').machinetags():
if tag.split('=')[0][:] == 'infoleak:automatic-detection':
r_serv_db.sadd('list_export_tags', tag)
r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"')
# ============ MAIN ============
def tag(self):
if not HAVE_PYTAX:
self.log('error', "Missing dependency, install PyTaxonomies (`pip install git+https://github.com/MISP/PyTaxonomies.git`)")
return
try:
taxonomies = Taxonomies(manifest_path=os.path.join(self.local_dir_taxonomies, 'MANIFEST.json'))
except Exception as e:
self.log('error', 'Unable to open the taxonomies, please fix the config file ([misp] - misp_taxonomies_directory): {}'.format(e))
return
if self.args.list:
self.log('table', dict(header=['Name', 'Description'], rows=[(title, tax.description)
for title, tax in taxonomies.items()]))
elif self.args.search:
matches = taxonomies.search(self.args.search)
if not matches:
self.log('error', 'No tags matching "{}".'.format(self.args.search))
return
self.log('success', 'Tags matching "{}":'.format(self.args.search))
for t in taxonomies.search(self.args.search):
self.log('item', t)
elif self.args.details:
taxonomy = taxonomies.get(self.args.details)
if not taxonomy:
self.log('error', 'No taxonomy called "{}".'.format(self.args.details))
return
if taxonomy.description:
self.log('info', taxonomy.description)
elif taxonomy.expanded:
self.log('info', taxonomy.expanded)
if taxonomy.refs:
self.log('info', 'References:')
for r in taxonomy.refs:
self.log('item', r)
if not taxonomy.has_entries():
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for p in taxonomy.predicates.values():
rows.append([p.description, p.predicate, taxonomy.make_machinetag(p)])
self.log('table', dict(header=header, rows=rows))
else:
for p in taxonomy.predicates.values():
if p.description:
self.log('info', p.description)
elif p.expanded:
self.log('info', p.expanded)
else:
self.log('info', p.predicate)
if not p.entries:
self.log('item', taxonomy.make_machinetag(p))
else:
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for e in p.entries.values():
if e.description:
descr = e.description
else:
descr = e.expanded
rows.append([descr, e.value, taxonomy.make_machinetag(p, e)])
self.log('table', dict(header=header, rows=rows))
elif self.args.event:
if not __sessions__.is_attached_misp():
return
try:
taxonomies.revert_machinetag(self.args.event)
except:
self.log('error', 'Not a valid machine tag available in misp-taxonomies: "{}".'.format(self.args.event))
return
__sessions__.current.misp_event.event.add_tag(self.args.event)
self._change_event()
elif self.args.attribute:
if not __sessions__.is_attached_misp():
return
identifier, tag = self.args.attribute
try:
taxonomies.revert_machinetag(tag)
except:
self.log('error', 'Not a valid machine tag available in misp-taxonomies: "{}".'.format(tag))
return
__sessions__.current.misp_event.event.add_attribute_tag(tag, identifier)
self._change_event()
return render_template("searchbox.html")
# ========== INITIAL taxonomies ============
r_serv_tags = redis.StrictRedis(
host=cfg.get("ARDB_Tags", "host"),
port=cfg.getint("ARDB_Tags", "port"),
db=cfg.getint("ARDB_Tags", "db"),
decode_responses=True)
# add default ail taxonomies
r_serv_tags.sadd('active_taxonomies', 'infoleak')
r_serv_tags.sadd('active_taxonomies', 'gdpr')
r_serv_tags.sadd('active_taxonomies', 'fpf')
# add default tags
taxonomies = Taxonomies()
for tag in taxonomies.get('infoleak').machinetags():
r_serv_tags.sadd('active_tag_infoleak', tag)
for tag in taxonomies.get('gdpr').machinetags():
r_serv_tags.sadd('active_tag_gdpr', tag)
for tag in taxonomies.get('fpf').machinetags():
r_serv_tags.sadd('active_tag_fpf', tag)
# ========== INITIAL tags auto export ============
r_serv_db = redis.StrictRedis(
host=cfg.get("ARDB_DB", "host"),
port=cfg.getint("ARDB_DB", "port"),
db=cfg.getint("ARDB_DB", "db"),
decode_responses=True)
infoleak_tags = taxonomies.get('infoleak').machinetags()
infoleak_automatic_tags = []
for tag in taxonomies.get('infoleak').machinetags():
