class TestPyTaxonomies(unittest.TestCase):
def setUp(self):
self.taxonomies_offline = Taxonomies()
self.loaded_tax = {}
for t in self.taxonomies_offline.manifest['taxonomies']:
with open('{}/{}/{}'.format(self.taxonomies_offline.url, t['name'], 'machinetag.json'), 'r') as f:
self.loaded_tax[t['name']] = json.load(f)
def test_compareOnlineOffilne(self):
taxonomies_online = Taxonomies(manifest_path=None)
for t_online, t_offline in zip(taxonomies_online.values(), self.taxonomies_offline.values()):
self.assertEqual(str(t_online), str(t_offline))
self.assertEqual(str(taxonomies_online), str(self.taxonomies_offline))
def test_expanded_machinetags(self):
self.taxonomies_offline.all_machinetags(expanded=True)
def test_machinetags(self):
self.taxonomies_offline.all_machinetags()
def test_dict(self):
len(self.taxonomies_offline)
for n, t in self.taxonomies_offline.items():
len(t)
for p, value in t.items():
continue
def test_search(self):
self.taxonomies_offline.search('phish')
def test_search_expanded(self):
self.taxonomies_offline.search('phish', expanded=True)
def test_print_classes(self):
for taxonomy in self.taxonomies_offline.values():
print(taxonomy)
for predicate in taxonomy.values():
print(predicate)
for entry in predicate.values():
print(entry)
def test_amountEntries(self):
for tax in self.taxonomies_offline.values():
tax.amount_entries()
def test_missingDependency(self):
pytaxonomies.api.HAS_REQUESTS = False
with self.assertRaises(Exception):
Taxonomies(manifest_path=None)
Taxonomies()
pytaxonomies.api.HAS_REQUESTS = True
def test_revert_machinetags(self):
for tax in self.taxonomies_offline.values():
for p in tax.values():
if tax.has_entries():
for e in p.values():
mt = tax.make_machinetag(p, e)
self.taxonomies_offline.revert_machinetag(mt)
else:
mt = tax.make_machinetag(p)
self.taxonomies_offline.revert_machinetag(mt)
def test_json(self):
for key, t in self.taxonomies_offline.items():
json.dumps(t, cls=EncodeTaxonomies)
def test_recreate_dump(self):
self.maxDiff = None
for key, t in self.taxonomies_offline.items():
out = t.to_dict()
self.assertDictEqual(out, self.loaded_tax[t.name])
def test_validate_schema(self):
self.taxonomies_offline.validate_with_schema()
class ReportGenerator():
def __init__(self, profile="daily_report"):
self.taxonomies = Taxonomies()
self.report = ''
profile_name = "profiles.{}".format(profile)
self.template = importlib.import_module(name=profile_name)
def from_remote(self, event_id):
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
misp = PyMISP(misp_url, misp_key, misp_verifycert)
result = misp.get(event_id)
self.misp_event = MISPEvent()
self.misp_event.load(result)
def from_file(self, path):
self.misp_event = MISPEvent()
self.misp_event.load_file(path)
def attributes(self):
if not self.misp_event.attributes:
return ''
list_attributes = []
for attribute in self.misp_event.attributes:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(defang(attribute.value)))
for obj in self.misp_event.Object:
if obj.name in self.template.objects_to_attach:
for attribute in obj.Attribute:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(
defang(attribute.value)))
return self.template.attributes.format(
list_attributes="\n".join(list_attributes))
def _get_tag_info(self, machinetag):
return self.taxonomies.revert_machinetag(machinetag)
def report_headers(self):
content = {'org_name': 'name', 'date': date.today().isoformat()}
self.report += self.template.headers.format(**content)
def event_level_tags(self):
if not self.misp_event.Tag:
return ''
for tag in self.misp_event.Tag:
# Only look for TLP for now
if tag['name'].startswith('tlp'):
tax, predicate = self._get_tag_info(tag['name'])
return self.template.event_level_tags.format(
value=predicate.predicate.upper(),
expanded=predicate.expanded)
def title(self):
internal_id = ''
summary = ''
# Get internal refs for report
for obj in self.misp_event.Object:
if obj.name != 'report':
continue
for a in obj.Attribute:
if a.object_relation == 'case-number':
internal_id = a.value
if a.object_relation == 'summary':
summary = a.value
return self.template.title.format(internal_id=internal_id,
title=self.misp_event.info,
summary=summary)
def asciidoc(self, lang='en'):
self.report += self.title()
self.report += self.event_level_tags()
self.report += self.attributes()
def tag(self):
if not HAVE_PYTAX:
self.log(
'error',
"Missing dependency, install PyTaxonomies (`pip install git+https://github.com/MISP/PyTaxonomies.git`)"
)
return
taxonomies = Taxonomies()
if self.args.list:
self.log(
'table',
dict(header=['Name', 'Description'],
rows=[(title, tax.description)
for title, tax in taxonomies.items()]))
elif self.args.search:
matches = taxonomies.search(self.args.search)
if not matches:
self.log('error',
'No tags matching "{}".'.format(self.args.search))
return
self.log('success', 'Tags matching "{}":'.format(self.args.search))
for t in taxonomies.search(self.args.search):
self.log('item', t)
elif self.args.details:
taxonomy = taxonomies.get(self.args.details)
if not taxonomy:
self.log('error',
'No taxonomy called "{}".'.format(self.args.details))
return
if taxonomy.description:
self.log('info', taxonomy.description)
elif taxonomy.expanded:
self.log('info', taxonomy.expanded)
if taxonomy.refs:
self.log('info', 'References:')
for r in taxonomy.refs:
self.log('item', r)
if not taxonomy.has_entries():
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for p in taxonomy.predicates.values():
rows.append(
[p.description, p.predicate,
taxonomy.make_machinetag(p)])
self.log('table', dict(header=header, rows=rows))
else:
for p in taxonomy.predicates.values():
if p.description:
self.log('info', p.description)
elif p.expanded:
self.log('info', p.expanded)
else:
self.log('info', p.predicate)
if not p.entries:
self.log('item', taxonomy.make_machinetag(p))
else:
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for e in p.entries.values():
if e.description:
descr = e.description
else:
descr = e.expanded
rows.append(
[descr, e.value,
taxonomy.make_machinetag(p, e)])
self.log('table', dict(header=header, rows=rows))
elif self.args.event:
if not __sessions__.is_attached_misp():
return
try:
taxonomies.revert_machinetag(self.args.event)
except Exception:
self.log(
'error',
'Not a valid machine tag available in misp-taxonomies: "{}".'.
format(self.args.event))
return
__sessions__.current.misp_event.event.add_tag(self.args.event)
self._change_event()
elif self.args.attribute:
if not __sessions__.is_attached_misp():
return
identifier, tag = self.args.attribute
try:
taxonomies.revert_machinetag(tag)
except Exception:
self.log(
'error',
'Not a valid machine tag available in misp-taxonomies: "{}".'.
format(tag))
return
__sessions__.current.misp_event.event.add_attribute_tag(
tag, identifier)
self._change_event()
class TestPyTaxonomies(unittest.TestCase):
def setUp(self):
self.taxonomies = Taxonomies()
self.manifest_path = "./misp-taxonomies/MANIFEST.json"
self.taxonomies_offline = Taxonomies(manifest_path=self.manifest_path)
self.json_load_taxonomies()
def __load_path(self, path):
with open(path, 'r') as f:
return json.load(f)
def json_load_taxonomies(self):
self.manifest = self.__load_path(self.manifest_path)
self.loaded_tax = {}
for t in self.manifest['taxonomies']:
path = '{}/{}/{}'.format(
os.path.dirname(os.path.realpath(self.manifest_path)),
t['name'], self.manifest['path'])
self.loaded_tax[t['name']] = self.__load_path(path)
def test_compareOnlineOffilne(self):
self.assertEqual(str(self.taxonomies), str(self.taxonomies_offline))
def test_expanded_machinetags(self):
self.taxonomies.all_machinetags(expanded=True)
def test_machinetags(self):
self.taxonomies.all_machinetags()
def test_dict(self):
len(self.taxonomies)
for n, t in self.taxonomies.items():
len(t)
for p, value in t.items():
continue
def test_search(self):
self.taxonomies.search('phish')
def test_search_expanded(self):
self.taxonomies.search('phish', expanded=True)
def test_print_classes(self):
for taxonomy in self.taxonomies.values():
print(taxonomy)
for predicate in taxonomy.values():
print(predicate)
for entry in predicate.values():
print(entry)
def test_amountEntries(self):
for tax in self.taxonomies.values():
tax.amount_entries()
def test_missingDependency(self):
pytaxonomies.api.HAS_REQUESTS = False
with self.assertRaises(Exception):
Taxonomies()
Taxonomies(manifest_path="./misp-taxonomies/MANIFEST.json")
pytaxonomies.api.HAS_REQUESTS = True
def test_revert_machinetags(self):
for tax in self.taxonomies.values():
for p in tax.values():
if tax.has_entries():
for e in p.values():
mt = tax.make_machinetag(p, e)
self.taxonomies.revert_machinetag(mt)
else:
mt = tax.make_machinetag(p)
self.taxonomies.revert_machinetag(mt)
def test_json(self):
for key, t in self.taxonomies.items():
json.dumps(t, cls=EncodeTaxonomies)
def test_recreate_dump(self):
self.maxDiff = None
for key, t in self.taxonomies.items():
out = t._json()
print(t.name)
self.assertCountEqual(out, self.loaded_tax[t.name])
def tag(self):
if not HAVE_PYTAX:
self.log('error', "Missing dependency, install PyTaxonomies (`pip install git+https://github.com/MISP/PyTaxonomies.git`)")
return
try:
taxonomies = Taxonomies(manifest_path=os.path.join(self.local_dir_taxonomies, 'MANIFEST.json'))
except Exception as e:
self.log('error', 'Unable to open the taxonomies, please fix the config file ([misp] - misp_taxonomies_directory): {}'.format(e))
return
if self.args.list:
self.log('table', dict(header=['Name', 'Description'], rows=[(title, tax.description)
for title, tax in taxonomies.items()]))
elif self.args.search:
matches = taxonomies.search(self.args.search)
if not matches:
self.log('error', 'No tags matching "{}".'.format(self.args.search))
return
self.log('success', 'Tags matching "{}":'.format(self.args.search))
for t in taxonomies.search(self.args.search):
self.log('item', t)
elif self.args.details:
taxonomy = taxonomies.get(self.args.details)
if not taxonomy:
self.log('error', 'No taxonomy called "{}".'.format(self.args.details))
return
if taxonomy.description:
self.log('info', taxonomy.description)
elif taxonomy.expanded:
self.log('info', taxonomy.expanded)
if taxonomy.refs:
self.log('info', 'References:')
for r in taxonomy.refs:
self.log('item', r)
if not taxonomy.has_entries():
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for p in taxonomy.predicates.values():
rows.append([p.description, p.predicate, taxonomy.make_machinetag(p)])
self.log('table', dict(header=header, rows=rows))
else:
for p in taxonomy.predicates.values():
if p.description:
self.log('info', p.description)
elif p.expanded:
self.log('info', p.expanded)
else:
self.log('info', p.predicate)
if not p.entries:
self.log('item', taxonomy.make_machinetag(p))
else:
header = ['Description', 'Predicate', 'Machinetag']
rows = []
for e in p.entries.values():
if e.description:
descr = e.description
else:
descr = e.expanded
rows.append([descr, e.value, taxonomy.make_machinetag(p, e)])
self.log('table', dict(header=header, rows=rows))
elif self.args.event:
if not __sessions__.is_attached_misp():
return
try:
taxonomies.revert_machinetag(self.args.event)
except:
self.log('error', 'Not a valid machine tag available in misp-taxonomies: "{}".'.format(self.args.event))
return
__sessions__.current.misp_event.event.add_tag(self.args.event)
self._change_event()
elif self.args.attribute:
if not __sessions__.is_attached_misp():
return
identifier, tag = self.args.attribute
try:
taxonomies.revert_machinetag(tag)
except:
self.log('error', 'Not a valid machine tag available in misp-taxonomies: "{}".'.format(tag))
return
__sessions__.current.misp_event.event.add_attribute_tag(tag, identifier)
self._change_event()
def revert_tag_from_taxonomies(tag):
taxonomies = Taxonomies()
try:
return taxonomies.revert_machinetag(tag)
except Exception:
return []
class ReportGenerator():
def __init__(self, profile="daily_report"):
self.taxonomies = Taxonomies()
self.report = ''
profile_name = "profiles.{}".format(profile)
self.template = importlib.import_module(name=profile_name)
def from_remote(self, event_id):
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
misp = PyMISP(misp_url, misp_key, misp_verifycert)
result = misp.get(event_id)
self.misp_event = MISPEvent()
self.misp_event.load(result)
def from_file(self, path):
self.misp_event = MISPEvent()
self.misp_event.load_file(path)
def attributes(self):
if not self.misp_event.attributes:
return ''
list_attributes = []
for attribute in self.misp_event.attributes:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(defang(attribute.value)))
for obj in self.misp_event.Object:
if obj.name in self.template.objects_to_attach:
for attribute in obj.Attribute:
if attribute.type in self.template.types_to_attach:
list_attributes.append("* {}".format(defang(attribute.value)))
return self.template.attributes.format(list_attributes="\n".join(list_attributes))
def _get_tag_info(self, machinetag):
return self.taxonomies.revert_machinetag(machinetag)
def report_headers(self):
content = {'org_name': 'name',
'date': date.today().isoformat()}
self.report += self.template.headers.format(**content)
def event_level_tags(self):
if not self.misp_event.Tag:
return ''
for tag in self.misp_event.Tag:
# Only look for TLP for now
if tag['name'].startswith('tlp'):
tax, predicate = self._get_tag_info(tag['name'])
return self.template.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded)
def title(self):
internal_id = ''
summary = ''
# Get internal refs for report
for obj in self.misp_event.Object:
if obj.name != 'report':
continue
for a in obj.Attribute:
if a.object_relation == 'case-number':
internal_id = a.value
if a.object_relation == 'summary':
summary = a.value
return self.template.title.format(internal_id=internal_id, title=self.misp_event.info,
summary=summary)
def asciidoc(self, lang='en'):
self.report += self.title()
self.report += self.event_level_tags()
self.report += self.attributes()
class TestPyTaxonomies(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.taxonomies_offline = Taxonomies()
self.loaded_tax = {}
for t in self.taxonomies_offline.manifest['taxonomies']:
with open(
'{}/{}/{}'.format(self.taxonomies_offline.url, t['name'],
'machinetag.json'), 'r') as f:
self.loaded_tax[t['name']] = json.load(f)
def test_compareOnlineOffilne(self):
taxonomies_online = Taxonomies(manifest_path=None)
for t_online, t_offline in zip(taxonomies_online.values(),
self.taxonomies_offline.values()):
self.assertEqual(str(t_online), str(t_offline))
self.assertEqual(str(taxonomies_online), str(self.taxonomies_offline))
def test_expanded_machinetags(self):
self.taxonomies_offline.all_machinetags(expanded=True)
def test_machinetags(self):
self.taxonomies_offline.all_machinetags()
def test_dict(self):
len(self.taxonomies_offline)
for n, t in self.taxonomies_offline.items():
len(t)
for p, value in t.items():
continue
def test_search(self):
self.taxonomies_offline.search('phish')
def test_search_expanded(self):
self.taxonomies_offline.search('phish', expanded=True)
def test_print_classes(self):
for taxonomy in self.taxonomies_offline.values():
print(taxonomy)
for predicate in taxonomy.values():
print(predicate)
for entry in predicate.values():
print(entry)
def test_amountEntries(self):
for tax in self.taxonomies_offline.values():
tax.amount_entries()
def test_missingDependency(self):
pytaxonomies.api.HAS_REQUESTS = False
with self.assertRaises(Exception):
Taxonomies(manifest_path=None)
Taxonomies()
pytaxonomies.api.HAS_REQUESTS = True
def test_revert_machinetags(self):
for tax in self.taxonomies_offline.values():
for p in tax.values():
if tax.has_entries():
for e in p.values():
mt = tax.make_machinetag(p, e)
self.taxonomies_offline.revert_machinetag(mt)
else:
mt = tax.make_machinetag(p)
self.taxonomies_offline.revert_machinetag(mt)
def test_json(self):
for key, t in self.taxonomies_offline.items():
t.to_json()
def test_recreate_dump(self):
self.maxDiff = None
for key, t in self.taxonomies_offline.items():
out = t.to_dict()
self.assertDictEqual(out, self.loaded_tax[t.name])
def test_validate_schema(self):
self.taxonomies_offline.validate_with_schema()
