class TestPyMISPWarningLists(unittest.TestCase):
def setUp(self):
self.warninglists = WarningLists()
def test_dump_warninglists(self):
warninglists_from_files = {}
for warninglist_file in glob(
os.path.join(self.warninglists.root_dir_warninglists, '*',
'list.json')):
with open(warninglist_file, 'r') as f:
warninglist = json.load(f)
warninglists_from_files[warninglist['name']] = warninglist
for name, w in self.warninglists.items():
out = w.to_dict()
self.assertDictEqual(out, warninglists_from_files[w.name])
def test_validate_schema_warninglists(self):
self.warninglists.validate_with_schema()
def test_json(self):
for w in self.warninglists.values():
w.to_json()
def test_search(self):
results = self.warninglists.search('8.8.8.8')
self.assertEqual(results[0].name,
'List of known IPv4 public DNS resolvers')
def test_slow_search(self):
self.warninglists = WarningLists(True)
results = self.warninglists.search('8.8.8.8')
self.assertEqual(results[0].name,
'List of known IPv4 public DNS resolvers')
results = self.warninglists.search('100.64.1.56')
self.assertEqual(results[0].name, 'List of RFC 6598 CIDR blocks')
results = self.warninglists.search('2001:DB8::34:1')
self.assertEqual(results[0].name, 'List of RFC 3849 CIDR blocks')
results = self.warninglists.search('1e100.net')
self.assertTrue(
'List of known google domains' in [r.name for r in results])
results = self.warninglists.search('something.files.1drv.com')
self.assertTrue(
'List of known microsoft domains' in [r.name for r in results])
results = self.warninglists.search('arbitrary-domain-1e100.net')
self.assertEqual(results, [])
results = self.warninglists.search('phishing.co.uk')
self.assertEqual(results, [])
class HygieneConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
self.warninglists = WarningLists()
# Create Hygiene Tag
self.label_hygiene = self.helper.api.label.create(
value="Hygiene", color="#fc0341"
)
def _process_observable(self, observable) -> str:
# Extract IPv4, IPv6 and Domain from entity data
observable_value = observable["observable_value"]
# Search in warninglist
result = self.warninglists.search(observable_value)
# Iterate over the hits
if result:
self.helper.log_info(
"Hit found for %s in warninglists" % (observable_value)
)
for hit in result:
self.helper.log_info(
"Type: %s | Name: %s | Version: %s | Descr: %s"
% (hit.type, hit.name, hit.version, hit.description)
)
# We set the score based on the number of warning list entries
if len(result) >= 5:
score = "5"
elif len(result) >= 3:
score = "10"
elif len(result) == 1:
score = "15"
else:
score = "20"
self.helper.log_info(
f"number of hits ({len(result)}) setting score to {score}"
)
self.helper.api.stix_cyber_observable.add_label(
id=observable["id"], label_id=self.label_hygiene["id"]
)
self.helper.api.stix_cyber_observable.update_field(
id=observable["id"], key="x_opencti_score", value=score
)
for indicator_id in observable["indicatorsIds"]:
self.helper.api.stix_domain_object.add_label(
id=indicator_id, label_id=self.label_hygiene["id"]
)
self.helper.api.stix_domain_object.update_field(
id=indicator_id, key="x_opencti_score", value=score
)
# Create external references
external_reference_id = self.helper.api.external_reference.create(
source_name="misp-warninglist",
url="https://github.com/MISP/misp-warninglists/tree/main/"
+ LIST_MAPPING[hit.name],
external_id=hit.name,
description=hit.description,
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=observable["id"],
external_reference_id=external_reference_id["id"],
)
return "Observable value found on warninglist and tagged accordingly"
def _process_message(self, data) -> str:
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
return self._process_observable(observable)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class HygieneConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.warninglists = WarningLists()
# Create Hygiene Tag
self.tag_hygiene = self.helper.api.tag.create(
tag_type="Hygiene",
value="Hygiene",
color="#fc0341",
)
def _process_observable(self, observable):
# Extract IPv4, IPv6 and Domain from entity data
observable_value = observable["observable_value"]
# Search in warninglist
result = self.warninglists.search(observable_value)
# Iterate over the hits
if result:
self.helper.log_info("Hit found for %s in warninglists" %
(observable_value))
for hit in result:
self.helper.log_info(
"Type: %s | Name: %s | Version: %s | Descr: %s" %
(hit.type, hit.name, hit.version, hit.description))
self.helper.api.stix_entity.add_tag(
id=observable["id"], tag_id=self.tag_hygiene["id"])
# Create external references
external_reference_id = self.helper.api.external_reference.create(
source_name="misp-warninglist",
url="https://github.com/MISP/misp-warninglists/tree/master"
+ LIST_MAPPING[hit.name],
external_id=hit.name,
description=hit.description,
)
self.helper.api.stix_entity.add_external_reference(
id=observable["id"],
external_reference_id=external_reference_id["id"],
)
return [
"observable value found on warninglist and tagged accordingly"
]
def _process_message(self, data):
entity_id = data["entity_id"]
observable = self.helper.api.stix_observable.read(id=entity_id)
return self._process_observable(observable)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
class HygieneConnector:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
warninglists_slow_search = bool(
get_config_variable(
"HYGIENE_WARNINGLISTS_SLOW_SEARCH",
["hygiene", "warninglists_slow_search"],
config,
default=False,
)
)
self.enrich_subdomains = bool(
get_config_variable(
"HYGIENE_ENRICH_SUBDOMAINS",
["hygiene", "enrich_subdomains"],
config,
default=False,
)
)
self.helper.log_info(f"Warning lists slow search: {warninglists_slow_search}")
self.warninglists = WarningLists(slow_search=warninglists_slow_search)
# Create Hygiene Tag
self.label_hygiene = self.helper.api.label.create(
value="Hygiene", color="#fc0341"
)
if self.enrich_subdomains:
self.label_hygiene_parent = self.helper.api.label.create(
value="Hygiene_parent", color="#fc0341"
)
def _process_observable(self, observable) -> str:
# Extract IPv4, IPv6 and Domain from entity data
observable_value = observable["observable_value"]
observable_type = observable["entity_type"]
# Search in warninglist
result = self.warninglists.search(observable_value)
# If not found and the domain is a subdomain, search with the parent.
use_parent = False
if not result and self.enrich_subdomains == True:
if observable_type == "Domain-Name":
ext = tldextract.extract(observable_value)
if observable_value != ext.domain + "." + ext.suffix:
result = self.warninglists.search(ext.domain + "." + ext.suffix)
use_parent = True
# Iterate over the hits
if result:
self.helper.log_info(
"Hit found for %s in warninglists" % (observable_value)
)
for hit in result:
self.helper.log_info(
"Type: %s | Name: %s | Version: %s | Descr: %s"
% (hit.type, hit.name, hit.version, hit.description)
)
# We set the score based on the number of warning list entries
if len(result) >= 5:
score = "5"
elif len(result) >= 3:
score = "10"
elif len(result) == 1:
score = "15"
else:
score = "20"
self.helper.log_info(
f"number of hits ({len(result)}) setting score to {score}"
)
self.helper.api.stix_cyber_observable.add_label(
id=observable["id"],
label_id=self.label_hygiene["id"]
if use_parent == False
else self.label_hygiene_parent["id"],
)
self.helper.api.stix_cyber_observable.update_field(
id=observable["id"],
input={"key": "x_opencti_score", "value": score},
)
for indicator_id in observable["indicatorsIds"]:
self.helper.api.stix_domain_object.add_label(
id=indicator_id,
label_id=self.label_hygiene["id"]
if use_parent == False
else self.label_hygiene_parent["id"],
)
self.helper.api.stix_domain_object.update_field(
id=indicator_id,
input={"key": "x_opencti_score", "value": score},
)
# Create external references
external_reference_id = self.helper.api.external_reference.create(
source_name="misp-warninglist",
url="https://github.com/MISP/misp-warninglists/tree/main/"
+ LIST_MAPPING[hit.name],
external_id=hit.name,
description=hit.description,
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=observable["id"],
external_reference_id=external_reference_id["id"],
)
return "Observable value found on warninglist and tagged accordingly"
def _process_message(self, data) -> str:
entity_id = data["entity_id"]
observable = self.helper.api.stix_cyber_observable.read(id=entity_id)
if observable is None:
raise ValueError(
"Observable not found (or the connector does not has access to this observable, check the group of the connector user)"
)
return self._process_observable(observable)
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys
from pymispwarninglists import WarningLists
if __name__ == '__main__':
warninglists = WarningLists(slow_search=True)
ioc_list = set()
for line in sys.stdin:
line = line.strip().split('\t')
ioc_list.add(line[-1])
for ioc in ioc_list:
r = warninglists.search(ioc)
if r:
continue
print(ioc)