class TestPyMISPWarningLists(unittest.TestCase): def setUp(self): self.warninglists = WarningLists() def test_dump_warninglists(self): warninglists_from_files = {} for warninglist_file in glob( os.path.join(self.warninglists.root_dir_warninglists, '*', 'list.json')): with open(warninglist_file, 'r') as f: warninglist = json.load(f) warninglists_from_files[warninglist['name']] = warninglist for name, w in self.warninglists.items(): out = w.to_dict() self.assertDictEqual(out, warninglists_from_files[w.name]) def test_validate_schema_warninglists(self): self.warninglists.validate_with_schema() def test_json(self): for w in self.warninglists.values(): w.to_json() def test_search(self): results = self.warninglists.search('8.8.8.8') self.assertEqual(results[0].name, 'List of known IPv4 public DNS resolvers') def test_slow_search(self): self.warninglists = WarningLists(True) results = self.warninglists.search('8.8.8.8') self.assertEqual(results[0].name, 'List of known IPv4 public DNS resolvers') results = self.warninglists.search('100.64.1.56') self.assertEqual(results[0].name, 'List of RFC 6598 CIDR blocks') results = self.warninglists.search('2001:DB8::34:1') self.assertEqual(results[0].name, 'List of RFC 3849 CIDR blocks') results = self.warninglists.search('1e100.net') self.assertTrue( 'List of known google domains' in [r.name for r in results]) results = self.warninglists.search('something.files.1drv.com') self.assertTrue( 'List of known microsoft domains' in [r.name for r in results]) results = self.warninglists.search('arbitrary-domain-1e100.net') self.assertEqual(results, []) results = self.warninglists.search('phishing.co.uk') self.assertEqual(results, [])
class HygieneConnector: def __init__(self): # Instantiate the connector helper from config config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml" config = ( yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {} ) self.helper = OpenCTIConnectorHelper(config) self.warninglists = WarningLists() # Create Hygiene Tag self.label_hygiene = self.helper.api.label.create( value="Hygiene", color="#fc0341" ) def _process_observable(self, observable) -> str: # Extract IPv4, IPv6 and Domain from entity data observable_value = observable["observable_value"] # Search in warninglist result = self.warninglists.search(observable_value) # Iterate over the hits if result: self.helper.log_info( "Hit found for %s in warninglists" % (observable_value) ) for hit in result: self.helper.log_info( "Type: %s | Name: %s | Version: %s | Descr: %s" % (hit.type, hit.name, hit.version, hit.description) ) # We set the score based on the number of warning list entries if len(result) >= 5: score = "5" elif len(result) >= 3: score = "10" elif len(result) == 1: score = "15" else: score = "20" self.helper.log_info( f"number of hits ({len(result)}) setting score to {score}" ) self.helper.api.stix_cyber_observable.add_label( id=observable["id"], label_id=self.label_hygiene["id"] ) self.helper.api.stix_cyber_observable.update_field( id=observable["id"], key="x_opencti_score", value=score ) for indicator_id in observable["indicatorsIds"]: self.helper.api.stix_domain_object.add_label( id=indicator_id, label_id=self.label_hygiene["id"] ) self.helper.api.stix_domain_object.update_field( id=indicator_id, key="x_opencti_score", value=score ) # Create external references external_reference_id = self.helper.api.external_reference.create( source_name="misp-warninglist", url="https://github.com/MISP/misp-warninglists/tree/main/" + LIST_MAPPING[hit.name], external_id=hit.name, description=hit.description, ) self.helper.api.stix_cyber_observable.add_external_reference( id=observable["id"], external_reference_id=external_reference_id["id"], ) return "Observable value found on warninglist and tagged accordingly" def _process_message(self, data) -> str: entity_id = data["entity_id"] observable = self.helper.api.stix_cyber_observable.read(id=entity_id) return self._process_observable(observable) # Start the main loop def start(self): self.helper.listen(self._process_message)
class HygieneConnector: def __init__(self): # Instantiate the connector helper from config config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.helper = OpenCTIConnectorHelper(config) self.warninglists = WarningLists() # Create Hygiene Tag self.tag_hygiene = self.helper.api.tag.create( tag_type="Hygiene", value="Hygiene", color="#fc0341", ) def _process_observable(self, observable): # Extract IPv4, IPv6 and Domain from entity data observable_value = observable["observable_value"] # Search in warninglist result = self.warninglists.search(observable_value) # Iterate over the hits if result: self.helper.log_info("Hit found for %s in warninglists" % (observable_value)) for hit in result: self.helper.log_info( "Type: %s | Name: %s | Version: %s | Descr: %s" % (hit.type, hit.name, hit.version, hit.description)) self.helper.api.stix_entity.add_tag( id=observable["id"], tag_id=self.tag_hygiene["id"]) # Create external references external_reference_id = self.helper.api.external_reference.create( source_name="misp-warninglist", url="https://github.com/MISP/misp-warninglists/tree/master" + LIST_MAPPING[hit.name], external_id=hit.name, description=hit.description, ) self.helper.api.stix_entity.add_external_reference( id=observable["id"], external_reference_id=external_reference_id["id"], ) return [ "observable value found on warninglist and tagged accordingly" ] def _process_message(self, data): entity_id = data["entity_id"] observable = self.helper.api.stix_observable.read(id=entity_id) return self._process_observable(observable) # Start the main loop def start(self): self.helper.listen(self._process_message)
class HygieneConnector: def __init__(self): # Instantiate the connector helper from config config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml" config = ( yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {} ) self.helper = OpenCTIConnectorHelper(config) warninglists_slow_search = bool( get_config_variable( "HYGIENE_WARNINGLISTS_SLOW_SEARCH", ["hygiene", "warninglists_slow_search"], config, default=False, ) ) self.enrich_subdomains = bool( get_config_variable( "HYGIENE_ENRICH_SUBDOMAINS", ["hygiene", "enrich_subdomains"], config, default=False, ) ) self.helper.log_info(f"Warning lists slow search: {warninglists_slow_search}") self.warninglists = WarningLists(slow_search=warninglists_slow_search) # Create Hygiene Tag self.label_hygiene = self.helper.api.label.create( value="Hygiene", color="#fc0341" ) if self.enrich_subdomains: self.label_hygiene_parent = self.helper.api.label.create( value="Hygiene_parent", color="#fc0341" ) def _process_observable(self, observable) -> str: # Extract IPv4, IPv6 and Domain from entity data observable_value = observable["observable_value"] observable_type = observable["entity_type"] # Search in warninglist result = self.warninglists.search(observable_value) # If not found and the domain is a subdomain, search with the parent. use_parent = False if not result and self.enrich_subdomains == True: if observable_type == "Domain-Name": ext = tldextract.extract(observable_value) if observable_value != ext.domain + "." + ext.suffix: result = self.warninglists.search(ext.domain + "." + ext.suffix) use_parent = True # Iterate over the hits if result: self.helper.log_info( "Hit found for %s in warninglists" % (observable_value) ) for hit in result: self.helper.log_info( "Type: %s | Name: %s | Version: %s | Descr: %s" % (hit.type, hit.name, hit.version, hit.description) ) # We set the score based on the number of warning list entries if len(result) >= 5: score = "5" elif len(result) >= 3: score = "10" elif len(result) == 1: score = "15" else: score = "20" self.helper.log_info( f"number of hits ({len(result)}) setting score to {score}" ) self.helper.api.stix_cyber_observable.add_label( id=observable["id"], label_id=self.label_hygiene["id"] if use_parent == False else self.label_hygiene_parent["id"], ) self.helper.api.stix_cyber_observable.update_field( id=observable["id"], input={"key": "x_opencti_score", "value": score}, ) for indicator_id in observable["indicatorsIds"]: self.helper.api.stix_domain_object.add_label( id=indicator_id, label_id=self.label_hygiene["id"] if use_parent == False else self.label_hygiene_parent["id"], ) self.helper.api.stix_domain_object.update_field( id=indicator_id, input={"key": "x_opencti_score", "value": score}, ) # Create external references external_reference_id = self.helper.api.external_reference.create( source_name="misp-warninglist", url="https://github.com/MISP/misp-warninglists/tree/main/" + LIST_MAPPING[hit.name], external_id=hit.name, description=hit.description, ) self.helper.api.stix_cyber_observable.add_external_reference( id=observable["id"], external_reference_id=external_reference_id["id"], ) return "Observable value found on warninglist and tagged accordingly" def _process_message(self, data) -> str: entity_id = data["entity_id"] observable = self.helper.api.stix_cyber_observable.read(id=entity_id) if observable is None: raise ValueError( "Observable not found (or the connector does not has access to this observable, check the group of the connector user)" ) return self._process_observable(observable) # Start the main loop def start(self): self.helper.listen(self._process_message)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- import sys from pymispwarninglists import WarningLists if __name__ == '__main__': warninglists = WarningLists(slow_search=True) ioc_list = set() for line in sys.stdin: line = line.strip().split('\t') ioc_list.add(line[-1]) for ioc in ioc_list: r = warninglists.search(ioc) if r: continue print(ioc)