def import_all(self, stations_short_names, interval, data_type): object_creator = getattr(self, f'{interval}_flask_{data_type}') if data_type == 'co2': base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/' elif data_type in ['c13', 'o18']: base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/' for station in stations_short_names: url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv' infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations' filepath = self.fetch(url) if not filepath: continue update = True event = self.get_existing_event_to_update(infofield) if event: location = event.get_objects_by_name('geolocation')[0] if not event: event = MISPEvent() event.info = infofield event.add_tag(getattr(self, f'tag_{station}')()) location = getattr(self, f'geolocation_{station}')() event.add_object(location) event.add_attribute('link', f'http://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}') update = False object_creator(event, location, filepath, update) if update: self.misp.update_event(event) else: self.misp.add_event(event)
class PassiveDNSParser(): def __init__(self, attribute, authentication): self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.attribute) self.pdns = pypdns.PyPDNS(basic_auth=authentication) def get_results(self): if hasattr(self, 'result'): return self.result event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return {'results': results} def parse(self, value): try: results = self.pdns.query(self.attribute.value) except Exception: self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'} return mapping = {'count': 'counter', 'origin': 'text', 'time_first': 'datetime', 'rrtype': 'text', 'rrname': 'text', 'rdata': 'text', 'time_last': 'datetime'} for result in results: pdns_object = MISPObject('passive-dns') for relation, attribute_type in mapping.items(): pdns_object.add_attribute(relation, type=attribute_type, value=result[relation]) pdns_object.add_reference(self.attribute.uuid, 'associated-to') self.misp_event.add_object(**pdns_object)
def create_list_of_objs_to_export(l_obj, r_type='json'): all_obj_to_export = set() set_relationship = set() for obj in l_obj: add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, obj, obj.get('lvl', 1)) # create MISP objects dict_misp_obj = create_all_misp_obj(all_obj_to_export, set_relationship) # create object relationships for obj_global_id_1, obj_global_id_2 in set_relationship: dict_relationship = get_relationship_between_global_obj( obj_global_id_1, obj_global_id_2) if dict_relationship: obj_src = dict_misp_obj[dict_relationship['src']] obj_dest = dict_misp_obj[dict_relationship['dest']] obj_src.add_reference(obj_dest.uuid, dict_relationship['relation'], 'add a comment') event = MISPEvent() event.info = 'AIL framework export' event.uuid = str(uuid.uuid4()) for obj_global_id in dict_misp_obj: misp_obj = dict_misp_obj[obj_global_id] AILObjects.create_map_obj_event_uuid(event.uuid, obj_global_id) AILObjects.create_map_obj_uuid_golbal_id(event.uuid, obj_global_id) if misp_obj: # add object to event event.add_object(dict_misp_obj[obj_global_id]) return event
def create_complex_event(self): event = MISPEvent() event.info = 'Complex Event' event.distribution = Distribution.all_communities event.add_tag('tlp:white') event.add_attribute('ip-src', '8.8.8.8') event.add_attribute('ip-dst', '8.8.8.9') event.add_attribute('domain', 'google.com') event.add_attribute('md5', '3c656da41f4645f77e3ec3281b63dd43') event.attributes[0].distribution = Distribution.your_organisation_only event.attributes[1].distribution = Distribution.this_community_only event.attributes[2].distribution = Distribution.connected_communities event.attributes[0].add_tag('tlp:red') event.attributes[1].add_tag('tlp:amber') event.attributes[2].add_tag('tlp:green') obj = MISPObject('file') obj.distribution = Distribution.connected_communities obj.add_attribute('filename', 'testfile') obj.add_attribute('md5', '3c656da41f4645f77e3ec3281b63dd44') obj.attributes[0].distribution = Distribution.your_organisation_only event.add_object(obj) return event
def main(): parser = argparse.ArgumentParser( description="Test of adding event to MISP") parser.add_argument("mwdb_user", help="Mwdb username") parser.add_argument("mwdb_pass", help="Mwdb password") parser.add_argument("config", help="Config") parser.add_argument("misp_url", help="Misp url") parser.add_argument("misp_key", help="Misp key") args = parser.parse_args() mwdb = Malwarecage() mwdb.login(args.mwdb_user, args.mwdb_pass) try: cfg = mwdb.query_config(args.config) iocs = parse(cfg.family, cfg.cfg) except FamilyNotSupportedYetError: logging.info("Family %s not supported yet...", cfg.family) return if not iocs: # Nothing actionable found - skip the config return event = MISPEvent() event.add_tag(f"mwdb:family:{cfg.family}") event.info = f"Malware configuration ({cfg.family})" for o in iocs.to_misp(): event.add_object(o) misp = PyMISP(args.misp_url, args.misp_key, False) misp.add_event(event)
def process(self, task: Task) -> None: # type: ignore config = task.get_payload("config") family = task.headers["family"] dhash = config_dhash(config) # Parse the config using iocextract library iocs = parse(family, config) if not iocs: # Nothing actionable found - skip the config return # Upload structured data to MISP event = MISPEvent() event.uuid = str(uuid5(self.CONFIG_NAMESPACE, dhash)) event.add_tag(f"mwdb:family:{family}") event.info = f"Malware configuration ({family})" if self.mwdb_url is not None: event.add_attribute("link", f"{self.mwdb_url}/config/{dhash}") for o in iocs.to_misp(): event.add_object(o) misp = ExpandedPyMISP(self.misp_url, self.misp_key, self.misp_verifycert) misp.add_event(event)
class PassiveSSLParser(): def __init__(self, attribute, authentication): self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.attribute) self.pssl = pypssl.PyPSSL(basic_auth=authentication) self.cert_hash = 'x509-fingerprint-sha1' self.cert_type = 'pem' self.mapping = {'issuer': ('text', 'issuer'), 'keylength': ('text', 'pubkey-info-size'), 'not_after': ('datetime', 'validity-not-after'), 'not_before': ('datetime', 'validity-not-before'), 'subject': ('text', 'subject')} def get_results(self): if hasattr(self, 'result'): return self.result event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return {'results': results} def parse(self): value = self.attribute.value.split('|')[0] if '|' in self.attribute.type else self.attribute.value try: results = self.pssl.query(value) except Exception: self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'} return if not results: self.result = {'error': 'Not found'} return for ip_address, certificates in results.items(): ip_uuid = self._handle_ip_attribute(ip_address) for certificate in certificates['certificates']: self._handle_certificate(certificate, ip_uuid) def _handle_certificate(self, certificate, ip_uuid): x509 = MISPObject('x509') x509.add_attribute(self.cert_hash, type=self.cert_hash, value=certificate) cert_details = self.pssl.fetch_cert(certificate) info = cert_details['info'] for feature, mapping in self.mapping.items(): attribute_type, object_relation = mapping x509.add_attribute(object_relation, type=attribute_type, value=info[feature]) x509.add_attribute(self.cert_type, type='text', value=self.cert_type) x509.add_reference(ip_uuid, 'seen-by') self.misp_event.add_object(**x509) def _handle_ip_attribute(self, ip_address): if ip_address == self.attribute.value: return self.attribute.uuid ip_attribute = MISPAttribute() ip_attribute.from_dict(**{'type': self.attribute.type, 'value': ip_address}) self.misp_event.add_attribute(**ip_attribute) return ip_attribute.uuid
def test_to_dict_json_format(self): misp_event = MISPEvent() av_signature_object = MISPObject("av-signature") av_signature_object.add_attribute("signature", "EICAR") av_signature_object.add_attribute("software", "ClamAv") misp_event.add_object(av_signature_object) self.assertEqual(json.loads(misp_event.to_json()), misp_event.to_dict(json_format=True))
def addNewObjectsExistingJson(mispVehicle, mispGeolocation): existing_event = MISPEvent() existing_event.load_file('data.json') existing_event.add_object(mispVehicle) existing_event.add_object(mispGeolocation) open("data.json", 'a').close() with open("data.json", "w") as fichier: fichier.write( existing_event.to_json(indent=2)) # écriture de l'évènement print("Nouveau véhicule ajouté au json existant")
class VulnerabilityParser(): def __init__(self, vulnerability): self.vulnerability = vulnerability self.misp_event = MISPEvent() self.vulnerability_mapping = { 'id': ('text', 'id'), 'summary': ('text', 'summary'), 'vulnerable_configuration_cpe_2_2': ('text', 'vulnerable_configuration'), 'Modified': ('datetime', 'modified'), 'Published': ('datetime', 'published'), 'references': ('link', 'references'), 'cvss': ('float', 'cvss-score') } def get_result(self): event = json.loads(self.misp_event.to_json())['Event'] results = { key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key]) } return {'results': results} def parse_vulnerability_information(self): vulnerability_object = MISPObject('vulnerability') for feature in ('id', 'summary', 'Modified', 'cvss'): value = self.vulnerability.get(feature) if value: attribute_type, relation = self.vulnerability_mapping[feature] vulnerability_object.add_attribute( relation, **{ 'type': attribute_type, 'value': value }) if 'Published' in self.vulnerability: vulnerability_object.add_attribute( 'published', **{ 'type': 'datetime', 'value': self.vulnerability['Published'] }) vulnerability_object.add_attribute( 'state', **{ 'type': 'text', 'value': 'Published' }) for feature in ('references', 'vulnerable_configuration_cpe_2_2'): if feature in self.vulnerability: attribute_type, relation = self.vulnerability_mapping[feature] for value in self.vulnerability[feature]: vulnerability_object.add_attribute( relation, **{ 'type': attribute_type, 'value': value }) self.misp_event.add_object(**vulnerability_object)
class VirusTotalParser(): def __init__(self): super(VirusTotalParser, self).__init__() self.misp_event = MISPEvent() def declare_variables(self, apikey, attribute): self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.apikey = apikey def get_result(self): event = json.loads(self.misp_event.to_json())['Event'] results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} return {'results': results} def parse_urls(self, query_result): for feature in ('detected_urls', 'undetected_urls'): if feature in query_result: for url in query_result[feature]: value = url['url'] if isinstance(url, dict) else url[0] self.misp_event.add_attribute('url', value) def parse_resolutions(self, resolutions, subdomains=None, uuids=None): domain_ip_object = MISPObject('domain-ip') if self.attribute.type == 'domain': domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value) attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address') else: domain_ip_object.add_attribute('ip', type='ip-dst', value=self.attribute.value) attribute_type, relation, key = ('domain', 'domain', 'hostname') for resolution in resolutions: domain_ip_object.add_attribute(relation, type=attribute_type, value=resolution[key]) if subdomains: for subdomain in subdomains: attribute = MISPAttribute() attribute.from_dict(**dict(type='domain', value=subdomain)) self.misp_event.add_attribute(**attribute) domain_ip_object.add_reference(attribute.uuid, 'subdomain') if uuids: for uuid in uuids: domain_ip_object.add_reference(uuid, 'sibling-of') self.misp_event.add_object(**domain_ip_object) def parse_vt_object(self, query_result): vt_object = MISPObject('virustotal-report') vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) self.misp_event.add_object(**vt_object) def get_query_result(self, query_type): params = {query_type: self.attribute.value, 'apikey': self.apikey} return requests.get(self.base_url, params=params)
def create_response(original_attribute: dict, software: str, signature: Optional[str] = None) -> dict: misp_event = MISPEvent() if signature: misp_event.add_attribute(**original_attribute) av_signature_object = MISPObject("av-signature") av_signature_object.add_attribute("signature", signature) av_signature_object.add_attribute("software", software) av_signature_object.add_reference(original_attribute["uuid"], "belongs-to") misp_event.add_object(av_signature_object) event = json.loads(misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} return {"results": results}
def export(self, cache: 'CaptureCache', is_public_instance: bool=False) -> MISPEvent: '''Export a capture in MISP format. You can POST the return of this method directly to a MISP instance and it will create an event.''' public_domain = get_config('generic', 'public_domain') event = MISPEvent() event.info = f'Lookyloo Capture ({cache.url})' lookyloo_link: MISPAttribute = event.add_attribute('link', f'https://{public_domain}/tree/{cache.uuid}') # type: ignore if not is_public_instance: lookyloo_link.distribution = 0 initial_url = URLObject(cache.url) initial_url.comment = 'Submitted URL' self.__misp_add_ips_to_URLObject(initial_url, cache.tree.root_hartree.hostname_tree) redirects: List[URLObject] = [] for nb, url in enumerate(cache.redirects): if url == cache.url: continue obj = URLObject(url) obj.comment = f'Redirect {nb}' self.__misp_add_ips_to_URLObject(obj, cache.tree.root_hartree.hostname_tree) redirects.append(obj) if redirects: redirects[-1].comment = f'Last redirect ({nb})' if redirects: prec_object = initial_url for u_object in redirects: prec_object.add_reference(u_object, 'redirects-to') prec_object = u_object initial_obj = event.add_object(initial_url) initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo') for u_object in redirects: event.add_object(u_object) final_redirect = event.objects[-1] try: fo = FileObject(pseudofile=cache.tree.root_hartree.rendered_node.body, filename=cache.tree.root_hartree.rendered_node.filename) fo.comment = 'Content received for the final redirect (before rendering)' fo.add_reference(final_redirect, 'loaded-by', 'URL loading that content') event.add_object(fo) except Har2TreeError: pass except AttributeError: # No `body` in rendered node pass return event
def parse_response(response): mapping = { 'file_name': { 'type': 'filename', 'object_relation': 'filename' }, 'file_size': { 'type': 'size-in-bytes', 'object_relation': 'size-in-bytes' }, 'file_type_mime': { 'type': 'mime-type', 'object_relation': 'mimetype' }, 'md5_hash': { 'type': 'md5', 'object_relation': 'md5' }, 'sha1_hash': { 'type': 'sha1', 'object_relation': 'sha1' }, 'sha256_hash': { 'type': 'sha256', 'object_relation': 'sha256' }, 'ssdeep': { 'type': 'ssdeep', 'object_relation': 'ssdeep' } } misp_event = MISPEvent() for data in response: misp_object = MISPObject('file') for feature, attribute in mapping.items(): if feature in data: misp_attribute = {'value': data[feature]} misp_attribute.update(attribute) misp_object.add_attribute(**misp_attribute) misp_event.add_object(**misp_object) return { 'results': { 'Object': [ json.loads(misp_object.to_json()) for misp_object in misp_event.objects ] } }
def parse_result(attribute, values): event = MISPEvent() initial_attribute = MISPAttribute() initial_attribute.from_dict(**attribute) event.add_attribute(**initial_attribute) mapping = {'asn': ('AS', 'asn'), 'prefix': ('ip-src', 'subnet-announced')} print(values) for last_seen, response in values['response'].items(): asn = MISPObject('asn') asn.add_attribute('last-seen', **{'type': 'datetime', 'value': last_seen}) for feature, attribute_fields in mapping.items(): attribute_type, object_relation = attribute_fields asn.add_attribute(object_relation, **{'type': attribute_type, 'value': response[feature]}) asn.add_reference(initial_attribute.uuid, 'related-to') event.add_object(**asn) event = json.loads(event.to_json()) return {key: event[key] for key in ('Attribute', 'Object')}
class HashlookupParser(): def __init__(self, attribute, hashlookupresult, api_url): self.attribute = attribute self.hashlookupresult = hashlookupresult self.api_url = api_url self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) self.references = defaultdict(list) def get_result(self): if self.references: self.__build_references() event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} return {'results': results} def parse_hashlookup_information(self): hashlookup_object = MISPObject('hashlookup') if 'source' in self.hashlookupresult: hashlookup_object.add_attribute('source', **{'type': 'text', 'value': self.hashlookupresult['source']}) if 'KnownMalicious' in self.hashlookupresult: hashlookup_object.add_attribute('KnownMalicious', **{'type': 'text', 'value': self.hashlookupresult['KnownMalicious']}) hashlookup_object.add_attribute('MD5', **{'type': 'md5', 'value': self.hashlookupresult['MD5']}) hashlookup_object.add_attribute('SHA-1', **{'type': 'sha1', 'value': self.hashlookupresult['SHA-1']}) if 'SSDEEP' in self.hashlookupresult: hashlookup_object.add_attribute('SSDEEP', **{'type': 'ssdeep', 'value': self.hashlookupresult['SSDEEP']}) if 'TLSH' in self.hashlookupresult: hashlookup_object.add_attribute('TLSH', **{'type': 'tlsh', 'value': self.hashlookupresult['TLSH']}) if 'FileName' in self.hashlookupresult: hashlookup_object.add_attribute('FileName', **{'type': 'filename', 'value': self.hashlookupresult['FileName']}) if 'FileSize' in self.hashlookupresult: hashlookup_object.add_attribute('FileSize', **{'type': 'size-in-bytes', 'value': self.hashlookupresult['FileSize']}) hashlookup_object.add_reference(self.attribute['uuid'], 'related-to') self.misp_event.add_object(hashlookup_object) def __build_references(self): for object_uuid, references in self.references.items(): for misp_object in self.misp_event.objects: if misp_object.uuid == object_uuid: for reference in references: misp_object.add_reference(**reference) break
class FarsightDnsdbParser(): def __init__(self, attribute): self.attribute = attribute self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) self.passivedns_mapping = { 'bailiwick': {'type': 'text', 'object_relation': 'bailiwick'}, 'count': {'type': 'counter', 'object_relation': 'count'}, 'raw_rdata': {'type': 'text', 'object_relation': 'raw_rdata'}, 'rdata': {'type': 'text', 'object_relation': 'rdata'}, 'rrname': {'type': 'text', 'object_relation': 'rrname'}, 'rrtype': {'type': 'text', 'object_relation': 'rrtype'}, 'time_first': {'type': 'datetime', 'object_relation': 'time_first'}, 'time_last': {'type': 'datetime', 'object_relation': 'time_last'}, 'zone_time_first': {'type': 'datetime', 'object_relation': 'zone_time_first'}, 'zone_time_last': {'type': 'datetime', 'object_relation': 'zone_time_last'} } self.comment = 'Result from an %s lookup on DNSDB about the %s: %s' def parse_passivedns_results(self, query_response): for query_type, results in query_response.items(): comment = self.comment % (query_type, TYPE_TO_FEATURE[self.attribute['type']], self.attribute['value']) for result in results: passivedns_object = MISPObject('passive-dns') if result.get('rdata') and isinstance(result['rdata'], list): for rdata in result.pop('rdata'): passivedns_object.add_attribute(**self._parse_attribute(comment, 'rdata', rdata)) for feature, value in result.items(): passivedns_object.add_attribute(**self._parse_attribute(comment, feature, value)) passivedns_object.add_reference(self.attribute['uuid'], 'related-to') self.misp_event.add_object(passivedns_object) def get_results(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return {'results': results} def _parse_attribute(self, comment, feature, value): attribute = {'value': value, 'comment': comment} attribute.update(self.passivedns_mapping[feature]) return attribute
def import_all(self, stations_short_names, interval, data_type): object_creator = getattr(self, f'{interval}_flask_{data_type}') if data_type == 'co2': base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/' elif data_type in ['c13', 'o18']: base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/' for station in stations_short_names: url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv' infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations' filepath = self.fetch(url) if not filepath: continue if infofield in self.scrippts_meta: event = MISPEvent() event.load_file(str(self.output_dir / self.scrippts_meta[infofield])) location = event.get_objects_by_name('geolocation')[0] update = True else: event = MISPEvent() event.uuid = str(uuid4()) event.info = infofield event.Orgc = self.misp_org event.add_tag(getattr(self, f'tag_{station}')()) location = getattr(self, f'geolocation_{station}')() event.add_object(location) event.add_attribute('link', f'https://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}') update = False with self.scrippts_meta_file.open('a') as f: writer = csv.writer(f) writer.writerow([infofield, f'{event.uuid}.json']) object_creator(event, location, filepath, update) if update: # Bump the publish timestamp event.publish_timestamp = datetime.datetime.timestamp(datetime.datetime.now()) feed_output = event.to_feed(with_meta=False) with (self.output_dir / f'{event.uuid}.json').open('w') as f: # json.dump(feed_output, f, indent=2, sort_keys=True) # For testing json.dump(feed_output, f)
def test_feed(self): me = MISPEvent() me.info = 'Test feed' org = MISPOrganisation() org.name = 'TestOrg' org.uuid = '123478' me.Orgc = org me.add_attribute('ip-dst', '8.8.8.8') obj = me.add_object(name='file') obj.add_attributes('filename', *['foo.exe', 'bar.exe']) h = hashlib.new('md5') h.update(b'8.8.8.8') hash_attr_val = h.hexdigest() feed = me.to_feed(with_meta=True) self.assertEqual(feed['Event']['_hashes'][0], hash_attr_val) self.assertEqual(feed['Event']['_manifest'][me.uuid]['info'], 'Test feed') self.assertEqual(len(feed['Event']['Object'][0]['Attribute']), 2)
class GoAmlParser(): def __init__(self): self.misp_event = MISPEvent() def read_xml(self, data): self.tree = ET.fromstring(data) def parse_xml(self): self.first_itteration() for t in self.tree.findall('transaction'): self.itterate(t, 'transaction') def first_itteration(self): submission_date = self.tree.find('submission_date').text.split('+')[0] self.misp_event.timestamp = int( time.mktime(time.strptime(submission_date, "%Y-%m-%dT%H:%M:%S"))) for node in goAMLobjects['report']['nodes']: element = self.tree.find(node) if element is not None: self.itterate(element, element.tag) def itterate(self, tree, aml_type, referencing_uuid=None, relationship_type=None): objects = goAMLobjects[aml_type] referenced_uuid = referencing_uuid rel = relationship_type if aml_type not in nodes_to_ignore: try: mapping = goAMLmapping[aml_type] misp_object = MISPObject(name=mapping['misp_name']) for leaf in objects['leaves']: element = tree.find(leaf) if element is not None: object_relation = mapping[element.tag] attribute = { 'object_relation': object_relation, 'value': element.text } misp_object.add_attribute(**attribute) if aml_type == 'transaction': for node in objects['nodes']: element = tree.find(node) if element is not None: self.fill_transaction(element, element.tag, misp_object) self.misp_event.add_object(misp_object) last_object = self.misp_event.objects[-1] referenced_uuid = last_object.uuid if referencing_uuid and relationship_type: referencing_object = self.misp_event.get_object_by_uuid( referencing_uuid) referencing_object.add_reference(referenced_uuid, rel, None, **last_object) except KeyError: pass for node in objects['nodes']: element = tree.find(node) if element is not None: tag = element.tag if tag in relationship_to_keep: rel = tag[2:] if tag.startswith('t_') else tag self.itterate(element, element.tag, referencing_uuid=referenced_uuid, relationship_type=rel) @staticmethod def fill_transaction(element, tag, misp_object): if 't_from' in tag: from_funds = element.find('from_funds_code').text from_funds_attribute = { 'object_relation': 'from-funds-code', 'value': from_funds } misp_object.add_attribute(**from_funds_attribute) from_country = element.find('from_country').text from_country_attribute = { 'object_relation': 'from-country', 'value': from_country } misp_object.add_attribute(**from_country_attribute) if 't_to' in tag: to_funds = element.find('to_funds_code').text to_funds_attribute = { 'object_relation': 'to-funds-code', 'value': to_funds } misp_object.add_attribute(**to_funds_attribute) to_country = element.find('to_country').text to_country_attribute = { 'object_relation': 'to-country', 'value': to_country } misp_object.add_attribute(**to_country_attribute)
fieldnames=args.fieldnames, has_fieldnames=has_fieldnames) objects = csv_loader.load() if args.dump: for o in objects: print(o.to_json()) else: if offline: print('You are in offline mode, quitting.') else: misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert) if args.new_event: event = MISPEvent() event.info = args.new_event for o in objects: event.add_object(**o) new_event = misp.add_event(event) if isinstance(new_event, str): print(new_event) elif 'id' in new_event: print(f'Created new event {new_event.id}') else: print('Something went wrong:') print(new_event) else: for o in objects: new_object = misp.add_object(args.update_event, o) if isinstance(new_object, str): print(new_object) elif new_object.attributes: print(f'New {new_object.name} object added to {args.update_event}')
class StixParser(): def __init__(self): self.misp_event = MISPEvent() self.event = [] self.misp_event['Galaxy'] = [] def loadEvent(self, args): try: filename = os.path.join(os.path.dirname(args[0]), args[1]) tempFile = open(filename, 'r', encoding='utf-8') self.filename = filename event = json.loads(tempFile.read()) self.stix_version = 'stix {}'.format(event.get('spec_version')) for o in event.get('objects'): try: try: self.event.append(stix2.parse(o, allow_custom=True)) except: self.parse_custom(o) except: pass if not self.event: print( json.dumps({ 'success': 0, 'message': 'There is no valid STIX object to import' })) sys.exit(1) try: event_distribution = args[2] if not isinstance(event_distribution, int): event_distribution = int( event_distribution) if event_distribution.isdigit( ) else 5 except: event_distribution = 5 try: attribute_distribution = args[3] if attribute_distribution != 'event' and not isinstance( attribute_distribution, int): attribute_distribution = int( attribute_distribution ) if attribute_distribution.isdigit() else 5 except: attribute_distribution = 5 self.misp_event.distribution = event_distribution self.__attribute_distribution = event_distribution if attribute_distribution == 'event' else attribute_distribution self.load_mapping() except: print( json.dumps({ 'success': 0, 'message': 'The STIX file could not be read' })) sys.exit(1) def parse_custom(self, obj): custom_object_type = obj.pop('type') labels = obj['labels'] try: @stix2.CustomObject( custom_object_type, [('id', stix2.properties.StringProperty(required=True)), ('x_misp_timestamp', stix2.properties.StringProperty(required=True)), ('labels', stix2.properties.ListProperty(labels, required=True)), ('x_misp_value', stix2.properties.StringProperty(required=True)), ('created_by_ref', stix2.properties.StringProperty(required=True)), ('x_misp_comment', stix2.properties.StringProperty()), ('x_misp_category', stix2.properties.StringProperty())]) class Custom(object): def __init__(self, **kwargs): return custom = Custom(**obj) except: @stix2.CustomObject( custom_object_type, [('id', stix2.properties.StringProperty(required=True)), ('x_misp_timestamp', stix2.properties.StringProperty(required=True)), ('labels', stix2.properties.ListProperty(labels, required=True)), ('x_misp_values', stix2.properties.DictionaryProperty(required=True)), ('created_by_ref', stix2.properties.StringProperty(required=True)), ('x_misp_comment', stix2.properties.StringProperty()), ('x_misp_category', stix2.properties.StringProperty())]) class Custom(object): def __init__(self, **kwargs): return custom = Custom(**obj) self.event.append(stix2.parse(custom)) def load_mapping(self): self.objects_mapping = { 'asn': { 'observable': observable_asn, 'pattern': pattern_asn }, 'domain-ip': { 'observable': observable_domain_ip, 'pattern': pattern_domain_ip }, 'email': { 'observable': self.observable_email, 'pattern': self.pattern_email }, 'file': { 'observable': observable_file, 'pattern': self.pattern_file }, 'ip-port': { 'observable': observable_ip_port, 'pattern': pattern_ip_port }, 'network-socket': { 'observable': observable_socket, 'pattern': pattern_socket }, 'process': { 'observable': observable_process, 'pattern': pattern_process }, 'registry-key': { 'observable': observable_regkey, 'pattern': pattern_regkey }, 'url': { 'observable': observable_url, 'pattern': pattern_url }, 'WindowsPEBinaryFile': { 'observable': self.observable_pe, 'pattern': self.pattern_pe }, 'x509': { 'observable': observable_x509, 'pattern': pattern_x509 } } def handler(self): self.outputname = '{}.stix2'.format(self.filename) if self.from_misp(): self.buildMispDict() else: self.version_attribute = { 'type': 'text', 'object_relation': 'version', 'value': self.stix_version } self.buildExternalDict() self.set_distribution() def from_misp(self): for o in self.event: if o._type == 'report' and 'misp:tool="misp2stix2"' in o.get( 'labels'): index = self.event.index(o) self.report = self.event.pop(index) return True return False def buildMispDict(self): self.parse_identity() self.parse_report() for o in self.event: try: object_type = o._type except: object_type = o['type'] labels = o.get('labels') if object_type in galaxy_types: self.parse_galaxy(o, labels) elif object_type == 'course-of-action': self.parse_course_of_action(o) elif 'x-misp-object' in object_type: if 'from_object' in labels: self.parse_custom_object(o) else: self.parse_custom_attribute(o, labels) else: if 'from_object' in labels: self.parse_object(o, labels) else: self.parse_attribute(o, labels) def parse_identity(self): identity = self.event.pop(0) org = {'name': identity.get('name')} self.misp_event['Org'] = org def parse_report(self): report = self.report self.misp_event.info = report.get('name') if report.get('published'): self.misp_event.publish_timestamp = self.getTimestampfromDate( report.get('published')) if hasattr(report, 'labels'): labels = report['labels'] for l in labels: self.misp_event.add_tag(l) if hasattr(report, 'external_references'): ext_refs = report['external_references'] for e in ext_refs: link = {"type": "link"} comment = e.get('source_name') try: comment = comment.split('url - ')[1] except: pass if comment: link['comment'] = comment link['value'] = e.get('url') self.misp_event.add_attribute(**link) def parse_galaxy(self, o, labels): galaxy_type = self.get_misp_type(labels) tag = labels[1] value = tag.split(':')[1].split('=')[1] galaxy_description, cluster_description = o.get('description').split( '|') galaxy = { 'type': galaxy_type, 'name': o.get('name'), 'description': galaxy_description, 'GalaxyCluster': [{ 'type': galaxy_type, 'value': value, 'tag_name': tag, 'description': cluster_description }] } self.misp_event['Galaxy'].append(galaxy) def parse_course_of_action(self, o): misp_object = MISPObject('course-of-action') if 'name' in o: attribute = { 'type': 'text', 'object_relation': 'name', 'value': o.get('name') } misp_object.add_attribute(**attribute) else: return if 'description' in o: attribute = { 'type': 'text', 'object_relation': 'description', 'value': o.get('description') } misp_object.add_attribute(**attribute) self.misp_event.add_object(**misp_object) def parse_custom_object(self, o): name = o.get('type').split('x-misp-object-')[1] timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp')) category = o.get('category') attributes = [] values = o.get('x_misp_values') for v in values: attribute_type, object_relation = v.split('_') attribute = { 'type': attribute_type, 'value': values.get(v), 'object_relation': object_relation } attributes.append(attribute) misp_object = { 'name': name, 'timestamp': timestamp, 'meta-category': category, 'Attribute': attributes } self.misp_event.add_object(**misp_object) def parse_custom_attribute(self, o, labels): attribute_type = o.get('type').split('x-misp-object-')[1] if attribute_type not in misp_types: attribute_type = attribute_type.replace('-', '|') timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp')) to_ids = bool(labels[1].split('=')[1]) value = o.get('x_misp_value') category = self.get_misp_category(labels) attribute = { 'type': attribute_type, 'timestamp': timestamp, 'to_ids': to_ids, 'value': value, 'category': category } self.misp_event.add_attribute(**attribute) def parse_object(self, o, labels): object_type = self.get_misp_type(labels) name = 'file' if object_type == 'WindowsPEBinaryFile' else object_type object_category = self.get_misp_category(labels) stix_type = o._type misp_object = MISPObject(name) misp_object['meta-category'] = object_category if stix_type == 'indicator': pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ') pattern[0] = pattern[0][1:] pattern[-1] = pattern[-1][:-1] attributes = self.objects_mapping[object_type]['pattern'](pattern) if stix_type == 'observed-data': observable = o.get('objects') attributes = self.objects_mapping[object_type]['observable']( observable) if isinstance(attributes, tuple): attributes, pe_uuid = attributes misp_object.add_reference(pe_uuid, 'included-in') for attribute in attributes: misp_object.add_attribute(**attribute) misp_object.to_ids = bool(labels[1].split('=')[1]) self.misp_event.add_object(**misp_object) def parse_attribute(self, o, labels): attribute_type = self.get_misp_type(labels) attribute_category = self.get_misp_category(labels) attribute = {'type': attribute_type, 'category': attribute_category} stix_type = o._type if stix_type == 'vulnerability': value = o.get('name') else: if stix_type == 'indicator': o_date = o.get('valid_from') pattern = o.get('pattern').replace('\\\\', '\\') value = self.parse_pattern_with_data( pattern) if attribute_type in ( 'malware-sample', 'attachment') else self.parse_pattern(pattern) attribute['to_ids'] = True else: o_date = o.get('first_observed') observable = o.get('objects') try: value = self.parse_observable(observable, attribute_type) except: print('{}: {}'.format(attribute_type, observable)) attribute['to_ids'] = False attribute['timestamp'] = self.getTimestampfromDate(o_date) if 'description' in o: attribute['comment'] = o.get('description') if isinstance(value, tuple): value, data = value attribute['data'] = io.BytesIO(data.encode()) attribute['value'] = value self.misp_event.add_attribute(**attribute) @staticmethod def observable_email(observable): attributes = [] addresses = {} files = {} for o_key, o_dict in observable.items(): part_type = o_dict._type if part_type == 'email-addr': addresses[o_key] = o_dict.get('value') elif part_type == 'file': files[o_key] = o_dict.get('name') else: message = dict(o_dict) attributes.append({ 'type': 'email-src', 'object_relation': 'from', 'value': addresses[message.pop('from_ref')], 'to_ids': False }) for ref in ('to_refs', 'cc_refs'): if ref in message: for item in message.pop(ref): mapping = email_mapping[ref] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': addresses[item], 'to_ids': False }) if 'body_multipart' in message: for f in message.pop('body_multipart'): attributes.append({ 'type': 'email-attachment', 'object_relation': 'attachment', 'value': files[f.get('body_raw_ref')], 'to_ids': False }) for m_key, m_value in message.items(): if m_key == 'additional_header_fields': for field_key, field_value in m_value.items(): mapping = email_mapping[field_key] if field_key == 'Reply-To': for rt in field_value: attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': rt, 'to_ids': False }) else: attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': field_value, 'to_ids': False }) else: try: mapping = email_mapping[m_key] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': m_value, 'to_ids': False }) except: if m_key.startswith("x_misp_attachment_"): attribute_type, relation = m_key.split( "x_misp_")[1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'to_ids': False, 'value': m_value['value'], 'data': io.BytesIO(m_value['data'].encode()) }) elif "x_misp_" in m_key: attribute_type, relation = m_key.split( "x_misp_")[1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': m_value, 'to_ids': False }) return attributes @staticmethod def pattern_email(pattern): attributes = [] attachments = defaultdict(dict) for p in pattern: p_type, p_value = p.split(' = ') try: mapping = email_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value[1:-1], 'to_ids': True }) except KeyError: if p_type.startswith("email-message:'x_misp_attachment_"): relation, field = p_type.split('.') relation = relation.split(':')[1][1:-1] attachments[relation][field] = p_value[1:-1] elif "x_misp_" in p_type: attribute_type, relation = p_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': p_value[1:-1], 'to_ids': True }) for a_key, a_dict in attachments.items(): _, _, attribute_type, relation = a_key.split('_') attributes.append({ 'type': attribute_type, 'object_relation': relation, 'to_ids': True, 'value': a_dict['value'], 'data': io.BytesIO(a_dict['data'].encode()) }) return attributes @staticmethod def pattern_file(pattern): attributes = [] malware_sample = {} for p in pattern: p_type, p_value = p.split(' = ') if p_type == 'artifact:payload_bin': malware_sample['data'] = p_value elif p_type in ("file:name", "file:hashes.'md5'"): try: mapping = file_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value[1:-1], 'to_ids': True }) malware_sample['filename'] = p_value[1:-1] except KeyError: attributes.append({ 'type': 'md5', 'object_relation': 'md5', 'value': p_value[1:-1], 'to_ids': True }) malware_sample['md5'] = p_value[1:-1] elif 'file:hashes.' in p_type: _, h = p_type.split('.') h = h[1:-1] attributes.append({ 'type': h, 'object_relation': h, 'value': p_value[1:-1] }) else: try: mapping = file_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value[1:-1], 'to_ids': True }) except KeyError: if "x_misp_" in p_type: attribute_type, relation = p_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': p_value[1:-1], 'to_ids': True }) if 'data' in malware_sample: value = "{}|{}".format(malware_sample['filename'], malware_sample['md5']) attributes.append({ 'type': 'malware-sample', 'object_relation': 'malware-sample', 'value': value, 'to_ids': True, 'data': io.BytesIO(malware_sample['data'].encode()) }) return attributes def observable_pe(self, observable): extension = observable['1']['extensions']['windows-pebinary-ext'] sections = extension['sections'] pe = MISPObject('pe') pe_uuid = str(uuid.uuid4()) pe.uuid = pe_uuid self.fill_object_attributes_observable(pe, pe_mapping, extension) for section in sections: pe_section = MISPObject('pe-section') if 'hashes' in section: for h_type, h_value in section['hashes'].items(): h_type = h_type.lower().replace('-', '') pe_section.add_attribute( **{ 'type': h_type, 'object_relation': h_type, 'value': h_value, 'to_ids': False }) self.fill_object_attributes_observable(pe_section, pe_section_mapping, section) section_uuid = str(uuid.uuid4()) pe_section.uuid = section_uuid pe.add_reference(section_uuid, 'included-in') self.misp_event.add_object(**pe_section) self.misp_event.add_object(**pe) return observable_file(observable), pe_uuid @staticmethod def fill_object_attributes_observable(misp_object, mapping_dict, stix_object): for stix_type, value in stix_object.items(): try: mapping = mapping_dict[stix_type] misp_object.add_attribute( **{ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': value, 'to_ids': False }) except KeyError: if stix_type.startswith("x_misp_"): attribute_type, relation = parse_custom_property(stix_type) misp_object.add_attribute( **{ 'type': attribute_type, 'object_relation': relation[:-1], 'value': value, 'to_ids': False }) def pattern_pe(self, pattern): attributes = [] sections = defaultdict(dict) pe = MISPObject('pe') pe_uuid = str(uuid.uuid4()) pe.uuid = pe_uuid for p in pattern: p_type, p_value = p.split(' = ') p_value = p_value[1:-1] if ':extensions.' in p_type: if '.sections[' in p_type: p_type_list = p_type.split('.') stix_type = "hashes.{}".format( p_type_list[4] [1:-1]) if '.hashes.' in p_type else p_type_list[3] sections[p_type_list[2]][stix_type] = p_value else: stix_type = p_type.split('.')[-1] try: mapping = pe_mapping[stix_type] pe.add_attribute( **{ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value, 'to_ids': True }) except KeyError: if stix_type.startswith("x_misp_"): attribute_type, relation = parse_custom_property( stix_type) pe.add_attribute( **{ 'type': attribute_type, 'object_relation': relation[:-2], 'value': p_value, 'to_ids': False }) else: if 'file:hashes.' in p_type: _, h = p_type.split('.') h = h[1:-1] attributes.append({ 'type': h, 'object_relation': h, 'value': p_value, 'to_ids': True }) else: try: mapping = file_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value, 'to_ids': True }) except KeyError: if "x_misp_" in p_type: attribute_type, relation = p_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': p_value, 'to_ids': True }) for _, section in sections.items(): pe_section = MISPObject('pe-section') for stix_type, value in section.items(): if 'hashes.' in stix_type: h_type = stix_type.split('.')[1] pe_section.add_attribute( **{ 'type': h_type, 'object_relation': h_type, 'value': value, 'to_ids': True }) else: try: mapping = pe_section_mapping[stix_type] pe_section.add_attribute( **{ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': value, 'to_ids': True }) except KeyError: if "x_misp_" in stix_type: attribute_type, relation = stix_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': value, 'to_ids': True }) section_uuid = str(uuid.uuid4()) pe_section.uuid = pe_uuid pe.add_reference(section_uuid, 'included-in') self.misp_event.add_object(**pe_section) self.misp_event.add_object(**pe) return attributes, pe_uuid def buildExternalDict(self): self.fetch_report() for o in self.event: object_type = o._type if object_type in ('relationship', 'report'): continue if object_type in galaxy_types: self.parse_external_galaxy(o) elif object_type == 'vulnerability': attribute = {'type': 'vulnerability', 'value': o.get('name')} if 'description' in o: attribute['comment'] = o.get('description') self.misp_event.add_attribute(**attribute) elif object_type == 'course-of-action': self.parse_course_of_action(o) elif object_type == 'indicator': pattern = o.get('pattern') self.parse_external_pattern(pattern) attribute = { 'type': 'stix2-pattern', 'object_relation': 'stix2-pattern', 'value': pattern } misp_object = { 'name': 'stix2-pattern', 'meta-category': 'stix2-pattern', 'Attribute': [self.version_attribute, attribute] } self.misp_event.add_object(**misp_object) def fetch_report(self): reports = [] for o in self.event: if o._type == 'report': reports.append(o) if len(reports) == 1: self.report = reports[0] self.parse_report() def parse_external_galaxy(self, o): galaxy = {'name': galaxy_types[o._type]} if 'kill_chain_phases' in o: galaxy['type'] = o['kill_chain_phases'][0].get('phase_name') cluster = defaultdict(dict) cluster['value'] = o.get('name') cluster['description'] = o.get('description') if 'aliases' in o: aliases = [] for a in o.get('aliases'): aliases.append(a) cluster['meta']['synonyms'] = aliases galaxy['GalaxyCluster'] = [cluster] self.misp_event['Galaxy'].append(galaxy) def parse_external_pattern(self, pattern): if ' OR ' in pattern and ' AND ' not in pattern: pattern = pattern.split('OR') for p in pattern: attribute = self.attribute_from_external_pattern(p) self.misp_event.add_attribute(**attribute) elif ' OR ' not in pattern and ' LIKE ' not in pattern: pattern = pattern.split('AND') if len(pattern) == 1: attribute = self.attribute_from_external_pattern(pattern[0]) self.misp_event.add_attribute(**attribute) @staticmethod def attribute_from_external_pattern(pattern): pattern_type, pattern_value = pattern.split(' = ') pattern_type, pattern_value = pattern_type[1:].strip( ), pattern_value[1:-2].strip() stix_type, value_type = pattern_type.split(':') if 'hashes' in value_type and 'x509' not in stix_type: h_type = value_type.split('.')[1] return {'type': h_type, 'value': pattern_value} else: # Might cause some issues, need more examples to test return { 'type': external_pattern_mapping[stix_type][value_type].get('type'), 'value': pattern_value } def set_distribution(self): for attribute in self.misp_event.attributes: attribute.distribution = self.__attribute_distribution for misp_object in self.misp_event.objects: misp_object.distribution = self.__attribute_distribution for attribute in misp_object.attributes: attribute.distribution = self.__attribute_distribution def saveFile(self): eventDict = self.misp_event.to_json() outputfile = '{}.stix2'.format(self.filename) with open(outputfile, 'w') as f: f.write(eventDict) @staticmethod def getTimestampfromDate(stix_date): try: return int(stix_date.timestamp()) except: return int( time.mktime( time.strptime( stix_date.split('+')[0], "%Y-%m-%d %H:%M:%S"))) @staticmethod def get_misp_type(labels): return labels[0].split('=')[1][1:-1] @staticmethod def get_misp_category(labels): return labels[1].split('=')[1][1:-1] @staticmethod def parse_observable(observable, attribute_type): return misp_types_mapping[attribute_type](observable, attribute_type) @staticmethod def parse_pattern(pattern): if ' AND ' in pattern: pattern_parts = pattern.split(' AND ') if len(pattern_parts) == 3: _, value1 = pattern_parts[2].split(' = ') _, value2 = pattern_parts[0].split(' = ') return '{}|{}'.format(value1[1:-2], value2[1:-1]) else: _, value1 = pattern_parts[0].split(' = ') _, value2 = pattern_parts[1].split(' = ') if value1 in ("'ipv4-addr'", "'ipv6-addr'"): return value2[1:-2] return '{}|{}'.format(value1[1:-1], value2[1:-2]) else: return pattern.split(' = ')[1][1:-2] def parse_pattern_with_data(self, pattern): if 'artifact:payload_bin' not in pattern: return self.parse_pattern(pattern) pattern_parts = pattern.split(' AND ') if len(pattern_parts) == 3: filename = pattern_parts[0].split(' = ')[1] md5 = pattern_parts[1].split(' = ')[1] return "{}|{}".format( filename[1:-1], md5[1:-1]), pattern_parts[2].split(' = ')[1][1:-2] else: return pattern_parts[0].split( ' = ')[1][1:-1], pattern_parts[1].split(' = ')[1][1:-2]
class AssemblyLineParser(): def __init__(self): self.misp_event = MISPEvent() self.results = {} self.attribute = {'to_ids': True} self._results_mapping = { 'NET_DOMAIN_NAME': 'domain', 'NET_FULL_URI': 'url', 'NET_IP': 'ip-dst' } self._file_mapping = { 'entropy': { 'type': 'float', 'object_relation': 'entropy' }, 'md5': { 'type': 'md5', 'object_relation': 'md5' }, 'mime': { 'type': 'mime-type', 'object_relation': 'mimetype' }, 'sha1': { 'type': 'sha1', 'object_relation': 'sha1' }, 'sha256': { 'type': 'sha256', 'object_relation': 'sha256' }, 'size': { 'type': 'size-in-bytes', 'object_relation': 'size-in-bytes' }, 'ssdeep': { 'type': 'ssdeep', 'object_relation': 'ssdeep' } } def get_submission(self, attribute, client): sid = attribute['value'].split('=')[-1] try: if not client.submission.is_completed(sid): self.results[ 'error'] = 'Submission not completed, please try again later.' return except Exception as e: self.results[ 'error'] = f'Something went wrong while trying to check if the submission in AssemblyLine is completed: {e.__str__()}' return try: submission = client.submission.full(sid) except Exception as e: self.results[ 'error'] = f"Something went wrong while getting the submission from AssemblyLine: {e.__str__()}" return self._parse_report(submission) def finalize_results(self): if 'error' in self.results: return self.results event = json.loads(self.misp_event.to_json()) results = { key: event[key] for key in ('Attribute', 'Object', 'Tag') if (key in event and event[key]) } return {'results': results} def _create_attribute(self, result, attribute_type): attribute = MISPAttribute() attribute.from_dict(type=attribute_type, value=result['value'], **self.attribute) if result['classification'] != 'UNCLASSIFIED': attribute.add_tag(result['classification'].lower()) self.misp_event.add_attribute(**attribute) return { 'referenced_uuid': attribute.uuid, 'relationship_type': '-'.join(result['context'].lower().split(' ')) } def _create_file_object(self, file_info): file_object = MISPObject('file') filename_attribute = {'type': 'filename'} filename_attribute.update(self.attribute) if file_info['classification'] != "UNCLASSIFIED": tag = {'Tag': [{'name': file_info['classification'].lower()}]} filename_attribute.update(tag) for feature, attribute in self._file_mapping.items(): attribute.update(tag) file_object.add_attribute(value=file_info[feature], **attribute) return filename_attribute, file_object for feature, attribute in self._file_mapping.items(): file_object.add_attribute(value=file_info[feature], **attribute) return filename_attribute, file_object @staticmethod def _get_results(submission_results): results = defaultdict(list) for k, values in submission_results.items(): h = k.split('.')[0] for t in values['result']['tags']: if t['context'] is not None: results[h].append(t) return results def _get_scores(self, file_tree): scores = {} for h, f in file_tree.items(): score = f['score'] if score > 0: scores[h] = {'name': f['name'], 'score': score} if f['children']: scores.update(self._get_scores(f['children'])) return scores def _parse_report(self, submission): if submission['classification'] != 'UNCLASSIFIED': self.misp_event.add_tag(submission['classification'].lower()) filtered_results = self._get_results(submission['results']) scores = self._get_scores(submission['file_tree']) for h, results in filtered_results.items(): if h in scores: attribute, file_object = self._create_file_object( submission['file_infos'][h]) print(file_object) for filename in scores[h]['name']: file_object.add_attribute('filename', value=filename, **attribute) for reference in self._parse_results(results): file_object.add_reference(**reference) self.misp_event.add_object(**file_object) def _parse_results(self, results): references = [] for result in results: try: attribute_type = self._results_mapping[result['type']] except KeyError: continue references.append(self._create_attribute(result, attribute_type)) return references
class TruSTARParser: ENTITY_TYPE_MAPPINGS = { 'BITCOIN_ADDRESS': "btc", 'CIDR_BLOCK': "ip-src", 'CVE': "vulnerability", 'URL': "url", 'EMAIL_ADDRESS': "email-src", 'SOFTWARE': "filename", 'IP': "ip-src", 'MALWARE': "malware-type", 'MD5': "md5", 'REGISTRY_KEY': "regkey", 'SHA1': "sha1", 'SHA256': "sha256" } # Relevant fields from each TruSTAR endpoint SUMMARY_FIELDS = ["severityLevel", "source", "score", "attributes"] METADATA_FIELDS = ["sightings", "firstSeen", "lastSeen", "tags"] REPORT_BASE_URL = "https://station.trustar.co/constellation/reports/{}" CLIENT_METATAG = f"MISP-{pymisp.__version__}" def __init__(self, attribute, config): config['enclave_ids'] = config.get('enclave_ids', "").strip().split(',') config['client_metatag'] = self.CLIENT_METATAG self.ts_client = TruStar(config=config) self.misp_event = MISPEvent() self.misp_attribute = MISPAttribute() self.misp_attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.misp_attribute) def get_results(self): """ Returns the MISP Event enriched with TruSTAR indicator summary data. """ try: event = json.loads(self.misp_event.to_json()) results = { key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key]) } return {'results': results} except Exception as e: misperrors[ 'error'] += f" -- Encountered issue serializing enrichment data -- {e}" return misperrors def generate_trustar_link(self, entity_type, entity_value): """ Generates link to TruSTAR report of entity. :param entity_type: <str> Type of entity. :param entity_value: <str> Value of entity. :return: <str> Link to indicator report in TruSTAR platform. """ report_id = b64encode( quote(f"{entity_type}|{entity_value}").encode()).decode() return self.REPORT_BASE_URL.format(report_id) @staticmethod def extract_tags(enrichment_report): """ Extracts tags from the enrichment report in order to add them to the TruSTAR MISP Object. Removes tags from report to avoid redundancy. :param: <OrderedDict> Enrichment data. :return: <list> List of tags. """ if enrichment_report and enrichment_report.get('tags'): return [tag.get('name') for tag in enrichment_report.pop('tags')] return None def generate_enrichment_report(self, summary, metadata): """ Extracts desired fields from summary and metadata reports and generates an enrichment report. :param summary: <trustar.IndicatorSummary> Indicator summary report. :param metadata: <trustar.Indicator> Indicator metadata report. :return: <str> Enrichment report. """ # Preserve order of fields as they exist in SUMMARY_FIELDS and METADATA_FIELDS enrichment_report = OrderedDict() if summary: summary_dict = summary.to_dict() enrichment_report.update({ field: summary_dict[field] for field in self.SUMMARY_FIELDS if summary_dict.get(field) }) if metadata: metadata_dict = metadata.to_dict() enrichment_report.update({ field: metadata_dict[field] for field in self.METADATA_FIELDS if metadata_dict.get(field) }) return enrichment_report def parse_indicator_summary(self, indicator, summary, metadata): """ Pulls enrichment data from the TruSTAR /indicators/summaries and /indicators/metadata endpoints and creates a MISP trustar_report. :param indicator: <str> Value of the attribute :summary: <trustar.IndicatorSummary> Indicator summary response object. :metadata: <trustar.Indicator> Indicator response object. """ # Verify that the indicator type is supported by TruSTAR if summary and summary.indicator_type in self.ENTITY_TYPE_MAPPINGS: indicator_type = summary.indicator_type elif metadata and metadata.type in self.ENTITY_TYPE_MAPPINGS: indicator_type = metadata.type else: misperrors['error'] += " -- Attribute not found or not supported" raise Exception try: # Extract most relevant fields from indicator summary and metadata responses enrichment_report = self.generate_enrichment_report( summary, metadata) tags = self.extract_tags(enrichment_report) if enrichment_report: # Create MISP trustar_report object and populate it with enrichment data trustar_obj = MISPObject('trustar_report') trustar_obj.add_attribute( indicator_type, attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type], value=indicator) trustar_obj.add_attribute("INDICATOR_SUMMARY", attribute_type="text", value=json.dumps(enrichment_report, indent=4)) report_link = self.generate_trustar_link( indicator_type, indicator) trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=report_link) self.misp_event.add_object(**trustar_obj) elif not tags: # If enrichment report is empty and there are no tags, nothing to add to attribute raise Exception("No relevant data found") if tags: for tag in tags: self.misp_event.add_attribute_tag(tag, indicator) except Exception as e: misperrors[ 'error'] += f" -- Error enriching attribute {indicator} -- {e}" raise e
def handler(q=False): if q is False: return False request = json.loads(q) if not request.get('attribute') or not check_input_attribute( request['attribute']): return { 'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.' } toquery = request['attribute'] if toquery['type'] not in mispattributes['input']: return {'error': 'Unsupported attribute type.'} bgpranking = BGPRanking() value_toquery = int( toquery['value'][2:]) if toquery['value'].startswith('AS') else int( toquery['value']) values = bgpranking.query(value_toquery, date=(date.today() - timedelta(1)).isoformat()) if not values['response'] or not values['response']['asn_description']: misperrors[ 'error'] = 'There is no result about this ASN in BGP Ranking' return misperrors event = MISPEvent() attribute = MISPAttribute() attribute.from_dict(**toquery) event.add_attribute(**attribute) asn_object = MISPObject('asn') asn_object.add_attribute(**{ 'type': 'AS', 'object_relation': 'asn', 'value': values['meta']['asn'] }) description, country = values['response']['asn_description'].split(', ') for relation, value in zip(('description', 'country'), (description, country)): asn_object.add_attribute(**{ 'type': 'text', 'object_relation': relation, 'value': value }) mapping = { 'address_family': { 'type': 'text', 'object_relation': 'address-family' }, 'date': { 'type': 'datetime', 'object_relation': 'date' }, 'position': { 'type': 'float', 'object_relation': 'position' }, 'rank': { 'type': 'float', 'object_relation': 'ranking' } } bgp_object = MISPObject('bgp-ranking') for feature in ('rank', 'position'): bgp_attribute = {'value': values['response']['ranking'][feature]} bgp_attribute.update(mapping[feature]) bgp_object.add_attribute(**bgp_attribute) date_attribute = { 'value': datetime.strptime(values['meta']['date'], '%Y-%m-%d') } date_attribute.update(mapping['date']) bgp_object.add_attribute(**date_attribute) address_attribute = {'value': values['meta']['address_family']} address_attribute.update(mapping['address_family']) bgp_object.add_attribute(**address_attribute) asn_object.add_reference(attribute.uuid, 'describes') asn_object.add_reference(bgp_object.uuid, 'ranked-with') event.add_object(asn_object) event.add_object(bgp_object) event = json.loads(event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return {'results': results}
class TestMISPEvent(unittest.TestCase): def setUp(self): self.maxDiff = None self.mispevent = MISPEvent() def init_event(self): self.mispevent.info = 'This is a test' self.mispevent.distribution = 1 self.mispevent.threat_level_id = 1 self.mispevent.analysis = 1 self.mispevent.set_date("2017-12-31") # test the set date method def test_simple(self): with open('tests/mispevent_testfiles/simple.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event(self): self.init_event() self.mispevent.publish() with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_loadfile(self): self.mispevent.load_file('tests/mispevent_testfiles/event.json') with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_tag(self): self.init_event() self.mispevent.add_tag('bar') self.mispevent.add_tag(name='baz') new_tag = MISPTag() new_tag.from_dict(name='foo') self.mispevent.add_tag(new_tag) with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_attribute(self): self.init_event() self.mispevent.add_attribute('filename', 'bar.exe') self.mispevent.add_attribute_tag('osint', 'bar.exe') attr_tags = self.mispevent.get_attribute_tag('bar.exe') self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint') self.assertEqual(attr_tags[0].name, 'osint') with open('tests/mispevent_testfiles/attribute.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) # Fake setting an attribute ID for testing self.mispevent.attributes[0].id = 42 self.mispevent.delete_attribute(42) with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_object_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}]) self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah') self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(['filename'])) self.assertEqual(len(self.mispevent.objects[0].get_attributes_by_relation('filename')), 1) self.mispevent.add_object(name='url', strict=True) self.mispevent.objects[1].add_attribute('url', value='https://www.circl.lu') self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' self.mispevent.objects[0].add_reference('b', 'baz', comment='foo') self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz') with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168") def test_object_level_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar') self.mispevent.objects[0].add_tag('osint') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_malware(self): with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.init_event() self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile) attribute = self.mispevent.attributes[0] self.assertEqual(attribute.malware_binary, pseudofile) with open('tests/mispevent_testfiles/malware.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_malware(self): self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json') with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.assertEqual( self.mispevent.objects[0].get_attributes_by_relation('malware-sample')[0].malware_binary.read(), pseudofile.read()) def test_sighting(self): sighting = MISPSighting() sighting.from_dict(value='1', type='bar', timestamp=11111111) with open('tests/mispevent_testfiles/sighting.json', 'r') as f: ref_json = json.load(f) self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_event(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes_existing(self): self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') with open('tests/mispevent_testfiles/shadow.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes(self): self.init_event() self.mispevent.add_proposal(type='filename', value='baz.jpg') self.mispevent.add_attribute('filename', 'bar.exe') self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf') with open('tests/mispevent_testfiles/proposals.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_default_attributes(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}]) self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent.objects[0].attributes[0]) self.mispevent.objects[1].add_attribute('filename', value='baz') self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_default_values(self): self.init_event() self.mispevent.add_object(name='whois', strict=True) self.mispevent.objects[0].add_attribute('registrar', value='registar.example.com') self.mispevent.objects[0].add_attribute('domain', value='domain.example.com') self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com') self.mispevent.objects[0].add_attribute('nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_not_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) def test_event_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.mispevent.info = 'blah' self.assertTrue(self.mispevent.edited) def test_event_tag_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.add_tag('foo') self.assertTrue(self.mispevent.edited) def test_event_attribute_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.mispevent.attributes[0].value = 'blah' self.assertTrue(self.mispevent.attributes[0].edited) self.assertFalse(self.mispevent.attributes[1].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].tags[0].name = 'blah' self.assertTrue(self.mispevent.attributes[0].tags[0].edited) self.assertFalse(self.mispevent.attributes[0].tags[1].edited) self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited_second(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].add_tag(name='blah') self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].edited) self.assertFalse(self.mispevent.objects[1].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited_tag(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].add_tag('blah') self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_by_id(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') misp_obj = self.mispevent.get_object_by_id(1556) self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f') def test_userdefined_object(self): self.init_event() self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles') with self.assertRaises(InvalidMISPObject) as e: # Fail on required self.mispevent.to_json() if sys.version_info >= (3, ): self.assertEqual(e.exception.message, '{\'member3\'} are required.') else: # Python2 bullshit self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.') self.mispevent.objects[0].add_attribute('member3', value='foo') with self.assertRaises(InvalidMISPObject) as e: # Fail on requiredOneOf self.mispevent.to_json() self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2') self.mispevent.objects[0].add_attribute('member1', value='bar') self.mispevent.objects[0].add_attribute('member1', value='baz') with self.assertRaises(InvalidMISPObject) as e: # member1 is not a multiple self.mispevent.to_json() self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2] self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
class SophosLabsApi(): def __init__(self, client_id, client_secret): self.misp_event = MISPEvent() self.client_id = client_id self.client_secret = client_secret self.authToken = f"{self.client_id}:{self.client_secret}" self.baseurl = 'de.api.labs.sophos.com' d = {'grant_type': 'client_credentials'} h = {'Authorization': f"Basic {base64.b64encode(self.authToken.encode('UTF-8')).decode('ascii')}",\ 'Content-Type': 'application/x-www-form-urlencoded'} r = requests.post('https://api.labs.sophos.com/oauth2/token', headers=h, data=d) if r.status_code == 200: j = json.loads(r.text) self.accessToken = j['access_token'] def get_result(self): event = json.loads(self.misp_event.to_json()) results = { key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key]) } return {'results': results} def hash_lookup(self, filehash): sophos_object = MISPObject('SOPHOSLabs Intelix SHA256 Report') h = {"Authorization": f"{self.accessToken}"} r = requests.get(f"https://{self.baseurl}/lookup/files/v1/{filehash}", headers=h) if r.status_code == 200: j = json.loads(r.text) if 'reputationScore' in j: sophos_object.add_attribute('Reputation Score', type='text', value=j['reputationScore']) if 0 <= j['reputationScore'] <= 19: sophos_object.add_attribute('Decision', type='text', value='This file is malicious') if 20 <= j['reputationScore'] <= 29: sophos_object.add_attribute( 'Decision', type='text', value='This file is potentially unwanted') if 30 <= j['reputationScore'] <= 69: sophos_object.add_attribute( 'Decision', type='text', value='This file is unknown and suspicious') if 70 <= j['reputationScore'] <= 100: sophos_object.add_attribute( 'Decision', type='text', value='This file is known good') if 'detectionName' in j: sophos_object.add_attribute('Detection Name', type='text', value=j['detectionName']) else: sophos_object.add_attribute( 'Detection Name', type='text', value='No name associated with this IoC') self.misp_event.add_object(**sophos_object) def ip_lookup(self, ip): sophos_object = MISPObject('SOPHOSLabs Intelix IP Category Lookup') h = {"Authorization": f"{self.accessToken}"} r = requests.get(f"https://{self.baseurl}/lookup/ips/v1/{ip}", headers=h) if r.status_code == 200: j = json.loads(r.text) if 'category' in j: for c in j['category']: sophos_object.add_attribute('IP Address Categorisation', type='text', value=c) else: sophos_object.add_attribute( 'IP Address Categorisation', type='text', value='No category assocaited with IoC') self.misp_event.add_object(**sophos_object) def url_lookup(self, url): sophos_object = MISPObject('SOPHOSLabs Intelix URL Lookup') h = {"Authorization": f"{self.accessToken}"} r = requests.get( f"https://{self.baseurl}/lookup/urls/v1/{quote(url, safe='')}", headers=h) if r.status_code == 200: j = json.loads(r.text) if 'productivityCategory' in j: sophos_object.add_attribute('URL Categorisation', type='text', value=j['productivityCategory']) else: sophos_object.add_attribute( 'URL Categorisation', type='text', value='No category assocaited with IoC') if 'riskLevel' in j: sophos_object.add_attribute('URL Risk Level', type='text', value=j['riskLevel']) else: sophos_object.add_attribute( 'URL Risk Level', type='text', value='No risk level associated with IoC') if 'securityCategory' in j: sophos_object.add_attribute('URL Security Category', type='text', value=j['securityCategory']) else: sophos_object.add_attribute( 'URL Security Category', type='text', value='No Security Category associated with IoC') self.misp_event.add_object(**sophos_object)
class VulnerabilityParser(): def __init__(self, attribute, vulnerability): self.attribute = attribute self.vulnerability = vulnerability self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) self.references = defaultdict(list) self.capec_features = ('id', 'name', 'summary', 'prerequisites', 'solutions') self.vulnerability_mapping = { 'id': ('text', 'id'), 'summary': ('text', 'summary'), 'vulnerable_configuration_cpe_2_2': ('text', 'vulnerable_configuration'), 'Modified': ('datetime', 'modified'), 'Published': ('datetime', 'published'), 'references': ('link', 'references'), 'cvss': ('float', 'cvss-score') } self.weakness_mapping = { 'name': 'name', 'description_summary': 'description', 'status': 'status', 'weaknessabs': 'weakness-abs' } def get_result(self): if self.references: self.__build_references() event = json.loads(self.misp_event.to_json()) results = { key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key]) } return {'results': results} def parse_vulnerability_information(self): vulnerability_object = MISPObject('vulnerability') for feature in ('id', 'summary', 'Modified', 'cvss'): value = self.vulnerability.get(feature) if value: attribute_type, relation = self.vulnerability_mapping[feature] vulnerability_object.add_attribute( relation, **{ 'type': attribute_type, 'value': value }) if 'Published' in self.vulnerability: vulnerability_object.add_attribute( 'published', **{ 'type': 'datetime', 'value': self.vulnerability['Published'] }) vulnerability_object.add_attribute( 'state', **{ 'type': 'text', 'value': 'Published' }) for feature in ('references', 'vulnerable_configuration_cpe_2_2'): if feature in self.vulnerability: attribute_type, relation = self.vulnerability_mapping[feature] for value in self.vulnerability[feature]: vulnerability_object.add_attribute( relation, **{ 'type': attribute_type, 'value': value }) vulnerability_object.add_reference(self.attribute['uuid'], 'related-to') self.misp_event.add_object(**vulnerability_object) if 'cwe' in self.vulnerability and self.vulnerability[ 'cwe'] != 'Unknown': self.__parse_weakness(vulnerability_object.uuid) if 'capec' in self.vulnerability: self.__parse_capec(vulnerability_object.uuid) def __build_references(self): for object_uuid, references in self.references.items(): for misp_object in self.misp_event.objects: if misp_object.uuid == object_uuid: for reference in references: misp_object.add_reference(**reference) break def __parse_capec(self, vulnerability_uuid): attribute_type = 'text' for capec in self.vulnerability['capec']: capec_object = MISPObject('attack-pattern') for feature in self.capec_features: capec_object.add_attribute( feature, **dict(type=attribute_type, value=capec[feature])) for related_weakness in capec['related_weakness']: attribute = dict(type='weakness', value="CWE-{}".format(related_weakness)) capec_object.add_attribute('related-weakness', **attribute) self.misp_event.add_object(**capec_object) self.references[vulnerability_uuid].append( dict(referenced_uuid=capec_object.uuid, relationship_type='targeted-by')) def __parse_weakness(self, vulnerability_uuid): attribute_type = 'text' cwe_string, cwe_id = self.vulnerability['cwe'].split('-') cwes = requests.get(cveapi_url.replace('/cve/', '/cwe')) if cwes.status_code == 200: for cwe in cwes.json(): if cwe['id'] == cwe_id: weakness_object = MISPObject('weakness') weakness_object.add_attribute( 'id', **dict(type=attribute_type, value='-'.join([cwe_string, cwe_id]))) for feature, relation in self.weakness_mapping.items(): if cwe.get(feature): weakness_object.add_attribute( relation, **dict(type=attribute_type, value=cwe[feature])) self.misp_event.add_object(**weakness_object) self.references[vulnerability_uuid].append( dict(referenced_uuid=weakness_object.uuid, relationship_type='weakened-by')) break
class Yeti(): def __init__(self, url, key, attribute): self.misp_mapping = { 'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url', 'AutonomousSystem': 'AS', 'File': 'sha256' } self.yeti_client = pyeti.YetiApi(url=url, api_key=key) self.attribute = attribute self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) def search(self, value): obs = self.yeti_client.observable_search(value=value) if obs: return obs[0] def get_neighboors(self, obs_id): neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: links_by_id = { link['dst']['id']: (link['description'], 'dst') for link in neighboors['links'] if link['dst']['id'] != obs_id } links_by_id.update({ link['src']['id']: (link['description'], 'src') for link in neighboors['links'] if link['src']['id'] != obs_id }) for n in neighboors['objs']: yield n, links_by_id[n['id']] def parse_yeti_result(self): obs = self.search(self.attribute['value']) for obs_to_add, link in self.get_neighboors(obs['id']): object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) if object_misp_domain_ip: self.misp_event.add_object(object_misp_domain_ip) continue object_misp_url = self.__get_object_url(obs_to_add) if object_misp_url: self.misp_event.add_object(object_misp_url) continue if link[0] == 'NS record': object_ns_record = self.__get_object_ns_record( obs_to_add, link[1]) if object_ns_record: self.misp_event.add_object(object_ns_record) continue self.__get_attribute(obs_to_add, link[0]) def get_result(self): event = json.loads(self.misp_event.to_json()) results = { key: event[key] for key in ('Attribute', 'Object') if key in event } return results def __get_attribute(self, obs_to_add, link): try: type_attr = self.misp_mapping[obs_to_add['type']] value = None if obs_to_add['type'] == 'File': value = obs_to_add['value'].split(':')[1] else: value = obs_to_add['value'] attr = self.misp_event.add_attribute(value=value, type=type_attr) attr.comment = '%s: %s' % (link, self.attribute['value']) except KeyError: logging.error('type not found %s' % obs_to_add['type']) return for t in obs_to_add['tags']: self.misp_event.add_attribute_tag(t['name'], attr['uuid']) def __get_object_domain_ip(self, obj_to_add): if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname', 'domain']) or \ (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add['value']) domain_ip_object.add_attribute( self.__get_relation(self.attribute, is_yeti_object=False), self.attribute['value']) domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') return domain_ip_object def __get_object_url(self, obj_to_add): if (obj_to_add['type'] == 'Url' and self.attribute['type'] in [ 'hostname', 'domain', 'ip-src', 'ip-dst' ]) or (obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url'): url_object = MISPObject('url') obj_relation = self.__get_relation(obj_to_add) if obj_relation: url_object.add_attribute(obj_relation, obj_to_add['value']) obj_relation = self.__get_relation(self.attribute, is_yeti_object=False) if obj_relation: url_object.add_attribute(obj_relation, self.attribute['value']) url_object.add_reference(self.attribute['uuid'], 'related_to') return url_object def __get_object_ns_record(self, obj_to_add, link): queried_domain = None ns_domain = None object_dns_record = MISPObject('dns-record') if link == 'dst': queried_domain = self.attribute['value'] ns_domain = obj_to_add['value'] elif link == 'src': queried_domain = obj_to_add['value'] ns_domain = self.attribute['value'] if queried_domain and ns_domain: object_dns_record.add_attribute('queried-domain', queried_domain) object_dns_record.add_attribute('ns-record', ns_domain) object_dns_record.add_reference(self.attribute['uuid'], 'related_to') return object_dns_record def __get_relation(self, obj, is_yeti_object=True): if is_yeti_object: type_attribute = self.misp_mapping[obj['type']] else: type_attribute = obj['type'] if type_attribute == 'ip-src' or type_attribute == 'ip-dst': return 'ip' elif 'domain' == type_attribute: return 'domain' elif 'hostname' == type_attribute: return 'domain' elif type_attribute == 'url': return type_attribute
class RFEnricher: """Class for enriching an attribute with data from Recorded Future. The enrichment data is returned as a custom MISP object. """ def __init__(self, attribute_props: dict): self.event = MISPEvent() self.enrichment_object = MISPObject("Recorded Future Enrichment") description = ("An object containing the enriched attribute and " "related entities from Recorded Future.") self.enrichment_object.from_dict(**{ "meta-category": "misc", "description": description, "distribution": 0 }) # Create a copy of enriched attribute to add tags to temp_attr = MISPAttribute() temp_attr.from_dict(**attribute_props) self.enriched_attribute = MISPAttribute() self.enriched_attribute.from_dict(**{ "value": temp_attr.value, "type": temp_attr.type, "distribution": 0 }) self.related_attributes: List[Tuple[str, MISPAttribute]] = [] self.color_picker = RFColors() self.galaxy_finder = GalaxyFinder() # Mapping from MISP-type to RF-type self.type_to_rf_category = { "ip": "ip", "ip-src": "ip", "ip-dst": "ip", "ip-src|port": "ip", "ip-dst|port": "ip", "domain": "domain", "hostname": "domain", "md5": "hash", "sha1": "hash", "sha256": "hash", "uri": "url", "url": "url", "vulnerability": "vulnerability", "weakness": "vulnerability", } # Related entities have 'Related' as part of the word and Links entities from RF # portrayed as related attributes in MISP self.related_attribute_types = [ "RelatedIpAddress", "RelatedInternetDomainName", "RelatedHash", "RelatedEmailAddress", "RelatedCyberVulnerability", "IpAddress", "InternetDomainName", "Hash", "EmailAddress", "CyberVulnerability", ] # Related entities have 'Related' as part of the word and and Links entities from RF portrayed as tags in MISP self.galaxy_tag_types = [ "RelatedMalware", "RelatedThreatActor", "Threat Actor", "MitreAttackIdentifier", "Malware", ] def enrich(self) -> None: """Run the enrichment.""" category = self.type_to_rf_category.get(self.enriched_attribute.type, "") enriched_attribute_value = self.enriched_attribute.value # If enriched attribute has a port we need to remove that port # since RF do not support enriching ip addresses with port if self.enriched_attribute.type in ["ip-src|port", "ip-dst|port"]: enriched_attribute_value = enriched_attribute_value.split("|")[0] json_response = GLOBAL_REQUEST_HANDLER.rf_lookup( category, enriched_attribute_value) response = json.loads(json_response.content) try: # Add risk score and risk rules as tags to the enriched attribute risk_score = response["data"]["risk"]["score"] hex_color = self.color_picker.riskscore_color(risk_score) tag_name = f'recorded-future:risk-score="{risk_score}"' self.add_tag(tag_name, hex_color) risk_criticality = response["data"]["risk"]["criticalityLabel"] hex_color = self.color_picker.criticality_color(risk_criticality) tag_name = f'recorded-future:criticality="{risk_criticality}"' self.add_tag(tag_name, hex_color) for evidence in response["data"]["risk"]["evidenceDetails"]: risk_rule = evidence["rule"] criticality = evidence["criticality"] hex_color = self.color_picker.riskrule_color(criticality) tag_name = f'recorded-future:risk-rule="{risk_rule}"' self.add_tag(tag_name, hex_color) links_data = response["data"].get("links", {}).get("hits") # Check if we have error in links response. If yes, then user do not have right module enabled in token links_access_error = response["data"].get("links", {}).get("error") galaxy_tags = [] if not links_access_error: for hit in links_data: for section in hit["sections"]: for sec_list in section["lists"]: entity_type = sec_list["type"]["name"] for entity in sec_list["entities"]: if entity_type in self.galaxy_tag_types: galaxy = self.galaxy_finder.find_galaxy_match( entity["name"], entity_type) if galaxy and galaxy not in galaxy_tags: galaxy_tags.append(galaxy) else: self.add_attribute(entity["name"], entity_type) else: # Retrieve related entities for related_entity in response["data"]["relatedEntities"]: related_type = related_entity["type"] if related_type in self.related_attribute_types: # Related entities returned as additional attributes for related in related_entity["entities"]: # filter those entities that have count bigger than 4, to reduce noise # because there can be a huge list of related entities if int(related["count"]) > 4: indicator = related["entity"]["name"] self.add_attribute(indicator, related_type) elif related_type in self.galaxy_tag_types: # Related entities added as galaxy-tags to the enriched attribute galaxy_tags = [] for related in related_entity["entities"]: # filter those entities that have count bigger than 4, to reduce noise # because there can be a huge list of related entities if int(related["count"]) > 4: indicator = related["entity"]["name"] galaxy = self.galaxy_finder.find_galaxy_match( indicator, related_type) # Handle deduplication of galaxy tags if galaxy and galaxy not in galaxy_tags: galaxy_tags.append(galaxy) for galaxy in galaxy_tags: self.add_tag(galaxy) except KeyError: misperrors[ "error"] = "Unexpected format in Recorded Future api response." raise def add_attribute(self, indicator: str, indicator_type: str) -> None: """Helper method for adding an indicator to the attribute list.""" out_type = self.get_output_type(indicator_type, indicator) attribute = MISPAttribute() attribute.from_dict(**{ "value": indicator, "type": out_type, "distribution": 0 }) self.related_attributes.append((indicator_type, attribute)) def add_tag(self, tag_name: str, hex_color: str = None) -> None: """Helper method for adding a tag to the enriched attribute.""" tag = MISPTag() tag_properties = {"name": tag_name} if hex_color: tag_properties["colour"] = hex_color tag.from_dict(**tag_properties) self.enriched_attribute.add_tag(tag) def get_output_type(self, related_type: str, indicator: str) -> str: """Helper method for translating a Recorded Future related type to a MISP output type.""" output_type = "text" if related_type in ["RelatedIpAddress", "IpAddress"]: output_type = "ip-dst" elif related_type in [ "RelatedInternetDomainName", "InternetDomainName" ]: output_type = "domain" elif related_type in ["RelatedHash", "Hash"]: hash_len = len(indicator) if hash_len == 64: output_type = "sha256" elif hash_len == 40: output_type = "sha1" elif hash_len == 32: output_type = "md5" elif related_type in ["RelatedEmailAddress", "EmailAddress"]: output_type = "email-src" elif related_type in [ "RelatedCyberVulnerability", "CyberVulnerability" ]: signature = indicator.split("-")[0] if signature == "CVE": output_type = "vulnerability" elif signature == "CWE": output_type = "weakness" elif related_type == "MalwareSignature": output_type = "malware-sample" elif related_type == "Organization": output_type = "target-org" elif related_type == "Username": output_type = "target-user" return output_type def get_results(self) -> dict: """Build and return the enrichment results.""" self.enrichment_object.add_attribute("Enriched attribute", **self.enriched_attribute) for related_type, attribute in self.related_attributes: self.enrichment_object.add_attribute(related_type, **attribute) self.event.add_object(**self.enrichment_object) event = json.loads(self.event.to_json()) result = {key: event[key] for key in ["Object"] if key in event} return {"results": result}
class APIVoidParser(): def __init__(self, attribute): self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.attribute) self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&' def get_results(self): if hasattr(self, 'result'): return self.result event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return {'results': results} def parse_domain(self, apikey): feature = 'dnslookup' if requests.get(f'{self.url.format(feature, apikey)}stats').json( )['credits_remained'] < 0.13: self.result = { 'error': 'You do not have enough APIVoid credits to proceed your request.' } return mapping = { 'A': 'resolution-of', 'MX': 'mail-server-of', 'NS': 'server-name-of' } dnslookup = requests.get( f'{self.url.format(feature, apikey)}action=dns-any&host={self.attribute.value}' ).json() for item in dnslookup['data']['records']['items']: record_type = item['type'] try: relationship = mapping[record_type] except KeyError: continue self._handle_dns_record(item, record_type, relationship) ssl = requests.get( f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}' ).json() self._parse_ssl_certificate(ssl['data']['certificate']) def _handle_dns_record(self, item, record_type, relationship): dns_record = MISPObject('dns-record') dns_record.add_attribute('queried-domain', type='domain', value=item['host']) attribute_type, feature = ('ip-dst', 'ip') if record_type == 'A' else ('domain', 'target') dns_record.add_attribute(f'{record_type.lower()}-record', type=attribute_type, value=item[feature]) dns_record.add_reference(self.attribute.uuid, relationship) self.misp_event.add_object(**dns_record) def _parse_ssl_certificate(self, certificate): x509 = MISPObject('x509') fingerprint = 'x509-fingerprint-sha1' x509.add_attribute(fingerprint, type=fingerprint, value=certificate['fingerprint']) x509_mapping = { 'subject': { 'name': ('text', 'subject') }, 'issuer': { 'common_name': ('text', 'issuer') }, 'signature': { 'serial': ('text', 'serial-number') }, 'validity': { 'valid_from': ('datetime', 'validity-not-before'), 'valid_to': ('datetime', 'validity-not-after') } } certificate = certificate['details'] for feature, subfeatures in x509_mapping.items(): for subfeature, mapping in subfeatures.items(): attribute_type, relation = mapping x509.add_attribute(relation, type=attribute_type, value=certificate[feature][subfeature]) x509.add_reference(self.attribute.uuid, 'seen-by') self.misp_event.add_object(**x509)
class VirusTotalParser(object): def __init__(self, apikey, limit): self.apikey = apikey self.limit = limit self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report" self.misp_event = MISPEvent() self.parsed_objects = {} self.input_types_mapping = { 'ip-src': self.parse_ip, 'ip-dst': self.parse_ip, 'domain': self.parse_domain, 'hostname': self.parse_domain, 'md5': self.parse_hash, 'sha1': self.parse_hash, 'sha256': self.parse_hash, 'url': self.parse_url } self.proxies = None def query_api(self, attribute): self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) return self.input_types_mapping[self.attribute.type]( self.attribute.value, recurse=True) def get_result(self): event = json.loads(self.misp_event.to_json()) results = { key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key]) } return {'results': results} ################################################################################ #### Main parsing functions #### # noqa ################################################################################ def parse_domain(self, domain, recurse=False): req = requests.get(self.base_url.format('domain'), params={ 'apikey': self.apikey, 'domain': domain }, proxies=self.proxies) if req.status_code != 200: return req.status_code req = req.json() hash_type = 'sha256' whois = 'whois' feature_types = { 'communicating': 'communicates-with', 'downloaded': 'downloaded-from', 'referrer': 'referring' } siblings = (self.parse_siblings(domain) for domain in req['domain_siblings']) uuid = self.parse_resolutions( req['resolutions'], req['subdomains'] if 'subdomains' in req else None, siblings) for feature_type, relationship in feature_types.items(): for feature in ('undetected_{}_samples', 'detected_{}_samples'): for sample in req.get(feature.format(feature_type), [])[:self.limit]: status_code = self.parse_hash(sample[hash_type], False, uuid, relationship) if status_code != 200: return status_code if req.get(whois): whois_object = MISPObject(whois) whois_object.add_attribute('text', type='text', value=req[whois]) self.misp_event.add_object(**whois_object) return self.parse_related_urls(req, recurse, uuid) def parse_hash(self, sample, recurse=False, uuid=None, relationship=None): req = requests.get(self.base_url.format('file'), params={ 'apikey': self.apikey, 'resource': sample }, proxies=self.proxies) status_code = req.status_code if req.status_code == 200: req = req.json() vt_uuid = self.parse_vt_object(req) file_attributes = [] for hash_type in ('md5', 'sha1', 'sha256'): if req.get(hash_type): file_attributes.append({ 'type': hash_type, 'object_relation': hash_type, 'value': req[hash_type] }) if file_attributes: file_object = MISPObject('file') for attribute in file_attributes: file_object.add_attribute(**attribute) file_object.add_reference(vt_uuid, 'analyzed-with') if uuid and relationship: file_object.add_reference(uuid, relationship) self.misp_event.add_object(**file_object) return status_code def parse_ip(self, ip, recurse=False): req = requests.get(self.base_url.format('ip-address'), params={ 'apikey': self.apikey, 'ip': ip }, proxies=self.proxies) if req.status_code != 200: return req.status_code req = req.json() if req.get('asn'): asn_mapping = { 'network': ('ip-src', 'subnet-announced'), 'country': ('text', 'country') } asn_object = MISPObject('asn') asn_object.add_attribute('asn', type='AS', value=req['asn']) for key, value in asn_mapping.items(): if req.get(key): attribute_type, relation = value asn_object.add_attribute(relation, type=attribute_type, value=req[key]) self.misp_event.add_object(**asn_object) uuid = self.parse_resolutions( req['resolutions']) if req.get('resolutions') else None return self.parse_related_urls(req, recurse, uuid) def parse_url(self, url, recurse=False, uuid=None): req = requests.get(self.base_url.format('url'), params={ 'apikey': self.apikey, 'resource': url }, proxies=self.proxies) status_code = req.status_code if req.status_code == 200: req = req.json() vt_uuid = self.parse_vt_object(req) if not recurse: feature = 'url' url_object = MISPObject(feature) url_object.add_attribute(feature, type=feature, value=url) url_object.add_reference(vt_uuid, 'analyzed-with') if uuid: url_object.add_reference(uuid, 'hosted-in') self.misp_event.add_object(**url_object) return status_code ################################################################################ #### Additional parsing functions #### # noqa ################################################################################ def parse_related_urls(self, query_result, recurse, uuid=None): if recurse: for feature in ('detected_urls', 'undetected_urls'): if feature in query_result: for url in query_result[feature]: value = url['url'] if isinstance(url, dict) else url[0] status_code = self.parse_url(value, False, uuid) if status_code != 200: return status_code else: for feature in ('detected_urls', 'undetected_urls'): if feature in query_result: for url in query_result[feature]: value = url['url'] if isinstance(url, dict) else url[0] self.misp_event.add_attribute('url', value) return 200 def parse_resolutions(self, resolutions, subdomains=None, uuids=None): domain_ip_object = MISPObject('domain-ip') if self.attribute.type in ('domain', 'hostname'): domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value) attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address') else: domain_ip_object.add_attribute('ip', type='ip-dst', value=self.attribute.value) attribute_type, relation, key = ('domain', 'domain', 'hostname') for resolution in resolutions: domain_ip_object.add_attribute(relation, type=attribute_type, value=resolution[key]) if subdomains: for subdomain in subdomains: attribute = MISPAttribute() attribute.from_dict(**dict(type='domain', value=subdomain)) self.misp_event.add_attribute(**attribute) domain_ip_object.add_reference(attribute.uuid, 'subdomain') if uuids: for uuid in uuids: domain_ip_object.add_reference(uuid, 'sibling-of') self.misp_event.add_object(**domain_ip_object) return domain_ip_object.uuid def parse_siblings(self, domain): attribute = MISPAttribute() attribute.from_dict(**dict(type='domain', value=domain)) self.misp_event.add_attribute(**attribute) return attribute.uuid def parse_vt_object(self, query_result): if query_result['response_code'] == 1: vt_object = MISPObject('virustotal-report') vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio, disable_correlation=True) self.misp_event.add_object(**vt_object) return vt_object.uuid def set_proxy_settings(self, config: dict) -> dict: """Returns proxy settings in the requests format. If no proxy settings are set, return None.""" proxies = None host = config.get('proxy_host') port = config.get('proxy_port') username = config.get('proxy_username') password = config.get('proxy_password') if host: if not port: misperrors['error'] = 'The virustotal_proxy_host config is set, ' \ 'please also set the virustotal_proxy_port.' raise KeyError parsed = urlparse(host) if 'http' in parsed.scheme: scheme = 'http' else: scheme = parsed.scheme netloc = parsed.netloc host = f'{netloc}:{port}' if username: if not password: misperrors['error'] = 'The virustotal_proxy_username config is set, ' \ 'please also set the virustotal_proxy_password.' raise KeyError auth = f'{username}:{password}' host = auth + '@' + host proxies = { 'http': f'{scheme}://{host}', 'https': f'{scheme}://{host}' } self.proxies = proxies return True