def test_first_last_seen(self):
me = MISPEvent()
me.info = 'Test First and Last Seen'
me.date = '2020.01.12'
self.assertEqual(me.date.day, 12)
me.add_attribute('ip-dst', '8.8.8.8', first_seen='06-21-1998', last_seen=1580213607.469571)
self.assertEqual(me.attributes[0].first_seen.year, 1998)
self.assertEqual(me.attributes[0].last_seen.year, 2020)
now = datetime.now().astimezone()
me.attributes[0].last_seen = now
today = date.today()
me.attributes[0].first_seen = today
self.assertEqual(me.attributes[0].first_seen.year, today.year)
self.assertEqual(me.attributes[0].last_seen, now)
def __init__(self, attribute, apikey, apipassword):
self.base_url = "https://api.xforce.ibmcloud.com"
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self._apikey = apikey
self._apipassword = apipassword
self.result = {}
self.objects = defaultdict(dict)
self.status_mapping = {
403:
"Access denied, please check if your authentication is valid and if you did not reach the limit of queries.",
404: "No result found for your query."
}
def processEvent(self, event):
mevent = MISPEvent()
mevent.from_dict(Event=event)
changed = False
for attr in mevent.attributes:
if (attr["type"] == "ip-dst" or attr["type"] == "ip-src") and attr["to_ids"]:
print("Removing IDS flag in event '{}' on attr '{}'".format(mevent.id, attr["value"]))
changed = True
attr["to_ids"] = False
self.misp.tag(mevent, self.expiredTag, True)
if changed:
res = self.misp.update_event(mevent.id, mevent)
def create_response(original_attribute: dict, software: str, signature: Optional[str] = None) -> dict:
misp_event = MISPEvent()
if signature:
misp_event.add_attribute(**original_attribute)
av_signature_object = MISPObject("av-signature")
av_signature_object.add_attribute("signature", signature)
av_signature_object.add_attribute("software", software)
av_signature_object.add_reference(original_attribute["uuid"], "belongs-to")
misp_event.add_object(av_signature_object)
event = json.loads(misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
return {"results": results}
def _parse_event(self, feature):
misp_event = MISPEvent()
misp_event.load_file(getattr(self, feature))
attributes = defaultdict(dict)
object_attributes = defaultdict(dict)
tags = defaultdict(set)
for attribute in misp_event.attributes:
attributes[attribute.type][attribute.uuid] = attribute.value
self.recovering[feature][attribute.uuid] = {
attribute.type: attribute.value
}
if attribute.tags:
tags[attribute.uuid].update(self._get_tags(attribute.tags))
if 'Galaxy' in attribute and attribute['Galaxy']:
tags[attribute.uuid].update(
self._get_galaxy(attribute['Galaxy']))
for misp_object in misp_event.objects:
object_dict = defaultdict(list)
for attribute in misp_object.attributes:
object_dict[attribute.object_relation].append(
self._get_datetime_value(attribute.value) if attribute.
type == 'datetime' else attribute.value)
simplified_attributes = self._jsonify(object_dict)
self.recovering[feature][misp_object.uuid] = {
misp_object.name: simplified_attributes
}
references = [{
'target': reference.referenced_uuid,
'relationship': reference.relationship_type
} for reference in misp_object.references]
object_attributes[misp_object.name][misp_object.uuid] = {
'Attribute': simplified_attributes,
'ObjectReference': references
}
if misp_event.tags:
tags[misp_event.uuid].update(self._get_tags(misp_event.tags))
if 'Galaxy' in misp_event and misp_event['Galaxy']:
tags[misp_event.uuid].update(self._get_galaxy(
misp_event['Galaxy']))
tags = {key: list(values) for key, values in tags.items()}
event = {
'Attribute': self._jsonify(attributes),
'Object': self._jsonify(object_attributes),
'Tag&Galaxy': self._jsonify(tags)
}
filename = f'{misp_event.uuid}_{feature}.json'
with open(filename, 'wt', encoding='utf-8') as f:
f.write(json.dumps(event, indent=4))
print(f'{filename} generated')
return filename
def __init__(self, event, offline=False):
if isinstance(event, MISPEvent):
self.event = event
else:
self.event = MISPEvent()
if isinstance(event, six.string_types) and os.path.exists(event):
self.event.load_file(event)
else:
self.event.load(event)
self.off = offline
if self.event.id:
self.current_dump_file = '{}.json'.format(self.event.id)
else:
self.current_dump_file = None
def __init__(self, attribute, api_key, client_id, client_secret):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.base_url = 'https://api.mvision.mcafee.com'
self.session = requests.Session()
self.api_key = api_key
auth = (client_id, client_secret)
self.logging()
self.auth(auth)
def export(self, cache: 'CaptureCache', is_public_instance: bool=False) -> MISPEvent:
'''Export a capture in MISP format. You can POST the return of this method
directly to a MISP instance and it will create an event.'''
public_domain = get_config('generic', 'public_domain')
event = MISPEvent()
event.info = f'Lookyloo Capture ({cache.url})'
lookyloo_link: MISPAttribute = event.add_attribute('link', f'https://{public_domain}/tree/{cache.uuid}') # type: ignore
if not is_public_instance:
lookyloo_link.distribution = 0
initial_url = URLObject(cache.url)
initial_url.comment = 'Submitted URL'
self.__misp_add_ips_to_URLObject(initial_url, cache.tree.root_hartree.hostname_tree)
redirects: List[URLObject] = []
for nb, url in enumerate(cache.redirects):
if url == cache.url:
continue
obj = URLObject(url)
obj.comment = f'Redirect {nb}'
self.__misp_add_ips_to_URLObject(obj, cache.tree.root_hartree.hostname_tree)
redirects.append(obj)
if redirects:
redirects[-1].comment = f'Last redirect ({nb})'
if redirects:
prec_object = initial_url
for u_object in redirects:
prec_object.add_reference(u_object, 'redirects-to')
prec_object = u_object
initial_obj = event.add_object(initial_url)
initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo')
for u_object in redirects:
event.add_object(u_object)
final_redirect = event.objects[-1]
try:
fo = FileObject(pseudofile=cache.tree.root_hartree.rendered_node.body, filename=cache.tree.root_hartree.rendered_node.filename)
fo.comment = 'Content received for the final redirect (before rendering)'
fo.add_reference(final_redirect, 'loaded-by', 'URL loading that content')
event.add_object(fo)
except Har2TreeError:
pass
except AttributeError:
# No `body` in rendered node
pass
return event
def __init__(self, attribute):
self.attribute = attribute
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
self.passivedns_mapping = {
'bailiwick': {
'type': 'text',
'object_relation': 'bailiwick'
},
'count': {
'type': 'counter',
'object_relation': 'count'
},
'rdata': {
'type': 'text',
'object_relation': 'rdata'
},
'rrname': {
'type': 'text',
'object_relation': 'rrname'
},
'rrtype': {
'type': 'text',
'object_relation': 'rrtype'
},
'time_first': {
'type': 'datetime',
'object_relation': 'time_first'
},
'time_last': {
'type': 'datetime',
'object_relation': 'time_last'
},
'zone_time_first': {
'type': 'datetime',
'object_relation': 'zone_time_first'
},
'zone_time_last': {
'type': 'datetime',
'object_relation': 'zone_time_last'
}
}
self.type_to_feature = {
'domain': 'domain name',
'hostname': 'hostname',
'ip-src': 'IP address',
'ip-dst': 'IP address'
}
self.comment = 'Result from an %s lookup on DNSDB about the %s: %s'
def _search_local_hashes(self, event, open_session=True):
local = []
samples_count = 0
if isinstance(event, MISPEvent):
misp_event = event
elif event.get('Event') is None:
self.log('error', event)
return
else:
misp_event = MISPEvent()
misp_event.load(event)
if not hasattr(misp_event, 'id'):
# The event doesn't exists upstream, breaking.
return
for a in misp_event.attributes:
row = None
if a.type == 'malware-sample':
samples_count += 1
if a.type in ('md5', 'sha1', 'sha256'):
row = Database().find(key=a.type, value=a.value)
elif a.type in ('filename|md5', 'filename|sha1',
'filename|sha256'):
row = Database().find(key=a.type.split('|')[1],
value=a.value.split('|')[1])
elif a.type == 'malware-sample':
row = Database().find(key='md5', value=a.value.split('|')[1])
if row:
local.append(row[0])
self.log(
'info',
'Event {} contains {} samples.'.format(misp_event.id,
samples_count))
if not open_session:
return
shas = set([l.sha256 for l in local])
if len(shas) == 1:
__sessions__.new(get_sample_path(shas.pop()),
MispEvent(misp_event, self.offline_mode))
elif len(shas) > 1:
self.log('success',
'The following samples are in this viper instance:')
__sessions__.new(
misp_event=MispEvent(misp_event, self.offline_mode))
for s in shas:
self.log('item', s)
else:
__sessions__.new(
misp_event=MispEvent(misp_event, self.offline_mode))
self.log('info', 'No known (in Viper) samples in that event.')
def __init__(self, attribute, authentication):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.pssl = pypssl.PyPSSL(basic_auth=authentication)
self.cert_hash = 'x509-fingerprint-sha1'
self.cert_type = 'pem'
self.mapping = {
'issuer': ('text', 'issuer'),
'keylength': ('text', 'pubkey-info-size'),
'not_after': ('datetime', 'validity-not-after'),
'not_before': ('datetime', 'validity-not-before'),
'subject': ('text', 'subject')
}
def __init__(self, client_id, client_secret):
self.misp_event = MISPEvent()
self.client_id = client_id
self.client_secret = client_secret
self.authToken = f"{self.client_id}:{self.client_secret}"
self.baseurl = 'de.api.labs.sophos.com'
d = {'grant_type': 'client_credentials'}
h = {'Authorization': f"Basic {base64.b64encode(self.authToken.encode('UTF-8')).decode('ascii')}",\
'Content-Type': 'application/x-www-form-urlencoded'}
r = requests.post('https://api.labs.sophos.com/oauth2/token',
headers=h,
data=d)
if r.status_code == 200:
j = json.loads(r.text)
self.accessToken = j['access_token']
def setUp(self):
self.maxDiff = None
self.mispevent = MISPEvent()
if not manual_testing:
self.root = "tests/"
else:
self.root = ""
self.test_folder = self.root + "reportlab_testfiles/"
self.test_batch_folder = self.root + "OSINT_output/"
self.storage_folder_OSINT = self.root + "OSINT_PDF/"
self.test_image_folder = self.root + "image_json/"
self.storage_folder = self.root + "reportlab_testoutputs/"
self.storage_image_folder = self.root + "reportlab_test_image_outputs/"
self.moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description",
"Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"]
def parse_response(response):
mapping = {
'file_name': {
'type': 'filename',
'object_relation': 'filename'
},
'file_size': {
'type': 'size-in-bytes',
'object_relation': 'size-in-bytes'
},
'file_type_mime': {
'type': 'mime-type',
'object_relation': 'mimetype'
},
'md5_hash': {
'type': 'md5',
'object_relation': 'md5'
},
'sha1_hash': {
'type': 'sha1',
'object_relation': 'sha1'
},
'sha256_hash': {
'type': 'sha256',
'object_relation': 'sha256'
},
'ssdeep': {
'type': 'ssdeep',
'object_relation': 'ssdeep'
}
}
misp_event = MISPEvent()
for data in response:
misp_object = MISPObject('file')
for feature, attribute in mapping.items():
if feature in data:
misp_attribute = {'value': data[feature]}
misp_attribute.update(attribute)
misp_object.add_attribute(**misp_attribute)
misp_event.add_object(**misp_object)
return {
'results': {
'Object': [
json.loads(misp_object.to_json())
for misp_object in misp_event.objects
]
}
}
def __init__(self, attribute_props: dict):
self.event = MISPEvent()
self.enrichment_object = MISPObject('Recorded Future Enrichment')
description = ('An object containing the enriched attribute and '
'related entities from Recorded Future.')
self.enrichment_object.from_dict(**{
'meta-category': 'misc',
'description': description,
'distribution': 0
})
# Create a copy of enriched attribute to add tags to
temp_attr = MISPAttribute()
temp_attr.from_dict(**attribute_props)
self.enriched_attribute = MISPAttribute()
self.enriched_attribute.from_dict(**{
'value': temp_attr.value,
'type': temp_attr.type,
'distribution': 0
})
self.related_attributes = []
self.color_picker = RFColors()
self.galaxy_finder = GalaxyFinder()
# Mapping from MISP-type to RF-type
self.type_to_rf_category = {
'ip': 'ip',
'ip-src': 'ip',
'ip-dst': 'ip',
'domain': 'domain',
'hostname': 'domain',
'md5': 'hash',
'sha1': 'hash',
'sha256': 'hash',
'uri': 'url',
'url': 'url',
'vulnerability': 'vulnerability',
'weakness': 'vulnerability'
}
# Related entities from RF portrayed as related attributes in MISP
self.related_attribute_types = [
'RelatedIpAddress', 'RelatedInternetDomainName', 'RelatedHash',
'RelatedEmailAddress', 'RelatedCyberVulnerability'
]
# Related entities from RF portrayed as tags in MISP
self.galaxy_tag_types = ['RelatedMalware', 'RelatedThreatActor']
def create_new_event(self, entry):
attribute = MISPAttribute()
attribute.type = "malware-sample"
attribute.value = entry["shasum"]
attribute.data = Path(entry["outfile"])
attribute.comment = "File uploaded to Cowrie ({})".format(entry["sensor"])
attribute.expand = "binary"
event = MISPEvent()
event.info = "File uploaded to Cowrie ({})".format(entry["sensor"])
event.attributes = [attribute]
event.run_expansions()
if self.publish:
event.publish()
result = self.misp_api.add_event(event)
if self.debug:
log.msg(f"Event creation result: \n{result}")
def __init__(self, attribute, vulnerability, api_url):
self.attribute = attribute
self.vulnerability = vulnerability
self.api_url = api_url
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
self.references = defaultdict(list)
self.capec_features = ('id', 'name', 'summary', 'prerequisites', 'solutions')
self.vulnerability_mapping = {
'id': ('text', 'id'), 'summary': ('text', 'summary'),
'vulnerable_configuration': ('text', 'vulnerable_configuration'),
'vulnerable_configuration_cpe_2_2': ('text', 'vulnerable_configuration'),
'Modified': ('datetime', 'modified'), 'Published': ('datetime', 'published'),
'references': ('link', 'references'), 'cvss': ('float', 'cvss-score')}
self.weakness_mapping = {'name': 'name', 'description_summary': 'description',
'status': 'status', 'weaknessabs': 'weakness-abs'}
def __init__(self, header, has_header, delimiter, data, from_misp,
MISPtypes):
self.misp_event = MISPEvent()
self.header = header
self.has_header = has_header
self.delimiter = delimiter
self.data = data
self.from_misp = from_misp
self.MISPtypes = MISPtypes
self.fields_number = len(self.header)
self.__score_mapping = {
0: self.__create_standard_misp,
1: self.__create_attribute_with_ids,
2: self.__create_attribute_with_tags,
3: self.__create_attribute_with_ids_and_tags
}
def _change_event(self):
if self.offline_mode:
self._dump()
else:
if __sessions__.current.misp_event.event.id:
event = self.misp.update(__sessions__.current.misp_event.event)
else:
event = self.misp.add_event(__sessions__.current.misp_event.event)
if self._has_error_message(event):
return
try:
me = MISPEvent()
me.load(event)
self._check_add(me)
except Exception as e:
self.log('error', e)
def __init__(self, apikey):
self.apikey = apikey
self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report"
self.misp_event = MISPEvent()
self.parsed_objects = {}
self.input_types_mapping = {
'ip-src': self.parse_ip,
'ip-dst': self.parse_ip,
'domain': self.parse_domain,
'hostname': self.parse_domain,
'md5': self.parse_hash,
'sha1': self.parse_hash,
'sha256': self.parse_hash,
'sha512': self.parse_hash,
'url': self.parse_url
}
def import_objs_from_file(filepath):
map_uuid_global_id = {}
event_to_import = MISPEvent()
try:
event_to_import.load_file(filepath)
except:
return map_uuid_global_id
for misp_obj in event_to_import.objects:
get_misp_import_fct(map_uuid_global_id, misp_obj)
for misp_obj in event_to_import.objects:
create_obj_relationships(map_uuid_global_id, misp_obj)
create_map_all_obj_uuid_golbal_id(map_uuid_global_id)
return map_uuid_global_id
def make_new_event(misp):
LOGGER.info('Creating new fixed event...')
event = MISPEvent()
timestamp = datetime.utcnow()
event_date = timestamp.strftime('%Y-%m-%d')
event.info = MISP_EVENT_TITLE
event.analysis = Analysis.completed
event.distribution = Distribution.your_organisation_only
event.threat_level_id = ThreatLevel.low
event.add_tag('type:OSINT')
event.add_tag('tlp:white')
LOGGER.info('Saving event...')
time.sleep(1)
return misp.add_event(event, pythonify=True)
def generate_event(self, modsecLog: ModsecLog) -> MISPEvent:
event = MISPEvent()
event.info = modsecLog.generateInfoLine()
[http_method, url, http_version] = modsecLog.getRequestLine()
event.add_attribute(type='ip-src|port', value=str(modsecLog.log['transaction']['remote_address']) + "|" + str(modsecLog.log['transaction']['remote_port']), comment="Attacker", pythonify=True)
event.add_attribute(type='ip-dst|port', value=str(modsecLog.log['transaction']['local_address']) + "|" + str(modsecLog.log['transaction']['local_port']), comment="Server", pythonify=True)
event.add_attribute(type='http-method', value=http_method, pythonify=True)
event.add_attribute(type='url', value=url, pythonify=True)
event.add_attribute(type='datetime', value=modsecLog.log['@timestamp'], pythonify=True)
event.add_attribute(type='other', value=modsecLog.log['audit_data']['producer'], comment="Producer", pythonify=True)
event.add_attribute(type='text', value=json.dumps(modsecLog.log, indent=2), comment="Json log", pythonify=True)
#event.add_attribute(type='vulnerability', value=json_log['transaction']['time'], pythonify=True)
for tag in self.tags:
event.add_tag(tag)
log.debug("elasticsearch data")
log.debug(json.dumps(modsecLog.log, indent=2))
return event
def parse_result(attribute, values):
event = MISPEvent()
initial_attribute = MISPAttribute()
initial_attribute.from_dict(**attribute)
event.add_attribute(**initial_attribute)
mapping = {'asn': ('AS', 'asn'), 'prefix': ('ip-src', 'subnet-announced')}
print(values)
for last_seen, response in values['response'].items():
asn = MISPObject('asn')
asn.add_attribute('last-seen', **{'type': 'datetime', 'value': last_seen})
for feature, attribute_fields in mapping.items():
attribute_type, object_relation = attribute_fields
asn.add_attribute(object_relation, **{'type': attribute_type, 'value': response[feature]})
asn.add_reference(initial_attribute.uuid, 'related-to')
event.add_object(**asn)
event = json.loads(event.to_json())
return {key: event[key] for key in ('Attribute', 'Object')}
def test_feed(self):
me = MISPEvent()
me.info = 'Test feed'
org = MISPOrganisation()
org.name = 'TestOrg'
org.uuid = '123478'
me.Orgc = org
me.add_attribute('ip-dst', '8.8.8.8')
obj = me.add_object(name='file')
obj.add_attributes('filename', *['foo.exe', 'bar.exe'])
h = hashlib.new('md5')
h.update(b'8.8.8.8')
hash_attr_val = h.hexdigest()
feed = me.to_feed(with_meta=True)
self.assertEqual(feed['Event']['_hashes'][0], hash_attr_val)
self.assertEqual(feed['Event']['_manifest'][me.uuid]['info'], 'Test feed')
self.assertEqual(len(feed['Event']['Object'][0]['Attribute']), 2)
def __init__(self, config):
self.log = logging.getLogger(__name__)
self.log.setLevel(logging.DEBUG)
self.misp_event = MISPEvent()
self.references = defaultdict(list)
self.attributes = defaultdict(lambda: defaultdict(set))
self.process_references = {}
self.import_pe = config["import_pe"]
self.create_mitre_attack = config["mitre_attack"]
self.import_malware_config = config["import_malware_config"]
self.import_network_interactions = config[
"import_network_interactions"]
self.import_dropped_files = config["import_dropped_files"]
self.import_registry_activities = config["import_registry_activities"]
self.import_system_behavior = config["import_system_behavior"]
self.import_network_behavior = config["import_network_behavior"]
def test_sync_community(self):
'''Simple event, this community only, pull from member of the community'''
event = MISPEvent()
event.info = 'Event created on first instance - test_sync_community'
event.distribution = Distribution.this_community_only
event.add_attribute('ip-src', '1.1.1.1')
try:
source = self.instances[0]
dest = self.instances[1]
event = source.org_admin_connector.add_event(event)
source.org_admin_connector.publish(event)
dest.site_admin_connector.server_pull(dest.sync_servers[0])
time.sleep(10)
dest_event = dest.org_admin_connector.get_event(event.uuid)
self.assertEqual(dest_event.distribution, 0)
finally:
source.org_admin_connector.delete_event(event)
dest.site_admin_connector.delete_event(dest_event)
def test_object_templates(self):
me = MISPEvent()
for template in glob.glob(str(me.misp_objects_path / '*' / 'definition.json')):
with open(template) as f:
t_json = json.load(f)
if 'requiredOneOf' in t_json:
obj_relations = set(t_json['attributes'].keys())
subset = set(t_json['requiredOneOf']).issubset(obj_relations)
self.assertTrue(subset, f'{t_json["name"]}')
if 'required' in t_json:
obj_relations = set(t_json['attributes'].keys())
subset = set(t_json['required']).issubset(obj_relations)
self.assertTrue(subset, f'{t_json["name"]}')
for obj_relation, entry in t_json['attributes'].items():
self.assertTrue(entry['misp-attribute'] in me.describe_types['types'])
if 'categories' in entry:
subset = set(entry['categories']).issubset(me.describe_types['categories'])
self.assertTrue(subset, f'{t_json["name"]} - {obj_relation}')
def feed_meta_generator(path: Path):
manifests = {}
hashes: List[str] = []
for f_name in path.glob('*.json'):
if str(f_name.name) == 'manifest.json':
continue
event = MISPEvent()
event.load_file(str(f_name))
manifests.update(event.manifest)
hashes += [f'{h},{event.uuid}' for h in event.attributes_hashes('md5')]
with (path / 'manifest.json').open('w') as f:
json.dump(manifests, f)
with (path / 'hashes.csv').open('w') as f:
for h in hashes:
f.write(f'{h}\n')
def _create_event(self, artifact: Type[Artifact]) -> MISPEvent:
"""Create an event in MISP, return an Event object."""
event = MISPEvent()
event.info = self.event_info.format(source_name=artifact.source_name)
# Add tags.
for tag in self.tags:
event.add_tag(tag)
# Add references.
if artifact.reference_link != "":
event.add_attribute("link", artifact.reference_link)
if artifact.reference_text != "":
event.add_attribute("text", artifact.reference_text)
if artifact.source_name != "" and self.include_artifact_source_name:
event.add_attribute("other", f"source:{artifact.source_name}")
return event
