def addDomain(domainValue):
eid = checkAge()
mispAttribute = MISPAttribute()
mispAttribute.type = 'domain'
mispAttribute.value = domainValue
misp.add_attribute(eid, mispAttribute)
returnSuccess("domain", domainValue, eid)
def misp_add_attribute(self, event, a_type, a_value):
# Create Attribute object in MISP
misp_attribute = MISPAttribute()
if a_type: misp_attribute.type = a_type
if a_value: misp_attribute.value = a_value
event.add_attribute(misp_attribute.type, misp_attribute.value)
class PayloadQuery(URLhaus):
def __init__(self, attribute):
super(PayloadQuery, self).__init__()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.url = 'https://urlhaus-api.abuse.ch/v1/payload/'
def query_api(self):
hash_type = self.attribute.type
file_object = MISPObject('file')
if self.attribute.event_id != '0':
file_object.id = self.attribute.object_id
response = requests.post(self.url, data={'{}_hash'.format(hash_type): self.attribute.value}).json()
other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256'
for key, relation in zip(('{}_hash'.format(other_hash_type), 'file_size'), (other_hash_type, 'size-in-bytes')):
if response[key]:
file_object.add_attribute(relation, **{'type': relation, 'value': response[key]})
if response['virustotal']:
vt_object = self._create_vt_object(response['virustotal'])
file_object.add_reference(vt_object.uuid, 'analyzed-with')
self.misp_event.add_object(**vt_object)
_filename_ = 'filename'
for url in response['urls']:
attribute = MISPAttribute()
attribute.from_dict(type='url', value=url['url'])
self.misp_event.add_attribute(**attribute)
file_object.add_reference(attribute.uuid, 'retrieved-from')
if url[_filename_]:
file_object.add_attribute(_filename_, **{'type': _filename_, 'value': url[_filename_]})
if any((file_object.attributes, file_object.references)):
self.misp_event.add_object(**file_object)
def _fill_object(self, obj_def, values):
empty_object = self.__new_empty_object(obj_def)
if self.links:
empty_object["ObjectReference"] = []
for link in self.links:
uuid, comment = link
empty_object['ObjectReference'].append({
'referenced_object_uuid':
uuid,
'comment':
comment
})
for object_type, value in values.items():
if value.get('value') is None:
continue
attribute = MISPAttribute(self.misp_event.describe_types)
value['type'] = obj_def['attributes'][object_type][
'misp-attribute']
if value.get('disable_correlation') is None:
value['disable_correlation'] = obj_def['attributes'][
object_type].get('disable_correlation')
if value.get('to_ids') is None:
value['to_ids'] = obj_def['attributes'][object_type].get(
'to_ids')
attribute.set_all_values(**value)
empty_object['ObjectAttribute'].append({
'type':
object_type,
'Attribute':
attribute._json()
})
return empty_object
def parse_resolutions(self, resolutions, subdomains=None, uuids=None):
domain_ip_object = MISPObject('domain-ip')
if self.attribute.type in ('domain', 'hostname'):
domain_ip_object.add_attribute('domain',
type='domain',
value=self.attribute.value)
attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address')
else:
domain_ip_object.add_attribute('ip',
type='ip-dst',
value=self.attribute.value)
attribute_type, relation, key = ('domain', 'domain', 'hostname')
for resolution in resolutions:
domain_ip_object.add_attribute(relation,
type=attribute_type,
value=resolution[key])
if subdomains:
for subdomain in subdomains:
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=subdomain))
self.misp_event.add_attribute(**attribute)
domain_ip_object.add_reference(attribute.uuid, 'subdomain')
if uuids:
for uuid in uuids:
domain_ip_object.add_reference(uuid, 'sibling-of')
self.misp_event.add_object(**domain_ip_object)
return domain_ip_object.uuid
def create_attr(self, raw_attr: dict) -> MISPAttribute:
# Create attribute and assign simple values
attr = MISPAttribute()
attr.type = 'url'
attr.value = raw_attr['url']
attr.disable_correlation = False
attr.__setattr__('first_seen', datetime.strptime(raw_attr['dateadded'], '%Y-%m-%d %H:%M:%S'))
# Add URLhaus tag
self.add_tag_to_attribute(attr, 'URLhaus')
# Add other tags
if raw_attr['tags']:
for tag in raw_attr['tags'].split(','):
self.add_tag_to_attribute(attr, tag.strip())
# Add online/offline tag
if not pandas.isna(raw_attr['url_status']):
if raw_attr['url_status'] == 'online':
attr.to_ids = True
else:
attr.to_ids = False
self.add_tag_to_attribute(attr, raw_attr['url_status'])
# Add reporter tag
if not pandas.isna(raw_attr['reporter']):
self.add_tag_to_attribute(attr, raw_attr['reporter'])
attr.comment = raw_attr['urlhaus_link']
return attr
class UrlQuery(URLhaus):
def __init__(self, attribute):
super(UrlQuery, self).__init__()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.url = 'https://urlhaus-api.abuse.ch/v1/url/'
@staticmethod
def _create_file_object(payload):
file_object = MISPObject('file')
for key, relation in zip(file_keys, file_relations):
if payload[key]:
file_object.add_attribute(
relation, **{
'type': relation,
'value': payload[key]
})
return file_object
def query_api(self):
response = requests.post(self.url, data={
'url': self.attribute.value
}).json()
if response['query_status'] != 'ok':
return self.parse_error(response['query_status'])
if 'payloads' in response and response['payloads']:
for payload in response['payloads']:
file_object = self._create_file_object(payload)
if payload['virustotal']:
vt_object = self._create_vt_object(payload['virustotal'])
file_object.add_reference(vt_object.uuid, 'analyzed-with')
self.misp_event.add_object(**vt_object)
if any((file_object.attributes, file_object.references)):
self.misp_event.add_object(**file_object)
return self.get_result()
class PassiveDNSParser():
def __init__(self, attribute, authentication):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.pdns = pypdns.PyPDNS(basic_auth=authentication)
def get_results(self):
if hasattr(self, 'result'):
return self.result
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
def parse(self, value):
try:
results = self.pdns.query(self.attribute.value)
except Exception:
self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'}
return
mapping = {'count': 'counter', 'origin': 'text',
'time_first': 'datetime', 'rrtype': 'text',
'rrname': 'text', 'rdata': 'text',
'time_last': 'datetime'}
for result in results:
pdns_object = MISPObject('passive-dns')
for relation, attribute_type in mapping.items():
pdns_object.add_attribute(relation, type=attribute_type, value=result[relation])
pdns_object.add_reference(self.attribute.uuid, 'associated-to')
self.misp_event.add_object(**pdns_object)
def _handle_ip_attribute(self, ip_address):
if ip_address == self.attribute.value:
return self.attribute.uuid
ip_attribute = MISPAttribute()
ip_attribute.from_dict(**{'type': self.attribute.type, 'value': ip_address})
self.misp_event.add_attribute(**ip_attribute)
return ip_attribute.uuid
def addIP(ipValue):
eid = checkAge()
mispAttribute = MISPAttribute()
mispAttribute.type = 'ip-dst'
mispAttribute.value = ipValue
misp.add_attribute(eid, mispAttribute)
returnSuccess("IP address", ipValue, eid)
def addEmail(emailValue):
eid = checkAge()
mispAttribute = MISPAttribute()
mispAttribute.type = 'email'
mispAttribute.value = emailValue
misp.add_attribute(eid, mispAttribute)
returnSuccess("email", emailValue, eid)
def generate_MISP_Event(deduplicated_observations, conf, tags, attr_tags):
dt = datetime.now()
event = MISPEvent()
event.info = dt.strftime("%Y%m%d ") + 'TIE'
event.publish_timestamp = dt.strftime("%s")
event.timestamp = dt.strftime("%s")
event['timestamp'] = dt.strftime("%s")
event.analysis = 2
event.published = conf.event_published
orgc = MISPOrganisation()
orgc.from_json(json.dumps({'name': conf.org_name, 'uuid': conf.org_uuid}))
event.orgc = orgc
event.threat_level_id = conf.event_base_thread_level
event.date = dt
event['uuid'] = str(uuid.uuid1())
if len(tags) > 0:
event['Tag'] = tags
attr_hashes = []
for key, attr in deduplicated_observations.items():
misp_attr = MISPAttribute()
misp_attr.timestamp = dt.strftime("%s")
misp_attr['timestamp'] = dt.strftime("%s")
misp_attr.type = get_Attribute_Type(attr)
misp_attr.value = get_MISP_Fitted_Value(attr["value"], misp_attr.type)
if 'c2-server' in attr['categories'] and attr_tags.c2tags:
misp_attr['Tag'] = attr_tags.c2tags
if 'malware' in attr['categories'] and attr_tags.malwaretags:
misp_attr['Tag'] = attr_tags.malwaretags
if 'espionage' in attr['categories'] and attr_tags.espionagetags:
misp_attr['Tag'] = attr_tags.espionagetags
if 'bot' in attr['categories'] and attr_tags.bottags:
misp_attr['Tag'] = attr_tags.bottags
if 'whitelist' in attr['categories'] and attr_tags.whitelisttags:
misp_attr['Tag'] = attr_tags.whitelisttags
if 'cybercrime' in attr['categories'] and attr_tags.cybercrimetags:
misp_attr['Tag'] = attr_tags.cybercrimetags
if 'phishing' in attr['categories'] and attr_tags.phishingtags:
misp_attr['Tag'] = attr_tags.phishingtags
misp_attr.category = get_Attribute_Category(attr)
if conf.attr_to_ids and attr[
'min_confidence'] >= conf.attr_to_ids_threshold:
misp_attr.to_ids = True
else:
misp_attr.to_ids = False
misp_attr['comment'] = 'categories: ' + str(attr['categories']) + ' actors: ' + str(attr['actors']) + \
' families: ' + str(attr['families']) + ' sources: ' + str(attr['sources']) + \
' severity: ' + str(attr['max_severity']) + \
' confidence: ' + str(attr['max_confidence'])
misp_attr.edited = False
event.add_attribute(**(misp_attr.to_dict()))
attr_hashes.append([
hashlib.md5(attr['value'].encode("utf-8")).hexdigest(),
event['uuid']
])
event.edited = False
return event, attr_hashes
def addBIC(bicValue):
eid = checkAge()
mispAttribute = MISPAttribute()
mispAttribute.type = 'bic'
mispAttribute.value = bicValue
misp.add_attribute(eid, mispAttribute)
returnSuccess("Bank Account", bicValue, eid)
class PassiveSSLParser():
def __init__(self, attribute, authentication):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.pssl = pypssl.PyPSSL(basic_auth=authentication)
self.cert_hash = 'x509-fingerprint-sha1'
self.cert_type = 'pem'
self.mapping = {'issuer': ('text', 'issuer'),
'keylength': ('text', 'pubkey-info-size'),
'not_after': ('datetime', 'validity-not-after'),
'not_before': ('datetime', 'validity-not-before'),
'subject': ('text', 'subject')}
def get_results(self):
if hasattr(self, 'result'):
return self.result
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
def parse(self):
value = self.attribute.value.split('|')[0] if '|' in self.attribute.type else self.attribute.value
try:
results = self.pssl.query(value)
except Exception:
self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'}
return
if not results:
self.result = {'error': 'Not found'}
return
for ip_address, certificates in results.items():
ip_uuid = self._handle_ip_attribute(ip_address)
for certificate in certificates['certificates']:
self._handle_certificate(certificate, ip_uuid)
def _handle_certificate(self, certificate, ip_uuid):
x509 = MISPObject('x509')
x509.add_attribute(self.cert_hash, type=self.cert_hash, value=certificate)
cert_details = self.pssl.fetch_cert(certificate)
info = cert_details['info']
for feature, mapping in self.mapping.items():
attribute_type, object_relation = mapping
x509.add_attribute(object_relation, type=attribute_type, value=info[feature])
x509.add_attribute(self.cert_type, type='text', value=self.cert_type)
x509.add_reference(ip_uuid, 'seen-by')
self.misp_event.add_object(**x509)
def _handle_ip_attribute(self, ip_address):
if ip_address == self.attribute.value:
return self.attribute.uuid
ip_attribute = MISPAttribute()
ip_attribute.from_dict(**{'type': self.attribute.type, 'value': ip_address})
self.misp_event.add_attribute(**ip_attribute)
return ip_attribute.uuid
def create_attribute(self, attribute_type, attribute_value):
attribute = MISPAttribute()
attribute.from_dict(**{
'type': attribute_type,
'value': attribute_value
})
self.misp_event.add_attribute(**attribute)
return attribute.uuid
def __misp_add_urlscan_to_event(self, capture_uuid: str, visibility: str) -> Optional[MISPAttribute]:
response = self.urlscan.url_submit(self.get_info(capture_uuid), visibility)
if 'result' in response:
attribute = MISPAttribute()
attribute.value = response['result']
attribute.type = 'link'
return attribute
return None
def add_related_attribute(self, indicator: str, related_type: str) -> None:
"""Helper method for adding an indicator to the related attribute list."""
out_type = self.get_output_type(related_type, indicator)
attribute = MISPAttribute()
attribute.from_dict(**{
'value': indicator,
'type': out_type,
'distribution': 0
})
self.related_attributes.append((related_type, attribute))
def create_misp_attribute(misp_client, misp_event_id, misp_attribute_type,
misp_attribute_value):
misp_event = MISPEvent()
misp_event.id = int(misp_event_id)
misp_event.uuid = get_event_uuid(misp_client, misp_event_id)
misp_attribute = MISPAttribute()
misp_attribute.type = misp_attribute_type
misp_attribute.value = misp_attribute_value
attribute_response = misp_client.add_attribute(misp_event, misp_attribute)
return attribute_response
def __init__(self, attribute, config):
config['enclave_ids'] = config.get('enclave_ids',
"").strip().split(',')
config['client_metatag'] = self.CLIENT_METATAG
self.ts_client = TruStar(config=config)
self.misp_event = MISPEvent()
self.misp_attribute = MISPAttribute()
self.misp_attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.misp_attribute)
def add_attribute(self, indicator: str, indicator_type: str) -> None:
"""Helper method for adding an indicator to the attribute list."""
out_type = self.get_output_type(indicator_type, indicator)
attribute = MISPAttribute()
attribute.from_dict(**{
"value": indicator,
"type": out_type,
"distribution": 0
})
self.related_attributes.append((indicator_type, attribute))
def __init__(self, attribute, apikey, apipassword):
self.base_url = "https://api.xforce.ibmcloud.com"
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self._apikey = apikey
self._apipassword = apipassword
self.result = {}
self.objects = defaultdict(dict)
self.status_mapping = {403: "Access denied, please check if your authentication is valid and if you did not reach the limit of queries.",
404: "No result found for your query."}
class HostQuery(URLhaus):
def __init__(self, attribute):
super(HostQuery, self).__init__()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.url = 'https://urlhaus-api.abuse.ch/v1/host/'
def query_api(self):
response = requests.post(self.url, data={'host': self.attribute.value}).json()
if 'urls' in response and response['urls']:
for url in response['urls']:
self.misp_event.add_attribute(type='url', value=url['url'])
def __init__(self, attribute, config_object):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.config_object = config_object
if self.config_object:
self.token = self.get_token()
else:
sys.exit('Missing configuration')
class VirusTotalParser():
def __init__(self):
super(VirusTotalParser, self).__init__()
self.misp_event = MISPEvent()
def declare_variables(self, apikey, attribute):
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.apikey = apikey
def get_result(self):
event = json.loads(self.misp_event.to_json())['Event']
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
return {'results': results}
def parse_urls(self, query_result):
for feature in ('detected_urls', 'undetected_urls'):
if feature in query_result:
for url in query_result[feature]:
value = url['url'] if isinstance(url, dict) else url[0]
self.misp_event.add_attribute('url', value)
def parse_resolutions(self, resolutions, subdomains=None, uuids=None):
domain_ip_object = MISPObject('domain-ip')
if self.attribute.type == 'domain':
domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value)
attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address')
else:
domain_ip_object.add_attribute('ip', type='ip-dst', value=self.attribute.value)
attribute_type, relation, key = ('domain', 'domain', 'hostname')
for resolution in resolutions:
domain_ip_object.add_attribute(relation, type=attribute_type, value=resolution[key])
if subdomains:
for subdomain in subdomains:
attribute = MISPAttribute()
attribute.from_dict(**dict(type='domain', value=subdomain))
self.misp_event.add_attribute(**attribute)
domain_ip_object.add_reference(attribute.uuid, 'subdomain')
if uuids:
for uuid in uuids:
domain_ip_object.add_reference(uuid, 'sibling-of')
self.misp_event.add_object(**domain_ip_object)
def parse_vt_object(self, query_result):
vt_object = MISPObject('virustotal-report')
vt_object.add_attribute('permalink', type='link', value=query_result['permalink'])
detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total'])
vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio)
self.misp_event.add_object(**vt_object)
def get_query_result(self, query_type):
params = {query_type: self.attribute.value, 'apikey': self.apikey}
return requests.get(self.base_url, params=params)
def __init__(self, attribute, authentication):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.pssl = pypssl.PyPSSL(basic_auth=authentication)
self.cert_hash = 'x509-fingerprint-sha1'
self.cert_type = 'pem'
self.mapping = {'issuer': ('text', 'issuer'),
'keylength': ('text', 'pubkey-info-size'),
'not_after': ('datetime', 'validity-not-after'),
'not_before': ('datetime', 'validity-not-before'),
'subject': ('text', 'subject')}
def __init__(self, attribute, api_key, client_id, client_secret):
self.misp_event = MISPEvent()
self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.attribute)
self.base_url = 'https://api.mvision.mcafee.com'
self.session = requests.Session()
self.api_key = api_key
auth = (client_id, client_secret)
self.logging()
self.auth(auth)
def map_misp_attribute_to_zeek(
attribute: MISPAttribute,
source: Union[Tuple[str], None] = None,
url: Union[str, None] = None,
description: Union[str, None] = None,
tags: Union[Tuple[str], None] = None,
confidence: Union[float, None] = None,
logger=None,
) -> Union[Tuple[defaultdict], None]:
"""
Maps a MISP attribute to Zeek intel items
@see https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type
@param attribute The MISPAttribute to convert
@return a list containing the Zeek intel dict(s) from the MISPAttribute object
"""
if logger is not None:
logger.debug(attribute.to_json())
results = []
# get matching Zeek intel type
if not (zeek_types := MISP_ZEEK_INTEL_TYPE_MAP.get(attribute.type, None)):
if logger is not None:
logger.warning(
f"No matching Zeek type found for MISP attribute type '{attribute.type}'"
)
return None
def fill_misp_object(self, item, name, to_ids=False):
try:
misp_object = MISPObject(name)
if to_ids:
observables = item.observable.observable_composition.observables
misp_object.timestamp = self.getTimestampfromDate(item.timestamp)
else:
observables = item.observable_composition.observables
for observable in observables:
properties = observable.object_.properties
misp_attribute = MISPAttribute()
misp_attribute.type, misp_attribute.value, misp_attribute.object_relation = self.handle_attribute_type(properties, is_object=True, observable_id=observable.id_)
misp_object.add_attribute(**misp_attribute)
self.misp_event.add_object(**misp_object)
except AttributeError:
properties = item.observable.object_.properties if to_ids else item.object_.properties
self.parse_observable(properties, to_ids)
def form_attr_obj(self, type, value, file=None):
try:
attr = MISPAttribute()
attr.type = type
attr.value = value
if file is not None:
path = Path(file)
attr.data = path
self.attributes.append(attr)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}"
.format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno,
error=str(e)))
def parse_result(attribute, values):
event = MISPEvent()
initial_attribute = MISPAttribute()
initial_attribute.from_dict(**attribute)
event.add_attribute(**initial_attribute)
mapping = {'asn': ('AS', 'asn'), 'prefix': ('ip-src', 'subnet-announced')}
print(values)
for last_seen, response in values['response'].items():
asn = MISPObject('asn')
asn.add_attribute('last-seen', **{'type': 'datetime', 'value': last_seen})
for feature, attribute_fields in mapping.items():
attribute_type, object_relation = attribute_fields
asn.add_attribute(object_relation, **{'type': attribute_type, 'value': response[feature]})
asn.add_reference(initial_attribute.uuid, 'related-to')
event.add_object(**asn)
event = json.loads(event.to_json())
return {key: event[key] for key in ('Attribute', 'Object')}
def find_hashes(htype):
r = mymisp.search(controller='attributes', type_attribute=htype)
echeck(r)
if not r.get('response'):
return
for a in r['response']['Attribute']:
attribute = MISPAttribute(mymisp.describe_types)
attribute.set_all_values(**a)
if '|' in attribute.type and '|' in attribute.value:
c, value = attribute.value.split('|')
comment = '{} - {}'.format(attribute.comment, c)
else:
comment = attribute.comment
value = attribute.value
mhash = value.replace(':', ';')
mfile = 'MISP event {} {}'.format(a['event_id'], comment.replace(':', ';').replace('\r', '').replace('\n', ''))
print('{}:*:{}:73'.format(mhash, mfile))
