def process(self, task: Task) -> None: # type: ignore
config = task.get_payload("config")
family = task.headers["family"]
dhash = config_dhash(config)
# Parse the config using iocextract library
iocs = parse(family, config)
if not iocs:
# Nothing actionable found - skip the config
return
# Upload structured data to MISP
event = MISPEvent()
event.uuid = str(uuid5(self.CONFIG_NAMESPACE, dhash))
event.add_tag(f"mwdb:family:{family}")
event.info = f"Malware configuration ({family})"
if self.mwdb_url is not None:
event.add_attribute("link", f"{self.mwdb_url}/config/{dhash}")
for o in iocs.to_misp():
event.add_object(o)
misp = ExpandedPyMISP(self.misp_url, self.misp_key, self.misp_verifycert)
misp.add_event(event)
def export_to_misp(user_details, report,indicators):
misp_url = user_details.get('misp_url')
misp_key = user_details.get('misp_api_key')
misp_verifycert = True
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
event = MISPEvent()
event.info = report.title
event = misp.add_event(event, pythonify=True)
print(event)
created = json.loads(event.to_json())
event_id = created.get('id')
report_id = report.id
for indicator in indicators:
indicator_value = indicator[1]
if indicator[2] == 'IP':
indicator_type = "ip-dst"
elif indicator[2] == 'Domain':
indicator_type = 'domain'
elif indicator[2] == 'Email':
indicator_type = 'email-src'
elif indicator[2] == 'CVE':
indicator_value = indicator[1].replace('_', '-')
indicator_type = 'vulnerability'
elif indicator[2] == 'MD5 Hash':
indicator_type = 'md5'
elif indicator[2] == 'SHA256 Hash':
indicator_type = 'sha256'
elif indicator[2] == 'URL':
indicator_type = 'url'
try:
misp.add_attribute(event_id,{'type':indicator_type,'value':indicator_value},pythonify=True)
except:
pass
def submit_tf_update(misp: ExpandedPyMISP, attributes: list) -> MISPEvent:
"""
create/update abuse.ch MISP-Event and append the new attributes
"""
eventinfo = event_info_template.format(
datetime.now().strftime(info_dateformat))
# logging.debug(eventinfo)
events = misp.search(controller='events',
eventinfo=eventinfo,
org=1,
pythonify=True)
if events: # current event exists already
event = events[0]
else: # create a new event
event = MISPEvent()
event.distribution = event_distribution
event.threat_level_id = event_threat_level
event.analysis = 2
event.info = eventinfo
for tag in tagging:
event.add_tag(tag)
event = misp.add_event(event, pythonify=True)
for att in attributes:
event.add_attribute(**att)
event.published = autopublish
return misp.update_event(event)
def save(self):
site_id = self.validated_data['id']
site = Site.objects.get(pk=site_id)
# Check if there is already an Event
if DnsTwisted.objects.filter(domain_name=site.domain_name):
dns_twisted = DnsTwisted.objects.get(domain_name=site.domain_name)
if site.misp_event_id is None:
site.misp_event_id = dns_twisted.misp_event_id
# Save the case id in database
Site.objects.filter(pk=site.pk).update(
misp_event_id=dns_twisted.misp_event_id)
# Test MISP instance connection
try:
requests.get(settings.MISP_URL, verify=settings.MISP_VERIFY_SSL)
except requests.exceptions.SSLError as e:
print(str(timezone.now()) + " - ", e)
raise AuthenticationFailed("SSL Error: " + settings.MISP_URL)
except requests.exceptions.RequestException as e:
print(str(timezone.now()) + " - ", e)
raise NotFound("Not Found: " + settings.MISP_URL)
misp_api = ExpandedPyMISP(settings.MISP_URL, settings.MISP_KEY,
settings.MISP_VERIFY_SSL)
if site.misp_event_id is not None:
# If the event already exist, then we update IOCs
update_attributes(misp_api, site)
else:
# If the event does not exist, then we create it
# Prepare MISP Event
event = MISPEvent()
event.distribution = 0
event.threat_level_id = 2
event.analysis = 0
event.info = "Suspicious domain name " + site.domain_name
event.tags = create_misp_tags(misp_api)
# Create MISP Event
print(str(timezone.now()) + " - " + 'Create MISP Event')
print('-----------------------------')
event = misp_api.add_event(event, pythonify=True)
# Store Event Id in database
Site.objects.filter(pk=site.pk).update(misp_event_id=event.id)
if DnsTwisted.objects.filter(domain_name=site.domain_name):
DnsTwisted.objects.filter(domain_name=site.domain_name).update(
misp_event_id=event.id)
# Create MISP Attributes
create_attributes(misp_api, event.id, site)
def register_misp(misp: ExpandedPyMISP, misp_event_dict: dict) -> None:
""" MISPイベントデータを受け取り、MISPに登録する """
misp_event = MISPEvent()
misp_event.from_dict(**misp_event_dict)
threat_level_id = misp_event.get('threat_level_id')
threat_level = f'anyrun:threat_level:{threat_level_id}'
misp_event.add_tag('anyrun')
misp_event.add_tag(threat_level)
retry_count = 0
while True:
try:
# pymispをインスタンス化してイベント登録
event_data = misp.add_event(misp_event)
if event_data.get('errors'):
raise Exception(event_data['errors'])
# MISPに登録されるイベントIDを取得し、出力
event_id = event_data['Event']['id']
print(f'新規に登録されたEvent_ID: {event_id}')
return
except:
except_return = traceback.format_exc()
# インポート済みの場合
if const.DUPLICATE_EVENT_CONFIRM_WORD in except_return:
print('Importを行おうとしたイベントは既にMISPに登録されています')
return
# リトライ回数チェック
retry_count += 1
if retry_count >= const.RETRY_MAXIMUM_LIMIT:
raise
# インターバル処理
print('MISPへのイベントインポートをリトライします')
time.sleep(const.COMMAND_INTERVAL_TIME)
class Output(cowrie.core.output.Output):
"""
MISP Upload Plugin for Cowrie.
This Plugin creates a new event for unseen file uploads
or adds sightings for previously seen files.
The decision is done by searching for the SHA 256 sum in all matching attributes.
"""
@ignore_warnings
def start(self):
"""
Start output plugin
"""
misp_url = CowrieConfig().get('output_misp', 'base_url')
misp_key = CowrieConfig().get('output_misp', 'api_key')
misp_verifycert = ("true" == CowrieConfig().get('output_misp', 'verify_cert').lower())
self.misp_api = PyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert, debug=False)
self.is_python2 = sys.version_info[0] < 3
self.debug = CowrieConfig().getboolean('output_misp', 'debug', fallback=False)
self.publish = CowrieConfig().getboolean('output_misp', 'publish_event', fallback=False)
def stop(self):
"""
Stop output plugin
"""
pass
def write(self, entry):
"""
Push file download to MISP
"""
if entry['eventid'] == 'cowrie.session.file_download':
file_sha_attrib = self.find_attribute("sha256", entry["shasum"])
if file_sha_attrib:
# file is known, add sighting!
if self.debug:
log.msg("File known, add sighting")
self.add_sighting(entry, file_sha_attrib)
else:
# file is unknown, new event with upload
if self.debug:
log.msg("File unknwon, add new event")
self.create_new_event(entry)
@ignore_warnings
def find_attribute(self, attribute_type, searchterm):
"""
Returns a matching attribute or None if nothing was found.
"""
result = self.misp_api.search(
controller="attributes",
type_attribute=attribute_type,
value=searchterm
)
# legacy PyMISP returns the Attribute wrapped in a response
if self.is_python2:
result = result["response"]
if result["Attribute"]:
return result["Attribute"][0]
else:
return None
@ignore_warnings
def create_new_event(self, entry):
if self.is_python2:
self.misp_api.upload_sample(
entry["shasum"],
entry["outfile"],
None,
distribution=1,
info="File uploaded to Cowrie ({})".format(entry["sensor"]),
analysis=0,
threat_level_id=2
)
else:
attribute = MISPAttribute()
attribute.type = "malware-sample"
attribute.value = entry["shasum"]
attribute.data = Path(entry["outfile"])
attribute.comment = "File uploaded to Cowrie ({})".format(entry["sensor"])
attribute.expand = "binary"
event = MISPEvent()
event.info = "File uploaded to Cowrie ({})".format(entry["sensor"])
event.attributes = [attribute]
event.run_expansions()
if self.publish:
event.publish()
result = self.misp_api.add_event(event)
if self.debug:
log.msg("Event creation result: \n%s" % result)
@ignore_warnings
def add_sighting(self, entry, attribute):
if self.is_python2:
self.misp_api.sighting(
uuid=attribute["uuid"],
source="{} (Cowrie)".format(entry["sensor"])
)
else:
sighting = MISPSighting()
sighting.source = "{} (Cowrie)".format(entry["sensor"])
self.misp_api.add_sighting(sighting, attribute)
class Mail2MISP():
def __init__(self,
misp_url,
misp_key,
verifycert,
config,
offline=False,
urlsonly=False):
self.offline = offline
if not self.offline:
self.misp = ExpandedPyMISP(misp_url,
misp_key,
verifycert,
debug=config.debug)
self.config = config
self.urlsonly = urlsonly
if not hasattr(self.config, 'enable_dns'):
setattr(self.config, 'enable_dns', True)
if self.urlsonly is False:
setattr(self.config, 'enable_dns', False)
self.debug = self.config.debug
self.config_from_email_body = {}
# Init Faup
self.f = Faup()
self.sightings_to_add = []
def load_email(self, pseudofile):
self.pseudofile = pseudofile
self.original_mail = message_from_bytes(self.pseudofile.getvalue(),
policy=policy.default)
self.subject = self.original_mail.get('Subject')
try:
self.sender = self.original_mail.get('From')
except:
self.sender = "<unknown sender>"
# Remove words from subject
for removeword in self.config.removelist:
self.subject = re.sub(removeword, "", self.subject).strip()
# Initialize the MISP event
self.misp_event = MISPEvent()
self.misp_event.info = f'{self.config.email_subject_prefix} - {self.subject}'
self.misp_event.distribution = self.config.default_distribution
self.misp_event.threat_level_id = self.config.default_threat_level
self.misp_event.analysis = self.config.default_analysis
def sighting(self, value, source):
if self.offline:
raise Exception('The script is running in offline mode, ')
'''Add a sighting'''
s = MISPSighting()
s.from_dict(value=value, source=source)
self.misp.add_sighting(s)
def _find_inline_forward(self):
'''Does the body contains a forwarded email?'''
for identifier in self.config.forward_identifiers:
if identifier in self.clean_email_body:
self.clean_email_body, fw_email = self.clean_email_body.split(
identifier)
return self.forwarded_email(
pseudofile=BytesIO(fw_email.encode()))
def _find_attached_forward(self):
forwarded_emails = []
for attachment in self.original_mail.iter_attachments():
attachment_content = attachment.get_content()
# Search for email forwarded as attachment
# I could have more than one, attaching everything.
if isinstance(attachment_content, message.EmailMessage):
forwarded_emails.append(
self.forwarded_email(
pseudofile=BytesIO(attachment_content.as_bytes())))
else:
if isinstance(attachment_content, str):
attachment_content = attachment_content.encode()
filename = attachment.get_filename()
if not filename:
filename = 'missing_filename'
if self.config_from_email_body.get(
'attachment'
) == self.config.m2m_benign_attachment_keyword:
# Attach sane file
self.misp_event.add_attribute(
'attachment',
value=filename,
data=BytesIO(attachment_content))
else:
f_object, main_object, sections = make_binary_objects(
pseudofile=BytesIO(attachment_content),
filename=filename,
standalone=False)
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
[
self.misp_event.add_object(section)
for section in sections
]
return forwarded_emails
def email_from_spamtrap(self):
'''The email comes from a spamtrap and should be attached as-is.'''
raw_body = self.original_mail.get_body(preferencelist=('html',
'plain'))
if raw_body:
self.clean_email_body = html.unescape(
raw_body.get_payload(decode=True).decode(
'utf8', 'surrogateescape'))
else:
self.clean_email_body = ''
return self.forwarded_email(self.pseudofile)
def forwarded_email(self, pseudofile: BytesIO):
'''Extracts all possible indicators out of an email and create a MISP event out of it.
* Gets all relevant Headers
* Attach the body
* Create MISP file objects (uses lief if possible)
* Set all references
'''
email_object = EMailObject(pseudofile=pseudofile,
attach_original_mail=True,
standalone=False)
if email_object.attachments:
# Create file objects for the attachments
for attachment_name, attachment in email_object.attachments:
if not attachment_name:
attachment_name = 'NameMissing.txt'
if self.config_from_email_body.get(
'attachment'
) == self.config.m2m_benign_attachment_keyword:
a = self.misp_event.add_attribute('attachment',
value=attachment_name,
data=attachment)
email_object.add_reference(a.uuid, 'related-to',
'Email attachment')
else:
f_object, main_object, sections = make_binary_objects(
pseudofile=attachment,
filename=attachment_name,
standalone=False)
if self.config.vt_key:
try:
vt_object = VTReportObject(
self.config.vt_key,
f_object.get_attributes_by_relation(
'sha256')[0].value,
standalone=False)
self.misp_event.add_object(vt_object)
f_object.add_reference(vt_object.uuid,
'analysed-with')
except InvalidMISPObject as e:
print(e)
pass
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
for section in sections:
self.misp_event.add_object(section)
email_object.add_reference(f_object.uuid, 'related-to',
'Email attachment')
self.process_body_iocs(email_object)
if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get(
'attach_original_mail'):
self.misp_event.add_object(email_object)
return email_object
def process_email_body(self):
mail_as_bytes = self.original_mail.get_body(
preferencelist=('html', 'plain')).get_payload(decode=True)
if mail_as_bytes:
self.clean_email_body = html.unescape(
mail_as_bytes.decode('utf8', 'surrogateescape'))
# Check if there are config lines in the body & convert them to a python dictionary:
# <config.body_config_prefix>:<key>:<value> => {<key>: <value>}
self.config_from_email_body = {
k.strip(): v.strip()
for k, v in re.findall(
f'{self.config.body_config_prefix}:(.*):(.*)',
self.clean_email_body)
}
if self.config_from_email_body:
# ... remove the config lines from the body
self.clean_email_body = re.sub(
rf'^{self.config.body_config_prefix}.*\n?',
'',
html.unescape(
self.original_mail.get_body(
preferencelist=('html', 'plain')).get_payload(
decode=True).decode('utf8',
'surrogateescape')),
flags=re.MULTILINE)
# Check if autopublish key is present and valid
if self.config_from_email_body.get(
'm2mkey') == self.config.m2m_key:
if self.config_from_email_body.get('distribution') is not None:
self.misp_event.distribution = self.config_from_email_body.get(
'distribution')
if self.config_from_email_body.get('threat_level') is not None:
self.misp_event.threat_level_id = self.config_from_email_body.get(
'threat_level')
if self.config_from_email_body.get('analysis') is not None:
self.misp_event.analysis = self.config_from_email_body.get(
'analysis')
if self.config_from_email_body.get('publish'):
self.misp_event.publish()
self._find_inline_forward()
else:
self.clean_email_body = ''
self._find_attached_forward()
def process_body_iocs(self, email_object=None):
if email_object:
body = html.unescape(
email_object.email.get_body(
preferencelist=('html',
'plain')).get_payload(decode=True).decode(
'utf8', 'surrogateescape'))
else:
body = self.clean_email_body
# Cleanup body content
# Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message
for ignoreline in self.config.ignorelist:
body = re.sub(rf'^{ignoreline}.*\n?', '', body, flags=re.MULTILINE)
# Remove everything after the stopword from the body
body = body.split(self.config.stopword, 1)[0]
# Add tags to the event if keywords are found in the mail
for tag in self.config.tlptags:
for alternativetag in self.config.tlptags[tag]:
if alternativetag in body.lower():
self.misp_event.add_tag(tag)
# Prepare extraction of IOCs
# Refang email data
body = refang(body)
# Extract and add hashes
contains_hash = False
for h in set(re.findall(hashmarker.MD5_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute(
'md5', h, enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sightings_to_add.append((h, self.config.sighting_source))
for h in set(re.findall(hashmarker.SHA1_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute(
'sha1', h, enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sightings_to_add.append((h, self.config.sighting_source))
for h in set(re.findall(hashmarker.SHA256_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute(
'sha256', h, enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sightings_to_add.append((h, self.config.sighting_source))
if contains_hash:
[
self.misp_event.add_tag(tag)
for tag in self.config.hash_only_tags
]
# # Extract network IOCs
urllist = []
urllist += re.findall(urlmarker.WEB_URL_REGEX, body)
urllist += re.findall(urlmarker.IP_REGEX, body)
if self.debug:
syslog.syslog(str(urllist))
hostname_processed = []
# Add IOCs and expanded information to MISP
for entry in set(urllist):
ids_flag = True
self.f.decode(entry)
domainname = self.f.get_domain()
if domainname in self.config.excludelist:
# Ignore the entry
continue
hostname = self.f.get_host()
scheme = self.f.get_scheme()
if scheme:
scheme = scheme
resource_path = self.f.get_resource_path()
if resource_path:
resource_path = resource_path
if self.debug:
syslog.syslog(domainname)
if domainname in self.config.internallist and self.urlsonly is False: # Add link to internal reference unless in urlsonly mode
attribute = self.misp_event.add_attribute(
'link',
entry,
category='Internal reference',
to_ids=False,
enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
elif domainname in self.config.externallist or self.urlsonly is False: # External analysis
attribute = self.misp_event.add_attribute(
'link',
entry,
category='External analysis',
to_ids=False,
enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
elif domainname in self.config.externallist or self.urlsonly: # External analysis
if self.urlsonly:
comment = self.subject + " (from: " + self.sender + ")"
else:
comment = ""
attribute = self.misp.add_attribute(
self.urlsonly, {
"type": 'link',
"value": entry,
"category": 'External analysis',
"to_ids": False,
"comment": comment
})
for tag in self.config.tlptags:
for alternativetag in self.config.tlptags[tag]:
if alternativetag in self.subject.lower():
self.misp.tag(attribute["uuid"], tag)
new_subject = comment.replace(alternativetag, '')
self.misp.change_comment(attribute["uuid"],
new_subject)
else: # The URL is probably an indicator.
comment = ""
if (domainname in self.config.noidsflaglist) or (
hostname in self.config.noidsflaglist):
ids_flag = False
comment = "Known host (mostly for connectivity test or IP lookup)"
if self.debug:
syslog.syslog(str(entry))
if scheme:
if is_ip(hostname):
attribute = self.misp_event.add_attribute(
'url',
entry,
to_ids=False,
enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid,
'contains')
else:
if resource_path: # URL has path, ignore warning list
attribute = self.misp_event.add_attribute(
'url',
entry,
to_ids=ids_flag,
enforceWarninglist=False,
comment=comment)
if email_object:
email_object.add_reference(
attribute.uuid, 'contains')
else: # URL has no path
attribute = self.misp_event.add_attribute(
'url',
entry,
to_ids=ids_flag,
enforceWarninglist=self.config.
enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(
attribute.uuid, 'contains')
if self.config.sighting:
self.sightings_to_add.append(
(entry, self.config.sighting_source))
if hostname in hostname_processed:
# Hostname already processed.
continue
hostname_processed.append(hostname)
if self.config.sighting:
self.sightings_to_add.append(
(hostname, self.config.sighting_source))
if self.debug:
syslog.syslog(hostname)
comment = ''
port = self.f.get_port()
if port:
port = port
comment = f'on port: {port}'
if is_ip(hostname):
attribute = self.misp_event.add_attribute(
'ip-dst',
hostname,
to_ids=ids_flag,
enforceWarninglist=self.config.enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else:
related_ips = []
if HAS_DNS and self.config.enable_dns:
try:
syslog.syslog(hostname)
for rdata in dns.resolver.query(hostname, 'A'):
if self.debug:
syslog.syslog(str(rdata))
related_ips.append(rdata.to_text())
except Exception as e:
if self.debug:
syslog.syslog(str(e))
if related_ips:
hip = MISPObject(name='ip-port')
hip.add_attribute(
'hostname',
value=hostname,
to_ids=ids_flag,
enforceWarninglist=self.config.enforcewarninglist,
comment=comment)
for ip in set(related_ips):
hip.add_attribute('ip',
type='ip-dst',
value=ip,
to_ids=False,
enforceWarninglist=self.config.
enforcewarninglist)
self.misp_event.add_object(hip)
if email_object:
email_object.add_reference(hip.uuid, 'contains')
else:
if self.urlsonly is False:
attribute = self.misp_event.add_attribute(
'hostname',
value=hostname,
to_ids=ids_flag,
enforceWarninglist=self.config.
enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(attribute.uuid,
'contains')
def add_event(self):
'''Add event on the remote MISP instance.'''
# Add additional tags depending on others
tags = []
for tag in [t.name for t in self.misp_event.tags]:
if self.config.dependingtags.get(tag):
tags += self.config.dependingtags.get(tag)
# Add additional tags according to configuration
for malware in self.config.malwaretags:
if malware.lower() in self.subject.lower():
tags += self.config.malwaretags.get(malware)
if tags:
[self.misp_event.add_tag(tag) for tag in tags]
has_tlp_tag = False
for tag in [t.name for t in self.misp_event.tags]:
if tag.lower().startswith('tlp'):
has_tlp_tag = True
if not has_tlp_tag:
self.misp_event.add_tag(self.config.tlptag_default)
if self.offline:
return self.misp_event.to_json()
event = self.misp.add_event(self.misp_event, pythonify=True)
if self.config.sighting:
for value, source in self.sightings_to_add:
self.sighting(value, source)
return event
class MispInstance:
ddosch_tag_name = 'DDoSCH'
ddosch_tag_colour = '#ff7dfd'
def __init__(self,
host: str,
token: str,
protocol: str,
verify_tls: bool,
sharing_group: str,
publish: bool = False):
self.host = host
self.token = token
self.protocol = protocol
self.verify_tls = verify_tls
self.misp: ExpandedPyMISP
self.sharing_group = sharing_group
self.publish = publish
try:
self.misp = ExpandedPyMISP(f'{self.protocol}://{self.host}',
self.token,
ssl=self.verify_tls,
tool='dissector')
except PyMISPError:
LOGGER.critical(
f'Could not connect to MISP instance at "{self.protocol}://{self.host}".'
)
self.misp = None
def search_misp_events(self, misp_filter: dict = None) -> Optional[dict]:
"""
Search for MISP events
:param misp_filter: fields by which to filter retrieved MISP events
:return: MISP events if found, else None
"""
LOGGER.debug(f'Searching MISP events with filter: {misp_filter}')
if not self.verify_tls:
urllib3.disable_warnings()
response = requests.post(f'{self.protocol}://{self.host}/events/index',
json=misp_filter or dict(),
headers={
'Authorization': self.token,
'Accept': 'application/json'
},
timeout=10,
verify=self.verify_tls)
try:
response.raise_for_status()
return response.json()
except requests.HTTPError:
LOGGER.critical(
f'Retrieving MISP events responded with status code:{response.status_code}'
)
return None
def get_misp_tag(self, tag_name) -> Optional[dict]:
"""
Search in MISP for a tag with tag_name, return the tag dict if it exists.
:param tag_name: Name of the tag to retrieve
:return: Tag or None
"""
LOGGER.info(f'Searching for {tag_name} tag in MISP')
if not self.verify_tls:
urllib3.disable_warnings()
response = requests.get(
f'{self.protocol}://{self.host}/tags/search/{self.ddosch_tag_name}',
headers={
'Authorization': self.token,
'Accept': 'application/json'
},
timeout=10,
verify=self.verify_tls)
try:
response.raise_for_status()
for item in response.json():
if item['Tag']['name'] == self.ddosch_tag_name:
return item
else:
LOGGER.info(
f'No tags found with name "{self.ddosch_tag_name}"')
return None
except requests.HTTPError:
LOGGER.critical(
f'Searching for MISP Tag responded with status code:{response.status_code}'
)
return None
def add_misp_tag(self, tag_name, tag_color) -> Optional[dict]:
"""
Create a new tag in MISP
:param tag_name: Name of the new tag
:param tag_color: Color of the new tag
:return: Server response if succesful, else None
"""
LOGGER.debug(f'Creating a {tag_name} tag in MISP')
if not self.verify_tls:
urllib3.disable_warnings()
response = requests.post(f'{self.protocol}://{self.host}/tags/add',
json={
'name': tag_name,
'colour': tag_color
},
headers={
'Authorization': self.token,
'Accept': 'application/json'
},
timeout=10,
verify=self.verify_tls)
try:
response.raise_for_status()
return response.json()
except requests.HTTPError:
LOGGER.critical(
f'Creating MISP Tag responded with status code:{response.status_code}'
)
return None
def add_misp_fingerprint(self, fingerprint_json: dict):
"""
Upload fingerprint to MISP
:param fingerprint_json: fingerprint to upload
:return:
"""
LOGGER.info('Uploading the fingerprint to MISP')
start = time.time()
# Maximum number of source IPs to include
# 0 means all (no limit)
source_ips_limit = 10
# Possible dicts in each attack_vector of the fingerprint
# that will be added as comments (with the dict as value) to the event (not the ddos objects)
attack_vector_dicts = [
'ttl',
'tcp_flags',
'fragmentation_offset',
'ethernet_type',
'frame_len',
'dns_query_name',
'dns_query_type',
'ICMP type',
'ntp_requestcode',
'http_uri',
'http_method',
'http_user_agent',
]
# Possible fields in each attack_vector of the fingerprint
# that will be added as comments to the event (not the ddos objects)
attack_vector_fields = [
'service',
'fraction_of_attack',
'nr_flows',
'nr_packets',
'nr_megabytes',
'time_start',
'duration_seconds',
]
# Possible fields in the fingerprint
# that will be added as comments to the event
fingerprint_fields = [
'time_start',
'time_end',
'duration_seconds',
'total_flows',
'total_megabytes',
'total_packets',
'total_ips',
'avg_bps',
'avg_pps',
'avg_Bpp',
]
# Create the DDoSCH tag (returns existing one if already present)
ddosch_tag = self.get_misp_tag(self.ddosch_tag_name)
if ddosch_tag is None:
ddosch_tag = self.add_misp_tag(self.ddosch_tag_name,
self.ddosch_tag_colour)
LOGGER.debug(ddosch_tag)
# Retrieve (or create) the sharing group if specified
misp_sharing_group = None
if self.sharing_group:
misp_sharing_group = [
sh_grp for sh_grp in self.misp.sharing_groups(pythonify=True)
if sh_grp.name == self.sharing_group
]
if len(misp_sharing_group) == 0:
misp_sharing_group = self.misp.add_sharing_group(
{'name': self.sharing_group}, pythonify=True)
else:
misp_sharing_group = misp_sharing_group[0]
# Create an event to link everything to
LOGGER.debug('Creating a new event for the fingerprint')
event = MISPEvent()
event.info = fingerprint_json['key']
# TARGET
event.add_attribute(category='Network activity',
type='ip-dst',
value=fingerprint_json['target'],
comment='target')
# KEY
event.add_attribute(category='Network activity',
type='md5',
value=fingerprint_json['key'],
comment='attack key')
LOGGER.debug('Adding fingerprint fields')
for fp_field in fingerprint_fields:
if fp_field in fingerprint_json:
event.add_attribute(category='Network activity',
type='comment',
value=fingerprint_json[fp_field],
comment=fp_field)
# TAGS
if 'tags' in fingerprint_json:
LOGGER.debug('Adding fingerprint tags')
for tag in fingerprint_json['tags']:
event.add_tag(tag=tag)
event.add_tag(tag='validated')
if ddosch_tag is not None:
event.add_tag(tag=self.ddosch_tag_name)
# Add each attack vector as a MISP object to the MISP event
for v_i, attack_vector in enumerate(
fingerprint_json['attack_vectors']):
LOGGER.debug(f'Processing Attack Vector #{v_i}')
ddos_object = MISPObject(name='ddos')
# ATTACK VECTOR PROTOCOL
ddos_object.add_attribute('protocol',
attack_vector['protocol'],
comment=f'vector {v_i}')
for av_dict in attack_vector_dicts:
if av_dict in attack_vector and type(
attack_vector[av_dict]) == dict:
LOGGER.debug(f'Adding dict {av_dict}')
event.add_attribute(
category='Network activity',
type='comment',
value=json.dumps(attack_vector[av_dict]),
comment=f'vector {v_i} {av_dict} ({av_dict}:fraction)')
for av_field in attack_vector_fields:
if av_field in attack_vector and attack_vector[
av_field] is not None:
LOGGER.debug(f'Adding field {av_field}')
event.add_attribute(category='Network activity',
type='comment',
value=attack_vector[av_field],
comment=f'vector {v_i} {av_field}')
# ATTACK VECTOR SOURCE_PORT
if type(attack_vector['source_port']) == int:
LOGGER.debug('Adding source ports')
ddos_object.add_attribute('src-port',
attack_vector['source_port'],
comment=f'vector {v_i} src-port')
# ATTACK VECTOR DESTINATION PORTS
if type(attack_vector['destination_ports']) == dict:
LOGGER.debug('Adding destination ports')
for port in attack_vector['destination_ports'].keys():
ddos_object.add_attribute(
'dst-port',
int(port),
comment=f'vector {v_i} destination port '
f'(fraction:{attack_vector["destination_ports"][port]}'
)
# ATTACK VECTOR DNS
if 'dns_query_name' in attack_vector or 'dns_query_type' in attack_vector:
ddos_object.add_attribute(
'type',
'dns',
comment=f'vector {v_i} type of attack vector')
ddos_object.add_attribute(
'type',
'dns-amplification',
comment=f'vector {v_i} type of attack vector')
# ATTACK VECTOR ICMP
if 'ICMP type' in attack_vector:
ddos_object.add_attribute(
'type',
'icmp',
comment=f'vector {v_i} type of attack vector')
# ATTACK VECTOR NTP
if 'ntp_requestcode' in attack_vector:
ddos_object.add_attribute(
'type',
'ntp-amplification',
comment=f'vector {v_i} type of attack vector')
# ATTACK VECTOR SOURCE IPS
if 'source_ips' in attack_vector:
for i, src_ip in enumerate(attack_vector['source_ips'],
start=1):
ddos_object.add_attribute(
'ip-src', src_ip, comment=f'vector {v_i} source IP')
if i >= source_ips_limit > 0:
break
event.add_object(ddos_object, pythonify=True)
if misp_sharing_group:
self.misp.change_sharing_group_on_entity(event,
misp_sharing_group.id,
pythonify=True)
if self.publish:
event.publish()
event = self.misp.add_event(event, pythonify=True)
LOGGER.info(f'event: {event}')
LOGGER.debug('That took {} seconds'.format(time.time() - start))
def save(self):
alert_id = self.validated_data['id']
alert = Alert.objects.get(pk=alert_id)
dns_twisted = DnsTwisted.objects.get(pk=alert.dns_twisted.pk)
# Getting IOCs related to the new twisted domain
if Site.objects.filter(domain_name=dns_twisted.domain_name):
already_in_monitoring = True
site = Site.objects.get(domain_name=dns_twisted.domain_name)
# Store Event Id in database
DnsTwisted.objects.filter(pk=dns_twisted.pk).update(misp_event_id=site.misp_event_id)
else:
already_in_monitoring = False
site = Site.objects.create(domain_name=dns_twisted.domain_name, rtir=-999999999)
monitoring_init(site)
site = Site.objects.get(pk=site.pk)
# We now hav the IOCs related to the domain, we can remove it from monitoring
if not already_in_monitoring:
Site.objects.filter(pk=site.pk).delete()
if site.misp_event_id is None:
site.misp_event_id = dns_twisted.misp_event_id
# Test MISP instance connection
try:
requests.get(settings.MISP_URL, verify=settings.MISP_VERIFY_SSL)
except requests.exceptions.SSLError as e:
print(str(timezone.now()) + " - ", e)
raise AuthenticationFailed("SSL Error: " + settings.MISP_URL)
except requests.exceptions.RequestException as e:
print(str(timezone.now()) + " - ", e)
raise NotFound("Not Found: " + settings.MISP_URL)
misp_api = ExpandedPyMISP(settings.MISP_URL, settings.MISP_KEY, settings.MISP_VERIFY_SSL)
if site.misp_event_id is not None:
# If the event already exist, then we update IOCs
update_attributes(misp_api, site)
else:
# If the event does not exist, then we create it
# Prepare MISP Event
event = MISPEvent()
event.distribution = 0
event.threat_level_id = 2
event.analysis = 0
event.info = "Suspicious domain name " + site.domain_name
event.tags = create_misp_tags(misp_api)
# Create MISP Event
print(str(timezone.now()) + " - " + 'Create MISP Event')
print('-----------------------------')
event = misp_api.add_event(event, pythonify=True)
# Store Event Id in database
DnsTwisted.objects.filter(pk=dns_twisted.pk).update(misp_event_id=event.id)
if Site.objects.filter(domain_name=dns_twisted.domain_name):
Site.objects.filter(pk=site.pk).update(misp_event_id=event.id)
# Create MISP Attributes
create_attributes(misp_api, event.id, site)
"to_ids": False,
"value": feed,
"disable_correlation": False,
"object_relation": "",
"type": "ip-src"
}
attr.append(attribute)
EventDict = {
"Event": {
"info": "Cyber Cure Known Bad",
"publish timestamp": ts,
"analysis": 2,
"Tag": {
"colour": "ff0000",
"exportable": True,
"name": "Malicious IPs"
},
"Attribute": attr,
"threat_level_id": "1",
"extends_uuid": "",
"published": False,
"date": now.strftime("%Y-%m-%d"),
"Orgc": {
"uuid": "Your UUID",
"name": "Cyber Cure"
},
}
}
misp.add_event(EventDict)
class MISP():
def __init__(self, query):
self.misp = ExpandedPyMISP(misp_url, misp_key, ssl=misp_verify, debug=False)
self.misp_tag = misp_tag
self.tags = self.misp.tags()
self.query = query
self.mainfile = query['Summary']['Subject']['Name']
self.attributes = []
def form_attr_obj(self, type, value, file=None):
try:
attr = MISPAttribute()
attr.type = type
attr.value = value
if file is not None:
path = Path(file)
attr.data = path
self.attributes.append(attr)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}"
.format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno,
error=str(e)))
def create_attr_obj(self):
try:
self.form_attr_obj('filename', self.mainfile)
atdip = self.query['Summary']['ATD IP']
if not atdip: pass
else: self.form_attr_obj('comment', 'ATD IP {0}'.format(atdip))
dstip = self.query['Summary']['Dst IP']
if not dstip: pass
else: self.form_attr_obj('ip-dst', dstip)
taskid = self.query['Summary']['TaskId']
if not taskid: pass
else: self.form_attr_obj('comment', 'ATD TaskID: {0}'.format(taskid))
md5 = self.query['Summary']['Subject']['md5']
if not md5: pass
else: self.form_attr_obj('md5', md5)
sha1 = self.query['Summary']['Subject']['sha-1']
if not sha1: pass
else: self.form_attr_obj('sha1', sha1)
sha256 = self.query['Summary']['Subject']['sha-256']
if not sha256: pass
else: self.form_attr_obj('sha256', sha256)
size = self.query['Summary']['Subject']['size']
if not size: pass
else: self.form_attr_obj('comment', 'File size is: {0}'.format(size))
verdict = self.query['Summary']['Verdict']['Description']
if not verdict: pass
else: self.form_attr_obj('comment', verdict)
# Add process information to MISP
try:
for processes in self.query['Summary']['Processes']:
name = processes['Name']
md5 = processes['Md5']
sha1 = processes['Sha1']
sha256 = processes['Sha256']
if not name: pass
else: self.form_attr_obj('filename', name)
if not md5: pass
else: self.form_attr_obj('md5', md5)
if not sha1: pass
else: self.form_attr_obj('sha1', sha1)
if not sha256: pass
else: self.form_attr_obj('sha256', sha256)
except:
pass
# Add files information to MISP
try:
for files in self.query['Summary']['Files']:
name = files['Name']
md5 = files['Md5']
sha1 = files['Sha1']
sha256 = files['Sha256']
if not name: pass
else: self.form_attr_obj('filename', name)
if not md5: pass
else: self.form_attr_obj('md5', md5)
if not sha1: pass
else: self.form_attr_obj('sha1', sha1)
if not sha256: pass
else: self.form_attr_obj('sha256', sha256)
except:
pass
# Add URL information to MISP
try:
for url in self.query['Summary']['Urls']:
url = url['Url']
if not url: pass
else: self.form_attr_obj('url', url)
except:
pass
# Add ips information to MISP
try:
for ips in self.query['Summary']['Ips']:
ipv4 = ips['Ipv4']
port = ips['Port']
if not ipv4: pass
else: self.form_attr_obj('ip_dst', ipv4)
if not port: pass
else:
self.form_attr_obj('port', port)
self.form_attr_obj('url','{0}:{1}'.format(ipv4, port))
except:
pass
# Add stats Information to MISP
try:
for stats in self.query['Summary']['Stats']:
category = stats['Category']
if not category: pass
else: self.form_attr_obj('comment', category)
except:
pass
# Add behaviour information to MISP
try:
for behave in self.query['Summary']['Behavior']:
behave = behave['Analysis']
if not behave: pass
else: self.form_attr_obj('comment', behave)
except:
pass
# Download original sample from ATD analysis
time.sleep(10)
atd = ATD()
atd.get_report(taskid, 'pdf')
atd.get_report(taskid, 'sample')
atd.logout()
pdf = '{0}.pdf'.format(taskid)
samplefile = '{0}.zip'.format(taskid)
# add attributes for analysis report
self.form_attr_obj('attachment', '{0}.pdf'.format(str(self.mainfile)), file=pdf)
# add attributes for malware sample
self.form_attr_obj('malware-sample', '{0}.zip'.format(str(self.mainfile)), file=samplefile)
# Delete original sample local - potentially nice to have locally as well
os.remove(pdf)
os.remove(samplefile)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}"
.format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno,
error=str(e)))
def add_event(self):
try:
event = MISPEvent()
event.distribution = 0
# ATD Threat mapping to MISP Threat Level
atd_threat_level = self.query['Summary']['Verdict']['Severity']
if not atd_threat_level:
pass
else:
if atd_threat_level == '3':
event.threat_level_id = 1
elif atd_threat_level == '4':
event.threat_level_id = 2
elif atd_threat_level == '5':
event.threat_level_id = 3
else:
event.threat_level_id = 0
event.analysis = 0 # initial
event.info = "ATD Analysis Report - {0}".format(self.mainfile)
event.attributes = self.attributes
event.Tag = 'ATD:Report'
event = self.misp.add_event(event, pythonify=True)
self.evenid = event.id
print('SUCCESS: New MISP Event got created with ID: {}'.format(str(event.id)))
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}"
.format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno,
error=str(e)))
def assign_tag(self):
try:
uuid = None
results = self.misp.search(eventid=self.evenid)
for event in results:
uuid = event['Event']['uuid']
self.misp.tag(uuid, self.misp_tag)
break
if uuid is None:
print('STATUS: Could not tag the MISP Event. Continuing...')
#MITRE tags assign to McAfee ATD findings
version = self.query['Summary']['SUMversion']
version = str(version).replace('.', '')
if int(version) >= 46021:
mitre_list = self.query['Summary']['Mitre']
for val in mitre_list:
mitre_tech = val['Techniques']
for tag in self.tags:
if 'mitre-attack-pattern="{0}'.format(mitre_tech) in tag['name']:
self.misp.tag(uuid, tag['id'])
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}"
.format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno,
error=str(e)))
def main(self):
self.create_attr_obj()
self.add_event()
self.assign_tag()
objects = csv_loader.load()
if args.dump:
for o in objects:
print(o.to_json())
else:
if offline:
print('You are in offline mode, quitting.')
else:
misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert)
if args.new_event:
event = MISPEvent()
event.info = args.new_event
for o in objects:
event.add_object(**o)
new_event = misp.add_event(event)
if isinstance(new_event, str):
print(new_event)
elif 'id' in new_event:
print(f'Created new event {new_event.id}')
else:
print('Something went wrong:')
print(new_event)
else:
for o in objects:
new_object = misp.add_object(args.update_event, o)
if isinstance(new_object, str):
print(new_object)
elif new_object.attributes:
print(f'New {new_object.name} object added to {args.update_event}')
else:
class MISPController(object):
''' MISP Controller '''
def __init__(self, misp_param, debug=False):
self.misp_param = misp_param
self.debug = debug
if misp_param.get('connect_immediately', False):
self._connect()
else:
self.misp = None
def import_event(self, event_data):
''' Import event '''
# Check registered same event info
print('importing: {}'.format(event_data['title']))
events = self._search_event(eventinfo=event_data['title'])
if events != None:
for event in events:
if event_data['title'] == event['Event']['info']:
self._remove_event(event['Event']['id'])
event = self._add_event(event_data)
if event:
print('created event: {}'.format(event.id))
else:
print("Import failed.Please retry: {}".format(event_data['title']))
def _connect(self):
self.debug_print('URL: {}'.format(self.misp_param['url']))
self.debug_print('authkey: {}'.format(self.misp_param['authkey']))
self.misp = ExpandedPyMISP(self.misp_param['url'],
self.misp_param['authkey'],
ssl=False,
debug=False)
self._registered_tags = self.misp.tags()
def _check_tag(self, target_tag):
if self.misp == None:
self._connect()
if self._registered_tags == None:
self._get_tags()
for tag_info in self._registered_tags:
if tag_info.get('name', '') == target_tag:
return True
self.debug_print('new tag: {}'.format(target_tag))
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
tmp = MISPTag()
tmp.from_dict(name=target_tag)
self.misp.add_tag(tmp)
self._get_tags()
return True
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('add new tag retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
return False
def _add_event(self, value):
for tag in value['event_tags']:
self._check_tag(tag)
for attribute in value['attributes']:
for tag in attribute['tags']:
self._check_tag(tag)
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
tmp = MISPEvent()
tmp.from_dict(
distribution=self.misp_param['distribution'],
threat_level_id=self.misp_param['threat_level_id'],
analysis=self.misp_param['analysis'],
info=value['title'],
date=value['date'],
published=False)
response = self.misp.add_event(tmp)
if response.get('errors'):
raise Exception(str(response['errors']))
event = MISPEvent()
event.load(response)
break
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('add new event retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
return None
self.debug_print(event.id)
for tag in value['event_tags']:
event.add_tag(tag)
for attribute in value['attributes']:
attribute_tags = []
event.add_attribute(type=attribute['type'],
value=attribute['value'],
category=attribute['category'],
comment=attribute.get('comment', ''),
distribution=self.misp_param['distribution'],
Tag=self._create_tags(attribute['tags']))
event.published = True
if self._update_event(event):
self.debug_print('completed')
return event
else:
self.debug_print('add failed')
return None
def _get_event(self, id):
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
self.debug_print('get event start: {}'.format(id))
event = self.misp.get_event(id)
if event.get('errors'):
raise Exception(str(event['errors']))
self.debug_print('get event end: {}'.format(id))
return event
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('get event retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
return None
def _remove_event(self, id):
if id:
print('delete event: {}'.format(id))
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
response = self.misp.delete_event(id)
if response.get('errors'):
raise Exception(str(response['errors']))
return True
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('remove event retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
return False
def _search_event(self, **cons):
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
self.debug_print('search event start')
response = self.misp.search_index(**cons)
self.debug_print('search event end')
results = []
for json in response:
if json.get('id', ''):
results.append(self._get_event(json['id']))
else:
print('no event ID')
print(json)
return results
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('search event retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
return None
def _update_event(self, event):
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
self.debug_print('event update start: {}'.format(event.id))
response = self.misp.update_event(event)
if response.get('errors'):
raise Exception(str(response['errors']))
self.debug_print('{} updated'.format(event.id))
return True
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
print('event update failed: {}'.format(event.info))
try:
self._remove_event(event.id)
except:
pass
return False
def _create_tags(self, values):
tags = []
for value in values:
if value:
tags.append({'name': value})
return tags
def debug_print(self, message):
if self.debug == False:
return
# nowstr = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
nowstr = datetime.datetime.now().strftime('%H:%M:%S')
print('{}\t{}'.format(nowstr, message))
class Scrippts:
def __init__(self):
self.misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
def geolocation_alt(self) -> MISPObject:
# Alert, NWT, Canada
location = MISPObject('geolocation', standalone=False)
location.add_attribute('latitude', 82.3)
location.add_attribute('longitude', 62.3)
location.add_attribute('altitude', 210)
location.add_attribute('text', 'Alert, NWT, Canada')
return location
def tag_alt(self) -> MISPTag:
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:ALT'
return tag
def geolocation_ptb(self):
# Point Barrow, Alaska
location = MISPObject('geolocation')
location.add_attribute('latitude', 71.3)
location.add_attribute('longitude', 156.6)
location.add_attribute('altitude', 11)
location.add_attribute('text', 'Point Barrow, Alaska')
return location
def tag_ptb(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:PTB'
return tag
def geolocation_stp(self) -> MISPObject:
# Station P
location = MISPObject('geolocation')
location.add_attribute('latitude', 50)
location.add_attribute('longitude', 145)
location.add_attribute('altitude', 0)
location.add_attribute('text', 'Station P')
return location
def tag_stp(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:STP'
return tag
def geolocation_ljo(self) -> MISPObject:
# La Jolla Pier, California
location = MISPObject('geolocation')
location.add_attribute('latitude', 32.9)
location.add_attribute('longitude', 117.3)
location.add_attribute('altitude', 10)
location.add_attribute('text', 'La Jolla Pier, California')
return location
def tag_ljo(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:LJO'
return tag
def geolocation_bcs(self) -> MISPObject:
# Baja California Sur, Mexico
location = MISPObject('geolocation')
location.add_attribute('latitude', 23.3)
location.add_attribute('longitude', 110.2)
location.add_attribute('altitude', 4)
location.add_attribute('text', 'Baja California Sur, Mexico')
return location
def tag_bcs(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:BCS'
return tag
def geolocation_mlo(self) -> MISPObject:
# Mauna Loa Observatory, Hawaii
location = MISPObject('geolocation')
location.add_attribute('latitude', 19.5)
location.add_attribute('longitude', 155.6)
location.add_attribute('altitude', 3397)
location.add_attribute('text', 'Mauna Loa Observatory, Hawaii')
return location
def tag_mlo(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:MLO'
return tag
def geolocation_kum(self) -> MISPObject:
# Cape Kumukahi, Hawaii
location = MISPObject('geolocation')
location.add_attribute('latitude', 19.5)
location.add_attribute('longitude', 154.8)
location.add_attribute('altitude', 3)
location.add_attribute('text', 'Cape Kumukahi, Hawaii')
return location
def tag_kum(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:KUM'
return tag
def geolocation_chr(self):
# Christmas Island, Fanning Island
location = MISPObject('geolocation')
location.add_attribute('latitude', 2)
location.add_attribute('longitude', 157.3)
location.add_attribute('altitude', 2)
location.add_attribute('text', 'Christmas Island, Fanning Island')
return location
def tag_chr(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:CHR'
return tag
def geolocation_sam(self):
# American Samoa
location = MISPObject('geolocation')
location.add_attribute('latitude', 14.2)
location.add_attribute('longitude', 170.6)
location.add_attribute('altitude', 30)
location.add_attribute('text', 'American Samoa')
return location
def tag_sam(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:SAM'
return tag
def geolocation_ker(self):
# Kermadec Islands, Raoul Island
location = MISPObject('geolocation')
location.add_attribute('latitude', 29.2)
location.add_attribute('longitude', 177.9)
location.add_attribute('altitude', 2)
location.add_attribute('text', 'Kermadec Islands, Raoul Island')
return location
def tag_ker(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:KER'
return tag
def geolocation_nzd(self):
# Baring Head, New Zealand
location = MISPObject('geolocation')
location.add_attribute('latitude', 41.4)
location.add_attribute('longitude', 174.9)
location.add_attribute('altitude', 85)
location.add_attribute('text', 'Baring Head, New Zealand')
return location
def tag_nzd(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:NZD'
return tag
def geolocation_psa(self):
# Palmer Station, Antarctica
location = MISPObject('geolocation')
location.add_attribute('latitude', 64.9)
location.add_attribute('longitude', 64)
location.add_attribute('altitude', 10)
location.add_attribute('text', 'Palmer Station, Antarctica')
return location
def tag_psa(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:PSA'
return tag
def geolocation_spo(self):
# South Pole
location = MISPObject('geolocation')
location.add_attribute('latitude', 90)
location.add_attribute('longitude', 0)
location.add_attribute('altitude', 2810)
location.add_attribute('text', 'South Pole')
return location
def tag_spo(self):
tag = MISPTag()
tag.name = 'scrippsco2-sampling-stations:SPO'
return tag
def fetch(self, url):
filepath = Path('scrippts') / Path(url).name
if filepath.exists():
return filepath
r = requests.get(url)
if r.status_code != 200 or r.text[0] != '"':
print(url)
return False
with filepath.open('w') as f:
f.write(r.text)
return filepath
def get_existing_event_to_update(self, infofield):
found = self.misp.search(eventinfo=infofield, pythonify=True)
if found:
event = found[0]
return event
return False
def import_all(self, stations_short_names, interval, data_type):
object_creator = getattr(self, f'{interval}_flask_{data_type}')
if data_type == 'co2':
base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/'
elif data_type in ['c13', 'o18']:
base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/'
for station in stations_short_names:
url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv'
infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations'
filepath = self.fetch(url)
if not filepath:
continue
update = True
event = self.get_existing_event_to_update(infofield)
if event:
location = event.get_objects_by_name('geolocation')[0]
if not event:
event = MISPEvent()
event.info = infofield
event.add_tag(getattr(self, f'tag_{station}')())
location = getattr(self, f'geolocation_{station}')()
event.add_object(location)
event.add_attribute('link', f'http://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}')
update = False
object_creator(event, location, filepath, update)
if update:
self.misp.update_event(event)
else:
self.misp.add_event(event)
def import_monthly_co2_all(self):
to_import = ['alt', 'ptb', 'stp', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd']
self.import_all(to_import, 'monthly', 'co2')
def import_monthly_c13_all(self):
to_import = ['alt', 'ptb', 'stp', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd', 'psa', 'spo']
self.import_all(to_import, 'monthly', 'c13')
def import_monthly_o18_all(self):
to_import = ['alt', 'ptb', 'stp', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd', 'spo']
self.import_all(to_import, 'monthly', 'o18')
def import_daily_co2_all(self):
to_import = ['alt', 'ptb', 'stp', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd']
self.import_all(to_import, 'daily', 'co2')
def import_daily_c13_all(self):
to_import = ['alt', 'ptb', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd', 'spo']
self.import_all(to_import, 'daily', 'c13')
def import_daily_o18_all(self):
to_import = ['alt', 'ptb', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd', 'spo']
self.import_all(to_import, 'daily', 'o18')
def split_data_comment(self, csv_file, update, event):
comment = ''
data = []
with csv_file.open() as f:
for line in f:
if line[0] == '"':
if update:
continue
if '----------' in line:
event.add_attribute('comment', comment, disable_correlation=True)
comment = ''
continue
comment += line[1:-1].strip()
else:
data.append(line)
if not update:
event.add_attribute('comment', comment, disable_correlation=True)
return data
def monthly_flask_co2(self, event, location, csv_file, update):
data = self.split_data_comment(csv_file, update, event)
dates_already_imported = []
if update:
# get all datetime from existing event
for obj in event.get_objects_by_name('scrippsco2-co2-monthly'):
date_attribute = obj.get_attributes_by_relation('sample-datetime')[0]
dates_already_imported.append(date_attribute.value)
reader = csv.reader(data)
for row in reader:
if not row[0].isdigit():
# This file has f****d up headers
continue
sample_date = parse(f'{row[0]}-{row[1]}-16T00:00:00')
if sample_date in dates_already_imported:
continue
obj = MISPObject('scrippsco2-co2-monthly', standalone=False)
obj.add_attribute('sample-datetime', sample_date)
obj.add_attribute('sample-date-excel', float(row[2]))
obj.add_attribute('sample-date-fractional', float(row[3]))
obj.add_attribute('monthly-co2', float(row[4]))
obj.add_attribute('monthly-co2-seasonal-adjustment', float(row[5]))
obj.add_attribute('monthly-co2-smoothed', float(row[6]))
obj.add_attribute('monthly-co2-smoothed-seasonal-adjustment', float(row[7]))
obj.add_reference(location, 'sampling-location')
event.add_object(obj)
def monthly_flask_c13(self, event, location, csv_file, update):
data = self.split_data_comment(csv_file, update, event)
dates_already_imported = []
if update:
# get all datetime from existing event
for obj in event.get_objects_by_name('scrippsco2-c13-monthly'):
date_attribute = obj.get_attributes_by_relation('sample-datetime')[0]
dates_already_imported.append(date_attribute.value)
reader = csv.reader(data)
for row in reader:
if not row[0].isdigit():
# This file has f****d up headers
continue
sample_date = parse(f'{row[0]}-{row[1]}-16T00:00:00')
if sample_date in dates_already_imported:
continue
obj = MISPObject('scrippsco2-c13-monthly', standalone=False)
obj.add_attribute('sample-datetime', sample_date)
obj.add_attribute('sample-date-excel', float(row[2]))
obj.add_attribute('sample-date-fractional', float(row[3]))
obj.add_attribute('monthly-c13', float(row[4]))
obj.add_attribute('monthly-c13-seasonal-adjustment', float(row[5]))
obj.add_attribute('monthly-c13-smoothed', float(row[6]))
obj.add_attribute('monthly-c13-smoothed-seasonal-adjustment', float(row[7]))
obj.add_reference(location, 'sampling-location')
event.add_object(obj)
def monthly_flask_o18(self, event, location, csv_file, update):
data = self.split_data_comment(csv_file, update, event)
dates_already_imported = []
if update:
# get all datetime from existing event
for obj in event.get_objects_by_name('scrippsco2-o18-monthly'):
date_attribute = obj.get_attributes_by_relation('sample-datetime')[0]
dates_already_imported.append(date_attribute.value)
reader = csv.reader(data)
for row in reader:
if not row[0].isdigit():
# This file has f****d up headers
continue
sample_date = parse(f'{row[0]}-{row[1]}-16T00:00:00')
if sample_date in dates_already_imported:
continue
obj = MISPObject('scrippsco2-o18-monthly', standalone=False)
obj.add_attribute('sample-datetime', sample_date)
obj.add_attribute('sample-date-excel', float(row[2]))
obj.add_attribute('sample-date-fractional', float(row[3]))
obj.add_attribute('monthly-o18', float(row[4]))
obj.add_attribute('monthly-o18-seasonal-adjustment', float(row[5]))
obj.add_attribute('monthly-o18-smoothed', float(row[6]))
obj.add_attribute('monthly-o18-smoothed-seasonal-adjustment', float(row[7]))
obj.add_reference(location, 'sampling-location')
event.add_object(obj)
def daily_flask_co2(self, event, location, csv_file, update):
data = self.split_data_comment(csv_file, update, event)
dates_already_imported = []
if update:
# get all datetime from existing event
for obj in event.get_objects_by_name('scrippsco2-co2-daily'):
date_attribute = obj.get_attributes_by_relation('sample-datetime')[0]
dates_already_imported.append(date_attribute.value)
reader = csv.reader(data)
for row in reader:
sample_date = parse(f'{row[0]}-{row[1]}')
if sample_date in dates_already_imported:
continue
obj = MISPObject('scrippsco2-co2-daily', standalone=False)
obj.add_attribute('sample-datetime', sample_date)
obj.add_attribute('sample-date-excel', float(row[2]))
obj.add_attribute('sample-date-fractional', float(row[3]))
obj.add_attribute('number-flask', int(row[4]))
obj.add_attribute('flag', int(row[5]))
attr = obj.add_attribute('co2-value', float(row[6]))
attr.add_tag(f'scrippsco2-fgc:{int(row[5])}')
obj.add_reference(location, 'sampling-location')
event.add_object(obj)
def daily_flask_c13(self, event, location, csv_file, update):
data = self.split_data_comment(csv_file, update, event)
dates_already_imported = []
if update:
# get all datetime from existing event
for obj in event.get_objects_by_name('scrippsco2-c13-daily'):
date_attribute = obj.get_attributes_by_relation('sample-datetime')[0]
dates_already_imported.append(date_attribute.value)
reader = csv.reader(data)
for row in reader:
sample_date = parse(f'{row[0]}-{row[1]}')
if sample_date in dates_already_imported:
continue
obj = MISPObject('scrippsco2-c13-daily', standalone=False)
obj.add_attribute('sample-datetime', sample_date)
obj.add_attribute('sample-date-excel', float(row[2]))
obj.add_attribute('sample-date-fractional', float(row[3]))
obj.add_attribute('number-flask', int(row[4]))
obj.add_attribute('flag', int(row[5]))
attr = obj.add_attribute('c13-value', float(row[6]))
attr.add_tag(f'scrippsco2-fgi:{int(row[5])}')
obj.add_reference(location, 'sampling-location')
event.add_object(obj)
def daily_flask_o18(self, event, location, csv_file, update):
data = self.split_data_comment(csv_file, update, event)
dates_already_imported = []
if update:
# get all datetime from existing event
for obj in event.get_objects_by_name('scrippsco2-o18-daily'):
date_attribute = obj.get_attributes_by_relation('sample-datetime')[0]
dates_already_imported.append(date_attribute.value)
reader = csv.reader(data)
for row in reader:
sample_date = parse(f'{row[0]}-{row[1]}')
if sample_date in dates_already_imported:
continue
obj = MISPObject('scrippsco2-o18-daily', standalone=False)
obj.add_attribute('sample-datetime', sample_date)
obj.add_attribute('sample-date-excel', float(row[2]))
obj.add_attribute('sample-date-fractional', float(row[3]))
obj.add_attribute('number-flask', int(row[4]))
obj.add_attribute('flag', int(row[5]))
attr = obj.add_attribute('o18-value', float(row[6]))
attr.add_tag(f'scrippsco2-fgi:{int(row[5])}')
obj.add_reference(location, 'sampling-location')
event.add_object(obj)
class MispHandler:
def __init__(self, config, logger):
self.logger = logger
MISP_KEY = config['MISP_KEY']
MISP_URL = config['MISP_URL']
MISP_VERIFYCERT = config['MISP_VERIFYCERT']
self.config = config
self.misp = ExpandedPyMISP(MISP_URL, MISP_KEY, MISP_VERIFYCERT)
self.orgc = MISPOrganisation()
self.orgc.name = config['MISP_ORG_NAME']
self.orgc.id = config['MISP_ORG_ID']
self.orgc.uuid = config['MISP_ORG_UUID']
self.tags = config['tags']
self.galaxy_synonyms = {}
self.enabled_clusters = self.config['galaxies']
self.galaxy_tags = {}
self._init_galaxies()
def get_event_id(self, event):
try:
return event['Event']['id']
except KeyError:
return event.id
except TypeError:
return event.id
def get_attributes(self, event):
try:
attributes = event['Event']['Attribute']
except KeyError:
attributes = event.attributes
return attributes
def add_sigthing(self, id):
sighting = MISPSighting()
self.misp.add_sighting(sighting, id)
def get_event(self, malware_type, feed_tag):
malware_tag = "malware:" + malware_type.lower()
res = self.misp.search(tags=[feed_tag], controller='events', pythonify=True)
for event in res:
for tag in event.tags:
if tag['name'].lower() == malware_tag.lower():
return event
return None
def _init_galaxies(self):
i = 1
cont = True
for cluster in self.enabled_clusters:
self.galaxy_tags[cluster] = []
while cont:
g = self.misp.get_galaxy(i)
try:
galaxy_cluster = g['Galaxy']['name']
except KeyError:
cont = False
continue
if galaxy_cluster.lower() in self.enabled_clusters:
elements = g['GalaxyCluster']
for element in elements:
self.galaxy_tags[galaxy_cluster.lower()].append(element['tag_name'])
for inner_element in element['GalaxyElement']:
if inner_element['key'] == 'synonyms':
if not element['tag_name'] in self.galaxy_synonyms:
self.galaxy_synonyms[element['tag_name']] = []
self.galaxy_synonyms[element['tag_name']].append(inner_element['value'])
i = i + 1
def get_galaxies(self, malware_tag):
res = []
for cluster in self.enabled_clusters:
for galaxy_tag in self.galaxy_tags[cluster]:
malware = malware_tag.split(':')[1].lower().replace(" ", "")
galaxy_value = galaxy_tag.split('"')[1].lower().replace(" ", "")
if malware == galaxy_value:
res.append(galaxy_tag)
break
else:
if galaxy_tag in self.galaxy_synonyms:
for synonym in self.galaxy_synonyms[galaxy_tag]:
galaxy_value = synonym.lower().replace(" ", "")
if malware == galaxy_value:
res.append(galaxy_tag)
break
return res
def get_file_taxonomy(self, ft):
if ft == 'exe':
return 'file-type:type="peexe"'
elif ft == 'dll':
return 'file-type:type="pedll"'
elif ft == 'zip':
return 'file-type:type="zip"'
elif ft == 'apk':
return 'file-type:type="android"'
elif ft == 'rar':
return 'file-type:type="rar"'
elif ft == 'xls':
return 'file-type:type="xls"'
elif ft == 'xlsx':
return 'file-type:type="xlsx"'
elif ft == 'doc':
return 'file-type:type="doc"'
elif ft == 'docx':
return 'file-type:type="docx"'
elif ft == '7z' or ft == '7zip':
return 'file-type:type="7zip"'
elif ft == 'gz' or ft == 'gzip':
return 'file-type:type="gzip"'
file_types = self.misp.get_taxonomy(52)['entries']
for file_type in file_types:
if ft == file_type['tag'].split('"')[1].strip():
return file_type['tag']
print("Unknown Filetype: " + ft)
return ''
def new_misp_event(self, malware_type, feed_tag, event_info, additional_tags=[],
info_cred='admirality-scale:information-credibility="2"'):
malware_tag = "malware:" + malware_type.lower()
misp_event_obj = MISPEvent()
misp_event_obj.info = event_info
misp_event_obj.add_tag(feed_tag)
misp_event_obj.add_tag(info_cred)
if len(malware_tag) > 0:
misp_event_obj.add_tag(malware_tag.lower())
for tag in self.config['tags']:
misp_event_obj.add_tag(tag)
for tag in additional_tags:
misp_event_obj.add_tag(tag)
misp_event_obj.orgc = self.orgc
galaxies = self.get_galaxies(malware_tag)
for galaxy in galaxies:
misp_event_obj.add_tag(galaxy)
misp_event = self.misp.add_event(misp_event_obj)
return misp_event
if last_event_date < date.today() or int(nb_attr) > 1000:
me = create_new_event()
else:
event_id = response[0].id
else:
me = create_new_event()
parameters = {'banned-ip': args.banned_ip, 'attack-type': args.attack_type}
if args.processing_timestamp:
parameters['processing-timestamp'] = args.processing_timestamp
if args.failures:
parameters['failures'] = args.failures
if args.sensor:
parameters['sensor'] = args.sensor
if args.victim:
parameters['victim'] = args.victim
if args.logline:
parameters['logline'] = b64decode(args.logline).decode()
if args.logfile:
with open(args.logfile, 'rb') as f:
parameters['logfile'] = {
'value': os.path.basename(args.logfile),
'data': BytesIO(f.read())
}
f2b = Fail2BanObject(parameters=parameters, standalone=False)
if me:
me.add_object(f2b)
pymisp.add_event(me)
elif event_id:
a = pymisp.add_object(event_id, f2b)
if args.is_malware:
arg_type = 'malware-sample'
else:
arg_type = 'attachment'
# Create attributes
attributes = []
for f in files:
a = MISPAttribute()
a.type = arg_type
a.value = f.name
a.data = f
a.comment = args.comment
a.distribution = args.distrib
if args.expand and arg_type == 'malware-sample':
a.expand = 'binary'
attributes.append(a)
if args.event:
for a in attributes:
misp.add_attribute(args.event, a)
else:
m = MISPEvent()
m.info = args.info
m.distribution = args.distrib
m.attributes = attributes
if args.expand and arg_type == 'malware-sample':
m.run_expansions()
misp.add_event(m)
class OutputMISPBot(Bot):
def init(self):
if ExpandedPyMISP is None:
raise ValueError('Could not import pymisp. Please install it.')
# Initialize MISP connection
self.misp = ExpandedPyMISP(self.parameters.misp_url,
self.parameters.misp_key,
self.parameters.misp_verify)
# Handle the InsecureRequestWarning
if not self.parameters.misp_verify: urllib3.disable_warnings()
self.logger.info ("Connected to MISP")
def process(self):
# Grab raw event
report = self.receive_message()
raw_report = utils.base64_decode(report.get('raw'))
atd_event = json.loads(raw_report)
if (atd_event['Summary']['Event_Type'] == 'ATD File Report'):
if (int(atd_event['Summary']['Verdict']['Severity']) >= self.parameters.atd_verdict_severity):
# Generate MISP Event
event = self.misp_add_event(atd_event)
# Add event to MISP instance
event = self.misp.add_event(event)
if self.parameters.atd_event_publish:
# Publish new MISP event
self.misp.publish(event)
self.acknowledge_message()
def misp_add_attribute(self, event, a_type, a_value):
# Create Attribute object in MISP
misp_attribute = MISPAttribute()
if a_type: misp_attribute.type = a_type
if a_value: misp_attribute.value = a_value
event.add_attribute(misp_attribute.type, misp_attribute.value)
def misp_add_tag(self, event, a_value):
# Create Tag object in MISP
misp_tag = MISPTag()
if a_value: misp_tag.name = a_value
event.add_tag(misp_tag)
def misp_add_fileObject (self, event, filename, md5, sha1, sha256):
# Create new MISPFileObject
misp_object = MISPObject (name = 'file')
# Add attributes
misp_object.add_attribute("filename", value = filename)
misp_object.add_attribute("md5", value = md5)
misp_object.add_attribute("sha1", value = sha1)
misp_object.add_attribute("sha256", value = sha256)
# Add MISPFileObject to event
event.add_object (misp_object)
def misp_add_event(self, query):
# Parse out all data from json
mainfile = query['Summary']['Subject']['Name']
# Set Distribution = Organization Only
distribution=self.parameters.misp_distribution
# Set Threat level = getting the threat level from ATD
threat_level_id=query['Summary']['Verdict']['Severity']
# Set Analysis status = completed
analysis_status=2
# Creat Event object in MISP
misp_event = MISPEvent()
misp_event.info = "McAfee ATD Sandbox Analysis Report - " + mainfile
misp_event.distribution = distribution
misp_event.threat_level_id = atd_to_misp_confidence(threat_level_id)
misp_event.analysis = analysis_status
# Add main Information to MISP
atdip = query['Summary']['ATD IP']
if not atdip: pass
else: self.misp_add_attribute(misp_event, "comment", "ATD IP " + atdip)
dstip = query['Summary']['Dst IP']
if not dstip: pass
else: self.misp_add_attribute(misp_event, "ip-dst", dstip)
taskid = query['Summary']['TaskId']
if not taskid: pass
else: self.misp_add_attribute(misp_event, "comment", "ATD TaskID: " + taskid)
size = query['Summary']['Subject']['size']
if not size: pass
else: self.misp_add_attribute(misp_event, "comment", "File size is " + size)
verdict = query['Summary']['Verdict']['Description']
if not verdict: pass
else: self.misp_add_attribute(misp_event, "comment", verdict)
# Add file object to MISP Event
self.misp_add_fileObject (misp_event, mainfile,
query['Summary']['Subject']['md5'],
query['Summary']['Subject']['sha-1'],
query['Summary']['Subject']['sha-256']
)
# Add process information to MISP
try:
for processes in query['Summary']['Processes']:
name = processes['Name']
md5 = processes['Md5']
sha1 = processes['Sha1']
sha256 = processes['Sha256']
if not name: pass
else: self.misp_add_attribute(misp_event, "filename", name)
if not md5: pass
else: self.misp_add_attribute(misp_event, "md5", md5)
if not sha1: pass
else: self.misp_add_attribute(misp_event, "sha1", sha1)
if not sha256: pass
else: self.misp_add_attribute(misp_event, "sha256", sha256)
except:
pass
# Add files information to MISP
try:
for files in query['Summary']['Files']:
# Evaluate attributes
name = files['Name']
md5 = files['Md5']
sha1 = files['Sha1']
sha256 = files['Sha256']
# Add attributes as FileObject to event
self.misp_add_fileObject (misp_event, name, md5, sha1, sha256)
except:
pass
# Add URL information to MISP
try:
for url in query['Summary']['Urls']:
url = url['Url']
if not url: pass
else: self.misp_add_attribute(misp_event, "url", url)
except:
pass
# Add ips information to MISP
try:
for ips in query['Summary']['Ips']:
ipv4 = ips['Ipv4']
port = ips['Port']
if not ipv4: pass
else: self.misp_add_attribute(misp_event, "ip-dst", ipv4)
if not port: pass
else: self.misp_add_attribute(misp_event, "url", ipv4 + ":" + port)
except:
pass
# Add stats Information to MISP
try:
for stats in query['Summary']['Stats']:
category = stats['Category']
if not category: pass
else: self.misp_add_attribute(misp_event, "comment", category)
except:
pass
# Add behaviour information to MISP
try:
for behave in query['Summary']['Behavior']:
behave = behave['Analysis']
if not category: pass
else: self.misp_add_attribute(misp_event, "comment", behave)
except:
pass
# Add Confidence level from ATD to MISP
self.misp_add_tag(misp_event, str(atd_to_veris_confidence(threat_level_id)))
# Add TLP info to MISP
self.misp_add_tag(misp_event, str("tlp:amber"))
self.misp_add_tag(misp_event, str("McAfee ATD Analysis"))
# Add tag to event
self.misp_add_tag(misp_event, str("cssa:origin=\"sandbox\""))
self.misp_add_tag(misp_event, str("cssa:sharing-class=\"unvetted\""))
# Add actual event to MISP instance
# Moved to calling routine
# misp_event = self.misp.add_event(misp_event)
return misp_event
customerUserId = event.add_attribute(
'other',
alert["identity"]["customerUserId"],
comment="Customer User Id")
if alert["identity"]["department"] is not None:
department = event.add_attribute(alert['other',
"identity"]["department"],
comment="Department")
if alert["identity"]["location"] is not None:
location = event.add_attribute('other',
alert["identity"]["location"],
comment="Location")
if alert["identity"]["name"] is not None:
name = event.add_attribute('target-user',
alert["identity"]["name"],
comment="Name")
if alert["identity"]["title"] is not None:
title = event.add_attribute('other',
alert["identity"]["title"],
comment="Title")
event.add_tag("VAP")
misp.add_event(event.to_json())
if args.dump:
for o in objects:
print(o.to_json())
else:
if offline:
print('You are in offline mode, quitting.')
else:
misp = ExpandedPyMISP(url=misp_url,
key=misp_key,
ssl=misp_verifycert)
if args.new_event:
event = MISPEvent()
event.info = args.new_event
for o in objects:
event.add_object(**o)
new_event = misp.add_event(event)
if isinstance(new_event, str):
print(new_event)
elif 'id' in new_event:
print(f'Created new event {new_event.id}')
else:
print('Something went wrong:')
print(new_event)
else:
for o in objects:
new_object = misp.add_object(args.update_event, o)
if isinstance(new_object, str):
print(new_object)
elif new_object.attributes:
print(
f'New {new_object.name} object added to {args.update_event}'
def main():
# Process arguments and initialize TRAP interface
try:
args, _ = process_arguments()
except SystemExit as e:
args = None # Error in arguments or help message
trap = pytrap.TrapCtx()
try:
trap.init(sys.argv,
1,
0,
module_name="Backscatter classifier common TRAP help")
# Allow trap to print it's own help but terminate script anyway due to error in arguments
if args is None:
sys.exit(PYTHON_ARGS_ERROR_EXIT)
trap.setRequiredFmt(0)
except Exception as e:
# Trap error message
print(e)
sys.exit(TRAP_ARGS_ERROR_EXIT)
# Logging settings
logger = logging.getLogger("backscatter_classifier")
logging.basicConfig(
level=logging.DEBUG,
filename=args.logfile,
filemode='w',
format=
"[%(levelname)s], %(asctime)s, %(name)s, %(funcName)s, line %(lineno)d: %(message)s"
)
# ASN and city databases
try:
geoip_db = Geoip2Wrapper(args.agp, args.cgp)
except Exception as e:
logger.error(e)
logger.error("Error while create GeoIP2 wrapper")
print(str(e), file=sys.stderr)
sys.exit(EXIT_FAILURE)
if args.export_to in ("c3isp", "c3isp-misp"):
c3isp_upload.read_config(args.c3isp_config)
if args.export_to == "misp":
# MISP instance
try:
misp_instance = ExpandedPyMISP(args.url, args.key, args.ssl)
except Exception as e:
logger.error(e)
logger.error("Error while creating MISP instance")
print(str(e), file=sys.stderr)
sys.exit(EXIT_FAILURE)
# DDoS model
ddos_model = pickle.load(args.model)
ddos_classifier = DDoSClassifier(ddos_model, args.min_threshold)
# *** MAIN PROCESSING LOOP ***
while True:
try:
# Receive data
try:
data = trap.recv()
except pytrap.FormatChanged as e:
# Automatically detect format
fmttype, fmtspec = trap.getDataFmt(0)
rec = pytrap.UnirecTemplate(fmtspec)
data = e.data
if len(data) <= 1:
# Terminating message
break
# Decode received data into python object
rec.setData(data)
# Only Ipv4
try:
victim_ip = IPv4Address(rec.SRC_IP)
except Exception as e:
logger.info("Received IPv6 address, skipping")
continue
# Predict class of backscatter like traffic
try:
duration = DDoSClassifier.DURATION(rec)
if duration < args.min_duration:
# attack is too short lived to be reported
continue
if duration > args.max_duration:
# attack is too long
continue
if rec.FLOW_COUNT < args.min_flows:
# attack does not have enough flows
continue
for subnet in CESNET_NET:
if victim_ip in subnet:
continue
ddos = ddos_classifier.predict(rec)
except Exception as e:
logger.error(e)
continue
# Report attack using MISP
try:
if ddos:
try:
domain = gethostbyaddr(str(victim_ip))
except herror as e:
# Do not report for unknown domains
continue
event = create_ddos_event(rec, geoip_db, victim_ip,
domain[0],
args.misp_templates_dir)
if args.export_to == "misp":
try:
event_id = misp_instance.add_event(
event)['Event']['id']
misp_instance.publish(event_id)
except Exception as e:
logger.error(e)
elif args.export_to in ("c3isp", "c3isp-misp"):
logger.debug(f"Uploading event to C3ISP platform.")
event_file_path = Path(
"misp_event_to_c3isp.json").absolute()
with temporary_open(event_file_path,
'w') as event_file:
json.dump(event.to_json(), event_file)
response = c3isp_upload.upload_to_c3isp(
event_file_path)
logger.debug(f"Response: {response}")
if not response or ('status' in response
and response['status'] == 'ERROR'):
logger.error("ERROR during upload!")
continue
if args.export_to == "c3isp-misp":
dpo_id = response['content'][
'additionalProperties']['dposId']
logger.debug(f'Exporting DPO {dpo_id} to MISP.')
c3isp_upload.export_misp(dpo_id, logger)
except Exception as e:
logger.error(str(e))
continue
except Exception as e:
# Log and re-raise exception
logger.error(e)
raise e
# *** END OF MAIN PROCESSING LOOP ***
trap.finalize()
from pymisp import ExpandedPyMISP, MISPEvent
from pymisp import MISPObject
from keys import misp_url, misp_key, misp_verifycert
from datetime import date
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
event = MISPEvent()
event.info = 'IoT malware' # Event Title
event.distribution = 1 # 0 = Your Organisation Only, 1 = Community
event.threat_level_id = 2 # 1 = High, 2 = Medium, 3 = Low
event.analysis = 2 # 0 (initial analysis), 1 (On-Going), 2 (Complete)
event.add_tag('malware_classification:malware-category="Botnet"')
event.add_tag('tlp:amber')
d = date.today()
event.set_date(d)
attribute_second = event.add_attribute('url',
'http://1.2.3.4/example',
disable_correlation=False,
comment="Botnet example text",
to_ids=False)
event = misp.add_event(event, pythonify=True)
# Publish event
event.publish()
org_new.sector = "Government"
org.id = api.add_organisation(org_new, pythonify=True).id
# Create the MISP event by loading the JSON file
# This will not add the attributes, but does add the event tags and galaxies
# We also add a random UUID for uniqueness
event = MISPEvent()
event.load_file(json_import)
event.uuid = event_import_uuid
if not event_import_info:
event_import_info = event.info
event.info = event_import_info
event.date = event_import_date
event.distribution = event_import_distribution
event.orgc = api.get_organisation(org, pythonify=True)
event = api.add_event(event, pythonify=True)
# Check if the event was created
if (int(event.id) > 0):
# Yes, read the content and add attributes and objects
# Include a sleep so for the scheduler
count_attributes = 0
count_objects = 0
with open(json_import) as json_file:
data = json.load(json_file)
if 'Attribute' in data.get("response")[0].get("Event"):
attributes = data.get("response")[0].get("Event").get("Attribute")
for attribute in attributes:
misp_tag = []
if 'Tag' in attribute:
class MISPCaseManagement:
def __init__(
self,
config_file='C:\\automation-hunting\\misp\\conf\\misp-case-provider.yaml',
debug=False):
self.misp_url = None
self.api_key = None
self.verify_cert = False
if not self.get_config_data(config_file):
raise Exception('Invalid Configuration File')
self.misp_api = ExpandedPyMISP(self.misp_url,
self.api_key,
self.verify_cert,
debug=debug)
self.add_ioc_functions = [
self.misp_api.add_ipdst, self.misp_api.add_ipsrc,
self.misp_api.add_internal_comment,
self.misp_api.add_internal_text, self.misp_api.add_email_subject,
self.misp_api.add_mutex, self.misp_api.add_filename,
self.misp_api.add_hostname, self.misp_api.add_domain,
self.misp_api.add_domain_ip, self.misp_api.add_email_src,
self.misp_api.add_email_dst, self.misp_api.add_hashes,
self.misp_api.add_object, self.misp_api.add_regkey,
self.misp_api.add_url, self.misp_api.add_user,
self.misp_api.add_useragent, self.misp_api.add_target_user
]
def get_config_data(self, yaml_file):
with open(yaml_file, 'r') as ymlfile:
cfg = yaml.load(ymlfile, Loader=yaml.FullLoader)
valid = False
if self.validate_cfg_yml(cfg):
self.misp_url = cfg['misp']['misp_url']
self.api_key = cfg['misp']['api_key']
self.verify_cert = cfg['misp']['verify_cert']
valid = True
return valid
@staticmethod
def validate_cfg_yml(cfg):
if 'misp' not in cfg:
print('Not main')
return False
else:
if 'misp_url' not in cfg['misp'] or 'api_key' not in cfg['misp']:
return False
return True
def create_full_event(
self,
info,
distribution: MISPDistribution = MISPDistribution.ORGANIZATION,
threat_level: MISPThreatLevel = MISPThreatLevel.MEDIUM,
analysis: MISPAnalysis = MISPAnalysis.INITIAL,
attributes: list = None,
tags: list = None):
new_event = MISPEvent()
new_event.distribution = distribution.value
new_event.threat_level_id = threat_level.value
new_event.analysis = analysis.value
new_event.info = info
if attributes is not None:
new_event.Attribute = list()
if tags is not None:
new_event.Tag = list()
event = self.misp_api.add_event(new_event)
self.misp_api.get_all_tags()
print(event.to_json())
return event
def search_tag(self,
tag_value,
operator: StringOperator = StringOperator.EQ,
sensitive=True,
**kwargs):
tags = self.misp_api.get_tags_list()
ret_tags = list()
for tag in tags:
found = False
if operator == StringOperator.EQ:
if sensitive:
if tag['name'] == tag_value:
found = True
else:
if tag['name'].lower() == tag_value.lower():
found = True
elif operator == StringOperator.CONTAINS:
if sensitive:
if tag_value in tag['name']:
found = True
else:
if tag['name'].lower() == tag_value.lower():
found = True
elif operator == StringOperator.REGEX:
if sensitive:
if re.search(tag_value, tag['name']) is not None:
found = True
else:
if re.search(tag_value.lower(),
tag['name'].lower()) is not None:
found = True
else:
print('Unsupported')
if found and len(kwargs) > 0:
if 'exportable' in kwargs:
if tag['exportable'] == kwargs['exportable']:
found = True
else:
found = False
if 'org_id' in kwargs:
if tag['org_id'] == str(kwargs['org_id']):
found = True
else:
found = False
if 'user_id' in kwargs:
if tag['user_id'] == str(kwargs['user_id']):
found = True
else:
found = False
if 'hide_tag' in kwargs:
if tag['hide_tag'] == kwargs['hide_tag']:
found = True
else:
found = False
if found:
ret_tags.append(tag)
return ret_tags
def create_tag(self, tag_data):
new_tag = self.misp_api.new_tag(name=tag_data['name'],
colour=tag_data['colour'],
exportable=tag_data['exportable'],
hide_tag=tag_data['hide_tag'])
return new_tag
def search_event(self, **kwargs):
result = None
if 'event_name' in kwargs:
result = self.misp_api.search(eventinfo=kwargs['event_name'])
elif 'event_id' in kwargs:
result = self.misp_api.search(eventid=kwargs['event_id'])
elif 'uuid' in kwargs:
result = self.misp_api.search(uuid=kwargs['uuid'])
else:
print('Unsupported search filter')
return result
def add_ioc(self, event, data_type: MISPDataType, **kwargs):
if data_type.value == 12:
if 'domain' in kwargs and 'ip' in kwargs:
self.add_ioc_functions[data_type.value](event,
kwargs['domain'],
kwargs['ip'])
else:
print('Missing parameters for: ' + data_type.name)
elif data_type.value == 9:
if 'md5' in kwargs:
self.add_ioc_functions[data_type.value](event,
md5=kwargs['md5'])
if 'sha1' in kwargs:
self.add_ioc_functions[data_type.value](event,
sha1=kwargs['sha1'])
if 'sha256' in kwargs:
self.add_ioc_functions[data_type.value](
event, sha256=kwargs['sha256'])
if 'ssdeep' in kwargs:
self.add_ioc_functions[data_type.value](
event, ssdeep=kwargs['ssdeep'])
else:
if 'value' in kwargs:
print('ADDING ATTRIBUTE')
self.add_ioc_functions[data_type.value](event, kwargs['value'])
else:
print('Missing parameters for: ' + data_type.name)
class Output(cowrie.core.output.Output):
"""
MISP Upload Plugin for Cowrie.
This Plugin creates a new event for unseen file uploads
or adds sightings for previously seen files.
The decision is done by searching for the SHA 256 sum in all matching attributes.
"""
@ignore_warnings
def start(self):
"""
Start output plugin
"""
misp_url = CowrieConfig.get("output_misp", "base_url")
misp_key = CowrieConfig.get("output_misp", "api_key")
misp_verifycert = ("true" == CowrieConfig.get("output_misp",
"verify_cert").lower())
self.misp_api = PyMISP(url=misp_url,
key=misp_key,
ssl=misp_verifycert,
debug=False)
self.debug = CowrieConfig.getboolean("output_misp",
"debug",
fallback=False)
self.publish = CowrieConfig.getboolean("output_misp",
"publish_event",
fallback=False)
def stop(self):
"""
Stop output plugin
"""
pass
def write(self, entry):
"""
Push file download to MISP
"""
if entry["eventid"] == "cowrie.session.file_download":
file_sha_attrib = self.find_attribute("sha256", entry["shasum"])
if file_sha_attrib:
# file is known, add sighting!
if self.debug:
log.msg("File known, add sighting")
self.add_sighting(entry, file_sha_attrib)
else:
# file is unknown, new event with upload
if self.debug:
log.msg("File unknwon, add new event")
self.create_new_event(entry)
@ignore_warnings
def find_attribute(self, attribute_type, searchterm):
"""
Returns a matching attribute or None if nothing was found.
"""
result = self.misp_api.search(controller="attributes",
type_attribute=attribute_type,
value=searchterm)
if result["Attribute"]:
return result["Attribute"][0]
else:
return None
@ignore_warnings
def create_new_event(self, entry):
attribute = MISPAttribute()
attribute.type = "malware-sample"
attribute.value = entry["shasum"]
attribute.data = Path(entry["outfile"])
attribute.comment = "File uploaded to Cowrie ({})".format(
entry["sensor"])
attribute.expand = "binary"
event = MISPEvent()
event.info = "File uploaded to Cowrie ({})".format(entry["sensor"])
event.attributes = [attribute]
event.run_expansions()
if self.publish:
event.publish()
result = self.misp_api.add_event(event)
if self.debug:
log.msg("Event creation result: \n%s" % result)
@ignore_warnings
def add_sighting(self, entry, attribute):
sighting = MISPSighting()
sighting.source = "{} (Cowrie)".format(entry["sensor"])
self.misp_api.add_sighting(sighting, attribute)
class MispConnector(): # TODO: Switch to ExpandedPyMISP
misp = None
tagsGenerated = False
tags = None
def __init__(self, url, apiKey):
self.misp = ExpandedPyMISP(url, apiKey, MISP_VERIFYCERT, debug=False)
self.tags = self.generate_misp_tags()
def generate_misp_tags(self) -> List[MISPTag]:
tagList = []
tags = [
{
'name': 'AutoGenerated',
'colour': '#00ace6',
'exportable': True
},
{
'name': 'HoneytrapEvent',
'colour': '#581845',
'exportable': True
},
{
'name': 'ModSecurity',
'colour': '#a04000',
'exportable': True
},
]
try:
for item in tags:
tag = MISPTag()
for key in item:
tag[key] = item[key]
self.misp.add_tag(tag, pythonify=True)
tagList.append(tag)
log.info("MISP tags generated")
return tagList
except PyMISPError as e:
raise RuntimeError("Failed to initialise MISP tags")
def send_misp_event(self, json_log):
logEvent = ModsecLog(json_log)
event = self.generate_event(logEvent)
self.misp.add_event(event)
log.debug(event)
def generate_event(self, modsecLog: ModsecLog) -> MISPEvent:
event = MISPEvent()
event.info = modsecLog.generateInfoLine()
[http_method, url, http_version] = modsecLog.getRequestLine()
event.add_attribute(
type='ip-src|port',
value=str(modsecLog.log['transaction']['remote_address']) + "|" +
str(modsecLog.log['transaction']['remote_port']),
comment="Attacker",
pythonify=True)
event.add_attribute(
type='ip-dst|port',
value=str(modsecLog.log['transaction']['local_address']) + "|" +
str(modsecLog.log['transaction']['local_port']),
comment="Server",
pythonify=True)
event.add_attribute(type='http-method',
value=http_method,
pythonify=True)
event.add_attribute(type='url', value=url, pythonify=True)
event.add_attribute(type='datetime',
value=modsecLog.log['@timestamp'],
pythonify=True)
event.add_attribute(type='other',
value=modsecLog.log['audit_data']['producer'],
comment="Producer",
pythonify=True)
event.add_attribute(type='text',
value=json.dumps(modsecLog.log, indent=2),
comment="Json log",
pythonify=True)
#event.add_attribute(type='vulnerability', value=json_log['transaction']['time'], pythonify=True)
for tag in self.tags:
event.add_tag(tag)
log.debug("elasticsearch data")
log.debug(json.dumps(modsecLog.log, indent=2))
return event
class MISPApi:
def __init__(self, api_url: str, api_key: str, verify_cert: bool = True):
self.pymisp = ExpandedPyMISP(api_url, api_key, verify_cert)
def search(self, controller: str = 'attributes', **kwargs):
return self.pymisp.search(controller, **kwargs)
def searchall(self,
value: str,
controller: str = 'attributes') -> List[Any]:
result = self.pymisp.search(controller, value=value, searchall=True)
assert isinstance(result, dict) # Please mypy
return result.get('Attribute', [])
def search_sightings(self,
context_id: str,
context: str = 'attribute',
source: Optional[str] = None):
return self.pymisp.search_sightings(context=context,
context_id=context_id,
source=source)
def org_name_id_mapping(self):
pass
def attr_search(self, attr: Attr) -> List[Any]:
result = []
for typ in attr.search_types:
result += self.search(type=typ.value,
value=attr.value).get('Attribute', [])
return result
def domain_name_search(
self,
domain_name: str,
searchall: bool = False,
publish_timestamp: Optional[datetime] = None,
limit: Optional[int] = None,
) -> List[Any]:
result = self.search(type='domain',
value=domain_name,
searchall=searchall,
publish_timestamp=publish_timestamp,
limit=limit)
return result.get('Attribute', [])
def url_search(self, url: str, searchall: bool = False) -> List[Any]:
result = self.search(type='url', value=url, searchall=searchall)
return result.get('Attribute', [])
def sighting_lookup(self,
attribute_id: str,
source: Optional[str] = None) -> List[Any]:
result = self.search_sightings(context_id=attribute_id, source=source)
return [item['Sighting'] for item in result]
def _get_report_category(self, attr: Attr) -> EventCategory:
if any(True for t in attr.report_types
if t in [AttrType.DOMAIN, AttrType.URL, AttrType.IP_SRC]):
return EventCategory.NETWORK_ACTIVITY
elif any(True for t in attr.report_types
if t in [AttrType.MD5, AttrType.SHA1]):
return EventCategory.PAYLOAD_DELIVERY
else:
raise NotImplementedError(
f'EventCategory for {attr.report_types} not implemented')
def add_event(
self,
attr_items: List[Attr],
info: str,
tags: List,
comment: str,
to_ids: bool,
reference: Optional[str],
ts: Optional[int] = None,
published: Optional[bool] = False,
):
attrs = []
for item in attr_items:
category = self._get_report_category(item)
for report_type in item.report_types:
report_attr = MISPAttribute()
report_attr.from_dict(
type=report_type.value,
category=category.value,
to_ids=to_ids,
value=item.value,
comment=comment,
timestamp=ts,
)
attrs.append(report_attr)
if reference:
reference_attr = MISPAttribute()
reference_attr.from_dict(
type='text',
category=EventCategory.INTERNAL_REFERENCE.value,
value=reference,
disable_correlation=True,
comment=comment,
timestamp=ts,
)
attrs.append(reference_attr)
event = MISPEvent()
event.from_dict(info=info,
Attribute=attrs,
Tag=tags,
date=date.today(),
published=published,
threat_level_id=2)
logger.debug(event)
return self.pymisp.add_event(event)
def add_sighting(self, attr: Attr, sighting_type: str,
source: str) -> MISPSighting:
sighting = MISPSighting()
sighting['value'] = attr.value
sighting['type'] = sighting_type
sighting['source'] = source
res = self.pymisp.add_sighting(sighting, pythonify=True)
assert isinstance(res, MISPSighting) # please mypy
return res
def remove_sighting(
self,
attr: Attr,
sighting_type: str,
source: str,
date_from: Optional[datetime] = None,
date_to: Optional[datetime] = None,
):
sightings: List[MISPSighting] = []
for item in self.attr_search(attr):
res = self.pymisp.search_sightings(
context='attribute',
context_id=item['id'],
type_sighting=sighting_type,
source=source,
date_from=date_from,
date_to=date_to,
pythonify=True,
)
# Can't get mypy to understand that sighting contains a MISPSighting
sightings.extend([
d['sighting'] for d in res if d.get('sighting') is not None
]) # type: ignore
# Please mypy
for sighting in sightings:
self.pymisp.delete_sighting(sighting)
