class MISPCollectorBot(CollectorBot):
def init(self):
if PyMISP is None:
self.logger.error('Could not import pymisp. Please install it.')
self.stop()
# Initialise MISP connection
self.misp = PyMISP(self.parameters.misp_url,
self.parameters.misp_key,
self.parameters.misp_verify)
# URLs used for deleting and adding MISP event tags
self.misp_add_tag_url = urljoin(self.parameters.misp_url,
'events/addTag')
self.misp_del_tag_url = urljoin(self.parameters.misp_url,
'events/removeTag')
def process(self):
# Grab the events from MISP
misp_result = self.misp.search(
tags=self.parameters.misp_tag_to_process
)
# Process the response and events
if 'response' in misp_result:
# Extract the MISP event details
for e in misp_result['response']:
misp_event = e['Event']
# Send the results to the parser
report = self.new_report()
report.add('raw', json.dumps(misp_event, sort_keys=True))
report.add('feed.url', self.parameters.misp_url)
self.send_message(report)
# Finally, update the tags on the MISP events.
# Note PyMISP does not currently support this so we use
# the API URLs directly with the requests module.
for misp_event in misp_result['response']:
# Remove the 'to be processed' tag
self.misp.remove_tag(misp_event,
self.parameters.misp_tag_to_process)
# Add a 'processed' tag to the event
self.misp.add_tag(misp_event,
self.parameters.misp_tag_processed)
class MISPCollectorBot(CollectorBot):
def init(self):
if PyMISP is None:
self.logger.error('Could not import pymisp. Please install it.')
self.stop()
# Initialise MISP connection
self.misp = PyMISP(self.parameters.misp_url, self.parameters.misp_key,
self.parameters.misp_verify)
# URLs used for deleting and adding MISP event tags
self.misp_add_tag_url = urljoin(self.parameters.misp_url,
'events/addTag')
self.misp_del_tag_url = urljoin(self.parameters.misp_url,
'events/removeTag')
def process(self):
# Grab the events from MISP
misp_result = self.misp.search(
tags=self.parameters.misp_tag_to_process)
# Process the response and events
if 'response' in misp_result:
# Extract the MISP event details
for e in misp_result['response']:
misp_event = e['Event']
# Send the results to the parser
report = self.new_report()
report.add('raw', json.dumps(misp_event, sort_keys=True))
report.add('feed.url', self.parameters.misp_url)
self.send_message(report)
# Finally, update the tags on the MISP events.
# Note PyMISP does not currently support this so we use
# the API URLs directly with the requests module.
for misp_event in misp_result['response']:
# Remove the 'to be processed' tag
self.misp.remove_tag(misp_event,
self.parameters.misp_tag_to_process)
# Add a 'processed' tag to the event
self.misp.add_tag(misp_event,
self.parameters.misp_tag_processed)
class generateEvents():
# generates a seperate event for every paste with more than 1 parsed IOC, after initialising a connection with the MISP instance.
def __init__(self, paste):
self.paste = paste
self.url = MISP_URL
self.key = MISP_KEY
def initMISP(self):
self.misp = PyMISP(self.url, self.key, False, 'json', debug=True)
def addEvents(self):
for i in range(len(self.paste)):
if len(self.paste[i].iocs) != 0:
logging.debug(
"Paste: {}, # of IOCs: {}. Creating an event.".format(
self.paste[i].title, len(self.paste[i].iocs)))
event = self.misp.new_event(distribution=2,
analysis=2,
info=self.paste[i].title)
self.misp.add_internal_link(event,
self.paste[i].URI,
category="External analysis")
self.misp.add_tag(event, "Type:OSINT")
self.misp.add_tag(event, 'osint:source-type="pastie-website"')
self.misp.add_tag(event, 'OSINT')
self.misp.add_tag(event, 'tlp:white')
for j in range(len(self.paste[i].iocs)):
if self.paste[i].iocs[j].kind == "IP":
self.misp.add_ipsrc(event, self.paste[i].iocs[j].value)
elif self.paste[i].iocs[j].kind == "uri":
self.misp.add_url(event, self.paste[i].iocs[j].value)
elif self.paste[i].iocs[j].kind == "md5":
self.misp.add_hashes(event,
md5=self.paste[i].iocs[j].value)
elif self.paste[i].iocs[j].kind == "sha1":
self.misp.add_hashes(event,
sha1=self.paste[i].iocs[j].value)
elif self.paste[i].iocs[j].kind == "sha256":
self.misp.add_hashes(
event, sha256=self.paste[i].iocs[j].value)
elif self.paste[i].iocs[j].kind == "CVE":
#self.misp.add_object(event, 63, self.paste[i].iocs[j].value)
pass
if self.paste[i].iocs[j].kind == "email":
self.misp.add_email_src(event,
self.paste[i].iocs[j].value)
if self.paste[i].iocs[j].kind == "filename":
self.misp.add_filename(event,
self.paste[i].iocs[j].value)
if PUBLISH_EVENTS:
self.misp.publish(event, alert=EMAIL_ALERTS)
class MISPHandler:
def __init__(self, config: dict):
self.url = config['misp_url']
self.key = config['misp_auth_key']
self.misp = PyMISP(self.url, self.key)
self.tag_list = self.create_tag_list()
self.logger = logging.getLogger('misp_handler')
self.logger.debug("URLhausHandler init done")
def create_tag_list(self) -> list:
tags = []
for item in self.misp.tags(pythonify=True):
tags.append(item.name)
return tags
def make_sure_tag_exists(self, tag: str) -> bool:
if tag in self.tag_list:
return True
else:
self.misp.add_tag({"name": tag}, pythonify=True)
self.tag_list = self.create_tag_list()
if tag in self.tag_list:
return True
else:
return False
def add_tag_to_attribute(self, attr: MISPAttribute, tag: str) -> MISPAttribute:
if self.make_sure_tag_exists(tag):
attr.add_tag(tag)
return attr
def create_attr(self, raw_attr: dict) -> MISPAttribute:
# Create attribute and assign simple values
attr = MISPAttribute()
attr.type = 'url'
attr.value = raw_attr['url']
attr.disable_correlation = False
attr.__setattr__('first_seen', datetime.strptime(raw_attr['dateadded'], '%Y-%m-%d %H:%M:%S'))
# Add URLhaus tag
self.add_tag_to_attribute(attr, 'URLhaus')
# Add other tags
if raw_attr['tags']:
for tag in raw_attr['tags'].split(','):
self.add_tag_to_attribute(attr, tag.strip())
# Add online/offline tag
if not pandas.isna(raw_attr['url_status']):
if raw_attr['url_status'] == 'online':
attr.to_ids = True
else:
attr.to_ids = False
self.add_tag_to_attribute(attr, raw_attr['url_status'])
# Add reporter tag
if not pandas.isna(raw_attr['reporter']):
self.add_tag_to_attribute(attr, raw_attr['reporter'])
attr.comment = raw_attr['urlhaus_link']
return attr
def create_attr_feodo(self, raw_attr: dict) -> MISPAttribute:
attr = MISPAttribute()
attr.type = 'ip-dst|port'
attr.value = f"{raw_attr['DstIP']}|{raw_attr['DstPort']}"
self.add_tag_to_attribute(attr, 'FeodoTracker')
self.add_tag_to_attribute(attr, raw_attr['Malware'])
attr.comment = 'Feodo tracker DST IP/port'
attr.__setattr__('first_seen', datetime.strptime(raw_attr['Firstseen'], '%Y-%m-%d %H:%M:%S'))
if not pandas.isna(raw_attr['LastOnline']):
last_seen_time = datetime.strptime(str(raw_attr['LastOnline']), '%Y-%m-%d').replace(tzinfo=pytz.utc)
first_seen_time = datetime.strptime(str(raw_attr["Firstseen"]), '%Y-%m-%d %H:%M:%S').replace(tzinfo=pytz.utc)
if first_seen_time > last_seen_time:
last_seen_time = first_seen_time + timedelta(seconds=1)
attr.__setattr__('last_seen', last_seen_time)
else:
last_seen_time = datetime.strptime(str(raw_attr['Firstseen']), '%Y-%m-%d %H:%M:%S').replace(tzinfo=pytz.utc)
attr.__setattr__('last_seen', last_seen_time)
attr.to_ids = False
attr.disable_correlation = False
return attr
def create_attr_azorult(self, raw_attr: dict) -> MISPAttribute:
attr_list = []
for type in [{'json': 'domain', 'misp': 'domain'},
{'json': 'ip', 'misp': 'ip-dst'},
{'json': 'panel_index', 'misp': 'url'}]:
if type['json'] in raw_attr:
attr = MISPAttribute()
self.add_tag_to_attribute(attr, 'AzorultTracker')
self.add_tag_to_attribute(attr, raw_attr['panel_version'])
self.add_tag_to_attribute(attr, raw_attr['feeder'])
self.add_tag_to_attribute(attr, raw_attr['status'])
attr.comment = f'Azorult panel {type["misp"]}'
attr.__setattr__('first_seen', datetime.fromtimestamp(raw_attr['first_seen']))
attr.to_ids = False
attr.disable_correlation = False
attr.type = type['misp']
attr.value = f"{raw_attr[type['json']]}"
attr_list.append(attr)
return attr_list
@staticmethod
def get_attribute_tag_list(self, tag_list: list) -> list:
tags = []
for item in tag_list:
tags.append(item['name'])
return tags
@staticmethod
def create_event(title: str, date_added: datetime) -> MISPEvent:
misp_event = MISPEvent()
misp_event.info = title
if date_added != '':
misp_event.date = date_added
return misp_event
def get_event(self, event_id):
return self.misp.get_event(event_id, pythonify=True)
def add_attr_to_event(self, event: MISPEvent, attribute: MISPAttribute):
event.attributes.append(attribute)
return event
def update_event(self, event: MISPEvent):
return self.misp.update_event(event, pythonify=True)
def get_day_event(self, day: str, source: str, date: str):
if source in ['URLHaus', 'FeodoTracker']:
misp_event = self.misp.search('events', 'json', org=1, eventinfo=f'{source} import day {day}',
pythonify=True)
elif source in ['AzorultTracker']:
misp_event = self.misp.search('events', 'json', org=1, eventinfo=f'{source} import panel {day}',
pythonify=True)
if len(misp_event) >= 1:
return misp_event[0]
else:
if source in ['URLHaus', 'FeodoTracker']:
misp_event = self.create_event(f"{source} import day {day}",
date_added=datetime.timestamp(
datetime.strptime(date, '%Y-%m-%d %H:%M:%S')))
elif source in ['AzorultTracker']:
misp_event = self.create_event(f"{source} import panel {day}",
date_added=datetime.fromtimestamp(date))
event_id = self.misp.add_event(misp_event)
self.misp.publish(event_id)
return self.get_event(event_id)
@staticmethod
def delete_attribute_by_value(search_value: str, event: MISPEvent):
found = False
for a in event.attributes:
if (hasattr(a, 'value') and a.value == search_value):
a.deleted = True
return event
class MISPInstance():
def __init__(self, misp_instance_dir: Path, secure_connection: bool):
with (misp_instance_dir / 'config.json').open() as f:
self.instance_config = json.load(f)
print('Initialize', self.instance_config['admin_orgname'])
self.secure_connection = secure_connection
self.synchronisations = {}
self.name = self.instance_config['admin_orgname']
# NOTE: never use that user again after initial config.
initial_user_connector = PyMISP(self.instance_config['baseurl'],
self.instance_config['admin_key'],
ssl=self.secure_connection,
debug=False)
# Set the default role (id 3 is normal user)
initial_user_connector.set_default_role(3)
initial_user_connector.toggle_global_pythonify()
self.baseurl = self.instance_config['baseurl']
self.external_baseurl = self.instance_config['external_baseurl']
# Create organisation
organisation = MISPOrganisation()
organisation.name = self.instance_config['admin_orgname']
self.host_org = initial_user_connector.add_organisation(organisation)
if not isinstance(self.host_org, MISPOrganisation):
# The organisation is probably already there
organisations = initial_user_connector.organisations()
for organisation in organisations:
if organisation.name == self.instance_config['admin_orgname']:
self.host_org = organisation
break
else:
raise Exception('Unable to find admin organisation')
# Create Site admin in new org
user = MISPUser()
user.email = self.instance_config['email_site_admin']
user.org_id = self.host_org.id
user.role_id = 1 # Site admin
self.host_site_admin = initial_user_connector.add_user(user)
if not isinstance(self.host_site_admin, MISPUser):
users = initial_user_connector.users()
for user in users:
if user.email == self.instance_config['email_site_admin']:
self.host_site_admin = user
break
else:
raise Exception('Unable to find admin user')
self.site_admin_connector = PyMISP(self.baseurl,
self.host_site_admin.authkey,
ssl=self.secure_connection,
debug=False)
self.site_admin_connector.toggle_global_pythonify()
# Setup external_baseurl
self.site_admin_connector.set_server_setting('MISP.external_baseurl',
self.external_baseurl,
force=True)
# Setup baseurl
self.site_admin_connector.set_server_setting('MISP.baseurl',
self.baseurl,
force=True)
# Setup host org
self.site_admin_connector.set_server_setting('MISP.host_org_id',
self.host_org.id)
# create other useful users
self.orgadmin = self.create_user(
self.instance_config['email_orgadmin'], 2)
self.user = self.create_user(self.instance_config['email_user'], 3)
# And connectors
self.org_admin_connector = PyMISP(self.baseurl,
self.orgadmin.authkey,
ssl=self.secure_connection,
debug=False)
self.org_admin_connector.toggle_global_pythonify()
self.user_connector = PyMISP(self.baseurl,
self.user.authkey,
ssl=self.secure_connection,
debug=False)
self.user_connector.toggle_global_pythonify()
def __repr__(self):
return f'<{self.__class__.__name__}(external={self.baseurl})>'
def create_user(self, email, role_id):
user = MISPUser()
user.email = email
user.org_id = self.host_org.id
user.role_id = role_id
new_user = self.site_admin_connector.add_user(user)
if not isinstance(new_user, MISPUser):
users = self.site_admin_connector.users()
for user in users:
if user.email == email:
new_user = user
break
else:
raise Exception('Unable to find admin user')
return new_user
def create_sync_user(self, organisation: MISPOrganisation) -> MISPServer:
sync_org = self.site_admin_connector.add_organisation(organisation)
if not isinstance(sync_org, MISPOrganisation):
# The organisation is probably already there
organisations = self.site_admin_connector.organisations(
scope='all')
for org in organisations:
if org.name == organisation.name:
if not org.local:
org.local = True
org = self.site_admin_connector.update_organisation(
org)
sync_org = org
break
else:
raise Exception('Unable to find sync organisation')
short_org_name = sync_org.name.lower().replace(' ', '-')
email = f"sync_user@{short_org_name}.local"
user = MISPUser()
user.email = email
user.org_id = sync_org.id
user.role_id = 5 # Sync user
sync_user = self.site_admin_connector.add_user(user)
if not isinstance(sync_user, MISPUser):
users = self.site_admin_connector.users()
for user in users:
if user.email == email:
sync_user = user
break
else:
raise Exception('Unable to find sync user')
sync_user_connector = PyMISP(self.site_admin_connector.root_url,
sync_user.authkey,
ssl=self.secure_connection,
debug=False)
return sync_user_connector.get_sync_config(pythonify=True)
def configure_sync(self, server_sync_config: MISPServer):
# Add sharing server
for s in self.site_admin_connector.servers():
if s.name == server_sync_config.name:
server = s
break
else:
server = self.site_admin_connector.import_server(
server_sync_config)
server.pull = True
server.push = False
server = self.site_admin_connector.update_server(server)
r = self.site_admin_connector.test_server(server)
if r['status'] != 1:
raise Exception(f'Sync test failed: {r}')
print(server)
print(server.to_json(indent=2))
# NOTE: this is dirty.
self.synchronisations[server_sync_config.name.replace(
'Sync with ', '')] = server
def add_tag_filter_sync(self, server_sync: MISPServer, name: str):
# Add tag to limit push
tag = MISPTag()
tag.name = name
tag.exportable = False
tag.org_id = self.host_org.id
tag = self.site_admin_connector.add_tag(tag)
if not isinstance(tag, MISPTag):
for t in self.site_admin_connector.tags():
if t.name == name:
tag = t
break
else:
raise Exception('Unable to find tag')
# Set limit on sync config
filter_tag_push = {
"tags": {
'OR': [tag.id],
'NOT': []
},
'orgs': {
'OR': [],
'NOT': []
}
}
# filter_tag_pull = {"tags": {'OR': [], 'NOT': []}, 'orgs': {'OR': [], 'NOT': []}}
server_sync.push_rules = json.dumps(filter_tag_push)
# server.pull_rules = json.dumps(filter_tag_pull)
server_sync = self.site_admin_connector.update_server(server_sync)
def add_sharing_group(self,
name: str,
releasibility: str = 'Whatever it is a test',
servers: List[MISPServer] = [],
organisations: List[MISPOrganisation] = []):
# Add sharing group
for sg in self.site_admin_connector.sharing_groups():
if sg.name == name:
self.sharing_group = sg
break
else:
sharing_group = MISPSharingGroup()
sharing_group.name = name
sharing_group.releasability = releasibility
self.sharing_group = self.site_admin_connector.add_sharing_group(
sharing_group)
for server in servers:
self.site_admin_connector.add_server_to_sharing_group(
self.sharing_group, server)
for organisation in organisations:
self.site_admin_connector.add_org_to_sharing_group(
self.sharing_group, organisation)
