def run(self):
if (not __sessions__.is_attached_misp()):
self.log("error", 'MISP session not attached')
return
cfg = Config()
key = cfg.misp.misp_key
url = cfg.misp.misp_url
pymisp = PyMISP(url,
key,
ssl=False,
proxies=None,
cert=('/opt/ssl/server/csp-internal.crt',
'/opt/ssl/server/csp-internal.key'))
xorSearch = XorSearch()
xorSearch.run()
event = pymisp.get_event(__sessions__.current.misp_event.event.id)
commentVal = ""
for out in xorSearch.output:
commentVal += out['data']
# if out['type'] == 'error':
# self.log("error", out['data'])
pymisp.add_named_attribute(
__sessions__.current.misp_event.event.id, "comment",
"File: " + __sessions__.current.file.path +
" -- XOR search out: " + commentVal)
def run(self):
if (not __sessions__.is_attached_misp()):
self.log("error", "MISP session not attached")
return
cfg = Config()
key = cfg.misp.misp_key
url = cfg.misp.misp_url
pymisp = PyMISP(url,
key,
ssl=False,
proxies=None,
cert=('/opt/ssl/server/csp-internal.crt',
'/opt/ssl/server/csp-internal.key'))
shellcode = Shellcode()
shellcode.run()
commentVal = ""
for out in shellcode.output:
commentVal += out['data']
self.log(
"info", "Updating MISP event " +
str(__sessions__.current.misp_event.event.id) + "...")
pymisp.add_named_attribute(__sessions__.current.misp_event.event.id,
"comment", "Shellcode out: " + commentVal)
class MISPIntegrator(object):
def __init__(self, logger, db):
"""
Declares some object attributes.
:param logger: A logger object.
:param db: A MongoDBConnection object.
"""
self.parser = get_main_config_parser()
self.misp_parser = get_misp_config_parser()
self.misp = PyMISP(self.parser.get('misp', 'url'),
self.parser.get('misp', 'key'),
ssl=bool(self.parser.get('misp', 'ssl')),
out_type=self.parser.get('misp', 'out_type'),
debug=bool(self.parser.get('misp', 'debug')))
self.distribution = self.parser.get('misp', 'distribution')
self.local_git = self.parser.get('git', 'local_git')
self.remote_git = self.parser.get('git', 'remote_git').rsplit('.',
1)[0]
self.attributes = dict(self.misp_parser.values()[1].items())
self.logger = logger
self.db = db
tag_name, tag_colour = self.parser.get('misp', 'tag').split(',')
all_tags = self._set_tag_id(tag_name, tag_colour)
taxonomies = self.parser.get('misp', 'taxonomies').split(',')
self._set_taxonomy_tag_ids(taxonomies, all_tags)
def _set_tag_id(self, tag_name, tag_colour):
"""
Determines the ID to a given tag. If the tag is not present in MISP
it will be created otherwise.
:param tag_name (str): Name of the tag to create if necessary.
:param tag_colour (str): A HTML colour code for the tag to create.
:return: List of all tags in MISP.
"""
all_tags = self.misp.get_tags_list()
exploit_tag = [tag for tag in all_tags if tag['name'] == tag_name]
if exploit_tag:
self.tag_ids = [exploit_tag[0]['id']]
else:
new_tag = self.misp.new_tag(name=tag_name, colour=tag_colour)
self.tag_ids = [new_tag['Tag']['id']]
return all_tags
def _set_taxonomy_tag_ids(self, taxonomies, all_tags):
"""
If the given taxonomies are not enabled yet, it will be enabled as well
as the specific tag. The IDs of the taxonomy tags are added to a list.
If a tag or taxonomy is not found, it will be logged.
:param taxonomies (list): A list of taxonomies as strings.
:param all_tags (list): List of all tags in MISP as strings.
"""
# TODO: it seems that get_taxonomies_list() does not fetch
# all existing taxonomies:
# https://github.com/MISP/PyMISP/issues/342
all_taxonomies = self.misp.get_taxonomies_list()['response']
for taxonomy in taxonomies:
namespace, _ = taxonomy.split(':')
tax = [
tax for tax in all_taxonomies
if namespace == tax['Taxonomy']['namespace']
]
if tax:
tax = tax[0]['Taxonomy']
if not tax['enabled']:
# TODO: it seems that it is not possible to enable a
# specific tag of a taxonomy, if it is not yet enabled:
# https://github.com/MISP/PyMISP/issues/343
# until this is not fixed, all tags of the taxonomy must
# be enabled. if it is fixed, remove the line with
# enable_taxonomy_tags() and uncomment the line with
# enable_tag() below
self.misp.enable_taxonomy(tax['id'])
self.misp.enable_taxonomy_tags(tax['id'])
tag = [tag for tag in all_tags if tag['name'] == taxonomy]
if tag:
tag_id = tag[0]['id']
# self.misp.enable_tag(tag_id)
self.tag_ids.append(tag_id)
else:
self.logger.warn(
'Could not find tag: {}.'.format(taxonomy))
else:
self.logger.warn(
'Could not find taxonomy: {}.'.format(taxonomy))
def _create_event(self, date, description):
"""
Creates a new event and tag it with the tag id of the config file.
"Threat Level" has not to be defined (:= no risk), because it is only
a POC analysis.
:param date (str): The date when the exploit was created.
:param description (str): Description of the new event.
:return: UUID of created event as string.
"""
uuid = self.misp.new_event(
date=date, info=description,
distribution=self.distribution)['Event']['uuid']
for tag_id in self.tag_ids:
response = self.misp.tag(uuid, tag_id)
if 'successfully attached to' not in response['message']:
self.logger.warn('Could not tag event {}: {}'.format(
uuid, response))
return uuid
def _add_attributes_to_event(self,
uuid,
misp_type,
values,
category,
new_event=True):
"""
Adds attributes to an event if it is new, otherwise it searches for
an attribute which has a given value. If not so, a new attribute will
be created for that event.
:param uuid (str): The uuid of the event.
:param misp_type (str): Type of an attribute.
:param values (list): List of values of the attribute as string.
:param category (str): Category of an attribute.
:param new_event (boolean): If the event was added before; default is
True.
"""
for val in values:
if new_event:
self.misp.add_named_attribute(uuid,
misp_type,
val,
category=category)
else:
# if we search for all values in the list, we do not know
# which attribute we need to add.
response = self.misp.search(
uuid=uuid,
type_attribute=misp_type,
category=category,
values=[val],
controller='attributes')['response']
if not response['Attribute']:
self.misp.add_named_attribute(uuid,
misp_type,
val,
category=category)
def _add_exploit_poc_template(self, uuid, exploit):
"""
Adds a new exploit-poc object template to a given event.
:param uuid (str): The uuid of the event.
:param exploit (dict): Dictionary of an exploit. It needs the following
keys: 'file', 'author' and 'description'.
"""
gen_obj = GenericObjectGenerator('exploit-poc')
attributes = []
exploit_file = exploit['file']
with open(os.path.join(self.local_git, exploit_file)) as f:
poc = f.read()
attributes.append({
'poc':
poc,
'references':
'{}/tree/master/{}'.format(self.remote_git, exploit_file),
'author':
exploit['author'],
'description':
exploit['description']
})
gen_obj.generate_attributes(attributes)
self.misp.add_object(uuid, gen_obj.template_uuid, gen_obj)
def integrate(self, exploits):
"""
It integrates the exploits as new events to MISP. If an unexcepted
exception occurs it will be logged.
:param exploits (list): List of exploit dictionaries.
"""
for exploit in exploits:
try:
# if this exploit alreadey exist, we do not need to add it
# again
if 'misp-uuid' not in exploit:
# encode uuid, otherwise the event is not found
uuid = self._create_event(
exploit['date'],
exploit['description']).encode('utf-8')
self.db.insert_or_update_exploits_by_id([{
'id':
exploit['id'],
'misp-uuid':
uuid
}])
event = None
else:
uuid = exploit['misp-uuid']
event = self.misp.get_event(uuid)
for tag, value in exploit.items():
category, misp_type = self.attributes.get(tag,
',').split(',')
if not value or tag not in exploit \
or (not category and not misp_type):
continue
# put value to a list, so that we do not need to differ if
# it is a list or not.
if not isinstance(value, list):
value = [value]
if event is None:
self._add_attributes_to_event(uuid, misp_type, value,
category)
else:
self._add_attributes_to_event(uuid,
misp_type,
value,
category,
new_event=False)
# only add poc if event is new
if event is None:
self._add_exploit_poc_template(uuid, exploit)
except Exception:
self.logger.warn(
'Unexpected exception while MISP integration occurred '
'for file: {}'.format(exploit['file']),
exc_info=True)
def main():
argparser = MISPImportArgumentParser()
argparser.add_argument("--url",
"-u",
default="https://localhost",
help="URL of MISP instance")
argparser.add_argument("--key", "-k", required=True, help="API key")
argparser.add_argument("--insecure",
"-I",
action="store_false",
help="Disable TLS certifcate validation.")
argparser.add_argument(
"--event",
"-e",
type=int,
help=
"Add Sigma rule to event with this ID. If not set, create new event.")
argparser.add_argument(
"--same-event",
"-s",
action="store_true",
help="Import all Sigma rules to the same event, if no event is set.")
argparser.add_argument(
"--info",
"-i",
default="Sigma import",
help="Event Information field for newly created MISP event.")
argparser.add_argument("--recursive",
"-r",
action="store_true",
help="Recursive traversal of directory")
argparser.add_argument("sigma",
nargs="+",
help="Sigma rule file that should be imported")
args = argparser.parse_args()
if args.recursive:
paths = [
p for pathname in args.sigma
for p in pathlib.Path(pathname).glob("**/*") if p.is_file()
]
else:
paths = [pathlib.Path(sigma) for sigma in args.sigma]
misp = PyMISP(args.url, args.key, args.insecure)
if args.event:
if hasattr(misp, "get"):
eventid = misp.get(args.event)["Event"]["id"]
else:
eventid = misp.get_event(args.event)["Event"]["id"]
first = True
for sigma in paths:
if not args.event and (first or not args.same_event):
eventid = create_new_event(args, misp)
print("Importing Sigma rule {} into MISP event {}...".format(sigma,
eventid,
end=""))
f = sigma.open("rt")
if hasattr(misp, "add_named_attribute"):
misp.add_named_attribute(eventid, "sigma", f.read())
else:
event = misp.get_event(eventid, pythonify=True)
event.add_attribute("sigma", f.read())
misp.update_event(event)
f.close()
first = False
class MispImport:
def __init__(self, logger):
self.api = PyMISP(misp_url,
misp_key,
misp_verifycert,
'json',
debug=False)
self.response = None
self.logger = logger
self.caching = Caching()
def import_data(self, data_to_push):
all_events = []
for k, data in data_to_push.items():
if not self.is_already_present(data['url_tweet']):
if data['retweet']:
self.logger.info('RT %s %s ' %
(data['retweet_id'], data['url_tweet']))
eid = self.caching.translate(data['retweet_id'])
self.logger.info('translate found %s %s' %
(eid, data['retweet_id']))
if eid:
event = self.api.get(int(eid))
else:
res = self.api.search(values=data['retweet_id'],
type_attribute='twitter-id')
if res['response']:
event = res['response'][0]
else:
self.logger.error('Event not found tweet %s ' %
data['retweet_id'])
continue
if 'Event' in event:
self.logger.info('Event has found %s' %
event['Event']['id'])
self.caching.caching(data['retweet_id'],
event['Event']['id'])
self.api.add_named_attribute(
event=event,
type_value='url',
category='External analysis',
value=data['url_tweet'])
self.api.add_named_attribute(event=event,
type_value="twitter-id",
category="Social network",
value=k)
all_tags = [t['name'] for t in event['Event']['Tag']]
if not 'toqualify' in all_tags and not 'toenrich' in all_tags:
self.api.tag(event['Event']['uuid'], 'topublish')
else:
self.logger.error(
'Event not found tweet %s error decode %s' %
(data['retweet_id'], event))
continue
elif data['quoted_tweet']:
self.logger.info('Tweet quoted %s' % data['quoted_tweet'])
eid = self.caching.translate(data['quoted_status_id'])
if eid:
event = self.api.get(int(eid))
else:
res = self.api.search(values=data['quoted_status_id'],
type_attribute='twitter-id')
if res['response']:
event = res['response'][0]
if 'Event' in event:
self.caching.caching(data['quoted_status_id'],
event['Event']['id'])
else:
self.logger.error('Event not found tweet %s ' %
data['quoted_status_id'])
continue
else:
self.logger.error('Quoted Tweet not found %s' %
data['url_tweet'])
continue
else:
event = self.api.new_event(distribution=0,
info=data['url_tweet'],
analysis=0,
threat_level_id=1)
self.caching.caching(k, event['Event']['id'])
self.logger.info('Event create %s' % event['Event']['id'])
if event:
self.add_tags(event, data['tags'])
self.api.add_named_attribute(event=event,
type_value='url',
category='External analysis',
value=data['url_tweet'])
self.logger.info('add url tweet %s at %s' %
(data['url_tweet'], event['Event']['id']))
self.api.add_named_attribute(event=event,
type_value='text',
category='External analysis',
value=data['tweet_text'])
self.api.add_named_attribute(event=event,
type_value="twitter-id",
category="Social network",
value=k)
for url in data['urls']:
self.api.add_named_attribute(
event=event,
type_value='url',
category="External analysis",
value=url)
self.logger.info('add externals url %s to %s' %
(url, event['Event']['id']))
self.api.freetext(event_id=event['Event']['id'],
string=data['tweet_text'],
adhereToWarninglists=True)
self.logger.debug(
'add text %s to %s' %
(data['tweet_text'], event['Event']['id']))
for d in data['data']:
if 'magic' in d.state_machine and d.state_machine[
'magic']['pe']:
hash_algo = sha256()
hash_algo.update(d.content_decoded)
self.api.add_named_attribute(
event=event,
type_value='sha256',
category='Payload delivery',
value=hash_algo.hexdigest())
self.add_object(event, d.content_decoded,
hash_algo.hexdigest())
self.add_tags(event, ['Malware'])
self.logger.info('add malware')
elif 'magic' in d.state_machine and d.state_machine[
'magic']['elf']:
self.api.add_object(event['Event']['id'], 13,
d.content_decoded)
else:
try:
self.api.freetext(
event_id=event['Event']['id'],
string=d.content_decoded.decode(),
adhereToWarninglists=True)
self.api.add_named_attribute(
event=event,
type_value='text',
category='External analysis',
value=d.content_decoded.decode())
except UnicodeDecodeError:
self.logger.Error('Error decoding')
pass
self.__remove_shortcut(event)
all_events.append(event['Event']['id'])
return all_events
def is_already_present(self, url_tweet):
try:
response = self.api.search(values=[url_tweet],
type_value='url',
category='External analysis')
self.response = response
return bool(response['response'])
except:
self.logger.error('Error search %s ' % url_tweet)
return True
def add_object(self, event, data, filename):
obj = make_binary_objects(pseudofile=BytesIO(data), filename=filename)
if obj[1]:
self.api.add_object(event['Event']['id'], 28, obj[1])
def add_tags(self, event, tags):
#self.api.add_tag(event,'OSINT')
for t in tags:
self.api.tag(event['Event']['uuid'], t)
self.api.tag(event['Event']['uuid'], 'OSINT')
self.api.tag(event['Event']['uuid'], 'tlp:white')
self.api.tag(event['Event']['uuid'], 'toenrich')
def __remove_shortcut(self, event):
self.logger.debug('delete attr function')
event = self.api.get_event(event['Event']['id'])
attrs = [
attr for attr in event['Event']['Attribute']
if attr['category'] == 'Network activity'
and 'https://t.co' in attr['value']
]
for attr in attrs:
self.logger.debug('try to connect to %s' % attr['value'])
r = requests.get(attr['value'].replace("'", ""))
if r.status_code == 200:
self.api.add_named_attribute(event=event,
type_value='url',
category='External analysis',
value=r.url)
self.api.delete_attribute(attr['id'])
self.logger.info('delete attr %s' % attr['id'])
def _misp_create_attribute_function(self, event, *args, **kwargs):
"""Function: """
try:
def get_config_option(option_name, optional=False):
"""Given option_name, checks if it is in app.config. Raises ValueError if a mandatory option is missing"""
option = self.options.get(option_name)
if option is None and optional is False:
err = "'{0}' is mandatory and is not set in ~/.resilient/app.config file. You must set this value to run this function".format(
option_name)
raise ValueError(err)
else:
return option
API_KEY = get_config_option("misp_key")
URL = get_config_option("misp_url")
VERIFY_CERT = True if get_config_option(
"verify_cert").lower() == "true" else False
# Get the function parameters:
misp_event_id = kwargs.get("misp_event_id") # number
misp_attribute_value = kwargs.get("misp_attribute_value") # text
misp_attribute_type = kwargs.get("misp_attribute_type") # text
log = logging.getLogger(__name__)
log.info("misp_event_id: %s", misp_event_id)
log.info("misp_attribute_value: %s", misp_attribute_value)
log.info("misp_attribute_type: %s", misp_attribute_type)
yield StatusMessage("Setting up connection to MISP")
misp_client = PyMISP(URL, API_KEY, VERIFY_CERT, 'json')
"""
default_map = {
"DNS Name": "domain",
"Email Attachment": "email-attachment",
"Email Body": "email-body",
"Email Recipient": "email-dst",
"Email Sender": "email-src",
"Email subject": "email-subject",
"File Name": "filename",
"DNS Name": "hostname",
"MAC Address": "mac-address",
"Malware MD5 Hash": "md5",
"Port": "port",
"Malware SHA-1 Hash": "sha1",
"Malware SHA-256 Hash": "sha256",
"URI Path": "uri",
"URL": "url",
"Threat CVE ID": "vulnerability",
"IP Address": "ip-dst"
}
"""
yield StatusMessage("Creating new misp attribute {} {}".format(
misp_attribute_type, misp_attribute_value))
attribute = misp_client.add_named_attribute(
misp_event_id, misp_attribute_type, misp_attribute_value)
log.info(event)
results = {"success": True, "content": attribute}
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception:
yield FunctionError()
def send_to_misp(misp_data, misp_configs, user):
debug_log=''
misp_key = misp_configs['MISP API Key']
misp_url = misp_configs['MISP URL']
ssl = False
proxies = ''
distribution = misp_data['misp_distro']
analysis = misp_data['misp_analysis']
threat_level = misp_data['misp_threat']
publish = misp_data['misp_pub']
tags = misp_data['misp_tags']
attributes = misp_data['attribs']
dt = datetime.utcnow()
event_date = dt.strftime('%Y-%m-%d')
'''
TODO:
+ Add other options from configs
(misp_configs['proxies'], misp_configs['ssl'], etc)
+ Get Event Date from CRITs instance, rather than today
'''
from pprint import pformat
# Load the PyMISP functions
misp = PyMISP(misp_url, misp_key, ssl, 'json', proxies=proxies)
# Build the event and tags if applicable
misp_title = misp_data['misp_info']
if misp_title=="None":
# Modify this to build a more-sane Event Info if none was given
for k,v in misp_data['attribs']:
misp_title=k
break
if misp_data['options']['misp_dedup_events']==True:
#Search for the event
event = ''
result = misp.search_index(eventinfo=misp_title)
#debug_log+=pformat(result)
if 'message' in result:
if result['message']=='No matches.':
event = misp.new_event(distribution, threat_level, analysis,
misp_title, date=event_date, published=publish)
else:
for evt in result['response']:
# If the event exists, set 'event' to the event
if evt['info']==misp_title:
event = {}
event['Event'] = evt
break
if event=='':
# Event not found, even though search results were returned
# Build new event
event = misp.new_event(distribution, threat_level, analysis,
misp_title, date=event_date, published=publish)
else:
event = misp.new_event(distribution, threat_level, analysis,
misp_title, date=event_date, published=publish)
misp_data['event']=event['Event']['id']
if tags!=[]:
for tag in tags:
misp.tag(event['Event']['uuid'], str(tag.strip()))
for k, v in attributes.iteritems():
if v['misp-submit']==True:
ind_kwargs = {}
attr = misp.add_named_attribute(event, v['misp-type'], v['ioc'],
category=v['misp-cat'], to_ids=v['misp-toids'],
**ind_kwargs)
#misp_data['debug']=attr
if 'response' in attr:
attrib_uuid = attr['response']['Attribute']['uuid']
elif 'message' in attr:
kwargs = {'uuid': str(event['Event']['uuid'])}
result = misp.search(controller='events', **kwargs)
for evt in result['response']:
if evt['Event']['info']==event['Event']['info']:
event=evt
break
single_attribute = (item for item in event['Event']['Attribute'] if item['value']==v['ioc']
and item['category']==v['misp-cat'] and item['type']==v['misp-type']).next()
attrib_uuid = single_attribute['uuid']
else:
v['tag']=''
#misp_data['debug']=attr
if v['tag']!='':
for t in v['tag']:
t=t.strip()
misp.tag(attrib_uuid, t)
return{
'misp_data': misp_data,
#'misp_configs': misp_configs,
#'user': user,
#'debug': debug_log,
}