def main():
parser = argparse.ArgumentParser(
description="Test of adding event to MISP")
parser.add_argument("mwdb_user", help="Mwdb username")
parser.add_argument("mwdb_pass", help="Mwdb password")
parser.add_argument("config", help="Config")
parser.add_argument("misp_url", help="Misp url")
parser.add_argument("misp_key", help="Misp key")
args = parser.parse_args()
mwdb = Malwarecage()
mwdb.login(args.mwdb_user, args.mwdb_pass)
try:
cfg = mwdb.query_config(args.config)
iocs = parse(cfg.family, cfg.cfg)
except FamilyNotSupportedYetError:
logging.info("Family %s not supported yet...", cfg.family)
return
if not iocs:
# Nothing actionable found - skip the config
return
event = MISPEvent()
event.add_tag(f"mwdb:family:{cfg.family}")
event.info = f"Malware configuration ({cfg.family})"
for o in iocs.to_misp():
event.add_object(o)
misp = PyMISP(args.misp_url, args.misp_key, False)
misp.add_event(event)
class MISP():
def __init__(self, config: Dict[str, Any]):
self.logger = logging.getLogger(f'{self.__class__.__name__}')
self.logger.setLevel(get_config('generic', 'loglevel'))
if not config.get('apikey'):
self.available = False
self.logger.info('Module not enabled.')
return
self.available = True
self.enable_lookup = False
self.enable_push = False
try:
self.client = PyMISP(url=config['url'],
key=config['apikey'],
ssl=config['verify_tls_cert'])
except Exception as e:
self.available = False
self.logger.warning(f'Unable to connect to MISP: {e}')
return
if config.get('enable_lookup'):
self.enable_lookup = True
if config.get('enable_push'):
self.enable_push = True
self.storage_dir_misp = get_homedir() / 'misp'
self.storage_dir_misp.mkdir(parents=True, exist_ok=True)
def push(self, event: MISPEvent) -> Union[MISPEvent, Dict]:
if self.available and self.enable_push:
return self.client.add_event(event, pythonify=True)
else:
return {'error': 'Module not available or push not enabled.'}
class MISPApi(object):
def __init__(self, config: dict):
self.pymisp = PyMISP(config['MISP_URL'], config['MISP_KEY'], config['MISP_VERIFYCERT'], 'json')
def search(self, controller='attributes', **kwargs):
return self.pymisp.search(controller, **kwargs)
def org_name_id_mapping(self):
pass
def domain_name_lookup(self, domain_name: str) -> List[Any]:
result = self.search(type='domain', value=domain_name)
return result['response'].get('Attribute', [])
def add_event(self, domain_names: list, info: str, tags: List, comment: str, to_ids: bool,
ts: Optional[int] = None, published: Optional[bool] = False):
attrs = []
for name in domain_names:
attr = MISPAttribute()
attr.from_dict(type='domain', category='Network activity', to_ids=to_ids, value=name,
comment=comment, timestamp=ts)
attrs.append(attr)
event = MISPEvent()
event.from_dict(info=info, Attribute=attrs, Tag=tags, date=datetime.date.today(), published=published)
logger.debug(event)
return self.pymisp.add_event(event)
def _misp_create_event_function(self, event, *args, **kwargs):
"""Function: create a MISP event from an incident """
try:
def get_config_option(option_name, optional=False):
"""Given option_name, checks if it is in app.config. Raises ValueError if a mandatory option is missing"""
option = self.options.get(option_name)
if option is None and optional is False:
err = "'{0}' is mandatory and is not set in ~/.resilient/app.config file. You must set this value to run this function".format(
option_name)
raise ValueError(err)
else:
return option
API_KEY = get_config_option("misp_key")
URL = get_config_option("misp_url")
VERIFY_CERT = True if get_config_option(
"verify_cert").lower() == "true" else False
# Get the function parameters:
misp_event_name = kwargs.get("misp_event_name") # text
misp_distribution = kwargs.get("misp_distribution") # number
misp_analysis_level = kwargs.get("misp_analysis_level") # number
misp_threat_level = kwargs.get("misp_threat_level") # number
log = logging.getLogger(__name__)
log.info("misp_event_name: %s", misp_event_name)
log.info("misp_distribution: %s", misp_distribution)
log.info("misp_analysis_level: %s", misp_analysis_level)
log.info("misp_threat_level: %s", misp_threat_level)
yield StatusMessage("Setting up connection to MISP")
misp_client = PyMISP(URL, API_KEY, VERIFY_CERT, 'json')
eventJson = {
"Event": {
"info": misp_event_name,
"analysis": misp_analysis_level,
"distribution": misp_distribution,
"threat_level_id": misp_threat_level
}
}
event = misp_client.add_event(eventJson)
log.info(event)
yield StatusMessage("Event has been created")
results = {"success": True, "content": event}
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception:
yield FunctionError()
def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, analysis=0, event_info=None):
if event_info:
event.info = event_info
event.distribution = sanitize_event_distribution(distribution)
event.threat_level_id = sanitize_event_threat_level_id(threat_level_id)
event.analysis = sanitize_event_analysis(analysis)
if publish:
event.publish()
# # TODO: handle multiple MISP instance
misp = PyMISP(misp_url, misp_key, misp_verifycert)
#print(event.to_json())
misp_event = misp.add_event(event, pythonify=True)
# # TODO: handle error
event_metadata = extract_event_metadata(misp_event)
return event_metadata
def pushtoMISP(ioc_value):
# Variable: MISP URL, Cert, Key
misp_url = os.getenv("MISP_URL")
misp_verifycert = False
misp_key = os.getenv("MISP_KEY")
# MISP connection
misp = PyMISP(misp_url, misp_key, misp_verifycert, 'json')
#date = "2018-02-28"
i = datetime.now()
date = i.strftime('%Y-%m-%d')
# Set the event name
Event_name = "ModSecurity - "+date+" - Honeypot"
# Prepare MISP attribute
misp_attributes = []
ioc_datetime = str(convertToEpoch(date))
comments = " "
misp_attribute = {
'category': ioc_category, 'type': 'ip-dst', 'value': ioc_value, 'distribution': '0',
'to_ids': True, 'comment': comments , 'timestamp': ioc_datetime
}
misp_attributes.append(misp_attribute)
# Prepare MISP event
misp_event = {
'Event': {
'info': Event_name, 'date': date, 'distribution': '0', 'threat_level_id': '1',
'analysis': '2', 'Tag': [
{'name': 'ModSecurity'},
{'name': 'AUTO'}
],
'Attribute': misp_attributes
}
}
# Insert Event + Attribute to MISP
misp_event_added = misp.add_event(misp_event)
# Publish (/!\ Bring in production)
misp.publish(misp_event_added)
class TestBasic(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.misp = PyMISP(url, key, True, 'json', True)
def _clean_event(self, event):
event['Event'].pop('orgc_id', None)
event['Event'].pop('uuid', None)
event['Event'].pop('sharing_group_id', None)
event['Event'].pop('timestamp', None)
event['Event'].pop('org_id', None)
event['Event'].pop('date', None)
event['Event'].pop('RelatedEvent', None)
event['Event'].pop('publish_timestamp', None)
if event['Event'].get('Attribute'):
for a in event['Event'].get('Attribute'):
a.pop('uuid', None)
a.pop('event_id', None)
a.pop('id', None)
a.pop('timestamp', None)
if event['Event'].get('Orgc'):
event['Event']['Orgc'].pop('uuid', None)
event['Event']['Orgc'].pop('id', None)
if event['Event'].get('Org'):
event['Event']['Org'].pop('uuid', None)
event['Event']['Org'].pop('id', None)
return event['Event'].pop('id', None)
def new_event(self):
event = self.misp.new_event(0, 1, 0, "This is a test")
event_id = self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': None, 'disable_correlation': False, u'analysis': u'0',
u'ShadowAttribute': [], u'published': False,
u'distribution': u'0', u'Attribute': [], u'proposal_email_lock': False,
u'Org': {u'name': u'ORGNAME'},
u'Orgc': {u'name': u'ORGNAME'},
u'Galaxy': [],
u'threat_level_id': u'1'}}
print(event)
self.assertEqual(event, to_check, 'Failed at creating a new Event')
return int(event_id)
def add_hashes(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.add_hashes(event, 'Payload installation', 'dll_installer.dll', '0a209ac0de4ac033f31d6ba9191a8f7a', '1f0ae54ac3f10d533013f74f48849de4e65817a7', '003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', 'Fanny modules', False, 2)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0',
u'Org': {u'name': u'ORGNAME'},
u'Orgc': {u'name': u'ORGNAME'},
u'Galaxy': [],
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at adding hashes')
def publish(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.publish(event)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': True, u'distribution': u'0',
u'Org': {u'name': u'ORGNAME'},
u'Orgc': {u'name': u'ORGNAME'},
u'Galaxy': [],
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at publishing event')
def delete(self, eventid):
event = self.misp.delete_event(eventid)
print(event)
def delete_attr(self, attrid):
event = self.misp.delete_attribute(attrid)
print(event)
def get(self, eventid):
event = self.misp.get_event(eventid)
print(event)
def get_stix(self, **kwargs):
event = self.misp.get_stix(kwargs)
print(event)
def add(self):
event = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0',
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
event = self.misp.add_event(event)
print(event)
def test_create_event(self):
eventid = self.new_event()
time.sleep(1)
self.delete(eventid)
def test_get_event(self):
eventid = self.new_event()
time.sleep(1)
self.get(eventid)
time.sleep(1)
self.delete(eventid)
def test_add_event(self):
self.add()
time.sleep(1)
self.delete(1)
def test_del_attr(self):
eventid = self.new_event()
time.sleep(1)
self.delete_attr(1)
time.sleep(1)
self.delete(eventid)
def test_one_or_more(self):
self.assertEqual(self.misp._one_or_more(1), (1,))
self.assertEqual(self.misp._one_or_more([1]), [1])
if province['deadCount']:
obj_province.add_attribute('total-death', province['deadCount'])
if province['comment']:
obj_province.add_attribute('comment', province['comment'])
for city in province['cities']:
obj_city = event.add_object(name='covid19-dxy-live-city',
standalone=False)
obj_city.add_attribute('city', city['cityName'])
obj_city.add_attribute('update', d)
if city['currentConfirmedCount']:
obj_city.add_attribute('current-confirmed',
city['currentConfirmedCount'])
if city['confirmedCount']:
obj_city.add_attribute('total-confirmed',
city['confirmedCount'])
if city['curedCount']:
obj_city.add_attribute('total-cured', city['curedCount'])
if city['deadCount']:
obj_city.add_attribute('total-death', city['deadCount'])
obj_city.add_reference(obj_province, 'part-of')
if make_feed:
with (Path('output') / f'{event.uuid}.json').open('w') as _w:
json.dump(event.to_feed(), _w)
else:
misp.add_event(event)
if make_feed:
feed_meta_generator(Path('output'))
with p.open() as f:
reader = DictReader(f)
for row in reader:
if aggregate_by_country:
country_aggregate(aggregate, row)
else:
obj = MISPObject(name='covid19-csse-daily-report')
add_detailed_object(obj, row)
event.add_object(obj)
if aggregate_by_country:
for country, values in aggregate.items():
obj = event.add_object(name='covid19-csse-daily-report',
standalone=False)
obj.add_attribute('country-region', country)
obj.add_attribute('update', values['update'])
obj.add_attribute('confirmed', values['confirmed'])
obj.add_attribute('death', values['death'])
obj.add_attribute('recovered', values['recovered'])
obj.add_attribute('active', values['active'])
if make_feed:
with (Path('output') / f'{event.uuid}.json').open('w') as _w:
json.dump(event.to_feed(), _w)
else:
event = misp.add_event(event)
misp.publish(event)
if make_feed:
feed_meta_generator(Path('output'))
csvfile = open("Ransomware.csv")
csvreader = csv.DictReader(csvfile)
lines = csvfile.readlines()
lasthash = ''
for line in lines:
if "BTC Address" in line:
attrs = line.split(",")
if lasthash != attrs[3]:
lasthash = attrs[3]
event_obj = MISPEvent()
event_obj.distribution = 1
event_obj.threat_level_id = 3
event_obj.analysis = 1
event_obj.info = "RansomCoin Ransomware Survey " + attrs[3]
event = misp.add_event(event_obj)
event_id = event["Event"]["id"]
print("Creating Event id: %s" % event_id)
event_dict = misp.get(event_id)['Event']
event = MISPEvent()
event.from_dict(**event_dict)
event.add_attribute(type='md5',
category='Artifacts dropped',
value=attrs[1],
disable_correlation=False,
to_ids=False,
proposal=False,
distribution=5)
event.add_attribute(type='sha1',
category='Artifacts dropped',
value=attrs[2],
if args.disable_new:
event_id = response['response'][0]['id']
else:
last_event_date = parse(response['response'][0]['date']).date()
nb_attr = response['response'][0]['attribute_count']
if last_event_date < date.today() or int(nb_attr) > 1000:
me = create_new_event()
else:
event_id = response['response'][0]['id']
else:
me = create_new_event()
parameters = {'banned-ip': args.banned_ip, 'attack-type': args.attack_type}
if args.processing_timestamp:
parameters['processing-timestamp'] = args.processing_timestamp
if args.failures:
parameters['failures'] = args.failures
if args.sensor:
parameters['sensor'] = args.sensor
if args.victim:
parameters['victim'] = args.victim
if args.logline:
parameters['logline'] = b64decode(args.logline).decode()
f2b = Fail2BanObject(parameters=parameters, standalone=False)
if me:
me.add_object(f2b)
pymisp.add_event(me)
elif event_id:
template_id = pymisp.get_object_template_id(f2b.template_uuid)
a = pymisp.add_object(event_id, template_id, f2b)
class TestBasic(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.misp = PyMISP(url, key, True, 'json')
def _clean_event(self, event):
event['Event'].pop('uuid', None)
event['Event'].pop('timestamp', None)
event['Event'].pop('date', None)
event['Event'].pop('org', None)
event['Event'].pop('orgc', None)
event['Event'].pop('RelatedEvent', None)
event['Event'].pop('publish_timestamp', None)
if event['Event'].get('Attribute'):
for a in event['Event'].get('Attribute'):
a.pop('uuid', None)
a.pop('event_id', None)
a.pop('id', None)
a.pop('timestamp', None)
return event['Event'].pop('id', None)
def new_event(self):
event = self.misp.new_event(0, 1, 0, "This is a test")
event_id = self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'0', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False,
u'distribution': u'0', u'Attribute': [], u'proposal_email_lock': False,
u'threat_level_id': u'1'}},
self.assertEqual(event, to_check, 'Failed at creating a new Event')
return int(event_id)
def add_hashes(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.add_hashes(event, 'Payload installation', 'dll_installer.dll', '0a209ac0de4ac033f31d6ba9191a8f7a', '1f0ae54ac3f10d533013f74f48849de4e65817a7', '003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', 'Fanny modules', False, 2)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0',
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at adding hashes')
def publish(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.publish(event)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': True, u'distribution': u'0',
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at publishing event')
def delete(self, eventid):
event = self.misp.delete_event(eventid)
print event.json()
def delete_attr(self, attrid):
event = self.misp.delete_attribute(attrid)
print event.json()
def get(self, eventid):
event = self.misp.get_event(eventid)
print event.json()
def add(self):
event = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0',
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
event = self.misp.add_event(event)
print event.json()
def test_create_event(self):
eventid = self.new_event()
time.sleep(1)
self.delete(eventid)
def test_get_event(self):
eventid = self.new_event()
time.sleep(1)
self.get(eventid)
time.sleep(1)
self.delete(eventid)
def test_add_event(self):
self.add()
time.sleep(1)
self.delete(1)
def test_del_attr(self):
eventid = self.new_event()
time.sleep(1)
self.delete_attr(1)
time.sleep(1)
self.delete(eventid)
class misp_custom:
def __init__(self, misp_url, misp_key, misp_ssl):
try:
self.misp = PyMISP(misp_url, misp_key, misp_ssl) #, 'json')
except Exception as err:
sys.exit('Batch Job Terminated: MISP connection error - \n' +
repr(err))
self.misp_logger = logging.getLogger('mispattruploader')
def submit_to_misp(self, misp, misp_event, misp_objects):
'''
Submit a list of MISP objects to a MISP event
:misp: PyMISP API object for interfacing with MISP
:misp_event: MISPEvent object
:misp_objects: List of MISPObject objects. Must be a list
'''
# go through round one and only add MISP objects
a = []
for misp_object in misp_objects:
self.misp_logger.debug(misp_object)
if len(misp_object.attributes) > 0:
if misp_object.name == 'network-connection':
template_id = 'af16764b-f8e5-4603-9de1-de34d272f80b'
else:
# self.misp_logger.debug(dir(pymisp.api))
# self.misp_logger.debug(dir(self.misp))
# exit()
self.misp_logger.debug(misp_object.template_uuid)
object_template = self.misp.get_object_template(
misp_object.template_uuid)
template_id = object_template['ObjectTemplate']['id']
self.misp_logger.debug(template_id)
self.misp_logger.debug(dir(misp_event))
self.misp_logger.debug(misp_event)
_a = misp.add_object(event=misp_event, misp_object=misp_object)
self.misp_logger.debug(_a)
a.append(_a)
# go through round two and add all the object references for each object
b = []
for misp_object in misp_objects:
for reference in misp_object.ObjectReference:
_b = misp.add_object_reference(reference)
b.append(_b)
return a, b
def check_object_length(self, misp_objects):
for misp_object in misp_objects:
self.misp_logger.info(misp_object.name)
self.misp_logger.info(dir(misp_object))
if len(misp_object.attributes) == 0:
self.misp_logger.error('failure to put in correct tags')
return False
return True
def get_comm_and_tags(self, strInput):
comment = None
str_comment = ""
tags = ["tlp:green"]
tag_type = None
for line in strInput.splitlines():
if ("comment:" in line.lower()):
vals = line.split(":", 1)
comment = vals[1:]
elif ("tag:" in line.lower()):
vals = line.split(":", 1)
value = vals[1].strip().lower()
if "tlp" in value:
tags.remove("tlp:green")
vals_str = "tlp:"
vals_split = vals[1].split(":")
vals_str += vals_split[1]
tags.append(vals_str)
elif ("type:" in line.lower()):
vals = line.split(":", 1)
value = vals[1].strip().lower()
if value == "phish":
tag_type = value
elif value == "malware":
tag_type = value
elif value == "bec/spam":
tag_type = value
elif value == "dump":
tag_type = value
elif (value == "apt") or (value == "APT"):
tag_type = value
if tag_type:
self.misp_logger.info('Setting tag to ir8: %s' % tag_type)
tag = "ir8:" + tag_type
tags.append(tag)
else:
tags = None
if comment != None:
for c in comment:
str_comment += c
else:
str_comment = comment
return str_comment, tags
####### SLACK ENTRY POINT! ######
def misp_send(self, strMISPEventID, strInput, strInfo, strUsername):
# Establish communication with MISP
# event = MISPEvent()
# event.info = 'Test event'
# event.analysis = 0
# event.distribution = 3
# event.threat_level_id = 2
# event.add_attribute('md5', '678ff97bf16d8e1c95679c4681834c41')
# #<add more attributes>
# self.misp.add_event(event)
# exit()
try:
objects = []
#get comments and tags from string input
str_comment, tags = self.get_comm_and_tags(strInput)
print(tags)
if tags == None:
self.misp_logger.info('Irate not in Tags: %s equals None' %
tags)
response = None
return response
#setup misp objects
mispobj_email = MISPObject(name="email")
mispobj_file = MISPObject(name="file")
mispobj_files = {}
mispobj_domainip = MISPObject(name="domain-ip")
url_no = 0
file_no = 0
mispobj_urls = {}
#process input
for line in strInput.splitlines():
if ("domain:" in line.lower()
): #Catch domain and add to domain/IP object
mispobj_domainip = MISPObject(name="domain-ip")
vals = line.split(":", 1)
mispobj_domainip.add_attribute("domain",
value=vals[1].strip(),
comment=str_comment)
objects.append(mispobj_domainip)
elif ("ip:" in line.lower()) or ("ip-dst:" in line.lower(
)) or ("ip-src:"
in line.lower()): #Catch IP and add to domain/IP object
if "domain:" in strInput.splitlines():
mispobj_domainip = MISPObject(name="domain-ip")
vals = line.split(":", 1)
mispobj_domainip.add_attribute("ip",
value=vals[1].strip(),
comment=str_comment)
objects.append(mispobj_domainip)
else:
mispobj_network_connection = MISPObject(
name="network-connection")
vals = line.split(":", 1)
if ("ip:" in line.lower()) or ("ip-dst:"
in line.lower()):
mispobj_network_connection.add_attribute(
"ip-dst",
type="ip-dst",
value=vals[1].strip(),
comment=str_comment)
else:
mispobj_network_connection.add_attribute(
"ip-src",
type="ip-src",
value=vals[1].strip(),
comment=str_comment)
objects.append(mispobj_network_connection)
elif ("source-email:"
in line.lower()) or ("email-source" in line.lower()) or (
"from:" in line.lower()
): #Catch email and add to email object
vals = line.split(":", 1)
mispobj_email.add_attribute("from",
value=vals[1].strip(),
comment=str_comment)
elif ("url:" in line.lower()) or (
('kit:' in line.lower() or ('creds:' in line.lower())) and
(('hxxp' in line.lower()) or ('http' in line.lower()))
): #Catch URL and add to URL object
vals = line.split(":", 1)
url = vals[1].strip()
url = refang(url)
parsed = urlparse(url)
mispobj_url = MISPObject(name="url")
mispobj_url.add_attribute("url",
value=parsed.geturl(),
category="Payload delivery",
comment=str_comment)
if parsed.hostname:
mispobj_url.add_attribute("host",
value=parsed.hostname,
comment=str_comment)
if parsed.scheme:
mispobj_url.add_attribute("scheme",
value=parsed.scheme,
comment=str_comment)
if parsed.port:
mispobj_url.add_attribute("port",
value=parsed.port,
comment=str_comment)
mispobj_urls[url_no] = mispobj_url
url_no += 1
#Catch different hashes and add to file object
elif ("sha1:" in line.lower()) or ("SHA1:" in line):
vals = line.split(":", 1)
mispobj_file.add_attribute("sha1",
value=vals[1].strip(),
comment=str_comment)
elif ("sha256:" in line.lower()) or ("SHA256:" in line):
vals = line.split(":", 1)
mispobj_file.add_attribute("sha256",
value=vals[1].strip(),
comment=str_comment)
elif ("md5:" in line.lower()) or ("MD5:" in line):
vals = line.split(":", 1)
mispobj_file.add_attribute("md5",
value=vals[1].strip(),
comment=str_comment)
elif (
"subject:" in line.lower()
): #or ("subject:" in line): #Catch subject and add to email object
self.misp_logger.info('adding subject')
vals = line.split(":", 1)
mispobj_email.add_attribute("subject",
value=vals[1].strip(),
comment=str_comment)
elif ("hash|filename:" in line.lower()
): #catch hash|filename pair and add to file object
vals = line.split(":", 1)
val = vals[1].split("|")
l_hash = val[0]
l_filename = val[1]
l_mispobj_file = MISPObject(name="file")
if len(re.findall(r"\b[a-fA-F\d]{32}\b", l_hash)) > 0:
l_mispobj_file.add_attribute("md5",
value=l_hash.strip(),
comment=str_comment)
l_mispobj_file.add_attribute("filename",
value=l_filename.strip(),
comment=str_comment)
mispobj_files[file_no] = l_mispobj_file
elif len(re.findall(r'\b[0-9a-f]{40}\b', l_hash)) > 0:
l_mispobj_file.add_attribute("sha1",
value=l_hash.strip(),
comment=str_comment)
l_mispobj_file.add_attribute("filename",
value=l_filename.strip(),
comment=str_comment)
mispobj_files[file_no] = l_mispobj_file
elif len(re.findall(r'\b[A-Fa-f0-9]{64}\b', l_hash)) > 0:
l_mispobj_file.add_attribute("sha256",
value=l_hash.strip(),
comment=str_comment)
l_mispobj_file.add_attribute("filename",
value=l_filename.strip(),
comment=str_comment)
mispobj_files[file_no] = l_mispobj_file
file_no += 1
#add all misp objects to List to be processed and submitted to MISP server as one.
if len(mispobj_file.attributes) > 0:
objects.append(mispobj_file)
if len(mispobj_email.attributes) > 0:
objects.append(mispobj_email)
for u_key, u_value in mispobj_urls.items():
if len(u_value.attributes) > 0:
objects.append(u_value)
for f_key, f_value in mispobj_files.items():
if len(f_value.attributes) > 0:
objects.append(f_value)
# Update timestamp and event
except Exception as e:
error = traceback.format_exc()
response = "Error occured when converting string to misp objects:\n %s" % error
self.misp_logger.error(response)
return response
if self.check_object_length(objects) != True:
self.misp_logger.error(
'Input from %s did not contain accepted tags.\n Input: \n%s' %
(strUsername, strInput))
return "Error in the tags you entered. Please see the guide for accepted tags."
try:
# self.misp_logger.error(dir(self.misp))
misp_event = MISPEvent()
misp_event.info = strInfo
misp_event.distribution = 0
misp_event.analysis = 2
misp_event.threat_level_id = 3
# event.add_attribute('md5', '678ff97bf16d8e1c95679c4681834c41')
#event = self.misp.new_event(info=strInfo, distribution='0', analysis='2', threat_level_id='3', published=False)
#misp_event = MISPEvent()
#misp_event.load(event)
add = self.misp.add_event(misp_event)
self.misp_logger.info("Added event %s" % add)
a, b = self.submit_to_misp(self.misp, misp_event, objects)
for tag in tags:
self.misp.tag(misp_event.uuid, tag)
#self.misp.add_internal_comment(misp_event.id, reference="Author: " + strUsername, comment=str_comment)
ccc = self.misp.publish(misp_event, alert=False)
self.misp_logger.info(ccc)
misp_event = self.misp.get_event(misp_event)
response = misp_event
#for response in misp_event:
if ('errors' in response and response['errors'] != None):
return ("Submission error: " + repr(response['errors']))
else:
if response['Event']['RelatedEvent']:
e_related = ""
for each in response['Event']['RelatedEvent']:
e_related = e_related + each['Event']['id'] + ", "
return "Created ID: " + str(
response['Event']
['id']) + "\nRelated Events: " + ''.join(e_related)
else:
return "Created ID: " + str(response['Event']['id'])
except Exception as e:
error = traceback.format_exc()
response = "Error occured when submitting to misp:\n %s" % error
self.misp_logger.error(response)
return response
class TestBasic(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.misp = PyMISP(url, key, True, 'json')
def _clean_event(self, event):
event['Event'].pop('orgc_id', None)
event['Event'].pop('uuid', None)
event['Event'].pop('sharing_group_id', None)
event['Event'].pop('timestamp', None)
event['Event'].pop('org_id', None)
event['Event'].pop('date', None)
event['Event'].pop('RelatedEvent', None)
event['Event'].pop('publish_timestamp', None)
if event['Event'].get('Attribute'):
for a in event['Event'].get('Attribute'):
a.pop('uuid', None)
a.pop('event_id', None)
a.pop('id', None)
a.pop('timestamp', None)
if event['Event'].get('Orgc'):
event['Event']['Orgc'].pop('uuid', None)
event['Event']['Orgc'].pop('id', None)
if event['Event'].get('Org'):
event['Event']['Org'].pop('uuid', None)
event['Event']['Org'].pop('id', None)
return event['Event'].pop('id', None)
def new_event(self):
event = self.misp.new_event(0, 1, 0, "This is a test")
event_id = self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'0', 'disable_correlation': False, u'analysis': u'0',
u'ShadowAttribute': [], u'published': False,
u'distribution': u'0', u'event_creator_email': u'*****@*****.**', u'Attribute': [], u'proposal_email_lock': False,
u'Object': [], u'Org': {u'name': u'ORGNAME'},
u'Orgc': {u'name': u'ORGNAME'},
u'Galaxy': [],
u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at creating a new Event')
return int(event_id)
def add_hashes(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.add_hashes(event, 'Payload installation', 'dll_installer.dll', '0a209ac0de4ac033f31d6ba9191a8f7a', '1f0ae54ac3f10d533013f74f48849de4e65817a7', '003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', 'Fanny modules', False, 2)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'*****@*****.**',
u'Org': {u'name': u'ORGNAME'},
u'Orgc': {u'name': u'ORGNAME'},
u'Galaxy': [],
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at adding hashes')
def publish(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.publish(event)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': True, u'distribution': u'0', u'event_creator_email': u'*****@*****.**',
u'Org': {u'name': u'ORGNAME'},
u'Orgc': {u'name': u'ORGNAME'},
u'Galaxy': [],
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at publishing event')
def delete(self, eventid):
event = self.misp.delete_event(eventid)
print(event)
def delete_attr(self, attrid):
event = self.misp.delete_attribute(attrid)
print(event)
def get(self, eventid):
event = self.misp.get_event(eventid)
print(event)
def get_stix(self, **kwargs):
event = self.misp.get_stix(kwargs)
print(event)
def add(self):
event = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'*****@*****.**',
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
event = self.misp.add_event(event)
print(event)
def add_user(self):
email = '*****@*****.**'
role_id = '5'
org_id = '1'
password = '******'
external_auth_required = False
external_auth_key = ''
enable_password = False
nids_sid = '1238717'
server_id = '1'
gpgkey = ''
certif_public = ''
autoalert = False
contactalert = False
disabled = False
change_pw = '0'
termsaccepted = False
newsread = '0'
authkey = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
to_check = {'User': {'email': email, 'org_id': org_id, 'role_id': role_id,
'password': password, 'external_auth_required': external_auth_required,
'external_auth_key': external_auth_key, 'enable_password': enable_password,
'nids_sid': nids_sid, 'server_id': server_id, 'gpgkey': gpgkey,
'certif_public': certif_public, 'autoalert': autoalert,
'contactalert': contactalert, 'disabled': disabled,
'change_pw': change_pw, 'termsaccepted': termsaccepted,
'newsread': newsread, 'authkey': authkey}}
user = self.misp.add_user(email=email,
role_id=role_id,
org_id=org_id,
password=password,
external_auth_required=external_auth_required,
external_auth_key=external_auth_key,
enable_password=enable_password,
nids_sid=nids_sid,
server_id=server_id,
gpgkey=gpgkey,
certif_public=certif_public,
autoalert=autoalert,
contactalert=contactalert,
disabled=disabled,
change_pw=change_pw,
termsaccepted=termsaccepted,
newsread=newsread,
authkey=authkey)
# delete user to allow reuse of test
uid = user.get('User').get('id')
self.misp.delete_user(uid)
# ----------------------------------
# test interesting keys only (some keys are modified(password) and some keys are added (lastlogin)
tested_keys = ['email', 'org_id', 'role_id', 'server_id', 'autoalert',
'authkey', 'gpgkey', 'certif_public', 'nids_sid', 'termsaccepted',
'newsread', 'contactalert', 'disabled']
for k in tested_keys:
self.assertEqual(user.get('User').get(k), to_check.get('User').get(k), "Failed to match input with output on key: {}".format(k))
def add_organisation(self):
name = 'Organisation tests'
description = 'This is a test organisation'
orgtype = 'Type is a string'
nationality = 'French'
sector = 'Bank sector'
uuid = '16fd2706-8baf-433b-82eb-8c7fada847da'
contacts = 'Text field with no limitations'
local = False
to_check = {'Organisation': {'name': name, 'description': description,
'type': orgtype, 'nationality': nationality,
'sector': sector, 'uuid': uuid, 'contacts': contacts,
'local': local}}
org = self.misp.add_organisation(name=name,
description=description,
type=orgtype,
nationality=nationality,
sector=sector,
uuid=uuid,
contacts=contacts,
local=local,
)
# delete organisation to allow reuse of test
oid = org.get('Organisation').get('id')
self.misp.delete_organisation(oid)
# ----------------------------------
tested_keys = ['anonymise', 'contacts', 'description', 'local', 'name',
'nationality', 'sector', 'type', 'uuid']
for k in tested_keys:
self.assertEqual(org.get('Organisation').get(k), to_check.get('Organisation').get(k), "Failed to match input with output on key: {}".format(k))
def test_create_event(self):
eventid = self.new_event()
time.sleep(1)
self.delete(eventid)
def test_get_event(self):
eventid = self.new_event()
time.sleep(1)
self.get(eventid)
time.sleep(1)
self.delete(eventid)
def test_add_event(self):
self.add()
time.sleep(1)
self.delete(1)
def test_del_attr(self):
eventid = self.new_event()
time.sleep(1)
self.delete_attr(1)
time.sleep(1)
self.delete(eventid)
def test_one_or_more(self):
self.assertEqual(self.misp._one_or_more(1), (1,))
self.assertEqual(self.misp._one_or_more([1]), [1])
def test_create_user(self):
self.add_user()
def test_create_organisation(self):
self.add_organisation()
def export(request):
"""Export extracteddump to misp"""
data = {}
if request.method == "POST":
extracted_dump = get_object_or_404(
ExtractedDump, pk=request.POST.get("selected_exdump"))
misp_info = get_object_or_404(Service, name=2)
# CREATE GENERIC EVENT
misp = PyMISP(misp_info.url,
misp_info.key,
False,
proxies=misp_info.proxy)
event = MISPEvent()
event.info = "From orochi: {}@{}".format(
extracted_dump.result.plugin.name, extracted_dump.result.dump.name)
# CREATE FILE OBJ
file_obj = FileObject(extracted_dump.path)
event.add_object(file_obj)
# ADD CLAMAV SIGNATURE
if extracted_dump.clamav:
clamav_obj = MISPObject("av-signature")
clamav_obj.add_attribute("signature", value=extracted_dump.clamav)
clamav_obj.add_attribute("software", value="clamav")
file_obj.add_reference(clamav_obj.uuid, "attributed-to")
event.add_object(clamav_obj)
# ADD VT SIGNATURE
if extracted_dump.vt_report:
vt_obj = MISPObject("virustotal-report")
vt_obj.add_attribute("last-submission",
value=extracted_dump.vt_report.get(
"scan_date", ""))
vt_obj.add_attribute(
"detection-ratio",
value="{}/{}".format(
extracted_dump.vt_report.get("positives", 0),
extracted_dump.vt_report.get("total", 0),
),
)
vt_obj.add_attribute("permalink",
value=extracted_dump.vt_report.get(
"permalink", ""))
file_obj.add_reference(vt_obj.uuid, "attributed-to")
event.add_object(vt_obj)
misp.add_event(event)
return JsonResponse({"success": True})
extracted_dump = get_object_or_404(ExtractedDump,
path=urllib.parse.unquote(
request.GET.get("path")))
form = MispExportForm(
instance=extracted_dump,
initial={
"selected_exdump": extracted_dump.pk,
"selected_index_name": extracted_dump.result.dump.name,
"selected_plugin_name": extracted_dump.result.plugin.name,
},
)
context = {"form": form}
data["html_form"] = render_to_string(
"website/partial_export.html",
context,
request=request,
)
return JsonResponse(data)
for sample in it:
raw_report = getFileDetail(apikey=api_key, resource=sample.id)
scandate = datetime.datetime.strptime(raw_report["scan_date"],
'%Y-%m-%d %H:%M:%S')
me = MISPEvent()
this_event_name = misp_event_name.format(scandate.strftime("%Y-%m-%d"))
search = pm.search(controller='events', eventinfo=this_event_name)
if (len(search) == 1):
me.load(search[0])
else:
me.info = this_event_name
pm.add_event(me)
vtreport = GenericObjectGenerator('virustotal-report')
vtreport.add_attribute("last-submission",
value=raw_report["scan_date"])
vtreport.add_attribute("permalink", value=raw_report["permalink"])
ratio = "{}/{}".format(raw_report["positives"], raw_report["total"])
vtreport.add_attribute("detection-ratio", value=ratio)
file_object = GenericObjectGenerator('file')
file_object.add_attribute("md5", value=raw_report["md5"])
file_object.add_attribute("sha1", value=raw_report["sha1"])
file_object.add_attribute("sha256", value=raw_report["sha256"])
file_object.add_attribute("ssdeep", value=raw_report["ssdeep"])
file_object.add_attribute("authentihash",
value=raw_report["authentihash"])
# # Load objects from the CSV file
objects = get_object_fields(args.csv, args.delim, args.quotechar, args.strictcsv)
if len(objects) == 0:
log.critical('No Objects to create! Are they commented out? Run with --verbose (-v) to see what\'s happening!')
exit(1)
# # Create a new Event
if args.info:
event = MISPEvent()
event.info = args.info
if args.distribution:
log.debug('Setting distribution level for Event: {}'.format(args.distribution))
event.distribution = args.distribution
if not args.dryrun:
new_event = pymisp.add_event(event)
if 'errors' in new_event.keys():
log.critical('Error creating the new event. {}'.format(new_event['errors'][2]))
exit(1)
# # Get the ID of the new event for later
args.event = new_event['Event']['uuid']
log.info('New event created: {}'.format(args.event))
# # Add Objects to existing Event
for i, o in enumerate(objects, 1):
misp_object = GenericObjectGenerator(o['object'], misp_objects_path_custom=args.custom_objects_path)
try:
misp_object.generate_attributes(o['attributes'])
except NewAttributeError as e:
class MISPHandler:
def __init__(self, config: dict):
self.url = config['misp_url']
self.key = config['misp_auth_key']
self.misp = PyMISP(self.url, self.key)
self.tag_list = self.create_tag_list()
self.logger = logging.getLogger('misp_handler')
self.logger.debug("URLhausHandler init done")
def create_tag_list(self) -> list:
tags = []
for item in self.misp.tags(pythonify=True):
tags.append(item.name)
return tags
def make_sure_tag_exists(self, tag: str) -> bool:
if tag in self.tag_list:
return True
else:
self.misp.add_tag({"name": tag}, pythonify=True)
self.tag_list = self.create_tag_list()
if tag in self.tag_list:
return True
else:
return False
def add_tag_to_attribute(self, attr: MISPAttribute, tag: str) -> MISPAttribute:
if self.make_sure_tag_exists(tag):
attr.add_tag(tag)
return attr
def create_attr(self, raw_attr: dict) -> MISPAttribute:
# Create attribute and assign simple values
attr = MISPAttribute()
attr.type = 'url'
attr.value = raw_attr['url']
attr.disable_correlation = False
attr.__setattr__('first_seen', datetime.strptime(raw_attr['dateadded'], '%Y-%m-%d %H:%M:%S'))
# Add URLhaus tag
self.add_tag_to_attribute(attr, 'URLhaus')
# Add other tags
if raw_attr['tags']:
for tag in raw_attr['tags'].split(','):
self.add_tag_to_attribute(attr, tag.strip())
# Add online/offline tag
if not pandas.isna(raw_attr['url_status']):
if raw_attr['url_status'] == 'online':
attr.to_ids = True
else:
attr.to_ids = False
self.add_tag_to_attribute(attr, raw_attr['url_status'])
# Add reporter tag
if not pandas.isna(raw_attr['reporter']):
self.add_tag_to_attribute(attr, raw_attr['reporter'])
attr.comment = raw_attr['urlhaus_link']
return attr
def create_attr_feodo(self, raw_attr: dict) -> MISPAttribute:
attr = MISPAttribute()
attr.type = 'ip-dst|port'
attr.value = f"{raw_attr['DstIP']}|{raw_attr['DstPort']}"
self.add_tag_to_attribute(attr, 'FeodoTracker')
self.add_tag_to_attribute(attr, raw_attr['Malware'])
attr.comment = 'Feodo tracker DST IP/port'
attr.__setattr__('first_seen', datetime.strptime(raw_attr['Firstseen'], '%Y-%m-%d %H:%M:%S'))
if not pandas.isna(raw_attr['LastOnline']):
last_seen_time = datetime.strptime(str(raw_attr['LastOnline']), '%Y-%m-%d').replace(tzinfo=pytz.utc)
first_seen_time = datetime.strptime(str(raw_attr["Firstseen"]), '%Y-%m-%d %H:%M:%S').replace(tzinfo=pytz.utc)
if first_seen_time > last_seen_time:
last_seen_time = first_seen_time + timedelta(seconds=1)
attr.__setattr__('last_seen', last_seen_time)
else:
last_seen_time = datetime.strptime(str(raw_attr['Firstseen']), '%Y-%m-%d %H:%M:%S').replace(tzinfo=pytz.utc)
attr.__setattr__('last_seen', last_seen_time)
attr.to_ids = False
attr.disable_correlation = False
return attr
def create_attr_azorult(self, raw_attr: dict) -> MISPAttribute:
attr_list = []
for type in [{'json': 'domain', 'misp': 'domain'},
{'json': 'ip', 'misp': 'ip-dst'},
{'json': 'panel_index', 'misp': 'url'}]:
if type['json'] in raw_attr:
attr = MISPAttribute()
self.add_tag_to_attribute(attr, 'AzorultTracker')
self.add_tag_to_attribute(attr, raw_attr['panel_version'])
self.add_tag_to_attribute(attr, raw_attr['feeder'])
self.add_tag_to_attribute(attr, raw_attr['status'])
attr.comment = f'Azorult panel {type["misp"]}'
attr.__setattr__('first_seen', datetime.fromtimestamp(raw_attr['first_seen']))
attr.to_ids = False
attr.disable_correlation = False
attr.type = type['misp']
attr.value = f"{raw_attr[type['json']]}"
attr_list.append(attr)
return attr_list
@staticmethod
def get_attribute_tag_list(self, tag_list: list) -> list:
tags = []
for item in tag_list:
tags.append(item['name'])
return tags
@staticmethod
def create_event(title: str, date_added: datetime) -> MISPEvent:
misp_event = MISPEvent()
misp_event.info = title
if date_added != '':
misp_event.date = date_added
return misp_event
def get_event(self, event_id):
return self.misp.get_event(event_id, pythonify=True)
def add_attr_to_event(self, event: MISPEvent, attribute: MISPAttribute):
event.attributes.append(attribute)
return event
def update_event(self, event: MISPEvent):
return self.misp.update_event(event, pythonify=True)
def get_day_event(self, day: str, source: str, date: str):
if source in ['URLHaus', 'FeodoTracker']:
misp_event = self.misp.search('events', 'json', org=1, eventinfo=f'{source} import day {day}',
pythonify=True)
elif source in ['AzorultTracker']:
misp_event = self.misp.search('events', 'json', org=1, eventinfo=f'{source} import panel {day}',
pythonify=True)
if len(misp_event) >= 1:
return misp_event[0]
else:
if source in ['URLHaus', 'FeodoTracker']:
misp_event = self.create_event(f"{source} import day {day}",
date_added=datetime.timestamp(
datetime.strptime(date, '%Y-%m-%d %H:%M:%S')))
elif source in ['AzorultTracker']:
misp_event = self.create_event(f"{source} import panel {day}",
date_added=datetime.fromtimestamp(date))
event_id = self.misp.add_event(misp_event)
self.misp.publish(event_id)
return self.get_event(event_id)
@staticmethod
def delete_attribute_by_value(search_value: str, event: MISPEvent):
found = False
for a in event.attributes:
if (hasattr(a, 'value') and a.value == search_value):
a.deleted = True
return event
class MISP(Report):
"""MISP Analyzer."""
order = 1
def malpedia(self, results, event, malfamily):
if malfamily in name_update_shema:
malfamily = name_update_shema[malfamily]
if malfamily in malpedia_json:
self.misp.tag(event["uuid"], 'misp-galaxy:malpedia="{}"'.format(malfamily))
def signature(self, results, event):
for ttp in results.get("ttps", []) or []:
for i in ttps_json.get("objects", []) or []:
try:
if i["external_references"][0]["external_id"] == ttp:
self.misp.tag(event, f'misp-galaxy:mitre-attack-pattern="{i["name"]}-{ttp}"')
except Exception:
pass
def sample_hashes(self, results, event):
if results.get("target", {}).get("file", {}):
f = results["target"]["file"]
misp_object = MISPObject("file")
misp_object.comment = "File submitted to CAPEv2"
misp_object.add_attribute("filename", value=f["name"], category="Payload delivery")
misp_object.add_attribute("md5", value=f["md5"], category="Payload delivery")
misp_object.add_attribute("sha1", value=f["sha1"], category="Payload delivery")
misp_object.add_attribute("sha256", value=f["sha256"], category="Payload delivery")
misp_object.add_attribute("ssdeep", value=f["ssdeep"], category="Payload delivery")
self.misp.add_object(event, misp_object)
def all_network(self, results, event):
"""All of the accessed URLS as per the PCAP."""
urls = set()
if self.options.get("network", False) and "network" in results.keys():
urls = set()
for req in results["network"].get("http", []):
if "uri" in req and req["uri"] not in whitelist:
urls.add(req["uri"])
if "user-agent" in req:
event.add_attribute("user-agent", req["user-agent"])
domains, ips = {}, set()
for domain in results.get("network", {}).get("domains", []):
if domain["domain"] not in whitelist and domain["ip"] not in whitelist:
domains[domain["domain"]] = domain["ip"]
ips.add(domain["ip"])
for block in results.get("network", {}).get("hosts", []):
if block["ip"] not in whitelist:
ips.add(block["ip"])
for block in results["network"].get("dns", []): # Added DNS
if block.get("request", "") and (block["request"] not in whitelist):
if block["request"] not in domains and block["request"] not in whitelist:
if block["answers"]:
domains[block["request"]] = block["answers"][0]["data"]
ips.add(domain[block["answers"][0]["data"]])
# Added CAPE Addresses
for section in results.get("CAPE", []) or []:
try:
if section.get("cape_config", {}).get("address", []) or []:
for ip in section["cape_config"]["address"]:
if ip not in ips:
ips.add(ip.split(":")[0])
except Exception as e:
print(e)
for url in sorted(list(urls)):
event.add_attribute("url", url)
for ip in sorted(list(ips)):
event.add_attribute("ip-dst", ip)
for domain, ips in domains.items():
obj = MISPObject("domain-ip")
obj.add_attribute("domain", domain)
for ip in ips:
obj.add_attribute("ip", ip)
event.add_object(obj)
self.misp.update_event(event)
def dropped_files(self, results, event):
"""
if self.options.get("dropped", False) and "dropped" in results:
for entry in results["dropped"]:
if entry["md5"] and entry["md5"] not in whitelist:
self.misper["iocs"].append({"md5": entry["md5"]})
self.misper["iocs"].append({"sha1": entry["sha1"]})
self.misper["iocs"].append({"sha256": entry["sha256"]})
"""
"""
Add all the dropped files as MISP attributes.
"""
# Upload all the dropped files at once
# TODO: Use expanded
for r in results.get("dropped", []) or []:
with open(r.get("path"), "rb") as f:
event.add_attribute("malware-sample", value=os.path.basename(r.get("path")), data=BytesIO(f.read()), expand="binary")
event.run_expansions()
self.misp.update_event(event)
"""
# Load the event from MISP (we cannot use event as it
# does not contain the sample uploaded above, nor it is
# a MISPEvent but a simple dict)
e = MISPEvent()
e.from_dict(Event=self.misp.get_event(event["Event"]["id"])["Event"])
dropped_files = {
f.get_attributes_by_relation("sha1")[0].value: f
for f in e.objects if f["name"] == "file"
}
# Add further details on the dropped files
for entry in results.get("dropped", []):
# Find the corresponding object
sha1 = entry.get("sha1")
obj = dropped_files[sha1]
# Add the real location of the dropped file (during the analysis)
real_filepath = entry.get("guest_paths")
obj.add_attribute("fullpath", real_filepath[0])
# Add Yara matches if any
for match in entry.get("yara", []):
desc = match["meta"]["description"]
obj.add_attribute("text", value=desc, comment="Yara match")
# Update the event
self.misp.update_event(event_id=event["Event"]["id"], event=e)
"""
def run(self, results):
"""Run analysis.
@return: MISP results dict.
"""
url = self.options.get("url", "")
apikey = self.options.get("apikey", "")
if not url or not apikey:
log.error("MISP URL or API key not configured.")
return
self.misp = PyMISP(url, apikey, False, "json")
self.threads = self.options.get("threads", "")
if not self.threads:
self.threads = 5
self.iocs = deque()
self.misper = dict()
try:
if self.options.get("upload_iocs", False) and results.get("malscore", 0) >= self.options.get("min_malscore", 0):
distribution = int(self.options.get("distribution", 0))
threat_level_id = int(self.options.get("threat_level_id", 4))
analysis = int(self.options.get("analysis", 0))
tag = self.options.get("tag") or "CAPEv2"
info = self.options.get("title", "")
upload_sample = self.options.get("upload_sample")
malfamily = ""
if results.get("detections", ""):
malfamily = results["detections"]
response = self.misp.search("attributes", value=results["target"]["file"]["sha256"], return_format="json", pythonify=True)
if response:
event = self.misp.get_event(response[0].event_id, pythonify=True)
else:
event = MISPEvent()
event.distribution = distribution
event.threat_level_id = threat_level_id
event.analysis = analysis
event.info = "{} {} - {}".format(info, malfamily, results.get("info", {}).get("id"))
event = self.misp.add_event(event, pythonify=True)
# Add a specific tag to flag Cuckoo's event
if tag:
self.misp.tag(event, tag)
# malpedia galaxy
if malpedia_json:
self.malpedia(results, event, malfamily)
# ToDo?
self.signature(results, event)
self.sample_hashes(results, event)
self.all_network(results, event)
self.dropped_files(results, event)
if upload_sample:
target = results.get("target", {})
f = target.get("file", {})
if target.get("category") == "file" and f:
with open(f["path"], "rb") as f:
event.add_attribute(
"malware-sample",
value=os.path.basename(f["path"]),
data=BytesIO(f.read()),
expand="binary",
comment="Sample run",
)
if results.get("target", {}).get("url", "") and results["target"]["url"] not in whitelist:
event.add_attribute("url", results["target"]["url"])
# ToDo migth be outdated!
# if self.options.get("ids_files", False) and "suricata" in results.keys():
# for surifile in results["suricata"]["files"]:
# if "file_info" in surifile.keys():
# self.misper["iocs"].append({"md5": surifile["file_info"]["md5"]})
# self.misper["iocs"].append({"sha1": surifile["file_info"]["sha1"]})
# self.misper["iocs"].append({"sha256": surifile["file_info"]["sha256"]})
if self.options.get("mutexes", False) and "behavior" in results and "summary" in results["behavior"]:
if "mutexes" in results.get("behavior", {}).get("summary", {}):
for mutex in results["behavior"]["summary"]["mutexes"]:
if mutex not in whitelist:
event.add_attribute("mutex", mutex)
if self.options.get("registry", False) and "behavior" in results and "summary" in results["behavior"]:
if "read_keys" in results["behavior"].get("summary", {}):
for regkey in results["behavior"]["summary"]["read_keys"]:
event.add_attribute("regkey", regkey)
event.run_expansions()
self.misp.update_event(event)
# Make event public
if self.options.get("published", True):
self.misp.publish(event)
except Exception as e:
log.error("Failed to generate JSON report: %s" % e, exc_info=True)
nb_attr = response['response'][0]['attribute_count']
if last_event_date < date.today() or int(nb_attr) > 1000:
me = create_new_event()
else:
event_id = response['response'][0]['id']
else:
me = create_new_event()
parameters = {'banned-ip': args.banned_ip, 'attack-type': args.attack_type}
if args.processing_timestamp:
parameters['processing-timestamp'] = args.processing_timestamp
if args.failures:
parameters['failures'] = args.failures
if args.sensor:
parameters['sensor'] = args.sensor
if args.victim:
parameters['victim'] = args.victim
if args.logline:
parameters['logline'] = b64decode(args.logline).decode()
if args.logfile:
with open(args.logfile, 'rb') as f:
parameters['logfile'] = {'value': os.path.basename(args.logfile),
'data': BytesIO(f.read())}
f2b = Fail2BanObject(parameters=parameters, standalone=False)
if me:
me.add_object(f2b)
pymisp.add_event(me)
elif event_id:
template_id = pymisp.get_object_template_id(f2b.template_uuid)
a = pymisp.add_object(event_id, template_id, f2b)
misp_event=MISPEvent()
search=pm.search(controller='events', eventinfo=misp_event_name)
if ( len(search) == 1):
misp_event.load(search[0])
#load 'samples' dictionary from misp_event
for obj in misp_event.get('Object'):
if (obj.name == "file"):
existing_hash = obj.get_attributes_by_relation("sha256")[0]['value']
samples.update({existing_hash : obj})
#reload 'attributes' dictionary from misp_event
for attr in misp_event.get('Attribute'):
attributes.update({attr.value : attr})
else:
misp_event.info=misp_event_name
pm.add_event(misp_event)
#bazaar setup
r = requests.post(bazaar_url, data=bazaar_query)
bazaardata = json.loads(r.text)
# add File objects/samples that don't currently exist
for sample in bazaardata.get('data'):
hash_id=sample["sha256_hash"]
if (hash_id not in samples):
addSampleByHash(hash_id, misp_event)
pm.update_event(misp_event)
class MISP():
"""
Handle manipulation of event into MISP
TODO:
- use MISPEvent for creating MISP events
check https://www.circl.lu/assets/files/misp-training/luxembourg2017/4.1-pymisp.pdf
- use PyTaxonomies for Tags
"""
api = ""
url = ""
misp = ""
proxies = None
sslCheck = False # Not recommended
debug = False # Enable debug mode
score = { # Map Facebook Threat Exchange "status" to a score to determine if event will be made in MISP
0 : "NON MALICIOUS",
1 : "UNKNOWN",
2 : "SUSPICIOUS",
3 : "MALICIOUS"
}
badness_threshold = 1 # Minimum score in score table to create a MISP event
published = False # Default state of MISP created object
# ThreatExchange type -> MISP
# see IndicatorType object
# https://developers.facebook.com/docs/threat-exchange/reference/apis/indicator-type/v2.8
type_map = {} # Map of Facebook Threat Exchange to MISP types
share_levels = {} # Map of sharing levels (TLP -> TLP)
extra_tag = None # Extra tag to add to all imported events (! no check of consistency !)
privacy_levels = {} # Map the privacy_type of Facebook Threat Exchange to Sharing Group ID of MISP
# ----------------------------------------------------------------------- #
def __init__(self, url, api, proxies=None, feed=False):
self.url = url
self.api = api
self.proxies = proxies
if not feed:
self.misp = PyMISP(self.url, self.api, ssl=self.sslCheck, out_type='json', debug=self.debug, proxies=self.proxies, cert=None)
else:
self.misp = None
return
def convertTEtoMISP(self, teevent):
"""
Convert a ThreatExchange entry to MISP entry
"""
# Create empty event
mispevt = MISPEvent()
mispevt.info = "[Facebook ThreatExchange]"
mispevt.distribution = 0
mispevt.sharing_group_id = self.privacy_levels[teevent["privacy_type"]]
# Check if event is to be kept
if "status" in teevent.keys() and teevent["status"] in self.score.keys() and self.score[teevent["status"]] < self.badness_threshold :
print("IGNORE EVENT %s due to status (%s)" % (teevent, teevent["status"]))
return None
# Add indicator to event
if "raw_indicator" in teevent.keys():
if "type" in teevent.keys():
if teevent["type"] in self.type_map.keys():
indicator = teevent["raw_indicator"].replace("\\", "")
mispevt.add_attribute(self.type_map[teevent["type"]] , indicator) # not to brutal??
else:
print("WARNING: TYPE %s SHOULD BE ADDED TO MAPPING" % teevent["type"])
else:
print("WARNING, event %s does not contains any indicator :(" % teevent)
return None # don't create event without content!
# Add a category
mispevt.category = "Network activity"
# Enrich description
if "description" in teevent.keys():
mispevt.info = mispevt.info + " - %s" % teevent["description"]
if "owner" in teevent.keys() and "name" in teevent["owner"].keys():
owner = teevent["owner"]["name"]
if("email" in teevent["owner"].keys()):
email = teevent["owner"]["email"].replace("\\u0040", "@")
else:
email = ""
mispevt.info = mispevt.info + " - by %s (%s)" % (owner, email)
# Add sharing indicators (tags)
if "share_level" in teevent.keys():
if teevent["share_level"] in self.share_levels.keys():
mispevt.Tag.append(self.share_levels[teevent["share_level"]])
else:
print("WARNING: SHARING LEVEL %s SHOULD BE ADDED TO MAPPING" % teevent["share_level"])
if self.extra_tag is not None:
mispevt.Tag.append(self.extra_tag)
evtid = teevent["id"]
return [evtid, mispevt]
def convertTEtoMISPTEST(self, teevents=[]):
"""
Convert a ThreatExchange entry to MISP entry
"""
# Create empty event
mispevt = MISPEvent()
mispevt.info = "[Facebook ThreatExchange]"
mispevt.distribution = 0
mispevt.category = "Network activity"
share_level = "WHITE"
evtids = []
for teevent in teevents:
# Set event visiblity to VISIBLE except if stated otherwise in event
if(self.privacy_levels[teevent["privacy_type"]] != self.privacy_levels["VISIBLE"]):
mispevt.sharing_group_id = self.privacy_levels[teevent["privacy_type"]]
else:
mispevt.sharing_group_id = self.privacy_levels["VISIBLE"]
# Check if event is to be kept
if "status" in teevent.keys() and teevent["status"] in self.score.keys() and self.score[teevent["status"]] < self.badness_threshold :
print("IGNORE EVENT %s due to status (%s)" % (teevent, teevent["status"]))
continue
# Add indicator to event
if "raw_indicator" in teevent.keys():
if "type" in teevent.keys():
if teevent["type"] in self.type_map.keys():
indicator = teevent["raw_indicator"].replace("\\", "")
mispevt.add_attribute(self.type_map[teevent["type"]] , indicator) # not to brutal??
else:
print("WARNING: TYPE %s SHOULD BE ADDED TO MAPPING" % teevent["type"])
# Enrich description - last will be kept :-S
if "description" in teevent.keys():
mispevt.info = mispevt.info + " - %s" % teevent["description"]
# Ownership - last will be kept :-S
if "owner" in teevent.keys() and "name" in teevent["owner"].keys():
owner = teevent["owner"]["name"]
if("email" in teevent["owner"].keys()):
email = teevent["owner"]["email"].replace("\\u0040", "@")
else:
email = ""
mispevt.info = mispevt.info + " - by %s (%s)" % (owner, email)
# Add sharing indicators (tags) - keep more strict
if "share_level" in teevent.keys():
if teevent["share_level"] in self.share_levels.keys(): # sharing level has to be reduced
if int(self.share_levels[share_level]["id"]) > int(self.share_levels[teevent["share_level"]]["id"]):
share_level = teevent["share_level"]
else:
print("WARNING: SHARING LEVEL %s SHOULD BE ADDED TO MAPPING" % teevent["share_level"])
# Add Extra Tags
if self.extra_tag is not None:
mispevt.Tag.append(self.extra_tag)
# Add ID to list of ID making this event
evtids.append(teevent["id"])
# Set share level
mispevt.Tag.append(self.share_levels[share_level])
# Return new MISP event ready for import
return [evtids, mispevt]
def createEvent(self, mispevent):
"""
Create a new event in MISP using a hash table structure describing the event
"""
if mispevent is None:
return None
# Not empty event
jevent = json.dumps(mispevent, cls=EncodeUpdate)
misp_event = self.misp.add_event(jevent)
mispid = misp_event["Event"]["id"]
return mispid
def saveMapping(self, mapfile="./mapping.json"):
"""
Save internal mapping definition
"""
mappings = {
"Sharing" : self.share_levels,
"Type" : self.type_map,
"Extra-Tag" : self.extra_tag,
"Privacy" : self.privacy_levels
}
try:
fd = open(mapfile, "w")
json.dump(mappings, fd, sort_keys=True,indent=4,separators=(',', ': '))
fd.close()
except Exception as e:
print("IMPOSSIBLE TO SAVE MAPPINGS to %s" % mapfile)
print(e)
return
def loadMapping(self, mapfile="./mapping.json"):
"""
Restore internal mapping from saved JSON file
"""
try:
fd = open(mapfile, "r")
mappings = json.load(fd)
if "Sharing" in mappings.keys():
self.share_levels = mappings["Sharing"]
if "Type" in mappings.keys():
self.type_map = mappings["Type"]
if "Extra-Tag" in mappings.keys():
self.extra_tag = mappings["Extra-Tag"]
if "Privacy" in mappings.keys():
self.privacy_levels = mappings["Privacy"]
fd.close()
except Exception as e:
print("IMPOSSIBLE TO LOAD MAPPINGS from %s" % mapfile)
print(e)
sys.exit(0)
return
dom, seen, score = row
seendate=datetime.datetime.strptime(seen, '%Y-%m-%d')
if (seendate >= filterdate):
curEvent=MISPEvent()
eventName = '[{}] - Domaintools +70 malscore'.format(seen)
if (eventName not in events) :
print("searching for: {}".format(eventName))
search=pm.search(controller='events', eventinfo=eventName)
if ( len(search) == 1):
curEvent.load(search[0])
events.update({eventName : curEvent})
elif ( len(search) == 0):
curEvent.info=eventName
pm.add_event(curEvent)
events.update({eventName : curEvent})
else:
curEvent=events[eventName]
curEvent=events[eventName]
attr=curEvent.add_attribute('domain', dom)
curEvent.add_attribute_tag("ifx-vetting:score=\"{}\"".format(score), attr.uuid)
curEvent.add_attribute_tag("tlp:white", attr.uuid)
curEvent.add_attribute_tag("pandemic:covid-19=\"cyber\"", attr.uuid)
if ( int(score) >= 95 ):
curEvent.add_attribute_tag("estimative-language:likelihood-probability=\"almost-certain\"", attr.uuid)
elif ( int(score) >= 80):
curEvent.add_attribute_tag("estimative-language:likelihood-probability=\"very-likely\"", attr.uuid)
assert server_test["status"] == 1
assert server_test["post"] == 1
# Get remote user
url = f'servers/getRemoteUser/{remote_server["id"]}'
remote_user = pymisp._check_json_response(pymisp._prepare_request('GET', url))
check_response(remote_user)
assert remote_user["Sync flag"] == "Yes"
assert remote_user["Role name"] == "admin"
assert remote_user["User"] == "*****@*****.**"
# Create testing event
event = MISPEvent()
event.load_file(os.path.dirname(os.path.realpath(__file__)) + "/event.json")
pymisp.delete_event_blocklist(event)
event = pymisp.add_event(event, metadata=True)
check_response(event)
# Publish that event
check_response(pymisp.publish(event))
# Preview event
url = f'servers/previewEvent/{remote_server["id"]}/{event.uuid}'
event_preview = pymisp._check_json_response(pymisp._prepare_request(
'GET', url))
check_response(event_preview)
assert event_preview["Event"]["uuid"] == event.uuid
# Test pull
url = f'servers/pull/{remote_server["id"]}/disable_background_processing:1'
pull_response = pymisp._check_json_response(pymisp._prepare_request(
class TestBasic(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.misp = PyMISP(url, key, True, 'json', True)
def _clean_event(self, event):
event['Event'].pop('orgc_id', None)
event['Event'].pop('uuid', None)
event['Event'].pop('sharing_group_id', None)
event['Event'].pop('timestamp', None)
event['Event'].pop('org_id', None)
event['Event'].pop('date', None)
event['Event'].pop('RelatedEvent', None)
event['Event'].pop('publish_timestamp', None)
if event['Event'].get('Attribute'):
for a in event['Event'].get('Attribute'):
a.pop('uuid', None)
a.pop('event_id', None)
a.pop('id', None)
a.pop('timestamp', None)
if event['Event'].get('Orgc'):
event['Event']['Orgc'].pop('uuid', None)
event['Event']['Orgc'].pop('id', None)
if event['Event'].get('Org'):
event['Event']['Org'].pop('uuid', None)
event['Event']['Org'].pop('id', None)
return event['Event'].pop('id', None)
def new_event(self):
event = self.misp.new_event(0, 1, 0, "This is a test")
event_id = self._clean_event(event)
to_check = {
u'Event': {
u'info': u'This is a test',
u'locked': False,
u'attribute_count': u'0',
'disable_correlation': False,
u'analysis': u'0',
u'ShadowAttribute': [],
u'published': False,
u'distribution': u'0',
u'event_creator_email': u'*****@*****.**',
u'Attribute': [],
u'proposal_email_lock': False,
u'Org': {
u'name': u'ORGNAME'
},
u'Orgc': {
u'name': u'ORGNAME'
},
u'Galaxy': [],
u'threat_level_id': u'1'
}
}
self.assertEqual(event, to_check, 'Failed at creating a new Event')
return int(event_id)
def add_hashes(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.add_hashes(
event, 'Payload installation', 'dll_installer.dll',
'0a209ac0de4ac033f31d6ba9191a8f7a',
'1f0ae54ac3f10d533013f74f48849de4e65817a7',
'003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
'Fanny modules', False, 2)
self._clean_event(event)
to_check = {
u'Event': {
u'info':
u'This is a test',
u'locked':
False,
u'attribute_count':
u'3',
u'analysis':
u'0',
u'ShadowAttribute': [],
u'published':
False,
u'distribution':
u'0',
u'event_creator_email':
u'*****@*****.**',
u'Org': {
u'name': u'ORGNAME'
},
u'Orgc': {
u'name': u'ORGNAME'
},
u'Galaxy': [],
u'Attribute': [{
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|md5'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha1'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha256'
}],
u'proposal_email_lock':
False,
u'threat_level_id':
u'1'
}
}
self.assertEqual(event, to_check, 'Failed at adding hashes')
def publish(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.publish(event)
self._clean_event(event)
to_check = {
u'Event': {
u'info':
u'This is a test',
u'locked':
False,
u'attribute_count':
u'3',
u'analysis':
u'0',
u'ShadowAttribute': [],
u'published':
True,
u'distribution':
u'0',
u'event_creator_email':
u'*****@*****.**',
u'Org': {
u'name': u'ORGNAME'
},
u'Orgc': {
u'name': u'ORGNAME'
},
u'Galaxy': [],
u'Attribute': [{
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|md5'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha1'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha256'
}],
u'proposal_email_lock':
False,
u'threat_level_id':
u'1'
}
}
self.assertEqual(event, to_check, 'Failed at publishing event')
def delete(self, eventid):
event = self.misp.delete_event(eventid)
print(event)
def delete_attr(self, attrid):
event = self.misp.delete_attribute(attrid)
print(event)
def get(self, eventid):
event = self.misp.get_event(eventid)
print(event)
def get_stix(self, **kwargs):
event = self.misp.get_stix(kwargs)
print(event)
def add(self):
event = {
u'Event': {
u'info':
u'This is a test',
u'locked':
False,
u'attribute_count':
u'3',
u'analysis':
u'0',
u'ShadowAttribute': [],
u'published':
False,
u'distribution':
u'0',
u'event_creator_email':
u'*****@*****.**',
u'Attribute': [{
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|md5'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha1'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha256'
}],
u'proposal_email_lock':
False,
u'threat_level_id':
u'1'
}
}
event = self.misp.add_event(event)
print(event)
def add_user(self):
email = '*****@*****.**'
role_id = '5'
org_id = '1'
password = '******'
external_auth_required = False
external_auth_key = ''
enable_password = False
nids_sid = '1238717'
server_id = '1'
gpgkey = ''
certif_public = ''
autoalert = False
contactalert = False
disabled = False
change_pw = '0'
termsaccepted = False
newsread = '0'
authkey = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
to_check = {
'User': {
'email': email,
'org_id': org_id,
'role_id': role_id,
'password': password,
'external_auth_required': external_auth_required,
'external_auth_key': external_auth_key,
'enable_password': enable_password,
'nids_sid': nids_sid,
'server_id': server_id,
'gpgkey': gpgkey,
'certif_public': certif_public,
'autoalert': autoalert,
'contactalert': contactalert,
'disabled': disabled,
'change_pw': change_pw,
'termsaccepted': termsaccepted,
'newsread': newsread,
'authkey': authkey
}
}
user = self.misp.add_user(
email=email,
role_id=role_id,
org_id=org_id,
password=password,
external_auth_required=external_auth_required,
external_auth_key=external_auth_key,
enable_password=enable_password,
nids_sid=nids_sid,
server_id=server_id,
gpgkey=gpgkey,
certif_public=certif_public,
autoalert=autoalert,
contactalert=contactalert,
disabled=disabled,
change_pw=change_pw,
termsaccepted=termsaccepted,
newsread=newsread,
authkey=authkey)
# delete user to allow reuse of test
uid = user.get('User').get('id')
self.misp.delete_user(uid)
# ----------------------------------
# test interesting keys only (some keys are modified(password) and some keys are added (lastlogin)
tested_keys = [
'email', 'org_id', 'role_id', 'server_id', 'autoalert', 'authkey',
'gpgkey', 'certif_public', 'nids_sid', 'termsaccepted', 'newsread',
'contactalert', 'disabled'
]
for k in tested_keys:
self.assertEqual(
user.get('User').get(k),
to_check.get('User').get(k),
"Failed to match input with output on key: {}".format(k))
def add_organisation(self):
name = 'Organisation tests'
description = 'This is a test organisation'
orgtype = 'Type is a string'
nationality = 'French'
sector = 'Bank sector'
uuid = '16fd2706-8baf-433b-82eb-8c7fada847da'
contacts = 'Text field with no limitations'
local = False
to_check = {
'Organisation': {
'name': name,
'description': description,
'type': orgtype,
'nationality': nationality,
'sector': sector,
'uuid': uuid,
'contacts': contacts,
'local': local
}
}
org = self.misp.add_organisation(
name=name,
description=description,
type=orgtype,
nationality=nationality,
sector=sector,
uuid=uuid,
contacts=contacts,
local=local,
)
# delete organisation to allow reuse of test
oid = org.get('Organisation').get('id')
self.misp.delete_organisation(oid)
# ----------------------------------
tested_keys = [
'anonymise', 'contacts', 'description', 'local', 'name',
'nationality', 'sector', 'type', 'uuid'
]
for k in tested_keys:
self.assertEqual(
org.get('Organisation').get(k),
to_check.get('Organisation').get(k),
"Failed to match input with output on key: {}".format(k))
def test_create_event(self):
eventid = self.new_event()
time.sleep(1)
self.delete(eventid)
def test_get_event(self):
eventid = self.new_event()
time.sleep(1)
self.get(eventid)
time.sleep(1)
self.delete(eventid)
def test_add_event(self):
self.add()
time.sleep(1)
self.delete(1)
def test_del_attr(self):
eventid = self.new_event()
time.sleep(1)
self.delete_attr(1)
time.sleep(1)
self.delete(eventid)
def test_one_or_more(self):
self.assertEqual(self.misp._one_or_more(1), (1, ))
self.assertEqual(self.misp._one_or_more([1]), [1])
def test_create_user(self):
self.add_user()
def test_create_organisation(self):
self.add_organisation()
attribute_values['value'] = value
else:
# No value, ignoring
continue
comment = extract_field(item, 'Comment')
if comment:
attribute_values["comment"] = '{} {}'.format(
attribute_values["comment"], comment)
# change value to composite
# 127.0.0.1:80 ip-* to 127.0.0.1|80 ip-*|port
if mapping['type'] in ['ip-src', 'ip-dst'] and value.count(':') == 1:
attribute_values['type'] = mapping['type'] + '|port'
attribute_values['value'] = attribute_values['value'].replace(
':', '|')
misp_event.add_attribute(**attribute_values)
return misp_event
if __name__ == '__main__':
# test file for composite
# https://github.com/fireeye/iocs/blob/master/BlogPosts/9cee306d-5441-4cd3-932d-f3119752634c.ioc
x = open('test.ioc', 'r')
mispEvent = load_openioc(x.read())
#~ print(mispEvent._json_full())
from pymisp import PyMISP
misp = PyMISP('http://misp.local', 'xxxxx')
r = misp.add_event(mispEvent)
print(r)
class MISP():
def __init__(self, config: Dict[str, Any]):
self.logger = logging.getLogger(f'{self.__class__.__name__}')
self.logger.setLevel(get_config('generic', 'loglevel'))
if not config.get('apikey'):
self.available = False
self.logger.info('Module not enabled.')
return
self.available = True
self.enable_lookup = False
self.enable_push = False
self.allow_auto_trigger = False
try:
self.client = PyMISP(url=config['url'], key=config['apikey'],
ssl=config['verify_tls_cert'], timeout=config['timeout'])
except Exception as e:
self.available = False
self.logger.warning(f'Unable to connect to MISP: {e}')
return
if config.get('enable_lookup'):
self.enable_lookup = True
if config.get('enable_push'):
self.enable_push = True
if config.get('allow_auto_trigger'):
self.allow_auto_trigger = True
self.default_tags: List[str] = config.get('default_tags') # type: ignore
self.auto_publish = config.get('auto_publish')
self.storage_dir_misp = get_homedir() / 'misp'
self.storage_dir_misp.mkdir(parents=True, exist_ok=True)
self.psl = get_public_suffix_list()
def get_fav_tags(self):
return self.client.tags(pythonify=True, favouritesOnly=1)
def _prepare_push(self, to_push: Union[List[MISPEvent], MISPEvent], allow_duplicates: bool=False, auto_publish: Optional[bool]=False) -> Union[List[MISPEvent], Dict]:
'''Adds the pre-configured information as required by the instance.
If duplicates aren't allowed, they will be automatically skiped and the
extends_uuid key in the next element in the list updated'''
if isinstance(to_push, MISPEvent):
events = [to_push]
else:
events = to_push
events_to_push = []
existing_uuid_to_extend = None
for event in events:
if not allow_duplicates:
existing_event = self.get_existing_event(event.attributes[0].value)
if existing_event:
existing_uuid_to_extend = existing_event.uuid
continue
if existing_uuid_to_extend:
event.extends_uuid = existing_uuid_to_extend
existing_uuid_to_extend = None
for tag in self.default_tags:
event.add_tag(tag)
if auto_publish:
event.publish()
events_to_push.append(event)
return events_to_push
def push(self, to_push: Union[List[MISPEvent], MISPEvent], allow_duplicates: bool=False, auto_publish: Optional[bool]=None) -> Union[List[MISPEvent], Dict]:
if auto_publish is None:
auto_publish = self.auto_publish
if self.available and self.enable_push:
events = self._prepare_push(to_push, allow_duplicates, auto_publish)
if not events:
return {'error': 'All the events are already on the MISP instance.'}
if isinstance(events, Dict):
return {'error': events}
to_return = []
for event in events:
try:
# NOTE: POST the event as published publishes inline, which can tak a long time.
# Here, we POST as not published, and trigger the publishing in a second call.
if hasattr(event, 'published'):
background_publish = event.published
else:
background_publish = False
if background_publish:
event.published = False
new_event = self.client.add_event(event, pythonify=True)
if background_publish and isinstance(new_event, MISPEvent):
self.client.publish(new_event)
except requests.exceptions.ReadTimeout:
return {'error': 'The connection to MISP timed out, try increasing the timeout in the config.'}
if isinstance(new_event, MISPEvent):
to_return.append(new_event)
else:
return {'error': new_event}
return to_return
else:
return {'error': 'Module not available or push not enabled.'}
def get_existing_event_url(self, permaurl: str) -> Optional[str]:
attributes = self.client.search('attributes', value=permaurl, limit=1, page=1, pythonify=True)
if not attributes or not isinstance(attributes[0], MISPAttribute):
return None
url = f'{self.client.root_url}/events/{attributes[0].event_id}'
return url
def get_existing_event(self, permaurl: str) -> Optional[MISPEvent]:
attributes = self.client.search('attributes', value=permaurl, limit=1, page=1, pythonify=True)
if not attributes or not isinstance(attributes[0], MISPAttribute):
return None
event = self.client.get_event(attributes[0].event_id, pythonify=True)
if isinstance(event, MISPEvent):
return event
return None
def lookup(self, node: URLNode, hostnode: HostNode) -> Union[Dict[str, Set[str]], Dict[str, Any]]:
if self.available and self.enable_lookup:
tld = self.psl.get_tld(hostnode.name)
domain = re.sub(f'.{tld}$', '', hostnode.name).split('.')[-1]
to_lookup = [node.name, hostnode.name, f'{domain}.{tld}'] + hostnode.resolved_ips
if hasattr(hostnode, 'cnames'):
to_lookup += hostnode.cnames
if not node.empty_response:
to_lookup.append(node.body_hash)
if attributes := self.client.search(controller='attributes', value=to_lookup,
enforce_warninglist=True, pythonify=True):
if isinstance(attributes, list):
to_return: Dict[str, Set[str]] = defaultdict(set)
# NOTE: We have MISPAttribute in that list
for a in attributes:
to_return[a.event_id].add(a.value) # type: ignore
return to_return
else:
# The request returned an error
return attributes # type: ignore
return {'info': 'No hits.'}
else:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import argparse
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
from pymisp.tools import load_openioc_file
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Convert an OpenIOC file to a MISPEvent. Optionnaly send it to MISP.')
parser.add_argument("-i", "--input", required=True, help="Input file")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-o", "--output", help="Output file")
group.add_argument("-m", "--misp", action='store_true', help="Create new event on MISP")
args = parser.parse_args()
misp_event = load_openioc_file(args.input)
if args.misp:
pymisp = PyMISP(misp_url, misp_key, misp_verifycert, debug=True)
pymisp.add_event(misp_event)
else:
with open(args.output, 'w') as f:
f.write(misp_event.to_json())
