def from_remote(self, event_id):
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
misp = PyMISP(misp_url, misp_key, misp_verifycert)
result = misp.get(event_id)
self.misp_event = MISPEvent()
self.misp_event.load(result)
def test_deleteEvent(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
d = pymisp.delete_event(2)
self.assertEqual(d, {'message': 'Event deleted.'})
d = pymisp.delete_event(3)
self.assertEqual(d, {'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'})
def test_getVersions(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
api_version = pymisp.get_api_version()
self.assertEqual(api_version, {"version": pm.__version__})
server_version = pymisp.get_version()
self.assertEqual(server_version, {"version": "2.4.56"})
def test_auth_error(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
error = pymisp.get(1)
response = self.auth_error_msg
response["errors"] = [response["message"]]
self.assertEqual(error, response)
def test_getEvent(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
e1 = pymisp.get_event(2)
e2 = pymisp.get(2)
self.assertEqual(e1, e2)
self.assertEqual(self.event, e2)
def test_tag_event(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
uuid = self.event["Event"]["uuid"]
pymisp.tag(uuid, "foo")
self.assertRaises(pm.PyMISPError, pymisp.tag, "test_uuid", "foo")
self.assertRaises(pm.PyMISPError, pymisp.tag, uuid.replace("a", "z"), "foo")
def test_freetext_offdomain(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
try:
pymisp.freetext(1, None, adhereToWarninglists='hard')
self.assertFalse('Exception required for off domain value')
except Exception:
pass
def test_change_toids_invalid(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
try:
pymisp.change_toids(self.key, 42)
self.assertFalse('Exception required for off domain value')
except Exception:
pass
def test_flatten_error_messages_singular(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
pymisp.get(1)
response = self.auth_error_msg
response['error'] = ['foo', 'bar', 'baz']
messages = pymisp.flatten_error_messages(response)
self.assertEqual(["foo", "bar", "baz"], messages)
def test_publish(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
e = pymisp.publish(self.event)
pub = self.event
pub['Event']['published'] = True
self.assertEqual(e, pub)
e = pymisp.publish(self.event)
self.assertEqual(e, {'error': 'Already published'})
def test_deleteEvent(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
d = pymisp.delete_event(2)
self.assertEqual(d, {"message": "Event deleted."})
d = pymisp.delete_event(3)
self.assertEqual(
d, {"errors": ["Invalid event"], "message": "Invalid event", "name": "Invalid event", "url": "/events/3"}
)
def test_flatten_error_messages_plural(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
error = pymisp.get(1)
self.assertIn("Authentication failed", error["message"])
response = self.auth_error_msg
response['errors'] = {'foo': 42, 'bar': False, 'baz': ['oo', 'ka']}
messages = pymisp.flatten_error_messages(response)
self.assertEqual(set(['42 (foo)', 'False (bar)', 'oo', 'ka']), set(messages))
def test_updateEvent(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
e0 = pymisp.update_event("5758ebf5-c898-48e6-9fe9-5665c0a83866", json.dumps(self.event))
e1 = pymisp.update_event("5758ebf5-c898-48e6-9fe9-5665c0a83866", self.event)
self.assertEqual(e0, e1)
e2 = pymisp.update(e0)
self.assertEqual(e1, e2)
self.assertEqual(self.event, e2)
def test_updateEvent(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
e0 = pymisp.update_event(2, json.dumps(self.event))
e1 = pymisp.update_event(2, self.event)
self.assertEqual(e0, e1)
e2 = pymisp.update(e0)
self.assertEqual(e1, e2)
self.assertEqual(self.event, e2)
def test_publish(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
e = pymisp.publish(self.event) # requests-mock always return the non-published event
pub = self.event
pub["Event"]["published"] = True
# self.assertEqual(e, pub) FIXME: broken test, not-published event returned
e = pymisp.publish(self.event)
self.assertEqual(e, {"error": "Already published"})
def test_flatten_error_messages_nested(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
error = pymisp.get(1)
self.assertIn("Authentication failed", error["message"])
response = self.auth_error_msg
response['errors'] = {
'fo': {'o': 42}, 'ba': {'r': True}, 'b': {'a': ['z']}, 'd': {'e': {'e': ['p']}}}
messages = pymisp.flatten_error_messages(response)
self.assertEqual(set(['Error in o: 42', 'Error in r: True', 'Error in a: z', "Error in e: {'e': ['p']}"]), set(messages))
def test_newEvent(self, m):
error_empty_info = {'message': 'The event could not be saved.', 'name': 'Add event failed.', 'errors': ['Error in info: Info cannot be empty.'], 'url': '/events/add'}
error_empty_info_flatten = {u'message': u'The event could not be saved.', u'name': u'Add event failed.', u'errors': [u"Error in info: Info cannot be empty."], u'url': u'/events/add'}
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
m.register_uri('POST', self.domain + 'events', json=error_empty_info)
# TODO Add test exception if info field isn't set
response = pymisp.new_event(0, 1, 0, 'Foo')
self.assertEqual(response, error_empty_info_flatten)
m.register_uri('POST', self.domain + 'events', json=self.new_misp_event)
response = pymisp.new_event(0, 1, 0, "This is a test.", '2016-08-26', False)
self.assertEqual(response, self.new_misp_event)
def test_newEvent(self, m):
error_empty_info = {'message': 'The event could not be saved.', 'name': 'Add event failed.', 'errors': ['Error in info: Info cannot be empty.'], 'url': '/events/add'}
error_empty_info_flatten = {u'message': u'The event could not be saved.', u'name': u'Add event failed.', u'errors': [u"Error in info: Info cannot be empty."], u'url': u'/events/add'}
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
m.register_uri('POST', self.domain + 'events', json=error_empty_info)
# TODO Add test exception if info field isn't set
response = pymisp.new_event(0, 1, 0, 'Foo')
self.assertEqual(response, error_empty_info_flatten)
m.register_uri('POST', self.domain + 'events', json=self.new_misp_event)
response = pymisp.new_event(0, 1, 0, "This is a test.", '2016-08-26', False)
self.assertEqual(response, self.new_misp_event)
def send(api: PyMISP,
request_type: str,
url: str,
data=None,
check_errors: bool = True) -> dict:
if data is None:
data = {}
response = api._prepare_request(request_type, url, data=data)
response = api._check_json_response(response)
if check_errors:
check_response(response)
return response
def create_or_update_site_admin(connector: PyMISP, user: MISPUser) -> MISPUser:
to_return_user = connector.add_user(user)
if isinstance(to_return_user, MISPUser):
return to_return_user
# The user already exists
for u in connector.users():
if u.email == user.email:
to_return_user = connector.update_user(user, u.id) # type: ignore
if isinstance(to_return_user, MISPUser):
return to_return_user
raise Exception(f'Unable to update {user.email}: {to_return_user}')
else:
raise Exception(f'Unable to create {user.email}: {to_return_user}')
def init(self):
if PyMISP is None:
raise ValueError('Could not import pymisp. Please install it.')
# Initialize MISP connection
self.misp = PyMISP(self.parameters.misp_url, self.parameters.misp_key,
self.parameters.misp_verify)
# URLs used for deleting and adding MISP event tags
self.misp_add_tag_url = urljoin(self.parameters.misp_url,
'events/addTag')
self.misp_del_tag_url = urljoin(self.parameters.misp_url,
'events/removeTag')
def __init__(self,
misp_key,
misp_url,
misp_verify_cert,
siem_mode=False,
debugon=False):
self.misp = PyMISP(misp_url, misp_key, misp_verify_cert, 'json')
self.debugon = debugon
if siem_mode:
self.siem_mode = True
self.separator = ","
self.use_headers = True
self.use_filename_regex = False
def test_deleteEvent(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
d = pymisp.delete_event(2)
self.assertEqual(d, {'message': 'Event deleted.'})
d = pymisp.delete_event(3)
self.assertEqual(
d, {
'errors': ['Invalid event'],
'message': 'Invalid event',
'name': 'Invalid event',
'url': '/events/3'
})
def pcap_analysis(analysis_id):
"""
Analyze the pcap file, first extract the IOCs then check in the MISP server
"""
analysis = Analysis.objects.get(pk=analysis_id)
print('analyzing : %s' % analysis_id)
try:
# Fix ME: log issues PyMISPError here
server = PyMISP(settings.MISP_SERVER, settings.MISP_KEY, True, 'json')
pc = Pcap(settings.TEMP_FOLDER + '/' + analysis_id + '.pcap')
malicious = False
results = []
pc.extract_indicators()
analysis.status = AnalysisStatus.SEARCH
analysis.save()
print('Searching for %i indicators' % len(pc.indicators))
for ind in pc.indicators:
events = server.search(values = [ind['value']])
if len(events['response']) > 0:
malicious = True
for event in events['response']:
results.append({
'indicator': ind['value'],
'event': event['Event']['info'],
'event_id': event['Event']['id']
})
except PyMISPError:
# TODO: create log here
analysis.status = AnalysisStatus.ERROR
analysis.result = AnalysisResult.ERROR
analysis.save()
else:
for alert in results:
a = Alert()
a.analysis = analysis
a.indicator = alert['indicator']
a.event_id = int(alert['event_id'])
a.event_name = alert['event']
a.save()
if malicious:
analysis.result = AnalysisResult.MALICIOUS
else:
analysis.result = AnalysisResult.NOTHING
analysis.status = AnalysisStatus.DONE
analysis.save()
# Securely delete the pcap file
subprocess.check_call([
'shred',
'-zu',
settings.TEMP_FOLDER + '/' + analysis_id + '.pcap']
)
def test_user_must_change_password_by_myself(self):
# Admin set that user must change password
updated_user = self.admin_misp_connector.update_user({'change_pw': 1},
self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "1")
# User try to change back trough API
logged_in = PyMISP(url, self.test_usr.authkey)
logged_in.update_user({'change_pw': 0}, self.test_usr)
updated_user = self.admin_misp_connector.get_user(self.test_usr)
# Should not be possible
self.assertEqual(updated_user.change_pw, "1")
def misp_connection(url, misp_key, proxies_usage):
try:
if proxies_usage:
proxies = {}
proxies ["http"] = config_parser("misp","http")
proxies ["https"] = config_parser("misp","https")
misp = PyMISP(url, misp_key, False, 'json', proxies=proxies)
else:
misp = PyMISP(url, misp_key, False, 'json',None)
except PyMISPError:
print("\t [!] Error connecting to MISP instance. Check if your MISP instance it's up!")
return None
return misp
def test_sample_upload(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
pymisp.upload_sample("tmux", "tests/viper-test-files/test_files/tmux",
1)
pymisp.upload_sample("tmux", "non_existing_file", 1)
pymisp.upload_sample("tmux", b"binblob", 1)
def _build_iterator(self, now):
"""
Queries specified MISP instance to get events, filter them according to given filters and returns events.
:param now: unused
:return: itarable. Holds the events returned by the API that matched the given filters.
"""
# Sanity checks
if self.automation_key is None:
raise RuntimeError('{} - MISP Automation Key not set'.format(
self.name))
if self.url is None:
raise RuntimeError('{} - MISP URL not set'.format(self.name))
kwargs = {'ssl': self.verify_cert}
if self.verify_cert and 'REQUESTS_CA_BUNDLE' in os.environ:
kwargs['ssl'] = os.environ['REQUESTS_CA_BUNDLE']
if self.client_cert_required:
kwargs['cert'] = (self.cert_file, self.key_file)
# Establish API connection
misp = PyMISP(self.url, self.automation_key, **kwargs)
filters = None
if self.filters is not None:
# Map self.filters in the way they are given by the configuration to the way they are required in order
# to actually filter the events returned by the API.
filters = self.filters.copy()
if 'datefrom' in filters:
filters['timestamp'] = filters.pop('datefrom')
du = filters.pop('dateuntil', None)
if du is not None:
filters['dateuntil'] = du
if 'event_tags' in filters and filters['event_tags']:
filters['tags'] = [
x.strip() for x in filters['event_tags'].split(",")
]
LOG.info('{} - query filters: {!r}'.format(self.name, filters))
# Get index from all events via API matching the given filter
r = misp.get_index(filters)
events = r['response']
return imap(partial(self._load_event, misp), events)
def __init__(self, config):
# Initialize config
self.config = config
# Initialize MISP
self.misp = PyMISP(self.config['misp']['url'], self.config['misp']['key'], False, 'json')
# Initialize OpenCTI client
self.opencti = OpenCTI(
self.config['opencti']['api_url'],
self.config['opencti']['api_key'],
os.path.dirname(os.path.abspath(__file__)) + '/misp.log',
True
)
def _misp_create_event_function(self, event, *args, **kwargs):
"""Function: create a MISP event from an incident """
try:
def get_config_option(option_name, optional=False):
"""Given option_name, checks if it is in app.config. Raises ValueError if a mandatory option is missing"""
option = self.options.get(option_name)
if option is None and optional is False:
err = "'{0}' is mandatory and is not set in ~/.resilient/app.config file. You must set this value to run this function".format(option_name)
raise ValueError(err)
else:
return option
API_KEY = get_config_option("misp_key")
URL = get_config_option("misp_url")
VERIFY_CERT = True if get_config_option("verify_cert").lower() == "true" else False
# Get the function parameters:
misp_event_name = kwargs.get("misp_event_name") # text
misp_distribution = kwargs.get("misp_distribution") # number
misp_analysis_level = kwargs.get("misp_analysis_level") # number
misp_threat_level = kwargs.get("misp_threat_level") # number
log = logging.getLogger(__name__)
log.info("misp_event_name: %s", misp_event_name)
log.info("misp_distribution: %s", misp_distribution)
log.info("misp_analysis_level: %s", misp_analysis_level)
log.info("misp_threat_level: %s", misp_threat_level)
yield StatusMessage("Setting up connection to MISP")
misp_client = PyMISP(URL, API_KEY, VERIFY_CERT, 'json')
event = misp_client.new_event(misp_distribution, misp_threat_level, misp_analysis_level, misp_event_name)
log.info(event)
yield StatusMessage("Event has been created")
results = {
"success": True,
"content": event
}
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception:
yield FunctionError()
def getMISPInfo(hash):
"""
Retrieves information from a MISP instance
:param hash: hash value
:return info: info object
"""
info = {'misp_available': False, 'misp_events': ''}
requests.packages.urllib3.disable_warnings() # I don't care
if MISP_API_KEY == "-":
return info
# Prepare API request
if args.debug:
print("[D] Querying MISP: %s" % MISP_URL)
misp = PyMISP(MISP_URL, MISP_API_KEY, args.verifycert, 'json')
try:
if args.debug:
print("[D] Query: values=%s" % hash)
result = misp.search('attributes', values=[hash])
if result['response']:
misp_info = []
misp_events = []
if args.debug:
print(json.dumps(result['response']))
for r in result['response']["Attribute"]:
# Try to get info on the events
event_info = ""
misp_events.append(r['event_id'])
e_result = misp.search('events', eventid=r['event_id'])
if e_result['response']:
event_info = e_result['response'][0]['Event']['info']
# too much
#if args.debug:
# print(json.dumps(e_result['response']))
# Create MISP info object
misp_info.append({
'event_info': event_info,
'event_id': r['event_id'],
'comment': r['comment'],
})
info['misp_info'] = misp_info
info['misp_events'] = ",".join(misp_events)
info['misp_available'] = True
else:
info['misp_available'] = False
except Exception as e:
if args.debug:
traceback.print_exc()
return info
class MISPCollectorBot(CollectorBot):
def init(self):
if PyMISP is None:
self.logger.error('Could not import pymisp. Please install it.')
self.stop()
# Initialise MISP connection
self.misp = PyMISP(self.parameters.misp_url,
self.parameters.misp_key,
self.parameters.misp_verify)
# URLs used for deleting and adding MISP event tags
self.misp_add_tag_url = urljoin(self.parameters.misp_url,
'events/addTag')
self.misp_del_tag_url = urljoin(self.parameters.misp_url,
'events/removeTag')
def process(self):
# Grab the events from MISP
misp_result = self.misp.search(
tags=self.parameters.misp_tag_to_process
)
# Process the response and events
if 'response' in misp_result:
# Extract the MISP event details
for e in misp_result['response']:
misp_event = e['Event']
# Send the results to the parser
report = self.new_report()
report.add('raw', json.dumps(misp_event, sort_keys=True))
report.add('feed.url', self.parameters.misp_url)
self.send_message(report)
# Finally, update the tags on the MISP events.
# Note PyMISP does not currently support this so we use
# the API URLs directly with the requests module.
for misp_event in misp_result['response']:
# Remove the 'to be processed' tag
self.misp.remove_tag(misp_event,
self.parameters.misp_tag_to_process)
# Add a 'processed' tag to the event
self.misp.add_tag(misp_event,
self.parameters.misp_tag_processed)
def each_with_type(self, target, file_type):
search_result = False
self.misp = PyMISP(self.misp_api_endpoint, self.misp_api_key, False)
if file_type == 'url':
misp_result = self.misp.search_all(keyword)
if 'response' in misp_result:
# Extract the MISP event details
for e in misp_result['response']:
search_result = True
misp_event = e['Event']
for tag in misp_event['Tag']:
self.add_tag(tag['name'])
Attribute = misp_event['Attribute']
self.results[misp_event['info']] = self.results.get(misp_event['info'], [])
for a in Attribute:
self.results[misp_event['info']].append({
'date':misp_event['date'],
'comment':a.get('comment'),
'value':a.get('value'),
'type':a.get('type'),
'category':a.get('category')
})
else:
with open(target) as f:
data = f.read()
md5 = hashlib.md5(data).hexdigest()
sha256 = hashlib.sha256(data).hexdigest()
search_content = [md5, sha256]
for keyword in search_content:
misp_result = self.misp.search_all(keyword)
# Process the response and events
if 'response' in misp_result:
# Extract the MISP event details
for e in misp_result['response']:
search_result = True
misp_event = e['Event']
for tag in misp_event['Tag']:
self.add_tag(tag['name'])
Attribute = misp_event['Attribute']
self.results[misp_event['info']] = self.results.get(misp_event['info'], [])
for a in Attribute:
self.results[misp_event['info']].append({
'date':misp_event['date'],
'comment':a.get('comment'),
'value':a.get('value'),
'type':a.get('type'),
'category':a.get('category')
})
return search_result
def getMISPData(self, since=None):
# Connect to your MISP API
misp = PyMISP(self.url, self.key, True, 'json')
since = since if since else "5d"
if since.lower() == "all": since = ""
misp_last = misp.download_last(since)
# Verify output
if 'message' in misp_last.keys():
if misp_last['message'].lower().startswith('no matches'):
return [] # No output
elif misp_last['message'].startswith('Authentication failed.'):
raise Exception("[-] MISP Authentication failed")
if not 'response' in misp_last:
raise Exception("[-] Error occured while fetching MISP data")
return misp_last['response']
def test_change_login_disabled(self):
with MISPSetting(self.admin_misp_connector,
"MISP.disable_user_login_change", True):
new_email = 'testusr@user' + random() + '.local'
logged_in = PyMISP(url, self.test_usr.authkey)
logged_in.global_pythonify = True
# Try to change email
updated_user = logged_in.update_user({'email': new_email},
self.test_usr)
check_response(updated_user)
# Change should be not successful
self.assertEqual(self.test_usr.email, updated_user.email)
def get_misp_instance():
"""
:return: MISP Instance
:rtype: PyMISP
"""
# Proxy settings are taken from the config file and converted to a dict.
if PySight_settings.MISP_PROXY:
misp_proxies = {
'http': str(PySight_settings.PROXY_URL),
'https': str(PySight_settings.PROXY_URL)
}
else:
misp_proxies = {}
try:
# URL of the MISP instance, API key and SSL certificate validation are taken from the config file.
return PyMISP(PySight_settings.MISP_URL,
PySight_settings.MISP_KEY,
PySight_settings.MISP_VERIFYCERT,
proxies=misp_proxies)
#return PyMISP(PySight_settings.MISP_URL, PySight_settings.MISP_KEY, PySight_settings.MISP_VERIFYCERT,
# proxies=misp_proxies)
except Exception:
PySight_settings.logger.error('Unexpected error in MISP init: %s',
sys.exc_info())
return False
def get_misp_instance():
"""
:return: MISP Instance
:rtype: PyMISP
"""
print('****hello****in get_misp_instance')
# Proxy settings are taken from the config file and converted to a dict.
if PySight_settings.USE_MISP_PROXY:
misp_proxies = {
'http': str(PySight_settings.proxy_address),
'https': str(PySight_settings.proxy_address)
}
else:
misp_proxies = {}
try:
# URL of the MISP instance, API key and SSL certificate validation are taken from the config file.
return ExpandedPyMISP(PySight_settings.misp_url,
PySight_settings.misp_key,
PySight_settings.misp_verifycert)
return PyMISP(PySight_settings.misp_url, PySight_settings.misp_key,
PySight_settings.misp_verifycert)
except Exception:
PySight_settings.logger.error('Unexpected error in MISP init: %s',
sys.exc_info())
return False
def init(self):
if PyMISP is None:
raise ValueError('Could not import pymisp. Please install it.')
# Initialize MISP connection
self.misp = PyMISP(self.parameters.misp_url, self.parameters.misp_key,
self.parameters.misp_verify)
def test_eventObject(self, m):
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
misp_event = MISPEvent(pymisp.describe_types)
with open('tests/57c4445b-c548-4654-af0b-4be3950d210f.json', 'r') as f:
misp_event.load(f.read())
json.dumps(misp_event, cls=MISPEncode)
class MISP():
def __init__(self, config: Dict[str, Any]):
self.logger = logging.getLogger(f'{self.__class__.__name__}')
self.logger.setLevel(get_config('generic', 'loglevel'))
if not config.get('apikey'):
self.available = False
self.logger.info('Module not enabled.')
return
self.available = True
self.enable_lookup = False
self.enable_push = False
try:
self.client = PyMISP(url=config['url'],
key=config['apikey'],
ssl=config['verify_tls_cert'])
except Exception as e:
self.available = False
self.logger.warning(f'Unable to connect to MISP: {e}')
return
if config.get('enable_lookup'):
self.enable_lookup = True
if config.get('enable_push'):
self.enable_push = True
self.storage_dir_misp = get_homedir() / 'misp'
self.storage_dir_misp.mkdir(parents=True, exist_ok=True)
def push(self, event: MISPEvent) -> Union[MISPEvent, Dict]:
if self.available and self.enable_push:
return self.client.add_event(event, pythonify=True)
else:
return {'error': 'Module not available or push not enabled.'}
def init():
"""
Initialize PyMISP
Get configuration settings from config file
"""
global source
source = PyMISP(misp_url, misp_key, misp_verifycert, 'json')
def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, analysis=0, event_info=None):
if event_info:
event.info = event_info
event.distribution = sanitize_event_distribution(distribution)
event.threat_level_id = sanitize_event_threat_level_id(threat_level_id)
event.analysis = sanitize_event_analysis(analysis)
if publish:
event.publish()
# # TODO: handle multiple MISP instance
misp = PyMISP(misp_url, misp_key, misp_verifycert)
#print(event.to_json())
misp_event = misp.add_event(event, pythonify=True)
# # TODO: handle error
event_metadata = extract_event_metadata(misp_event)
return event_metadata
def get_misp_connection(config=None):
global misp_connection
if misp_connection:
return misp_connection
if not config:
raise MaltegoException(
"ERROR: MISP connection not yet established, and config not provided as parameter."
)
if config['MISP_maltego.local.misp_verify'] in [
'True', 'true', 1, 'yes', 'Yes'
]:
misp_verify = True
else:
misp_verify = False
if config['MISP_maltego.local.misp_debug'] in [
'True', 'true', 1, 'yes', 'Yes'
]:
misp_debug = True
else:
misp_debug = False
try:
misp_connection = PyMISP(config['MISP_maltego.local.misp_url'],
config['MISP_maltego.local.misp_key'],
misp_verify, 'json', misp_debug)
except Exception:
raise MaltegoException(
"ERROR: Cannot connect to MISP server. Please verify your MISP_Maltego.conf settings"
)
return misp_connection
def test_user_must_change_password(self):
updated_user = self.admin_misp_connector.update_user({'change_pw': 1},
self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "1")
# Try to login, should still work because key is still valid
PyMISP(url, self.test_usr.authkey)
updated_user = self.admin_misp_connector.update_user({'change_pw': 0},
self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "0")
# Try to login, should also still works
PyMISP(url, self.test_usr.authkey)
def ping_misp():
try:
PyMISP(misp_url, misp_key, misp_verifycert)
return True
except Exception as e:
print(e)
return False
def get_misp_connection(config=None, parameters=None):
global misp_connection
if misp_connection:
return misp_connection
if not config:
raise MaltegoException("ERROR: MISP connection not yet established, and config not provided as parameter.")
misp_verify = True
misp_debug = False
misp_url = None
misp_key = None
try:
if is_local_exec_mode():
misp_url = config['MISP_maltego.local.misp_url']
misp_key = config['MISP_maltego.local.misp_key']
if config['MISP_maltego.local.misp_verify'] in ['False', 'false', 0, 'no', 'No']:
misp_verify = False
if config['MISP_maltego.local.misp_debug'] in ['True', 'true', 1, 'yes', 'Yes']:
misp_debug = True
if is_remote_exec_mode():
try:
misp_url = parameters['mispurl'].value
misp_key = parameters['mispkey'].value
except AttributeError:
raise MaltegoException("ERROR: mispurl and mispkey need to be set to something valid")
misp_connection = PyMISP(misp_url, misp_key, misp_verify, 'json', misp_debug, tool='misp_maltego')
except Exception:
if is_local_exec_mode():
raise MaltegoException("ERROR: Cannot connect to MISP server. Please verify your MISP_Maltego.conf settings.")
if is_remote_exec_mode():
raise MaltegoException("ERROR: Cannot connect to MISP server. Please verify your settings (MISP URL and API key), and ensure the MISP server is reachable from the internet.")
return misp_connection
def init(misp_url, misp_key):
return PyMISP(misp_url,
misp_key,
misp_verifycert,
'json',
debug=False,
proxies=proxies)
class MISPCollectorBot(CollectorBot):
def init(self):
if PyMISP is None:
self.logger.error('Could not import pymisp. Please install it.')
self.stop()
# Initialise MISP connection
self.misp = PyMISP(self.parameters.misp_url, self.parameters.misp_key,
self.parameters.misp_verify)
# URLs used for deleting and adding MISP event tags
self.misp_add_tag_url = urljoin(self.parameters.misp_url,
'events/addTag')
self.misp_del_tag_url = urljoin(self.parameters.misp_url,
'events/removeTag')
def process(self):
# Grab the events from MISP
misp_result = self.misp.search(
tags=self.parameters.misp_tag_to_process)
# Process the response and events
if 'response' in misp_result:
# Extract the MISP event details
for e in misp_result['response']:
misp_event = e['Event']
# Send the results to the parser
report = self.new_report()
report.add('raw', json.dumps(misp_event, sort_keys=True))
report.add('feed.url', self.parameters.misp_url)
self.send_message(report)
# Finally, update the tags on the MISP events.
# Note PyMISP does not currently support this so we use
# the API URLs directly with the requests module.
for misp_event in misp_result['response']:
# Remove the 'to be processed' tag
self.misp.remove_tag(misp_event,
self.parameters.misp_tag_to_process)
# Add a 'processed' tag to the event
self.misp.add_tag(misp_event,
self.parameters.misp_tag_processed)
def __init__(self, misp_key, misp_url, misp_verify_cert, siem_mode=False, debugon=False):
self.misp = PyMISP(misp_url, misp_key, misp_verify_cert, 'json')
self.debugon = debugon
if siem_mode:
self.siem_mode = True
self.separator = ","
self.use_headers = True
self.use_filename_regex = False
def onDatabaseUpdate(self):
lastUpdate = db.p_readSetting(self.collectionName, "last_update")
now = datetime.utcnow().replace(tzinfo = pytz.utc)
if lastUpdate:
last = dateutil.parser.parse(lastUpdate)
delta = now - last
since = "%sm"%math.ceil(delta.total_seconds()/60)
else:
since = ""
if self.url and self.key:
try:
# Misp interface
misp = PyMISP(self.url, self.key, True, 'json')
except:
return "[-] Failed to connect to MISP. Wrong URL?"
try:
# Fetch data
misp_last = misp.download_last(since)
# Check data
if 'message' in misp_last.keys():
if misp_last['message'].lower().startswith('no matches'): return "[+] MISP collection updated (0 updates)"
elif misp_last['message'].startswith('Authentication failed.'): return "[-] MISP Authentication failed"
if not 'response' in misp_last: print(misp_last); return "[-] Error occured while fetching MISP data"
# Nothing wrong so far, so let's continue
bulk =[]
for entry in progressbar(misp_last['response']):
# Get info
attrs=entry['Event']['Attribute']
CVEs= [x['value'] for x in attrs if x['type'] == 'vulnerability']
if len(CVEs) == 0: continue
threats= [x['value'] for x in attrs if x['category'] == 'Attribution' and x['type'] == 'threat-actor']
tags = [x['value'] for x in attrs if x['category'] == 'Other' and x['type'] == 'text']
tags.extend([x['value'] for x in attrs if x['category'] == 'External analysis' and x['type'] == 'text'])
# Add info to each CVE
for cve in CVEs:
item={'id':cve}
if len(threats) !=0: item['threats'] = threats
if len(tags) !=0: item['tags'] = tags
if len(item.keys())>1: bulk.append(item) # Avoid empty collections
db.p_bulkUpdate(self.collectionName, "id", bulk)
#update database info after successful program-run
db.p_writeSetting(self.collectionName, "last_update", now.strftime("%a, %d %h %Y %H:%M:%S %Z"))
return "[+] MISP collection updated (%s updates)"%len(bulk)
except Exception as e: print(e);print(e);return "[-] Something went wrong..."
else: return "[-] MISP credentials not specified"
def init(self):
# Initialise MISP connection
self.misp = PyMISP(self.parameters.misp_url,
self.parameters.misp_key, 'json')
# URLs used for deleting and adding MISP event tags
self.misp_add_tag_url = urljoin(self.parameters.misp_url,
'events/addTag')
self.misp_del_tag_url = urljoin(self.parameters.misp_url,
'events/removeTag')
def test_newEvent(self, m):
error_empty_info = {'message': 'The event could not be saved.', 'name': 'Add event failed.', 'errors': {'Event': {'info': ['Info cannot be empty.']}}, 'url': '/events/add'}
error_empty_info_flatten = {u'message': u'The event could not be saved.', u'name': u'Add event failed.', u'errors': [u"Error in info: Info cannot be empty."], u'url': u'/events/add'}
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
with self.assertRaises(pm.api.NewEventError):
pymisp.new_event()
with self.assertRaises(pm.api.NewEventError):
pymisp.new_event(0)
with self.assertRaises(pm.api.NewEventError):
pymisp.new_event(0, 1)
m.register_uri('POST', self.domain + 'events', json=error_empty_info)
response = pymisp.new_event(0, 1, 0)
self.assertEqual(response, error_empty_info_flatten)
m.register_uri('POST', self.domain + 'events', json=self.new_misp_event)
response = pymisp.new_event(0, 1, 0, "This is a test.", '2016-08-26', False)
self.assertEqual(response, self.new_misp_event)
def test_newEvent(self, m):
error_empty_info = {
"message": "The event could not be saved.",
"name": "Add event failed.",
"errors": {"Event": {"info": ["Info cannot be empty."]}},
"url": "/events/add",
}
error_empty_info_flatten = {
u"message": u"The event could not be saved.",
u"name": u"Add event failed.",
u"errors": [u"Error in info: Info cannot be empty."],
u"url": u"/events/add",
}
self.initURI(m)
pymisp = PyMISP(self.domain, self.key)
m.register_uri("POST", self.domain + "events", json=error_empty_info)
# TODO Add test exception if info field isn't set
response = pymisp.new_event(0, 1, 0, "Foo")
self.assertEqual(response, error_empty_info_flatten)
m.register_uri("POST", self.domain + "events", json=self.new_misp_event)
response = pymisp.new_event(0, 1, 0, "This is a test.", "2016-08-26", False)
self.assertEqual(response, self.new_misp_event)
def mispextract(fpath):
# md5="626576e5f0f85d77c460a322a92bb267"
url="https://misppriv.circl.lu"
apikey="Enter your API key HERE"
conn=internet_on()
if conn:
api=PyMISP(url,apikey,ssl=True, out_type='json' ,debug=False, proxies=None)
md5=prelim(fpath)
response = api.search_all(md5)
length=len(response["response"])
retu={"connection":True,
"positives":length
}
return retu
#print(len(list(root)))
else:
print("No connectivity with MISP")
retu={"connection":False,
"positives":0
}
return retu
def init(self):
if PyMISP is None:
raise ValueError('Could not import pymisp. Please install it.')
# Initialize MISP connection
self.misp = PyMISP(self.parameters.misp_url,
self.parameters.misp_key,
self.parameters.misp_verify)
# URLs used for deleting and adding MISP event tags
self.misp_add_tag_url = urljoin(self.parameters.misp_url,
'events/addTag')
self.misp_del_tag_url = urljoin(self.parameters.misp_url,
'events/removeTag')
class MISPReceiver():
hash_iocs = {}
filename_iocs = {}
c2_iocs = {}
yara_rules = {}
debugon = False
# Output
siem_mode = False
separator = ";"
use_headers = False
use_filename_regex = True
def __init__(self, misp_key, misp_url, misp_verify_cert, siem_mode=False, debugon=False):
self.misp = PyMISP(misp_url, misp_key, misp_verify_cert, 'json')
self.debugon = debugon
if siem_mode:
self.siem_mode = True
self.separator = ","
self.use_headers = True
self.use_filename_regex = False
def get_iocs_last(self, last):
# Retrieve events from MISP
result = self.misp.download_last(last)
self.events = result['response']
# Process each element (without related eevents)
for event_element in self.events:
event = event_element["Event"]
# Info for Comment
info = event['info']
uuid = event['uuid']
comment = "{0} - UUID: {1}".format(info.encode('unicode_escape'), uuid)
# Event data
for attribute in event['Attribute']:
# Skip iocs that are not meant for ioc detection
if attribute['to_ids'] == False:
continue
# Value
value = attribute['value']
# Non split type
if '|' not in attribute['type']:
self.add_ioc(attribute['type'], value, comment, uuid, info)
# Split type
else:
# Prepare values
type1, type2 = attribute['type'].split('|')
value1, value2 = value.split('|')
# self.add_ioc(type1, value1, comment)
self.add_ioc(type2, value2, comment, uuid, info)
def add_ioc(self, ioc_type, value, comment, uuid, info):
# Cleanup value
value = value.encode('unicode_escape')
# Debug
if self.debugon:
print "{0} = {1}".format(ioc_type, value)
# C2s
if ioc_type in ('hostname', 'ip-dst', 'domain'):
if value == '127.0.0.1':
return
self.c2_iocs[value] = comment
# Hash
if ioc_type in ('md5', 'sha1', 'sha256'):
# No empty files
if value == 'd41d8cd98f00b204e9800998ecf8427e' or \
value == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' or \
value == 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855':
return
self.hash_iocs[value] = comment
# Filenames
if ioc_type in ('filename', 'filepath'):
# Add prefix to filenames
if not re.search(r'^([a-zA-Z]:|%)', value):
if not self.siem_mode:
value = "\\\\{0}".format(value)
if self.use_filename_regex:
self.filename_iocs[my_escape(value)] = comment
else:
self.filename_iocs[value.decode('string_escape')] = comment
# Yara
if ioc_type in ('yara'):
self.add_yara_rule(value, uuid, info)
def add_yara_rule(self, yara_rule, uuid, info):
identifier = generate_identifier(info)
self.yara_rules[identifier] = ur'%s' % repair_yara_rule(yara_rule.decode('string_escape'), uuid)
def write_iocs(self, output_path, output_path_yara):
# Write C2 IOCs
self.write_file(os.path.join(output_path, "misp-c2-iocs.txt"), self.c2_iocs, "c2")
# Write Filename IOCs
self.write_file(os.path.join(output_path, "misp-filename-iocs.txt"), self.filename_iocs, "filename")
# Write Hash IOCs
self.write_file(os.path.join(output_path, "misp-hash-iocs.txt"), self.hash_iocs, "hash")
# Yara
if len(self.yara_rules) > 0:
# Create dir if not exists
if not os.path.exists(output_path_yara):
os.makedirs(output_path_yara)
# Loop through rules (keys are identifiers used for file names)
for yara_rule in self.yara_rules:
output_rule_filename = os.path.join(output_path_yara, "%s.yar" % yara_rule)
self.write_yara_rule(output_rule_filename, self.yara_rules[yara_rule])
print "{0} YARA rules written to directory {1}".format(len(self.yara_rules), output_path_yara)
def write_file(self, ioc_file, iocs, ioc_type):
with open(ioc_file, 'w') as file:
if self.use_headers:
file.write("{0}{1}description\n".format(ioc_type, self.separator))
for ioc in iocs:
file.write("{0}{2}{1}\n".format(ioc,iocs[ioc],self.separator))
print "{0} IOCs written to file {1}".format(len(iocs), ioc_file)
def write_yara_rule(self, yara_file, yara_rule):
# Write the YARA rule
with io.open(yara_file, 'wb') as fh:
fh.write(ur'%s' % yara_rule)
def setUp(self):
self.maxDiff = None
self.misp = PyMISP(url, key, True, 'json')
class TestBasic(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.misp = PyMISP(url, key, True, 'json')
def _clean_event(self, event):
event['Event'].pop('uuid', None)
event['Event'].pop('timestamp', None)
event['Event'].pop('date', None)
event['Event'].pop('org', None)
event['Event'].pop('orgc', None)
event['Event'].pop('RelatedEvent', None)
event['Event'].pop('publish_timestamp', None)
if event['Event'].get('Attribute'):
for a in event['Event'].get('Attribute'):
a.pop('uuid', None)
a.pop('event_id', None)
a.pop('id', None)
a.pop('timestamp', None)
return event['Event'].pop('id', None)
def new_event(self):
event = self.misp.new_event(0, 1, 0, "This is a test")
event_id = self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'0', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False,
u'distribution': u'0', u'Attribute': [], u'proposal_email_lock': False,
u'threat_level_id': u'1'}},
self.assertEqual(event, to_check, 'Failed at creating a new Event')
return int(event_id)
def add_hashes(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.add_hashes(event, 'Payload installation', 'dll_installer.dll', '0a209ac0de4ac033f31d6ba9191a8f7a', '1f0ae54ac3f10d533013f74f48849de4e65817a7', '003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', 'Fanny modules', False, 2)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0',
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at adding hashes')
def publish(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.publish(event)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': True, u'distribution': u'0',
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at publishing event')
def delete(self, eventid):
event = self.misp.delete_event(eventid)
print event.json()
def delete_attr(self, attrid):
event = self.misp.delete_attribute(attrid)
print event.json()
def get(self, eventid):
event = self.misp.get_event(eventid)
print event.json()
def add(self):
event = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0',
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
event = self.misp.add_event(event)
print event.json()
def test_create_event(self):
eventid = self.new_event()
time.sleep(1)
self.delete(eventid)
def test_get_event(self):
eventid = self.new_event()
time.sleep(1)
self.get(eventid)
time.sleep(1)
self.delete(eventid)
def test_add_event(self):
self.add()
time.sleep(1)
self.delete(1)
def test_del_attr(self):
eventid = self.new_event()
time.sleep(1)
self.delete_attr(1)
time.sleep(1)
self.delete(eventid)
import numpy
import tools
import date_tools
import bokeh_tools
import time
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Show the evolution of trend of tags.')
parser.add_argument("-d", "--days", type=int, required=True, help='')
parser.add_argument("-s", "--begindate", required=True, help='format yyyy-mm-dd')
parser.add_argument("-e", "--enddate", required=True, help='format yyyy-mm-dd')
args = parser.parse_args()
misp = PyMISP(misp_url, misp_key, misp_verifycert)
result = misp.search(date_from=args.begindate, date_to=args.enddate, metadata=False)
# Getting data
if 'response' in result:
events = tools.eventsListBuildFromArray(result)
NbTags = []
dates = []
enddate = date_tools.toDatetime(args.enddate)
begindate = date_tools.toDatetime(args.begindate)
for i in range(round(date_tools.days_between(enddate, begindate)/args.days)):
begindate = date_tools.getNDaysBefore(enddate, args.days)
eventstemp = tools.selectInRange(events, begindate, enddate)
