def run(self, conf, args, plugins):
server = ExpandedPyMISP(conf['Misp']['url'], conf['Misp']['key'],
not args.no_tls)
if args.list:
# List events
events = server.events(pythonify=True)
for event in sorted(events, key=lambda x: x.id):
print("%i : %s" % (event.id, event.info))
elif args.event is not None:
event = server.get_event(args.event, pythonify=True)
if args.attr is None and args.type is None:
if args.raw:
for a in event.attributes:
print(a.value)
else:
print("Event {} : {}".format(event.id, event.info))
print("Tags : {}".format(", ".join(
map(lambda x: str(x.name), event.tags))))
print("{} Attributes including:".format(
len(event.attributes)))
attrs = Counter(map(lambda x: x.type, event.attributes))
attrs_ids = Counter(
map(lambda x: x.type,
filter(lambda x: x.to_ids, event.attributes)))
for type in attrs:
print("- %i %s (%i for detection)" %
(attrs[type], type, attrs_ids[type]))
else:
if args.type is not None:
# Display all attributes from this type
for attr in event.attributes:
if attr.type == args.type:
if args.raw:
print("%s" % attr.value)
else:
print("{:20}{:10}{:40}{}{}".format(
attr.category, attr.type, attr.value,
attr.comment, attr.to_ids))
elif args.attr is not None:
# search by attribute value
for attr in event.attributes:
if args.attr in str(attr.value):
print("%s\t%s\t%s\t%s\t%s" %
(attr.category, attr.type, attr.value,
attr.comment, attr.to_ids))
elif args.attr is not None:
res = server.search('attributes', value=args.attr)
if len(res['Attribute']) == 0:
print("Search %s: no results" % args.attr)
else:
print("Search %s, result founds" % args.attr)
for attr in res['Attribute']:
print('{} - {}'.format(attr['Event']['id'],
attr['Event']['info']))
else:
self.parser.print_help()
class VMRayAutomation:
def __init__(
self,
misp_url: str,
misp_key: str,
verify_cert: bool = False,
debug: bool = False,
) -> None:
# setup logging
log_level = logging.DEBUG if debug else logging.INFO
log_format = "%(asctime)s - %(name)s - %(levelname)8s - %(message)s"
logging.basicConfig(level=log_level, format=log_format)
logging.getLogger("pymisp").setLevel(log_level)
self.logger = logging.getLogger(self.__class__.__name__)
self.misp_url = misp_url.rstrip("/")
self.misp_key = misp_key
self.verifycert = verify_cert
self.misp = ExpandedPyMISP(misp_url,
misp_key,
ssl=verify_cert,
debug=debug)
self.config = {}
self.tag_incomplete = 'workflow:state="incomplete"'
@staticmethod
def _setting_enabled(value: bool) -> bool:
if not value:
raise VMRayAutomationException(
"VMRay import is disabled. "
"Please enable `vmray_import` in the MISP settings.")
return True
@staticmethod
def _setting_apikey(value: str) -> str:
if not value:
raise VMRayAutomationException(
"VMRay API key not set. Please set the API key in the MISP settings."
)
return value
@staticmethod
def _setting_url(value: str) -> str:
if not value:
raise VMRayAutomationException(
"VMRay URL not set. Please set the URL in the MISP settings.")
if not is_url(value):
raise VMRayAutomationException("Not a valid URL")
return value
@staticmethod
def _setting_disabled(value: str) -> bool:
return value.lower() in ["no", "false"]
@staticmethod
def _services_port(value: int) -> bool:
if value == 0:
return 6666
return value
@staticmethod
def services_url(value: str) -> bool:
if not is_url(value):
raise VMRayAutomationException("Services URL is not valid.")
return value
@property
def vmray_settings(self) -> Dict[str, Any]:
return {
"vmray_import_enabled": self._setting_enabled,
"vmray_import_apikey": self._setting_apikey,
"vmray_import_url": self._setting_url,
"vmray_import_disable_tags": self._setting_disabled,
"vmray_import_disable_misp_objects": self._setting_disabled,
"vmray_import_ignore_analysis_finished": self._setting_disabled,
"services_port": self._services_port,
"services_url": self.services_url,
}
def _get_misp_settings(self) -> List[Dict[str, Any]]:
misp_headers = {
"Content-Type": "application/json",
"Accept": "application/json",
"Authorization": self.misp_key,
}
response = requests.get(
f"{self.misp_url}/servers/serverSettings.json",
verify=self.verifycert,
headers=misp_headers,
)
if response.status_code == 200:
settings = response.json()
if "finalSettings" in settings:
return settings["finalSettings"]
raise VMRayAutomationException(
"Could not get settings from MISP server.")
def get_config(self) -> None:
self.logger.debug("Loading confing...")
# get settings from MISP server
settings = self._get_misp_settings()
for setting in settings:
config_name = setting["setting"].replace("Plugin.Import_", "")
if config_name in self.vmray_settings:
func = self.vmray_settings[config_name]
value = func(setting["value"])
self.config[config_name] = value
# set default `vmray_import` settings
self.config.setdefault("VTI", "1")
self.config.setdefault("IOCs", "1")
self.config.setdefault("Artifacts", "0")
self.config.setdefault("Analysis Details", "1")
self.logger.info("Loading config: Done.")
def overwrite_config(self, config: Dict[str, Any]) -> None:
self.config.update(config)
def _get_sample_id(self, value: str) -> Optional[int]:
vmray_sample_id_text = "VMRay Sample ID: "
if not value.startswith(vmray_sample_id_text):
self.logger.warning("Invalid Sample ID: %s.", value)
return None
return int(value.replace(vmray_sample_id_text, ""))
def _call_vmray_import(self, sample_id: int,
event_id: str) -> Dict[str, Any]:
url = f"{self.config['services_url']}:{self.config['services_port']}/query"
config = {"Sample ID": sample_id}
for key, value in self.config.items():
vmray_config_key = key.replace("vmray_import_", "")
config[vmray_config_key] = str(value)
data = {
"module": "vmray_import",
"event_id": event_id,
"config": config,
"data": "",
}
self.logger.debug("calling `vmray_import`: url=%s, config=%s", url,
config)
response = requests.post(url, json=data)
if response.status_code != 200:
raise VMRayAutomationException(
f"MISP modules returned status code `{response.status_code}`")
json_response = response.json()
if "error" in json_response:
error = json_response["error"]
raise VMRayAutomationException(
f"MISP modules returned error: {error}")
return json_response
def _add_event_attributes(self, event_id: int,
attributes: Dict[str, Any]) -> None:
event = self.misp.get_event(event_id, pythonify=True)
for attr in attributes["Attribute"]:
event.add_attribute(**attr)
self.misp.update_event(event)
def _add_event_objects(self, event_id: int, objects: Dict[str,
Any]) -> None:
event = self.misp.get_event(event_id, pythonify=True)
for obj in objects["Object"]:
event.add_object(**obj)
if "Tag" in objects:
for tag in objects["Tag"]:
event.add_tag(tag["name"])
self.misp.update_event(event)
def _add_misp_event(self, event_id: int, response: Dict[str, Any]) -> None:
if self.config["vmray_import_disable_misp_objects"]:
self._add_event_attributes(event_id, response["results"])
else:
self._add_event_objects(event_id, response["results"])
def import_incomplete_analyses(self) -> None:
self.logger.info("Searching for attributes with tag='%s'",
self.tag_incomplete)
result = self.misp.search("attributes", tags=self.tag_incomplete)
attributes = result["Attribute"]
for attr in attributes:
event_id = int(attr["event_id"])
self.logger.info("Processing event ID `%d`.", event_id)
sample_id = self._get_sample_id(attr["value"])
if not sample_id:
continue
response = self._call_vmray_import(sample_id, event_id)
self._add_misp_event(event_id, response)
self.misp.untag(attr["uuid"], self.tag_incomplete)
'http': 'http://127.0.0.1:8123',
'https': 'http://127.0.0.1:8123',
}
proxies = None
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Get an event from a MISP instance.')
parser.add_argument("-e",
"--event",
required=True,
help="Event ID to get.")
parser.add_argument("-o", "--output", help="Output file")
args = parser.parse_args()
if args.output is not None and os.path.exists(args.output):
print('Output file already exists, abort.')
exit(0)
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, proxies=proxies)
event = misp.get_event(args.event, pythonify=True)
if args.output:
with open(args.output, 'w') as f:
f.write(event.to_json())
else:
print(event.to_json())
class MISPController(object):
''' MISP Controller '''
def __init__(self, misp_param, debug=False):
self.misp_param = misp_param
self.debug = debug
if misp_param.get('connect_immediately', False):
self._connect()
else:
self.misp = None
def import_event(self, event_data):
''' Import event '''
# Check registered same event info
print('importing: {}'.format(event_data['title']))
events = self._search_event(eventinfo=event_data['title'])
if events != None:
for event in events:
if event_data['title'] == event['Event']['info']:
self._remove_event(event['Event']['id'])
event = self._add_event(event_data)
if event:
print('created event: {}'.format(event.id))
else:
print("Import failed.Please retry: {}".format(event_data['title']))
def _connect(self):
self.debug_print('URL: {}'.format(self.misp_param['url']))
self.debug_print('authkey: {}'.format(self.misp_param['authkey']))
self.misp = ExpandedPyMISP(self.misp_param['url'],
self.misp_param['authkey'],
ssl=False,
debug=False)
self._registered_tags = self.misp.tags()
def _check_tag(self, target_tag):
if self.misp == None:
self._connect()
if self._registered_tags == None:
self._get_tags()
for tag_info in self._registered_tags:
if tag_info.get('name', '') == target_tag:
return True
self.debug_print('new tag: {}'.format(target_tag))
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
tmp = MISPTag()
tmp.from_dict(name=target_tag)
self.misp.add_tag(tmp)
self._get_tags()
return True
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('add new tag retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
return False
def _add_event(self, value):
for tag in value['event_tags']:
self._check_tag(tag)
for attribute in value['attributes']:
for tag in attribute['tags']:
self._check_tag(tag)
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
tmp = MISPEvent()
tmp.from_dict(
distribution=self.misp_param['distribution'],
threat_level_id=self.misp_param['threat_level_id'],
analysis=self.misp_param['analysis'],
info=value['title'],
date=value['date'],
published=False)
response = self.misp.add_event(tmp)
if response.get('errors'):
raise Exception(str(response['errors']))
event = MISPEvent()
event.load(response)
break
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('add new event retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
return None
self.debug_print(event.id)
for tag in value['event_tags']:
event.add_tag(tag)
for attribute in value['attributes']:
attribute_tags = []
event.add_attribute(type=attribute['type'],
value=attribute['value'],
category=attribute['category'],
comment=attribute.get('comment', ''),
distribution=self.misp_param['distribution'],
Tag=self._create_tags(attribute['tags']))
event.published = True
if self._update_event(event):
self.debug_print('completed')
return event
else:
self.debug_print('add failed')
return None
def _get_event(self, id):
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
self.debug_print('get event start: {}'.format(id))
event = self.misp.get_event(id)
if event.get('errors'):
raise Exception(str(event['errors']))
self.debug_print('get event end: {}'.format(id))
return event
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('get event retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
return None
def _remove_event(self, id):
if id:
print('delete event: {}'.format(id))
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
response = self.misp.delete_event(id)
if response.get('errors'):
raise Exception(str(response['errors']))
return True
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('remove event retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
return False
def _search_event(self, **cons):
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
self.debug_print('search event start')
response = self.misp.search_index(**cons)
self.debug_print('search event end')
results = []
for json in response:
if json.get('id', ''):
results.append(self._get_event(json['id']))
else:
print('no event ID')
print(json)
return results
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('search event retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
return None
def _update_event(self, event):
cnt = 0
while True:
try:
if self.misp == None:
self._connect()
self.debug_print('event update start: {}'.format(event.id))
response = self.misp.update_event(event)
if response.get('errors'):
raise Exception(str(response['errors']))
self.debug_print('{} updated'.format(event.id))
return True
except:
print(traceback.format_exc())
if cnt < int(self.misp_param.get('max_retry_count', '0')):
print('retry: {}'.format(cnt))
cnt = cnt + 1
time.sleep(10)
else:
print('event update failed: {}'.format(event.info))
try:
self._remove_event(event.id)
except:
pass
return False
def _create_tags(self, values):
tags = []
for value in values:
if value:
tags.append({'name': value})
return tags
def debug_print(self, message):
if self.debug == False:
return
# nowstr = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
nowstr = datetime.datetime.now().strftime('%H:%M:%S')
print('{}\t{}'.format(nowstr, message))
