def prepare_alert(helper, app_name): instance = helper.get_param("misp_instance") sessionKey = helper.settings['session_key'] splunkService = client.connect(token=sessionKey) storage = splunkService.storage_passwords config_args = prepare_config(helper, app_name, instance, storage) if config_args is None: return None alert_args = dict() # Get string values from alert form alert_args['mode'] = str(helper.get_param("mode")) alert_args['type'] = int(helper.get_param("type")) alert_args['source'] = str(helper.get_param("source")) if not helper.get_param("unique"): alert_args['unique'] = "no_timestamp_field" else: alert_args['unique'] = str(helper.get_param("unique")) config_args.update(alert_args) return config_args
def prepare_alert(helper, app_name): # Get parameters for calling prepare_config instance = helper.get_param("misp_instance") sessionKey = helper.settings['session_key'] splunkService = client.connect(token=sessionKey) storage = splunkService.storage_passwords helper.log_debug("[AL-PA-001] successfully retrieved storage object") config_args = prepare_config(helper, app_name, instance, storage, sessionKey) if config_args is None: return None alert_args = dict() # Get string values from alert form alert_args['tlp'] = str(helper.get_param("tlp").replace('_', ':')) alert_args['pap'] = str(helper.get_param("pap").replace('_', ':')) if not helper.get_param("eventid"): alert_args['eventid'] = "0" else: alert_args['eventid'] = str(helper.get_param("eventid")) if not helper.get_param("unique"): alert_args['eventkey'] = "oneEvent" else: alert_args['eventkey'] = str(helper.get_param("unique")) if not helper.get_param("info"): alert_args['info'] = "notable event" else: alert_args['info'] = str(helper.get_param("info")) published = helper.get_param("publish_on_creation") if published == "1": alert_args['published'] = True else: alert_args['published'] = False if not helper.get_param("tags"): alert_args['tags'] = None else: alert_args['tags'] = str(helper.get_param("tags")) # Get numeric values from alert form alert_args['analysis'] = int(helper.get_param("analysis")) alert_args['threatlevel'] = int(helper.get_param("threatlevel")) alert_args['distribution'] = int(helper.get_param("distribution")) config_args.update(alert_args) return config_args
def stream(self, records): # self.self.logging.debug('mispgetioc.reduce') # Generate args my_args = prepare_config(self, 'misp42splunk') # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' fieldname = str(self.field) search_url = my_args['misp_url'] + '/attributes/restSearch' sight_url = my_args['misp_url'] + \ '/sightings/restSearch/attribute' for record in records: if fieldname in record: value = record.get(fieldname, None) if value is not None: search_dict = {"returnFormat": "json"} search_dict['value'] = str(value) search_dict['withAttachments'] = "false", search_body = json.dumps(search_dict) sight_dict = {"returnFormat": "json"} misp_value = '' misp_fp = False misp_fp_ts = 0 misp_fp_id = '' ms_seen = False ms = { 'count': 0, 'first': 0, 'f_id': 0, 'last': 0, 'l_id': 0 } # search logging.debug('mispsight request body: %s', search_body) rs = requests.post(search_url, headers=headers, data=search_body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is rs.raise_for_status() # response is 200 by this point or we would # have thrown an exception response = rs.json() logging.info( "MISP REST API %s has got a response with status code 200", search_url) if 'response' in response: if 'Attribute' in response['response']: r_number = len(response['response']['Attribute']) logging.debug( "MISP REST API %s: response: with %s records" % (search_url, str(r_number))) for a in response['response']['Attribute']: if misp_value == '': misp_value = str(a['value']) if misp_fp is False: sight_dict['id'] = str(a['id']) sight_body = json.dumps(sight_dict) rt = requests.post( sight_url, headers=headers, data=sight_body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything # other than 200; throw an exception rt.raise_for_status() # response is 200 by this point or we # would have thrown an exception sight = rt.json() logging.info( "MISP REST API %s has got a response with status code 200", sight_url) logging.debug( "MISP REST API %s has got a response: with %s records" % (sight_url, len(sight))) if 'response' in sight: for s in sight['response']: if 'Sighting' in s: # true sighting ty = s['Sighting']['type'] ds = int(s['Sighting'] ['date_sighting']) ev = str( s['Sighting']['event_id']) if int(ty) == 0: ms_seen = True ms['count'] = \ ms['count'] + 1 if ms['first'] == 0 or \ ms['first'] > ds: ms['first'] = ds ms['f_id'] = ev if ms['last'] < int(ds): ms['last'] = int(ds) ms['l_id'] = ev # false positive elif int(ty) == 1: misp_fp = True misp_fp_ts = ds misp_fp_id = ev if misp_fp is True: record['misp_value'] = misp_value record['misp_fp'] = "True" record['misp_fp_timestamp'] = str(misp_fp_ts) record['misp_fp_event_id'] = str(misp_fp_id) if ms_seen is True: record['misp_value'] = misp_value record['misp_count'] = str(ms['count']) record['misp_first'] = str(ms['first']) record['misp_first_event_id'] = str(ms['f_id']) record['misp_last'] = str(ms['last']) record['misp_last_event_id'] = str(ms['l_id']) yield record
def reduce(self, records): # Phase 1: Preparation my_args = prepare_config(self, 'misp42splunk') my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch' # check that ONE of mandatory fields is present mandatory_arg = 0 if self.json_request is not None: mandatory_arg = mandatory_arg + 1 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: logging.error('Missing "json_request", eventid", \ "last" or "date" argument') raise Exception('Missing "json_request", "eventid", \ "last" or "date" argument') elif mandatory_arg > 1: logging.error('Options "json_request", eventid", "last" \ and "date" are mutually exclusive') raise Exception('Options "json_request", "eventid", "last" \ and "date" are mutually exclusive') body_dict = dict() # Only ONE combination was provided if self.json_request is not None: body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') elif self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set with %s', json.dumps(body_dict['eventid'])) elif self.last: body_dict['last'] = self.last logging.info('Option "last" set with %s', str(body_dict['last'])) else: body_dict['date'] = self.date.split() logging.info('Option "date" set with %s', json.dumps(body_dict['date'])) # Force some values on JSON request body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True if self.limit is not None: limit = int(self.limit) elif 'limit' in body_dict: limit = int(body_dict['limit']) else: limit = 1000 if limit == 0: pagination = False if self.page is not None: page = int(self.page) elif 'page' in body_dict: page = body_dict['page'] else: page = 1 if self.published is True: body_dict['published'] = True elif self.published is False: body_dict['published'] = False if self.category is not None: if "," in self.category: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria else: body_dict['category'] = self.category if self.type is not None: if "," in self.type: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria else: body_dict['type'] = self.type if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria # output filter parameters if self.getioc is True: getioc = True else: getioc = False if self.pipesplit is True: pipesplit = True else: pipesplit = False if self.output is not None: output = self.output else: output = "default" if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.error('MispGetEventCommand request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() if output == "raw": if 'response' in response: for r_item in response['response']: if 'Event' in r_item: for e in list(r_item.values()): yield e else: results = [] # add colums for each type in results typelist = [] format_output_table(response, results, typelist, getioc, pipesplit) logging.info('typelist is %s', json.dumps(typelist)) # relevant_cat = ['Artifacts dropped', 'Financial fraud', # 'Network activity','Payload delivery','Payload installation'] # logging.debug('results is %s', json.dumps(results)) logging.debug('results contains %s records', str(len(results))) if getioc is False: for e in results: # logging.debug('event is %s', json.dumps(e)) yield e else: output_dict = {} for e in results: if 'Attribute' in e: for r in e['Attribute']: if int(r['misp_object_id']) == 0: # not an object key = str(e['misp_event_id']) + '_' \ + r['misp_attribute_id'] is_object_member = False else: # this is a MISP object key = str(e['misp_event_id']) + \ '_object_' + str(r['misp_object_id']) is_object_member = True if key not in output_dict: v = init_misp_output(e, r) for t in typelist: misp_t = 'misp_' \ + t.replace('-', '_')\ .replace('|', '_p_') v[misp_t] = [] if t == r['misp_type']: v[misp_t].append(r['misp_value']) v['misp_to_ids'] = [] v['misp_to_ids'].append(r['misp_to_ids']) v['misp_category'] = [] v['misp_category'].append(r['misp_category']) a_tag = [] if r['misp_attribute_tag'] is not None: for t in r['misp_attribute_tag']: a_tag.append(t) v['misp_attribute_tag'] = a_tag comment = [] comment.append(r['misp_comment']) v['misp_comment'] = comment v['misp_attribute_uuid'] = [] v['misp_attribute_uuid']\ .append(r['misp_attribute_uuid']) v['misp_attribute_id'] = [] v['misp_attribute_id']\ .append(r['misp_attribute_id']) if is_object_member is True: v['misp_type'] = v['misp_object_name'] v['misp_value'] = v['misp_object_id'] output_dict[key] = dict(v) else: v = dict(output_dict[key]) misp_t = 'misp_' + r['misp_type']\ .replace('-', '_') misp_value = v[misp_t] misp_value.append(r['misp_value']) v[misp_t] = misp_value to_ids = v['misp_to_ids'] if r['misp_to_ids'] not in to_ids: to_ids.append(r['misp_to_ids']) v['misp_to_ids'] = to_ids category = v['misp_category'] # append if r['misp_category'] not in category: category.append(r['misp_category']) v['misp_category'] = category v['misp_attribute_uuid']\ .append(r['misp_attribute_uuid']) v['misp_attribute_id']\ .append(r['misp_attribute_id']) if r['misp_attribute_tag'] is not None: a_tag = v['misp_attribute_tag'] for t in r['misp_attribute_tag']: if t not in a_tag: a_tag.append(t) v['misp_attribute_tag'] = a_tag comment = v['misp_comment'] if r['misp_comment'] not in comment: comment.append(r['misp_comment']) v['misp_comment'] = comment if is_object_member is False: misp_type = r['misp_type'] \ + '|' + v['misp_type'] v['misp_type'] = misp_type misp_value = r['misp_value'] + \ '|' + v['misp_value'] v['misp_value'] = misp_value output_dict[key] = dict(v) if output_dict is not None: for k, v in list(output_dict.items()): yield v
def stream(self, records): # Generate args my_args = prepare_config(self) my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' fieldname = str(self.field) if self.gettag is True: get_tag = True else: get_tag = False pagination = True if self.limit is not None: if int(self.limit) == 0: pagination = False else: limit = int(self.limit) else: limit = 1000 if self.page is not None: page = int(self.page) else: page = 1 if self.json_request is not None: body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False if 'limit' in body_dict: limit = int(body_dict['limit']) if limit == 0: pagination = False if 'page' in body_dict: page = body_dict['page'] pagination = False else: # build search JSON object body_dict = {"returnFormat": "json", "withAttachments": False} if self.onlyids is True: body_dict['to_ids'] = "True" if self.includeEventUuid is not None: body_dict['includeEventUuid'] = self.includeEventUuid if self.includeEventTags is not None: body_dict['includeEventTags'] = self.includeEventTags if self.last is not None: body_dict['last'] = self.last for record in records: if fieldname in record: value = record.get(fieldname, None) if value is not None: body_dict['value'] = str(value) misp_category = [] misp_event_id = [] misp_event_uuid = [] misp_orgc_id = [] misp_to_ids = [] misp_tag = [] misp_type = [] misp_value = [] misp_uuid = [] # search if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.debug('mispsearch request body: %s', body) r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception # print >> sys.stderr, "DEBUG MISP REST API response: %s" % response.json() response = r.json() if 'response' in response: if 'Attribute' in response['response']: for a in response['response']['Attribute']: if str(a['type']) not in misp_type: misp_type.append(str(a['type'])) if str(a['value']) not in misp_value: misp_value.append(str(a['value'])) if str(a['to_ids']) not in misp_to_ids: misp_to_ids.append(str(a['to_ids'])) if str(a['category']) not in misp_category: misp_category.append(str(a['category'])) if str(a['uuid']) not in misp_uuid: misp_uuid.append(str(a['uuid'])) if str(a['event_id']) not in misp_event_id: misp_event_id.append(str(a['event_id'])) if 'Tag' in a: for tag in a['Tag']: if str(tag['name']) not in misp_tag: misp_tag.append(str(tag['name'])) if 'Event' in a: if a['Event'][ 'uuid'] not in misp_event_uuid: misp_event_uuid.append( str(a['Event']['uuid'])) if a['Event'][ 'orgc_id'] not in misp_orgc_id: misp_orgc_id.append( str(a['Event']['orgc_id'])) record['misp_type'] = misp_type record['misp_value'] = misp_value record['misp_to_ids'] = misp_to_ids record['misp_category'] = misp_category record['misp_attribute_uuid'] = misp_uuid record['misp_event_id'] = misp_event_id record['misp_event_uuid'] = misp_event_uuid record['misp_orgc_id'] = misp_orgc_id record['misp_tag'] = misp_tag yield record
def generate(self): # Phase 1: Preparation misp_instance = self.misp_instance storage = self.service.storage_passwords my_args = prepare_config(self, 'misp42splunk', misp_instance, storage) if my_args is None: raise Exception( "Sorry, no configuration for misp_instance={}".format( misp_instance)) my_args['host'] = my_args['misp_url'].replace('https://', '') if self.target not in [None, '']: my_args['misp_url'] = my_args['misp_url'] + self.target if self.json_request not in [None, '']: body_dict = json.loads(self.json_request) logging.debug('[MR-201] body_dict is {}'.format(body_dict)) else: body_dict = {} # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' if self.method == "GET": r = requests.get(my_args['misp_url'], headers=headers, params=body_dict, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) elif self.method == "POST": r = requests.post(my_args['misp_url'], headers=headers, data=json.dumps(body_dict), verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) elif self.method == "DELETE": r = requests.delete(my_args['misp_url'], headers=headers, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) else: raise Exception( "Sorry, no valid method provided (GET/POST//DELETE)." " it was {}.".format(self.method)) # check if status is anything other than 200; # throw an exception if it is if r.status_code in (200, 201, 204): logging.info("[RE301] INFO mispcollect successful. " "url={}, HTTP status={}".format( my_args['misp_url'], r.status_code)) else: logging.error("[RE302] ERROR mispcollect failed. " "url={}, data={}, HTTP Error={}, content={}".format( my_args['misp_url'], body_dict, r.status_code, r.text)) raise Exception( "[RE302] ERROR mispcollect failed. " "url={}, data={}, HTTP Error={}, content={}".format( my_args['misp_url'], body_dict, r.status_code, r.text)) # response is 200 by this point or we would have thrown an exception data = {'_time': time.time(), '_raw': json.dumps(r.json())} yield data
def stream(self, records): # Phase 1: Preparation misp_instance = self.misp_instance storage = self.service.storage_passwords my_args = prepare_config(self, 'misp42splunk', misp_instance, storage) if my_args is None: raise Exception("Sorry, no configuration for misp_instance={}".format(misp_instance)) # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' fieldname = str(self.field) search_url = my_args['misp_url'] + '/attributes/restSearch' sight_url = my_args['misp_url'] + \ '/sightings/restSearch/attribute' for record in records: if fieldname in record: value = record.get(fieldname, None) if value is not None: search_dict = {"returnFormat": "json"} search_dict['value'] = str(value) search_dict['withAttachments'] = "false", search_body = json.dumps(search_dict) sight_dict = {"returnFormat": "json"} misp_value = '' misp_fp = False misp_fp_ts = 0 misp_fp_id = '' ms_seen = False ms = { 'count': 0, 'first': 0, 'f_id': 0, 'last': 0, 'l_id': 0 } # search rs = requests.post( search_url, headers=headers, data=search_body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies'] ) # check if status is anything other than 200; # throw an exception if it is # check if status is anything other than 200; # throw an exception if it is if rs.status_code in (200, 201, 204): logging.info( "[SI301] INFO mispsight part 1 successful. " "url={}, HTTP status={}".format(my_args['misp_url'], rs.status_code) ) else: logging.error( "[SI302] ERROR mispsight part 1 failed. " "url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], search_body, rs.status_code, rs.text) ) raise Exception( "[SI302] ERROR mispsight part 1 failed. " "url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], search_body, rs.status_code, rs.text) ) # response is 200 by this point or we would # have thrown an exception response = rs.json() if 'response' in response: if 'Attribute' in response['response']: r_number = len(response['response']['Attribute']) logging.info( "MISP REST API %s: response: with %s records" % (search_url, str(r_number)) ) for a in response['response']['Attribute']: if misp_value == '': misp_value = str(a['value']) if misp_fp is False: sight_dict['id'] = str(a['id']) sight_body = json.dumps(sight_dict) rt = requests.post( sight_url, headers=headers, data=sight_body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies'] ) # check if status is anything # other than 200; throw an exception if rt.status_code in (200, 201, 204): logging.info( "[SI301] INFO mispsight part 2 successful. " "url={}, HTTP status={}".format(my_args['misp_url'], rt.status_code) ) else: logging.error( "[SI302] ERROR mispsight part 2 failed. " "url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], sight_body, rt.status_code, rt.text) ) raise Exception( "[SI302] ERROR mispsight part 2 failed. " "url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], sight_body, rt.status_code, rt.text) ) # response is 200 by this point or we # would have thrown an exception sight = rt.json() if 'response' in sight: for s in sight['response']: if 'Sighting' in s: # true sighting ty = s['Sighting']['type'] ds = int( s['Sighting'] ['date_sighting'] ) ev = str( s['Sighting'] ['event_id'] ) if int(ty) == 0: ms_seen = True ms['count'] = \ ms['count'] + 1 if ms['first'] == 0 or \ ms['first'] > ds: ms['first'] = ds ms['f_id'] = ev if ms['last'] < int(ds): ms['last'] = int(ds) ms['l_id'] = ev # false positive elif int(ty) == 1: misp_fp = True misp_fp_ts = ds misp_fp_id = ev if misp_fp is True: record['misp_value'] = misp_value record['misp_fp'] = "True" record['misp_fp_timestamp'] = str( misp_fp_ts ) record['misp_fp_event_id'] = str( misp_fp_id ) if ms_seen is True: record['misp_value'] = misp_value record['misp_count'] = str(ms['count']) record['misp_first'] = str(ms['first']) record['misp_first_event_id'] = str( ms['f_id'] ) record['misp_last'] = str(ms['last']) record['misp_last_event_id'] = str( ms['l_id'] ) yield record
def stream(self, records): # loggging self.set_log_level() # Phase 1: Preparation misp_instance = self.misp_instance storage = self.service.storage_passwords my_args = prepare_config(self, 'misp42splunk', misp_instance, storage) if my_args is None: raise Exception( "Sorry, no configuration for misp_instance={}".format( misp_instance)) # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' fieldname = str(self.field) search_url = my_args['misp_url'] + '/attributes/restSearch' sight_url = my_args['misp_url'] + \ '/sightings/restSearch/attribute' # iterate through records from SPL for record in records: # if a record contains the field passed to mispsight command # search by value in MISP to get attribute local ID. if fieldname in record: value = record.get(fieldname, None) if value is not None: search_dict = dict(returnFormat='json', value=str(value), withAttachments="false") search_body = json.dumps(search_dict) # search rs = requests.post(search_url, headers=headers, data=search_body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is # check if status is anything other than 200; # throw an exception if it is if rs.status_code in (200, 201, 204): self.log_info( "[SI301] INFO mispsight part 1 successful. url={}, HTTP status={}" .format(my_args['misp_url'], rs.status_code)) else: self.log_error( "[SI302] ERROR mispsight part 1 failed. url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], search_body, rs.status_code, rs.text)) raise Exception( "[SI302] ERROR mispsight part 1 failed for url={} with HTTP Error={}. Check search.log for details" .format(my_args['misp_url'], rs.status_code)) # response is 200 by this point or we would # have thrown an exception response = rs.json() if 'response' in response: if 'Attribute' in response['response']: # MISP API returned a JSON response r_number = len(response['response']['Attribute']) self.log_info( "MISP REST API {}: response: with {} records". format(search_url, str(r_number))) sightings = dict() sight_types = ['t0', 't1', 't2'] metrics = [ 'et', 'lt', 'first_a_id', 'last_a_id', 'first_e_id', 'last_e_id', 'first_org_id', 'last_org_id', 'first_source', 'last_source' ] sight_counter = { 'misp_value': '' # 'misp_sightings': [] } for stype in sight_types: skey = 'misp_sight_' + stype + '_count' sight_counter[skey] = 0 for metric in metrics: skey = 'misp_sight_' + stype + '_' + metric sight_counter[skey] = '' # iterate through results to get sighting counters for a in response['response']['Attribute']: misp_value = str(a['value']) if misp_value in sightings: a_sight = sightings[misp_value] else: a_sight = sight_counter.copy() a_sight['misp_value'] = misp_value sight_dict = {"returnFormat": "json"} sight_dict['id'] = str(a['id']) sight_body = json.dumps(sight_dict) rt = requests.post( sight_url, headers=headers, data=sight_body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything # other than 200; throw an exception if rt.status_code in (200, 201, 204): self.log_info( "[SI301] INFO mispsight part 2 successful. url={}, HTTP status={}" .format(my_args['misp_url'], rt.status_code)) else: self.log_error( "[SI302] ERROR mispsight part 2 failed. url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], sight_body, rt.status_code, rt.text)) raise Exception( "[SI302] ERROR mispsight part 2 failed for url={} with HTTP Error={}. Check search.log for details" .format(my_args['misp_url'], rt.status_code)) # response is 200 by this point or we # would have thrown an exception sight = rt.json() if 'response' in sight: for s in sight['response']: if 'Sighting' in s: # true sighting sr = s['Sighting'] ty = int(sr['type']) if ty == 0: MispSightCommand._sight_metric( a_sight, 't0', sr) elif ty == 1: MispSightCommand._sight_metric( a_sight, 't1', sr) elif ty == 2: MispSightCommand._sight_metric( a_sight, 't2', sr) sightings[misp_value] = a_sight init_record = True for srec in sightings.values(): self.log_debug("[SI401] srec={}".format( json.dumps(srec))) if init_record is True: for key, value in sorted(srec.items()): record[key] = [value] init_record = False else: for key, value in sorted(srec.items()): record[key].append(value) yield record
def reduce(self, records): # Phase 1: Preparation my_args = prepare_config(self) my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' # build search JSON object body_dict = {"returnFormat": "json", "withAttachments": False, "deleted": False } # check that ONE of mandatory fields is present mandatory_arg = 0 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date_from: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: logging.error('Missing "eventid", "last" or "date_from" argument') raise Exception('Missing "eventid", "last" or "date_from" argument') elif mandatory_arg > 1: logging.error('Options "eventid", "last" and "date_from" are mutually exclusive') raise Exception('Options "eventid", "last" and "date_from" are mutually exclusive') # Only ONE combination was provided if self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set') elif self.last: body_dict['last'] = self.last logging.info('Option "last" set with %s', body_dict['last']) else: body_dict['from'] = self.date_from logging.info('Option "date_from" set with %s', body_dict['from']) if self.date_to: body_dict['to'] = self.date_to logging.info('Option "date_to" set with %s', body_dict['to']) else: logging.info('Option "date_to" will be set to now().') # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True other_page = True page = 1 page_length = 0 if self.limit is not None: if int(self.limit) == 0: pagination = False else: limit = int(self.limit) else: limit = 10000 # Search parameters: boolean and filter if self.to_ids is True: body_dict['to_ids'] = True body_dict['enforceWarninglist'] = True elif self.to_ids is False: body_dict['to_ids'] = False if self.warning_list is True: body_dict['enforceWarninglist'] = True elif self.warning_list is False: body_dict['enforceWarninglist'] = False if self.published is True: body_dict['published'] = True elif self.published is False: body_dict['published'] = False if self.geteventtag is True: body_dict['includeEventTags'] = True if self.category is not None: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria if self.type is not None: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria if self.threat_level_id is not None: body_dict['threat_level_id'] = int(self.threat_level_id) # output filter parameters if self.getuuid is True: my_args['getuuid'] = True else: my_args['getuuid'] = False if self.getorg is True: my_args['getorg'] = True else: my_args['getorg'] = False if self.pipesplit is True: my_args['pipe'] = True else: my_args['pipe'] = False if self.add_description is True: my_args['add_desc'] = True else: my_args['add_desc'] = False results = [] # add colums for each type in results typelist = [] while other_page: if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.debug('mispgetioc request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() if 'response' in response: if 'Attribute' in response['response']: page_lenth = len(response['response']['Attribute']) for a in response['response']['Attribute']: v = {} v['misp_category'] = str(a['category']) v['misp_attribute_id'] = str(a['id']) v['misp_event_id'] = str(a['event_id']) v['misp_timestamp'] = str(a['timestamp']) v['misp_to_ids'] = str(a['to_ids']) tag_list = [] if 'Tag' in a: for tag in a['Tag']: try: tag_list.append(str(tag['name'])) except Exception: pass v['misp_tag'] = tag_list # include ID of the organisation that created the attribute if requested # in previous version this was the ORG name ==> create lookup if 'Event' in a and my_args['getorg']: v['misp_orgc_id'] = str(a['Event']['orgc_id']) # include attribute UUID if requested if my_args['getuuid']: v['misp_attribute_uuid'] = str(a['uuid']) # handle object and multivalue attributes v['misp_object_id'] = str(a['object_id']) if my_args['add_desc'] is True: if int(a['object_id']) == 0: v['misp_description'] = 'MISP e' + str(a['event_id']) + ' attribute ' \ + str(a['uuid']) + ' of type "' + str(a['type']) \ + '" in category "' + str(a['category']) \ + '" (to_ids:' + str(a['to_ids']) + ')' else: v['misp_description'] = 'MISP e' + str(a['event_id']) \ + ' attribute ' + str(a['uuid']) + ' of type "' \ + str(a['type']) + '" in category "' \ + str(a['category']) \ + '" (to_ids:' + str(a['to_ids']) \ + ' - o' + str(a['object_id']) + ' )' current_type = str(a['type']) # combined: not part of an object AND multivalue attribute QND to be split if int(a['object_id']) == 0 and '|' in current_type and my_args['pipe'] is True: mv_type_list = current_type.split('|') mv_value_list = str(a['value']).split('|') left_v = v.copy() left_v['misp_type'] = mv_type_list.pop() left_v['misp_value'] = mv_value_list.pop() results.append(left_v) if left_v['misp_type'] not in typelist: typelist.append(left_v['misp_type']) right_v = v.copy() right_v['misp_type'] = mv_type_list.pop() right_v['misp_value'] = mv_value_list.pop() results.append(right_v) if right_v['misp_type'] not in typelist: typelist.append(right_v['misp_type']) else: v['misp_type'] = current_type v['misp_value'] = str(a['value']) results.append(v) if current_type not in typelist: typelist.append(current_type) if pagination is True: if page_length < limit: other_page = False else: page = page + 1 else: other_page = False logging.info(json.dumps(typelist)) output_dict = {} # relevant_cat = ['Artifacts dropped', 'Financial fraud', 'Network activity','Payload delivery','Payload installation'] for r in results: if int(r['misp_object_id']) == 0: # not an object key = str(r['misp_event_id']) + '_' + r['misp_attribute_id'] is_object_member = False else: # this is a MISP object key = str(r['misp_event_id']) + '_object_' + str(r['misp_object_id']) is_object_member = True if key not in output_dict: v = dict(r) for t in typelist: misp_t = 'misp_' + t.replace('-', '_').replace('|', '_p_') if t == r['misp_type']: v[misp_t] = r['misp_value'] else: v[misp_t] = '' to_ids = [] to_ids.append(r['misp_to_ids']) v['misp_to_ids'] = to_ids category = [] category.append(r['misp_category']) v['misp_category'] = category if my_args['add_desc'] is True: description = [] description.append(r['misp_description']) v['misp_description'] = description if my_args['getuuid'] is True: attribute_uuid = [] attribute_uuid.append(r['misp_attribute_uuid']) v['misp_attribute_uuid'] = attribute_uuid if is_object_member is True: v['misp_type'] = 'misp_object' v['misp_value'] = r['misp_object_id'] output_dict[key] = dict(v) else: v = dict(output_dict[key]) misp_t = 'misp_' + r['misp_type'].replace('-', '_') v[misp_t] = r['misp_value'] # set value for relevant type to_ids = v['misp_to_ids'] if r['misp_to_ids'] not in to_ids: to_ids.append(r['misp_to_ids']) v['misp_to_ids'] = to_ids category = v['misp_category'] if r['misp_category'] not in category: # append category category.append(r['misp_category']) v['misp_category'] = category if my_args['add_desc'] is True: description = v['misp_description'] if r['misp_description'] not in description: description.append(r['misp_description']) v['misp_description'] = description if my_args['getuuid'] is True: attribute_uuid = v['misp_attribute_uuid'] if r['misp_attribute_uuid'] not in attribute_uuid: attribute_uuid.append(r['misp_attribute_uuid']) v['misp_attribute_uuid'] = attribute_uuid if is_object_member is False: misp_type = r['misp_type'] + '|' + v['misp_type'] v['misp_type'] = misp_type misp_value = r['misp_value'] + '|' + v['misp_value'] v['misp_value'] = misp_value output_dict[key] = dict(v) for k, v in output_dict.items(): yield v
def generate(self): # Phase 1: Preparation misp_instance = self.misp_instance storage = self.service.storage_passwords my_args = prepare_config(self, 'misp42splunk', misp_instance, storage) if my_args is None: raise Exception("Sorry, no configuration for misp_instance={}".format(misp_instance)) my_args['host'] = my_args['misp_url'].replace('https://', '') # check that ONE of mandatory fields is present mandatory_arg = 0 if self.json_request is not None: mandatory_arg = mandatory_arg + 1 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: raise Exception('Missing "json_request", "eventid", "last" or "date" argument') elif mandatory_arg > 1: raise Exception('Options "json_request", "eventid", "last" and "date" are mutually exclusive') body_dict = dict() # Only ONE combination was provided if self.json_request is not None: body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') elif self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set with %s', json.dumps(body_dict['eventid'])) elif self.last: body_dict['last'] = self.last logging.info('Option "last" set with %s', str(body_dict['last'])) else: body_dict['date'] = self.date.split() logging.info('Option "date" set with %s', json.dumps(body_dict['date'])) # Force some values on JSON request body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False body_dict['deleted'] = False body_dict['includeEventUuid'] = True # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True if self.limit is not None: limit = int(self.limit) elif 'limit' in body_dict: limit = int(body_dict['limit']) else: limit = 1000 if limit == 0: pagination = False if self.page is not None: page = int(self.page) elif 'page' in body_dict: page = body_dict['page'] else: page = 1 # Search parameters: boolean and filter # manage to_ids and enforceWarninglist # to avoid FP enforceWarninglist is set to True if # to_ids is set to True (search criterion) if self.category is not None: if "," in self.category: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria else: body_dict['category'] = self.category if self.endpoint == 'events': my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch' else: my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' if self.geteventtag is True: body_dict['includeEventTags'] = True if self.keep_related is True: keep_related = True else: keep_related = False if self.to_ids is True: body_dict['to_ids'] = True body_dict['enforceWarninglist'] = True # protection elif self.to_ids is False: body_dict['to_ids'] = False if self.type is not None: if "," in self.type: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria else: body_dict['type'] = self.type if self.warning_list is True: body_dict['enforceWarninglist'] = True elif self.warning_list is False: body_dict['enforceWarninglist'] = False if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.debug('mispgetioc request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is if r.status_code in (200, 201, 204): logging.info( "[CO301] INFO mispcollect successful. " "url={}, HTTP status={}".format(my_args['misp_url'], r.status_code) ) else: logging.error( "[CO302] ERROR mispcollect failed. " "url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], body, r.status_code, r.text) ) raise Exception( "[CO302] ERROR mispcollect failed. " "url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], body, r.status_code, r.text) ) # response is 200 by this point or we would have thrown an exception response = r.json() encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':')) if self.endpoint == "events": if 'response' in response: for r_item in response['response']: if 'Event' in r_item: attribute_names = [] serial_number = 0 for e in list(r_item.values()): if keep_related is False: e.pop('RelatedEvent', None) if serial_number == 0: for k in list(e.keys()): attribute_names.append(k) yield MispCollectCommand._record( serial_number, e['timestamp'], my_args['host'], e, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush else: if 'response' in response: if 'Attribute' in response['response']: attribute_names = [] serial_number = 0 for a in response['response']['Attribute']: if serial_number == 0: for k in list(a.keys()): attribute_names.append(k) yield MispCollectCommand._record( serial_number, a['timestamp'], my_args['host'], a, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush
def generate(self): # Phase 1: Preparation my_args = prepare_config(self, 'misp42splunk') my_args['host'] = my_args['misp_url'].replace('https://', '') my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' # check that ONE of mandatory fields is present mandatory_arg = 0 if self.json_request is not None: mandatory_arg = mandatory_arg + 1 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: logging.error('Missing "json_request", eventid", \ "last" or "date" argument') raise Exception('Missing "json_request", "eventid", \ "last" or "date" argument') elif mandatory_arg > 1: logging.error('Options "json_request", eventid", "last" \ and "date" are mutually exclusive') raise Exception('Options "json_request", "eventid", "last" \ and "date" are mutually exclusive') body_dict = dict() # Only ONE combination was provided if self.json_request is not None: body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') elif self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set with %s', json.dumps(body_dict['eventid'])) elif self.last: body_dict['last'] = self.last logging.info('Option "last" set with %s', str(body_dict['last'])) else: body_dict['date'] = self.date.split() logging.info('Option "date" set with %s', json.dumps(body_dict['date'])) # Force some values on JSON request body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False body_dict['deleted'] = False body_dict['includeEventUuid'] = True # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True if self.limit is not None: limit = int(self.limit) elif 'limit' in body_dict: limit = int(body_dict['limit']) else: limit = 1000 if limit == 0: pagination = False if self.page is not None: page = int(self.page) elif 'page' in body_dict: page = body_dict['page'] else: page = 1 # Search parameters: boolean and filter # manage to_ids and enforceWarninglist # to avoid FP enforceWarninglist is set to True if # to_ids is set to True (search criterion) if self.to_ids is True: body_dict['to_ids'] = True body_dict['enforceWarninglist'] = True # protection elif self.to_ids is False: body_dict['to_ids'] = False if self.warning_list is True: body_dict['enforceWarninglist'] = True elif self.warning_list is False: body_dict['enforceWarninglist'] = False if self.geteventtag is True: body_dict['includeEventTags'] = True if self.category is not None: if "," in self.category: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria else: body_dict['category'] = self.category if self.type is not None: if "," in self.type: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria else: body_dict['type'] = self.type if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria # output filter parameters if self.getuuid is True: my_args['getuuid'] = True else: my_args['getuuid'] = False if self.getorg is True: my_args['getorg'] = True else: my_args['getorg'] = False if self.pipesplit is True: my_args['pipe'] = True else: my_args['pipe'] = False if self.add_description is True: my_args['add_desc'] = True else: my_args['add_desc'] = False if self.output is not None: my_args['output'] = self.output else: my_args['output'] = "default" results = [] # add colums for each type in results typelist = [] if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.debug('mispgetioc request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':')) if my_args['output'] == "raw": if 'response' in response: if 'Attribute' in response['response']: attribute_names = [] serial_number = 0 for a in response['response']['Attribute']: yield MispGetIocCommand._record( serial_number, a['timestamp'], my_args['host'], a, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush else: if 'response' in response: if 'Attribute' in response['response']: for a in response['response']['Attribute']: v = {} v['misp_category'] = str(a['category']) v['misp_attribute_id'] = str(a['id']) v['misp_event_id'] = str(a['event_id']) v['misp_timestamp'] = str(a['timestamp']) v['misp_to_ids'] = str(a['to_ids']) v['misp_comment'] = str(a['comment']) tag_list = [] if 'Tag' in a: for tag in a['Tag']: try: tag_list.append(str(tag['name'])) except Exception: pass v['misp_tag'] = tag_list # include ID of the organisation that # created the attribute if requested if 'Event' in a: v['misp_event_uuid'] = str(a['Event']['uuid']) if my_args['getorg']: v['misp_orgc_id'] = str(a['Event']['orgc_id']) if my_args['add_desc'] is True: v['misp_event_info'] = str(a['Event']['info']) # include attribute UUID if requested if my_args['getuuid']: v['misp_attribute_uuid'] = str(a['uuid']) # handle object and multivalue attributes v['misp_object_id'] = str(a['object_id']) if my_args['add_desc'] is True: if int(a['object_id']) == 0: v['misp_description'] = 'MISP e' \ + str(a['event_id']) + ' attribute ' \ + str(a['uuid']) + ' of type "' \ + str(a['type']) \ + '" in category "' + str(a['category']) \ + '" (to_ids:' + str(a['to_ids']) + ')' else: v['misp_description'] = 'MISP e' \ + str(a['event_id']) + ' attribute ' \ + str(a['uuid']) + ' of type "' \ + str(a['type']) + '" in category "' \ + str(a['category']) \ + '" (to_ids:' + str(a['to_ids']) \ + ' - o' + str(a['object_id']) + ' )' current_type = str(a['type']) # combined: not part of an object # AND multivalue attribute AND to be split # logging.debug('misp_event: %s', json.dumps(v)) if int(a['object_id']) == 0 and '|' in current_type \ and my_args['pipe'] is True: mv_type_list = current_type.split('|') mv_value_list = str(a['value']).split('|') left_v = v.copy() left_v['misp_type'] = mv_type_list.pop() left_v['misp_value'] = mv_value_list.pop() results.append(left_v) if left_v['misp_type'] not in typelist: typelist.append(left_v['misp_type']) right_v = v.copy() right_v['misp_type'] = mv_type_list.pop() right_v['misp_value'] = mv_value_list.pop() results.append(right_v) if right_v['misp_type'] not in typelist: typelist.append(right_v['misp_type']) else: v['misp_type'] = current_type v['misp_value'] = str(a['value']) results.append(v) if current_type not in typelist: typelist.append(current_type) logging.info(json.dumps(typelist)) output_dict = {} attribute_names = [ 'misp_attribute_id', 'misp_attribute_uuid', 'misp_category', 'misp_comment', 'misp_description', 'misp_event_id', 'misp_tag', 'misp_timestamp', 'misp_to_ids', 'misp_type', 'misp_value' ] if my_args['add_desc'] is True: attribute_names.append('misp_event_info') for t in typelist: misp_t = 'misp_' + \ t.replace('-', '_').replace('|', '_p_') if misp_t not in attribute_names: attribute_names.append(misp_t) for r in results: if int(r['misp_object_id']) == 0: # not an object key = str(r['misp_event_id']) + \ '_' + r['misp_attribute_id'] is_object_member = False else: # this is a MISP object key = str(r['misp_event_id']) \ + '_object_' + str(r['misp_object_id']) is_object_member = True if key not in output_dict: v = dict(r) for t in typelist: misp_t = 'misp_' + t.replace('-', '_')\ .replace('|', '_p_') v[misp_t] = [] if t == r['misp_type']: v[misp_t].append(r['misp_value']) v['misp_to_ids'] = [] v['misp_to_ids'].append(r['misp_to_ids']) v['misp_category'] = [] v['misp_category'].append(r['misp_category']) v['misp_attribute_id'] = [] v['misp_attribute_id']\ .append(r['misp_attribute_id']) if my_args['getuuid'] is True: v['misp_attribute_uuid'] = [] v['misp_attribute_uuid']\ .append(r['misp_attribute_uuid']) if my_args['add_desc'] is True: description = [] description.append(r['misp_description']) v['misp_description'] = description if is_object_member is True: v['misp_type'] = 'misp_object' v['misp_value'] = r['misp_object_id'] output_dict[key] = dict(v) else: v = dict(output_dict[key]) misp_t = 'misp_' + r['misp_type'].replace('-', '_') v[misp_t].append(r['misp_value']) # set value for type v['misp_to_ids'].append(r['misp_to_ids']) v['misp_category'].append(r['misp_category']) tag_list = v['misp_tag'] for tag in r['misp_tag']: if tag not in tag_list: tag_list.append(tag) v['misp_tag'] = tag_list if my_args['add_desc'] is True: description = v['misp_description'] if r['misp_description'] not in description: description.append(r['misp_description']) v['misp_description'] = description v['misp_attribute_id']\ .append(r['misp_attribute_id']) if my_args['getuuid'] is True: v['misp_attribute_uuid']\ .append(r['misp_attribute_uuid']) if is_object_member is False: misp_type = r['misp_type'] + '|' + v['misp_type'] v['misp_type'] = misp_type misp_value = r['misp_value'] + '|' + v['misp_value'] v['misp_value'] = misp_value output_dict[key] = dict(v) serial_number = 0 logging.debug(json.dumps(attribute_names)) for k, v in list(output_dict.items()): yield MispGetIocCommand._record(serial_number, v['misp_timestamp'], my_args['host'], v, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush
def stream(self, records): # self.logger.debug('mispgetioc.reduce') # Generate args my_args = prepare_config(self) # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' fieldname = str(self.field) for record in records: if fieldname in record: value = record.get(fieldname, None) if value is not None: search_url = my_args['misp_url'] + '/attributes/restSearch' search_dict = {"returnFormat": "json"} search_dict['value'] = str(value) search_dict['withAttachments'] = "false", search_body = json.dumps(search_dict) sight_url = my_args[ 'misp_url'] + '/sightings/restSearch/attribute' sight_dict = {"returnFormat": "json"} misp_value = '' misp_fp = False misp_fp_timestamp = 0 misp_fp_event_id = '' misp_sight_seen = False misp_sight = { 'count': 0, 'first': 0, 'first_event_id': 0, 'last': 0, 'last_event_id': 0 } # search r = requests.post(search_url, headers=headers, data=search_body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() logging.info("INFO MISP REST API %s has response: %s" % (search_url, r.json())) if 'response' in response: if 'Attribute' in response['response']: for a in response['response']['Attribute']: if misp_value == '': misp_value = str(a['value']) if misp_fp == False: sight_dict['id'] = str(a['id']) sight_body = json.dumps(sight_dict) s = requests.post( sight_url, headers=headers, data=sight_body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is s.raise_for_status() # response is 200 by this point or we would have thrown an exception sight = s.json() logging.info( "INFO MISP REST API %s has response: %s" % (sight_url, s.json())) if 'response' in sight: for se in sight['response']: if 'Sighting' in se: if int(se['Sighting']['type'] ) == 0: #true sighting misp_sight_seen = True misp_sight[ 'count'] = misp_sight[ 'count'] + 1 if misp_sight['first'] == 0 or \ misp_sight['first'] > int(se['Sighting']['date_sighting']): misp_sight[ 'first'] = int( se['Sighting'] ['date_sighting'] ) misp_sight[ 'first_event_id'] = se[ 'Sighting'][ 'event_id'] if misp_sight['last'] < int( se['Sighting'] ['date_sighting']): misp_sight['last'] = int( se['Sighting'] ['date_sighting']) misp_sight[ 'last_event_id'] = se[ 'Sighting'][ 'event_id'] elif int( se['Sighting']['type'] ) == 1: #false positive misp_fp = True misp_fp_timestamp = int( se['Sighting'] ['date_sighting']) misp_fp_event_id = se[ 'Sighting']['event_id'] if misp_fp == True: record['misp_value'] = misp_value record['misp_fp'] = "True" record['misp_fp_timestamp'] = str( misp_fp_timestamp) record['misp_fp_event_id'] = str( misp_fp_event_id) if misp_sight_seen == True: record['misp_value'] = misp_value record['misp_sight_count'] = str( misp_sight['count']) record['misp_sight_first'] = str( misp_sight['first']) record['misp_sight_first_event_id'] = str( misp_sight['first_event_id']) record['misp_sight_last'] = str( misp_sight['last']) record['misp_sight_last_event_id'] = str( misp_sight['last_event_id']) yield record
def generate(self): # Phase 1: Preparation my_args = prepare_config(self, 'misp42splunk') my_args['host'] = my_args['misp_url'].replace('https://', '') my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch' # check that ONE of mandatory fields is present mandatory_arg = 0 if self.json_request is not None: mandatory_arg = mandatory_arg + 1 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: logging.error('Missing "json_request", eventid", \ "last" or "date" argument') raise Exception('Missing "json_request", "eventid", \ "last" or "date" argument') elif mandatory_arg > 1: logging.error('Options "json_request", eventid", "last" \ and "date" are mutually exclusive') raise Exception('Options "json_request", "eventid", "last" \ and "date" are mutually exclusive') body_dict = dict() # Only ONE combination was provided if self.json_request is not None: body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') elif self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set with %s', json.dumps(body_dict['eventid'])) elif self.last: body_dict['last'] = self.last logging.info('Option "last" set with %s', str(body_dict['last'])) else: body_dict['date'] = self.date.split() logging.info('Option "date" set with %s', json.dumps(body_dict['date'])) # Force some values on JSON request body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True if self.limit is not None: limit = int(self.limit) elif 'limit' in body_dict: limit = int(body_dict['limit']) else: limit = 1000 if limit == 0: pagination = False if self.page is not None: page = int(self.page) elif 'page' in body_dict: page = body_dict['page'] else: page = 1 if self.published is True: body_dict['published'] = True elif self.published is False: body_dict['published'] = False # Search parameters: boolean and filter # manage enforceWarninglist if self.warning_list is True: body_dict['enforceWarninglist'] = True elif self.warning_list is False: body_dict['enforceWarninglist'] = False if self.category is not None: if "," in self.category: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria else: body_dict['category'] = self.category if self.type is not None: if "," in self.type: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria else: body_dict['type'] = self.type if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria # output filter parameters if self.getioc is True: getioc = True else: getioc = False if self.pipesplit is True: pipesplit = True else: pipesplit = False if self.output is not None: output = self.output else: output = "default" if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.error('MispGetEventCommand request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':')) if output == "raw": if 'response' in response: attribute_names = list() serial_number = 0 for r_item in response['response']: if 'Event' in r_item: for e in r_item.values(): yield MispGetEventCommand._record( serial_number, e['timestamp'], my_args['host'], e, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush else: # build output table and list of types events = [] typelist = [] column_list = format_output_table(response, events, typelist, getioc, pipesplit) logging.info('typelist containss %s values', str(len(typelist))) logging.debug('typelist is %s', json.dumps(typelist)) logging.info('results contains %s records', str(len(events))) if getioc is False: attribute_names = list() init_attribute_names = True serial_number = 0 for e in events: # logging.debug('event is %s', json.dumps(e)) if init_attribute_names is True: for key in e.keys(): if key not in attribute_names: attribute_names.append(key) attribute_names.sort() init_attribute_names = False logging.debug(json.dumps(attribute_names)) yield MispGetEventCommand._record(serial_number, e['misp_timestamp'], my_args['host'], e, attribute_names, encoder, True) serial_number += 1 GeneratingCommand.flush else: output_dict = {} for e in events: if 'Attribute' in e: for a in e['Attribute']: if int(a['misp_object_id']) == 0: # not an object key = str(e['misp_event_id']) + '_' \ + str(a['misp_attribute_id']) is_object_member = False else: # this is a MISP object key = str(e['misp_event_id']) + \ '_object_' + str(a['misp_object_id']) is_object_member = True if key not in output_dict: v = init_misp_output(e, a, column_list) for t in typelist: misp_t = 'misp_' \ + t.replace('-', '_')\ .replace('|', '_p_') v[misp_t] = [] if t == a['misp_type']: v[misp_t].append(a['misp_value']) if is_object_member is True: v['misp_type'] = v['misp_object_name'] v['misp_value'] = v['misp_object_id'] output_dict[key] = dict(v) else: v = dict(output_dict[key]) misp_t = 'misp_' + a['misp_type']\ .replace('-', '_').replace('|', '_p_') v[misp_t].append(a['misp_value']) if a['misp_to_ids'] not in v['misp_to_ids']: v['misp_to_ids'].append(a['misp_to_ids']) if a['misp_category'] not in v[ 'misp_category']: v['misp_category'].append( a['misp_category']) v['misp_attribute_uuid']\ .append(a['misp_attribute_uuid']) v['misp_attribute_id']\ .append(a['misp_attribute_id']) if a['misp_attribute_tag'] is not None: a_tag = v['misp_attribute_tag'] for t in a['misp_attribute_tag']: if t not in a_tag: a_tag.append(t) v['misp_attribute_tag'] = a_tag if a['misp_comment'] not in v['misp_comment']: v['misp_comment'].append(a['misp_comment']) if is_object_member is False: misp_type = a['misp_type'] \ + '|' + v['misp_type'] v['misp_type'] = misp_type misp_value = a['misp_value'] + \ '|' + v['misp_value'] v['misp_value'] = misp_value output_dict[key] = dict(v) if output_dict is not None: attribute_names = list() init_attribute_names = True serial_number = 0 for v in output_dict.values(): if init_attribute_names is True: for key in v.keys(): if key not in attribute_names: attribute_names.append(key) attribute_names.sort() init_attribute_names = False logging.debug(json.dumps(attribute_names)) yield MispGetEventCommand._record( serial_number, v['misp_timestamp'], my_args['host'], v, attribute_names, encoder, True) serial_number += 1 GeneratingCommand.flush
def generate(self): # loggging self.set_log_level() # Phase 1: Preparation misp_instance = self.misp_instance storage = self.service.storage_passwords my_args = prepare_config(self, 'misp42splunk', misp_instance, storage) if my_args is None: raise Exception( "Sorry, no configuration for misp_instance={}".format( misp_instance)) my_args['host'] = my_args['misp_url'].replace('https://', '') my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' # check that ONE of mandatory fields is present mandatory_arg = 0 if self.json_request is not None: mandatory_arg = mandatory_arg + 1 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: self.log_error( 'Missing "json_request", eventid", "last" or "date" argument') raise Exception( 'Missing "json_request", "eventid", "last" or "date" argument') elif mandatory_arg > 1: self.log_error( 'Options "json_request", eventid", "last" and "date" are mutually exclusive' ) raise Exception( 'Options "json_request", "eventid", "last" and "date" are mutually exclusive' ) body_dict = dict() # Only ONE combination was provided if self.json_request is not None: body_dict = json.loads(self.json_request) self.log_info('Option "json_request" set') elif self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid self.log_info('Option "eventid" set with {}'.format( json.dumps(body_dict['eventid']))) elif self.last: body_dict['last'] = self.last self.log_info('Option "last" set with {}'.format( body_dict['last'])) else: body_dict['date'] = self.date.split() self.log_info('Option "date" set with {}'.format( json.dumps(body_dict['date']))) # Force some values on JSON request body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False body_dict['deleted'] = False body_dict['includeEventUuid'] = True # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True if self.limit is not None: limit = int(self.limit) elif 'limit' in body_dict: limit = int(body_dict['limit']) else: limit = 1000 if limit == 0: pagination = False if self.page is not None: page = int(self.page) elif 'page' in body_dict: page = body_dict['page'] else: page = 1 # Search parameters: boolean and filter # manage to_ids and enforceWarninglist # to avoid FP enforceWarninglist is set to True if # to_ids is set to True (search criterion) if self.to_ids is True: body_dict['to_ids'] = True body_dict['enforceWarninglist'] = True # protection elif self.to_ids is False: body_dict['to_ids'] = False if self.warning_list is True: body_dict['enforceWarninglist'] = True elif self.warning_list is False: body_dict['enforceWarninglist'] = False if self.geteventtag is True: body_dict['includeEventTags'] = True if self.category is not None: if "," in self.category: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria else: body_dict['category'] = self.category if self.type is not None: if "," in self.type: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria else: body_dict['type'] = self.type if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria # output filter parameters if self.add_description is True: my_args['add_desc'] = True else: my_args['add_desc'] = False if self.expand_object is True: my_args['expand'] = True else: my_args['expand'] = False if self.getorg is True: my_args['getorg'] = True else: my_args['getorg'] = False if self.getuuid is True: my_args['getuuid'] = True else: my_args['getuuid'] = False if self.pipesplit is True: my_args['pipe'] = True else: my_args['pipe'] = False if self.output is not None: my_args['output'] = self.output else: my_args['output'] = "default" # add colums for each type in results results = [] typelist = [] if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is if r.status_code in (200, 201, 204): self.log_info( "[IO301] INFO mispgetioc successful. url={}, HTTP status={}". format(my_args['misp_url'], r.status_code)) else: self.log_error( "[IO302] ERROR mispgetioc failed. url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], body, r.status_code, r.text)) raise Exception( "[IO302] ERROR mispgetioc failed for url={} with HTTP Error={}. Check search.log for details" .format(my_args['misp_url'], r.status_code)) # response is 200 by this point or we would have thrown an exception response = r.json() encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':')) # if raw output, returns JSON 1st-level keys as columns if my_args['output'] == "raw": if 'response' in response: if 'Attribute' in response['response']: attribute_names = list() serial_number = 0 for a in response['response']['Attribute']: yield MispGetIocCommand._record( serial_number, a['timestamp'], my_args['host'], a, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush # default output: extract some values from JSON attributes else: if 'response' in response: if 'Attribute' in response['response']: for a in response['response']['Attribute']: v = {} v['misp_category'] = str(a['category']) v['misp_attribute_id'] = str(a['id']) v['misp_event_id'] = str(a['event_id']) v['misp_timestamp'] = str(a['timestamp']) v['misp_to_ids'] = str(a['to_ids']) v['misp_comment'] = str(a['comment']) tag_list = [] if 'Tag' in a: for tag in a['Tag']: try: tag_list.append(str(tag['name'])) except Exception: pass v['misp_tag'] = tag_list # include ID of the organisation that # created the attribute if requested if 'Event' in a: v['misp_event_uuid'] = str(a['Event']['uuid']) if my_args['getorg']: v['misp_orgc_id'] = str(a['Event']['orgc_id']) if my_args['add_desc'] is True: v['misp_event_info'] = str(a['Event']['info']) # include attribute UUID if requested if my_args['getuuid']: v['misp_attribute_uuid'] = str(a['uuid']) # handle object and multivalue attributes v['misp_object_id'] = str(a['object_id']) if my_args['add_desc'] is True: if int(a['object_id']) == 0: v['misp_description'] = 'MISP e' \ + str(a['event_id']) + ' attribute ' \ + str(a['uuid']) + ' of type "' \ + str(a['type']) \ + '" in category "' + str(a['category']) \ + '" (to_ids:' + str(a['to_ids']) + ')' else: v['misp_description'] = 'MISP e' \ + str(a['event_id']) + ' attribute ' \ + str(a['uuid']) + ' of type "' \ + str(a['type']) + '" in category "' \ + str(a['category']) \ + '" (to_ids:' + str(a['to_ids']) \ + ' - o' + str(a['object_id']) + ' )' current_type = str(a['type']) # combined: not part of an object # AND multivalue attribute AND to be split if int(a['object_id']) == 0 and '|' in current_type \ and my_args['pipe'] is True: mv_type_list = current_type.split('|') mv_value_list = str(a['value']).split('|') left_v = v.copy() left_v['misp_type'] = mv_type_list.pop() left_v['misp_value'] = mv_value_list.pop() results.append(left_v) if left_v['misp_type'] not in typelist: typelist.append(left_v['misp_type']) right_v = v.copy() right_v['misp_type'] = mv_type_list.pop() right_v['misp_value'] = mv_value_list.pop() results.append(right_v) if right_v['misp_type'] not in typelist: typelist.append(right_v['misp_type']) else: v['misp_type'] = current_type v['misp_value'] = str(a['value']) results.append(v) if current_type not in typelist: typelist.append(current_type) self.log_info(json.dumps(typelist)) # consolidate attribute values under output table if my_args['expand'] is True: output_dict = {} for r in results: key = str(r['misp_event_id']) + \ '_' + str(r['misp_attribute_id']) if key not in output_dict: v = dict(r) for t in typelist: misp_t = 'misp_' + t.replace('-', '_').replace( '|', '_p_') v[misp_t] = [] if t == r['misp_type']: v[misp_t].append(r['misp_value']) output_dict[key] = dict(v) else: v = dict(output_dict[key]) misp_t = 'misp_' + r['misp_type'].replace( '-', '_').replace('|', '_p_') v[misp_t].append(r['misp_value']) # set value for type misp_type = r['misp_type'] + '|' + v['misp_type'] v['misp_type'] = misp_type misp_value = str(r['misp_value']) + '|' + str( v['misp_value']) v['misp_value'] = misp_value output_dict[key] = dict(v) else: output_dict = {} for r in results: if int(r['misp_object_id']) == 0: # not an object key = str(r['misp_event_id']) + \ '_' + str(r['misp_attribute_id']) is_object_member = False else: # this is a MISP object key = str(r['misp_event_id']) \ + '_object_' + str(r['misp_object_id']) is_object_member = True if key not in output_dict: v = dict(r) for t in typelist: misp_t = 'misp_' + t.replace('-', '_').replace( '|', '_p_') v[misp_t] = [] if t == r['misp_type']: v[misp_t].append(r['misp_value']) v['misp_to_ids'] = [] v['misp_to_ids'].append(r['misp_to_ids']) v['misp_category'] = [] v['misp_category'].append(r['misp_category']) v['misp_attribute_id'] = [] v['misp_attribute_id'].append(r['misp_attribute_id']) if my_args['getuuid'] is True: v['misp_attribute_uuid'] = [] v['misp_attribute_uuid'].append( r['misp_attribute_uuid']) if my_args['add_desc'] is True: v['misp_description'] = [] v['misp_description'].append(r['misp_description']) v['misp_type'] = [] v['misp_type'].append(r['misp_type']) v['misp_value'] = [] v['misp_value'].append(str(r['misp_value'])) output_dict[key] = dict(v) else: v = dict(output_dict[key]) misp_t = 'misp_' + r['misp_type'].replace( '-', '_').replace('|', '_p_') v[misp_t].append(r['misp_value']) # set value for type v['misp_to_ids'].append(r['misp_to_ids']) v['misp_category'].append(r['misp_category']) tag_list = v['misp_tag'] for tag in r['misp_tag']: if tag not in tag_list: tag_list.append(tag) v['misp_tag'] = tag_list if my_args['add_desc'] is True: description = v['misp_description'] if r['misp_description'] not in description: description.append(r['misp_description']) v['misp_description'] = description if is_object_member is False: if r['misp_attribute_id'] not in v[ 'misp_attribute_id']: v['misp_attribute_id'].append( r['misp_attribute_id']) if my_args['getuuid'] is True: if r['misp_attribute_uuid'] not in v[ 'misp_attribute_uuid']: v['misp_attribute_uuid'].append( r['misp_attribute_uuid']) misp_type = [] misp_type.append(r['misp_type'] + '|' + v['misp_type'][0]) v['misp_type'] = misp_type misp_value = [] misp_value.append( str(r['misp_value']) + '|' + str(v['misp_value'][0])) v['misp_value'] = misp_value else: v['misp_attribute_id'].append( r['misp_attribute_id']) if my_args['getuuid'] is True: v['misp_attribute_uuid'].append( r['misp_attribute_uuid']) v['misp_type'].append(r['misp_type']) v['misp_value'].append(r['misp_value']) output_dict[key] = dict(v) # return output table attribute_names = list() init_attribute_names = True serial_number = 0 for v in output_dict.values(): if init_attribute_names is True: for key in v.keys(): if key not in attribute_names: attribute_names.append(key) attribute_names.sort() init_attribute_names = False yield MispGetIocCommand._record(serial_number, v['misp_timestamp'], my_args['host'], v, attribute_names, encoder, True) serial_number += 1 GeneratingCommand.flush
def stream(self, records): # loggging self.set_log_level() # Phase 1: Preparation misp_instance = self.misp_instance storage = self.service.storage_passwords my_args = prepare_config(self, 'misp42splunk', misp_instance, storage) if my_args is None: raise Exception( "Sorry, no configuration for misp_instance={}".format( misp_instance)) my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' fieldname = str(self.field) pagination = True if self.limit is not None: if int(self.limit) == 0: pagination = False else: limit = int(self.limit) else: limit = 1000 if self.page is not None: page = int(self.page) else: page = 1 if self.json_request is not None: body_dict = json.loads(self.json_request) self.log_info('Option "json_request" set') body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False if 'limit' in body_dict: limit = int(body_dict['limit']) if limit == 0: pagination = False if 'page' in body_dict: page = body_dict['page'] pagination = False else: # build search JSON object body_dict = {"returnFormat": "json", "withAttachments": False} if self.to_ids is True: body_dict['to_ids'] = "True" if self.includeEventUuid is not None: body_dict['includeEventUuid'] = self.includeEventUuid if self.includeEventTags is not None: body_dict['includeEventTags'] = self.includeEventTags if self.last is not None: body_dict['last'] = self.last for record in records: if fieldname in record: value = record.get(fieldname, None) if value is not None: body_dict['value'] = str(value) misp_category = [] misp_event_id = [] misp_event_uuid = [] misp_orgc_id = [] misp_to_ids = [] misp_comment = [] misp_tag = [] misp_type = [] misp_value = [] misp_uuid = [] # search if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is # check if status is anything other than 200; # throw an exception if it is if r.status_code in (200, 201, 204): self.log_info( "[SE301] INFO mispsearch successful. url={}, HTTP status={}" .format(my_args['misp_url'], r.status_code)) else: self.log_error( "[SE302] ERROR mispsearch failed. url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], body, r.status_code, r.text)) raise Exception( "[SE302] ERROR mispsearch failed for url={} with HTTP Error={}. Check search.log for details" .format(my_args['misp_url'], r.status_code)) # response is 200 by this point or we would have thrown an exception response = r.json() if 'response' in response: if 'Attribute' in response['response']: for a in response['response']['Attribute']: if str(a['type']) not in misp_type: misp_type.append(str(a['type'])) if str(a['value']) not in misp_value: misp_value.append(str(a['value'])) if str(a['to_ids']) not in misp_to_ids: misp_to_ids.append(str(a['to_ids'])) if str(a['comment']) not in misp_comment: misp_comment.append(str(a['comment'])) if str(a['category']) not in misp_category: misp_category.append(str(a['category'])) if str(a['uuid']) not in misp_uuid: misp_uuid.append(str(a['uuid'])) if str(a['event_id']) not in misp_event_id: misp_event_id.append(str(a['event_id'])) if 'Tag' in a: for tag in a['Tag']: if str(tag['name']) not in misp_tag: misp_tag.append(str(tag['name'])) if 'Event' in a: if a['Event']['uuid'] \ not in misp_event_uuid: misp_event_uuid.append( str(a['Event']['uuid'])) if a['Event']['orgc_id'] \ not in misp_orgc_id: misp_orgc_id.append( str(a['Event']['orgc_id'])) record['misp_type'] = misp_type record['misp_value'] = misp_value record['misp_to_ids'] = misp_to_ids record['misp_comment'] = misp_comment record['misp_category'] = misp_category record['misp_attribute_uuid'] = misp_uuid record['misp_event_id'] = misp_event_id record['misp_event_uuid'] = misp_event_uuid record['misp_orgc_id'] = misp_orgc_id record['misp_tag'] = misp_tag yield record
def reduce(self, records): # Phase 1: Preparation my_args = prepare_config(self) my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch' # check that ONE of mandatory fields is present mandatory_arg = 0 if self.json_request is not None: mandatory_arg = mandatory_arg + 1 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: logging.error('Missing "json_request", eventid", \ "last" or "date" argument') raise Exception('Missing "json_request", "eventid", \ "last" or "date" argument') elif mandatory_arg > 1: logging.error('Options "json_request", eventid", "last" \ and "date" are mutually exclusive') raise Exception('Options "json_request", "eventid", "last" \ and "date" are mutually exclusive') body_dict = dict() # Only ONE combination was provided if self.json_request is not None: body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') elif self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set with %s', json.dumps(body_dict['eventid'])) elif self.last: body_dict['last'] = self.last logging.info('Option "last" set with %s', str(body_dict['last'])) else: body_dict['date'] = self.date.split() logging.info('Option "date" set with %s', json.dumps(body_dict['date'])) # Force some values on JSON request body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True if self.limit is not None: limit = int(self.limit) elif 'limit' in body_dict: limit = int(body_dict['limit']) else: limit = 1000 if limit == 0: pagination = False if self.page is not None: page = int(self.page) elif 'page' in body_dict: page = body_dict['page'] else: page = 1 if self.published is True: body_dict['published'] = True elif self.published is False: body_dict['published'] = False if self.category is not None: if "," in self.category: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria else: body_dict['category'] = self.category if self.type is not None: if "," in self.type: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria else: body_dict['type'] = self.type if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria # output filter parameters if self.getioc is True: my_args['getioc'] = True else: my_args['getioc'] = False if self.pipesplit is True: my_args['pipe'] = True else: my_args['pipe'] = False results = [] # add colums for each type in results typelist = [] if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.error('mispgetevent request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() if 'response' in response: for r_item in response['response']: if 'Event' in r_item: for a in list(r_item.values()): v = {} v['misp_event_id'] = str(a['id']) v['misp_orgc_id'] = str(a['orgc_id']) v['misp_event_date'] = str(a['date']) v['threat_level_id'] = str(a['threat_level_id']) v['misp_event_info'] = a['info'] v['misp_event_published'] = str(a['published']) v['misp_event_uuid'] = str(a['uuid']) v['misp_attribute_count'] = str(a['attribute_count']) v['misp_analysis'] = str(a['analysis']) v['misp_timestamp'] = str(a['timestamp']) v['misp_distribution'] = str(a['distribution']) v['misp_publish_timestamp'] = \ str(a['publish_timestamp']) v['misp_sharing_group_id'] = str(a['sharing_group_id']) v['misp_extends_uuid'] = str(a['extends_uuid']) if 'Orgc' in a: v['misp_orgc_name'] = str(a['Orgc']['name']) v['misp_orgc_uuid'] = str(a['Orgc']['uuid']) tag_list = [] if 'Tag' in a: for tag in a['Tag']: try: tag_list.append(str(tag['name'])) except Exception: pass v['misp_tag'] = tag_list if my_args['getioc'] is True: v['Attribute'] = list() v['misp_attribute_count'] = 0 if 'Attribute' in a: v['misp_attribute_count'] = \ v['misp_attribute_count'] + len(a['Attribute']) if my_args['getioc'] is True: for attribute in a['Attribute']: # combined: not part of an object AND # multivalue attribute AND to be split if int(attribute['object_id']) == 0 \ and '|' in attribute['type'] \ and my_args['pipe'] is True: mv_type_list = \ attribute['type'].split('|') mv_value_list = \ str(attribute['value']).split('|') left_a = attribute.copy() left_a['type'] = mv_type_list.pop() left_a['value'] = mv_value_list.pop() v['Attribute'].append( getioc(left_a, typelist, my_args['pipe'], left_a['object_id'])) right_a = attribute.copy() right_a['type'] = mv_type_list.pop() right_a['value'] = mv_value_list.pop() v['Attribute'].append( getioc(right_a, typelist, my_args['pipe'], right_a['object_id'])) else: v['Attribute'].append( getioc(attribute, typelist, my_args['pipe'], attribute['object_id'])) if 'Object' in a: for misp_o in a['Object']: if 'Attribute' in misp_o: v['misp_attribute_count'] = \ v['misp_attribute_count'] \ + len(misp_o['Attribute']) if my_args['getioc'] is True: object_id = misp_o['id'] object_name = misp_o['name'] object_comment = misp_o['comment'] for attribute in misp_o['Attribute']: v['Attribute'].append( getioc(attribute, typelist, my_args['pipe'], object_id, object_name, object_comment)) logging.debug('event is %s', json.dumps(v)) results.append(v) logging.info('typelist is %s', json.dumps(typelist)) # relevant_cat = ['Artifacts dropped', 'Financial fraud', # 'Network activity','Payload delivery','Payload installation'] logging.debug('results is %s', json.dumps(results)) if my_args['getioc'] is False: for e in results: yield e else: output_dict = {} for e in results: if 'Attribute' in e: for r in e['Attribute']: if int(r['misp_object_id']) == 0: # not an object key = str(e['misp_event_id']) + '_' \ + r['misp_attribute_id'] is_object_member = False else: # this is a MISP object key = str(e['misp_event_id']) + \ '_object_' + str(r['misp_object_id']) is_object_member = True if key not in output_dict: v = init_misp_output(e, r) for t in typelist: misp_t = 'misp_' \ + t.replace('-', '_').replace('|', '_p_') if t == r['misp_type']: v[misp_t] = r['misp_value'] else: v[misp_t] = '' to_ids = [] to_ids.append(r['misp_to_ids']) v['misp_to_ids'] = to_ids category = [] category.append(r['misp_category']) v['misp_category'] = category attribute_uuid = [] attribute_uuid.append(r['misp_attribute_uuid']) v['misp_attribute_uuid'] = attribute_uuid if is_object_member is True: v['misp_type'] = v['misp_object_name'] v['misp_value'] = v['misp_object_id'] output_dict[key] = dict(v) else: v = dict(output_dict[key]) misp_t = 'misp_' + r['misp_type'].replace('-', '_') v[misp_t] = r['misp_value'] # set value for type to_ids = v['misp_to_ids'] if r['misp_to_ids'] not in to_ids: to_ids.append(r['misp_to_ids']) v['misp_to_ids'] = to_ids category = v['misp_category'] # append if r['misp_category'] not in category: category.append(r['misp_category']) v['misp_category'] = category attribute_uuid = v['misp_attribute_uuid'] if r['misp_attribute_uuid'] not in attribute_uuid: attribute_uuid.append(r['misp_attribute_uuid']) v['misp_attribute_uuid'] = attribute_uuid if is_object_member is False: misp_type = r['misp_type'] \ + '|' + v['misp_type'] v['misp_type'] = misp_type misp_value = r['misp_value'] + \ '|' + v['misp_value'] v['misp_value'] = misp_value output_dict[key] = dict(v) for k, v in list(output_dict.items()): yield v
def reduce(self, records): # Phase 1: Preparation my_args = prepare_config(self) my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' jsonmode = False if self.mode is not None: if 'j' in self.mode and self.json_request is not None: jsonmode = True if jsonmode is True: pagination = True other_page = True body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False if 'limit' in body_dict: limit = int(body_dict['limit']) if limit == 0: pagination = False else: limit = 10000 if 'page' in body_dict: page = body_dict['page'] else: page = 1 page_length = 0 else: # build search JSON object body_dict = {"returnFormat": "json", "withAttachments": False} # add provided parameters to JSON request body # specific formats if self.last is not None: body_dict['last'] = self.last logging.info('Option "last" set with %s', body_dict['last']) if self.date_from is not None: body_dict['from'] = self.date_from logging.info('Option "date_from" set with %s', body_dict['from']) if self.date_to is not None: body_dict['to'] = self.date_to logging.info('Option "date_to" set with %s', body_dict['to']) else: logging.info('Option "date_to" will be set to now().') if self.threat_level_id is not None: body_dict['threat_level_id'] = self.threat_level_id logging.info('Option "threat_level_id" set with %s', body_dict['threat_level_id']) if self.org is not None: body_dict['org'] = self.org logging.info('Option "org" set') if self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set') if self.value is not None: body_dict['value'] = self.value logging.info('Option "value" set') if self.eventinfo is not None: body_dict['eventinfo'] = self.eventinfo logging.info('Option "eventinfo" set') # CSV strings if self.category is not None: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria if self.type is not None: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria if self.uuid is not None: uuid_criteria = {} uuid_list = self.uuid.split(",") uuid_criteria['OR'] = uuid_list body_dict['uuid'] = uuid_criteria # Booleans if self.to_ids is not None: body_dict['to_ids'] = self.to_ids logging.info('Option "to_ids" set with %s', body_dict['to_ids']) if self.enforceWarninglist is not None: body_dict['enforceWarninglist'] = self.enforceWarninglist logging.info('Option "enforceWarninglist" set with %s', body_dict['enforceWarninglist']) if self.deleted is not None: body_dict['deleted'] = self.deleted logging.info('Option "deleted" set with %s', body_dict['deleted']) if self.includeEventUuid is not None: body_dict['includeEventUuid'] = self.includeEventUuid logging.info('Option "includeEventUuid" set with %s', body_dict['includeEventUuid']) if self.includeEventTags is not None: body_dict['includeEventTags'] = self.includeEventTags logging.info('Option "includeEventTags" set with %s', body_dict['includeEventTags']) # Search pagination pagination = True other_page = True if self.page: page = self.page else: page = 1 page_length = 0 if self.limit is not None: if int(self.limit) == 0: pagination = False else: limit = int(self.limit) else: limit = 10000 # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' results = [] # add colums for each type in results while other_page: if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.debug('mispapireport request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() if 'response' in response: if 'Attribute' in response['response']: page_length = len(response['response']['Attribute']) for a in response['response']['Attribute']: v = {} v['misp_Object'] = "-" if self.includeEventTags is True: v['misp_tag'] = "-" for ak, av in a.items(): if ak == 'Event': json_event = a['Event'] for ek, ev in json_event.items(): key = 'misp_event_' + ek v[key] = str(ev) elif ak == 'Tag': tag_list = [] for tag in a['Tag']: try: tag_list.append(str(tag['name'])) except Exception: pass v['misp_tag'] = tag_list else: vkey = 'misp_' + ak v[vkey] = av results.append(v) if pagination is True: if page_length < limit: other_page = False else: page = page + 1 else: other_page = False # add colums for each type in results typelist = [] for r in results: if r['misp_type'] not in typelist: typelist.append(r['misp_type']) output_dict = {} increment = 1 for r in results: key = str(r['misp_event_id']) + '_' + str(increment) increment = increment + 1 v = r for t in typelist: misp_t = 'misp_' + t.replace('-', '_').replace('|', '_p_') if t == r['misp_type']: v[misp_t] = r['misp_value'] else: v[misp_t] = '' output_dict[key] = v for k, v in output_dict.items(): yield v
def stream(self, records): # Generate args my_args = prepare_config(self) my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' fieldname = str(self.field) if self.onlyids is True: to_ids = True else: to_ids = False if self.gettag is True: get_tag = True else: get_tag = False for record in records: if fieldname in record: value = record.get(fieldname, None) if value is not None: body_dict = {"returnFormat": "json"} body_dict['value'] = str(value) body_dict['withAttachments'] = "false" if to_ids: body_dict['to_ids'] = "True" body = json.dumps(body_dict) misp_category = [] misp_event_id = [] misp_to_ids = [] misp_tag = [] misp_type = [] misp_value = [] misp_uuid = [] # search logging.debug('mispsearch request body: %s', body) r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception # print >> sys.stderr, "DEBUG MISP REST API response: %s" % response.json() response = r.json() if 'response' in response: if 'Attribute' in response['response']: for a in response['response']['Attribute']: if str(a['type']) not in misp_type: misp_type.append(str(a['type'])) if str(a['value']) not in misp_value: misp_value.append(str(a['value'])) if str(a['to_ids']) not in misp_to_ids: misp_to_ids.append(str(a['to_ids'])) if str(a['category']) not in misp_category: misp_category.append(str(a['category'])) if str(a['uuid']) not in misp_uuid: misp_uuid.append(str(a['uuid'])) if str(a['event_id']) not in misp_event_id: misp_event_id.append(str(a['event_id'])) if get_tag and 'Tag' in a: for tag in a['Tag']: if str(tag['name']) not in misp_tag: misp_tag.append(str(tag['name'])) record['misp_type'] = misp_type record['misp_value'] = misp_value record['misp_to_ids'] = misp_to_ids record['misp_category'] = misp_category record['misp_attribute_uuid'] = misp_uuid record['misp_event_id'] = misp_event_id if get_tag: record['misp_tag'] = misp_tag yield record