def test_match(self): validator = validators.Match('social security number', r'\d{3}-\d{2}-\d{4}') self.assertEqual(validator.__call__('123-45-6789'), '123-45-6789') self.assertRaises(ValueError, validator.__call__, 'foo') self.assertEqual(validator.__call__(None), None) self.assertEqual(validator.format(None), None) self.assertEqual(validator.format('123-45-6789'), '123-45-6789')
class mispapireport(ReportingCommand): """ MISP API wrapper for endpoint /attributes/restSearch. return format is JSON for the momemnt ##Syntax use paramater names to set values in the POST request body below. .. code-block:: | mispapireport misp_instance=<input> page=<int> limit=<int> value=string type=CSVstring category=CSVstring org=string tags=CSVstring not_tags=CSVstrings date_from=date_string date_to=date_string last=<int>(d|h|m) eventid=CSVint uuid=CSVuuid_string enforceWarninglist=True|False to_ids=True|False deleted=True|False includeEventUuid=True|False includeEventTags==True|False threat_level_id=<int> eventinfo=string forced parameters: "returnFormat": "json" withAttachments: False not handled parameters: "publish_timestamp": "optional", "timestamp": "optional", "event_timestamp": "optional", ##Description { "returnFormat": "mandatory", "page": "optional", "limit": "optional", "value": "optional", "type": "optional", "category": "optional", "org": "optional", "tags": "optional", "from": "optional", "to": "optional", "last": "optional", "eventid": "optional", "withAttachments": "optional", "uuid": "optional", "publish_timestamp": "optional", "timestamp": "optional", "enforceWarninglist": "optional", "to_ids": "optional", "deleted": "optional", "includeEventUuid": "optional", "includeEventTags": "optional", "event_timestamp": "optional", "threat_level_id": "optional", "eventinfo": "optional", "includeProposals": "optional" } # status for mode=p "returnFormat": forced to json, "page": param, "limit": param, "value": param, "type": param, CSV string, "category": param, CSV string, "org": param, CSV string, "tags": param with not_tags, "from": param, "to": param, "last": param, "eventid": param, "withAttachments": forced to false, "uuid": param, "publish_timestamp": not managed, "timestamp": not managed, "enforceWarninglist": param, "to_ids": param, "deleted": param, "includeEventUuid": param, "includeEventTags": param, "event_timestamp": not managed, "threat_level_id": param, "eventinfo": param, "includeProposals": not managed } """ # Superseede MISP instance for this search misp_instance = Option(doc=''' **Syntax:** **misp_instance=instance_name* **Description:**MISP instance parameters as described in local/inputs.conf.''', require=True) # mode: p - give parameters one by one / j provide a complete JSON string # default is mode=p mode = Option(doc=''' **Syntax:** **mode=***p|j<AUTH_KEY>* **Description:**mode to build the JSON request.''', require=False, validate=validators.Match("mode", r"^(p|j)$")) # if mode=j a complete JSON request has to be provided json_request = Option(doc=''' **Syntax:** **json_request=***valid JSON request* **Description:**Valid JSON request''', require=False) # specific formats last = Option(doc=''' **Syntax:** **last=***<int>d|h|m* **Description:**publication duration in day(s), hour(s) or minute(s).''', require=False, validate=validators.Match("last", r"^[0-9]+[hdm]$")) date_from = Option(doc=''' **Syntax:** **date_from=***date_string"* **Description:**starting date.''', require=False) date_to = Option(doc=''' **Syntax:** **date_to=***date_string"* **Description:**(optional)ending date in searches with date_from. if not set default is now''', require=False) threat_level_id = Option(doc=''' **Syntax:** **threat_level_id=***1-4* **Description:**Threat level.''', require=False, validate=validators.Match("threat_level_id", r"^[1-4]$")) org = Option(doc=''' **Syntax:** **org=***CSV string* **Description:**Comma(,)-separated string of org name(s), id(s), uuid(s).''', require=False) # CSV numeric list eventid = Option(doc=''' **Syntax:** **eventid=***id1(,id2,...)* **Description:**list of event ID(s).''', require=False, validate=validators.Match("eventid", r"^[0-9,]+$")) # strings value = Option(doc=''' **Syntax:** **value=***string* **Description:**value.''', require=False) eventinfo = Option(doc=''' **Syntax:** **eventinfo=***string* **Description:**eventinfo string''', require=False) # numeric values limit = Option(doc=''' **Syntax:** **limit=***<int>* **Description:**define the limit for each MISP search; default 10000. 0 = no pagination.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) page = Option(doc=''' **Syntax:** **page=***<int>* **Description:**define the page of result to get.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) # CSV strings uuid = Option(doc=''' **Syntax:** **uuid=***id1(,id2,...)* **Description:**list of event UUID(s).''', require=False) type = Option(doc=''' **Syntax:** **type=***CSV string* **Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''', require=False) category = Option(doc=''' **Syntax:** **category=***CSV string* **Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''', require=False) tags = Option(doc=''' **Syntax:** **tags=***CSV string* **Description:**Comma(,)-separated string of tags to search for. Wildcard is %.''', require=False) not_tags = Option(doc=''' **Syntax:** **not_tags=***CSV string* **Description:**Comma(,)-separated string of tags to exclude from results. Wildcard is %.''', require=False) # Booleans to_ids = Option(doc=''' **Syntax:** **to_ids=***y|Y|1|true|True|n|N|0|false|False* **Description:**Boolean to search only attributes with the flag "to_ids" set to true.''', require=False, validate=validators.Boolean()) enforceWarninglist = Option(doc=''' **Syntax:** **enforceWarninglist=***y|Y|1|true|True|n|N|0|false|False* **Description:**Boolean to apply warning lists to results.''', require=False, validate=validators.Boolean()) deleted = Option(doc=''' **Syntax:** **deleted=***y|Y|1|true|True|n|N|0|false|False* **Description:**Boolean to include deleted attributes to results.''', require=False, validate=validators.Boolean()) includeEventUuid = Option(doc=''' **Syntax:** **includeEventUuid=***y|Y|1|true|True|n|N|0|false|False* **Description:**Boolean to include event UUID(s) to results.''', require=False, validate=validators.Boolean()) includeEventTags = Option(doc=''' **Syntax:** **includeEventTags=***y|Y|1|true|True|n|N|0|false|False* **Description:**Boolean to include event UUID(s) to results.''', require=False, validate=validators.Boolean()) pipesplit = Option(doc=''' **Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to split multivalue attributes into 2 attributes.''', require=False, validate=validators.Boolean()) @Configuration() def map(self, records): # self.logger.debug('mispgetioc.map') return records def reduce(self, records): # Phase 1: Preparation my_args = prepare_config(self) my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' jsonmode = False if self.mode is not None: if 'j' in self.mode and self.json_request is not None: jsonmode = True if jsonmode is True: pagination = True other_page = True body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False if 'limit' in body_dict: limit = int(body_dict['limit']) if limit == 0: pagination = False else: limit = 10000 if 'page' in body_dict: page = body_dict['page'] else: page = 1 page_length = 0 else: # build search JSON object body_dict = {"returnFormat": "json", "withAttachments": False} # add provided parameters to JSON request body # specific formats if self.last is not None: body_dict['last'] = self.last logging.info('Option "last" set with %s', body_dict['last']) if self.date_from is not None: body_dict['from'] = self.date_from logging.info('Option "date_from" set with %s', body_dict['from']) if self.date_to is not None: body_dict['to'] = self.date_to logging.info('Option "date_to" set with %s', body_dict['to']) else: logging.info('Option "date_to" will be set to now().') if self.threat_level_id is not None: body_dict['threat_level_id'] = self.threat_level_id logging.info('Option "threat_level_id" set with %s', body_dict['threat_level_id']) if self.org is not None: body_dict['org'] = self.org logging.info('Option "org" set') if self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set') if self.value is not None: body_dict['value'] = self.value logging.info('Option "value" set') if self.eventinfo is not None: body_dict['eventinfo'] = self.eventinfo logging.info('Option "eventinfo" set') # CSV strings if self.category is not None: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria if self.type is not None: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria if self.uuid is not None: uuid_criteria = {} uuid_list = self.uuid.split(",") uuid_criteria['OR'] = uuid_list body_dict['uuid'] = uuid_criteria # Booleans if self.to_ids is not None: body_dict['to_ids'] = self.to_ids logging.info('Option "to_ids" set with %s', body_dict['to_ids']) if self.enforceWarninglist is not None: body_dict['enforceWarninglist'] = self.enforceWarninglist logging.info('Option "enforceWarninglist" set with %s', body_dict['enforceWarninglist']) if self.deleted is not None: body_dict['deleted'] = self.deleted logging.info('Option "deleted" set with %s', body_dict['deleted']) if self.includeEventUuid is not None: body_dict['includeEventUuid'] = self.includeEventUuid logging.info('Option "includeEventUuid" set with %s', body_dict['includeEventUuid']) if self.includeEventTags is not None: body_dict['includeEventTags'] = self.includeEventTags logging.info('Option "includeEventTags" set with %s', body_dict['includeEventTags']) # Search pagination pagination = True other_page = True if self.page: page = self.page else: page = 1 page_length = 0 if self.limit is not None: if int(self.limit) == 0: pagination = False else: limit = int(self.limit) else: limit = 10000 # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' results = [] # add colums for each type in results while other_page: if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.debug('mispapireport request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() if 'response' in response: if 'Attribute' in response['response']: page_length = len(response['response']['Attribute']) for a in response['response']['Attribute']: v = {} v['misp_Object'] = "-" if self.includeEventTags is True: v['misp_tag'] = "-" for ak, av in a.items(): if ak == 'Event': json_event = a['Event'] for ek, ev in json_event.items(): key = 'misp_event_' + ek v[key] = str(ev) elif ak == 'Tag': tag_list = [] for tag in a['Tag']: try: tag_list.append(str(tag['name'])) except Exception: pass v['misp_tag'] = tag_list else: vkey = 'misp_' + ak v[vkey] = av results.append(v) if pagination is True: if page_length < limit: other_page = False else: page = page + 1 else: other_page = False # add colums for each type in results typelist = [] for r in results: if r['misp_type'] not in typelist: typelist.append(r['misp_type']) output_dict = {} increment = 1 for r in results: key = str(r['misp_event_id']) + '_' + str(increment) increment = increment + 1 v = r for t in typelist: misp_t = 'misp_' + t.replace('-', '_').replace('|', '_p_') if t == r['misp_type']: v[misp_t] = r['misp_value'] else: v[misp_t] = '' output_dict[key] = v for k, v in output_dict.items(): yield v
class MispCollectCommand(GeneratingCommand): """ get the attributes from a MISP instance. ##Syntax .. code-block:: | mispgetioc misp_instance=<input> last=<int>(d|h|m) | mispgetioc misp_instance=<input> event=<id1>(,<id2>,...) | mispgetioc misp_instance=<input> date=<<YYYY-MM-DD> (date_to=<YYYY-MM-DD>) ##Description { "returnFormat": "mandatory", "page": "optional", "limit": "optional", "value": "optional", "type": "optional", "category": "optional", "org": "optional", "tags": "optional", "date": "optional", "last": "optional", "eventid": "optional", "withAttachments": "optional", "uuid": "optional", "publish_timestamp": "optional", "timestamp": "optional", "enforceWarninglist": "optional", "to_ids": "optional", "deleted": "optional", "includeEventUuid": "optional", "includeEventTags": "optional", "event_timestamp": "optional", "threat_level_id": "optional", "eventinfo": "optional", "includeProposals": "optional", "includeDecayScore": "optional", "includeFullModel": "optional", "decayingModel": "optional", "excludeDecayed": "optional", "score": "optional" } # status "returnFormat": forced to json, "page": param, "limit": param, "value": not managed, "type": param, CSV string, "category": param, CSV string, "org": not managed, "tags": param, see also not_tags "date": param, "last": param, "eventid": param, "withAttachments": forced to false, "uuid": not managed, "publish_timestamp": managed via param last "timestamp": not managed, "enforceWarninglist": param, "to_ids": param, "deleted": forced to False, "includeEventUuid": set to True, "includeEventTags": param, "event_timestamp": not managed, "threat_level_id": not managed, "eventinfo": not managed, "includeProposals": not managed "includeDecayScore": not managed, "includeFullModel": not managed, "decayingModel": not managed, "excludeDecayed": not managed, "score": not managed } """ # MANDATORY MISP instance for this search misp_instance = Option( doc=''' **Syntax:** **misp_instance=instance_name* **Description:** MISP instance parameters as described in local/misp42splunk_instances.conf.''', require=True) # MANDATORY: json_request XOR eventid XOR last XOR date json_request = Option( doc=''' **Syntax:** **json_request=***valid JSON request* **Description:**Valid JSON request''', require=False) eventid = Option( doc=''' **Syntax:** **eventid=***id1(,id2,...)* **Description:**list of event ID(s) or event UUID(s).''', require=False, validate=validators.Match("eventid", r"^[0-9a-f,\-]+$")) last = Option( doc=''' **Syntax:** **last=***<int>d|h|m* **Description:** publication duration in day(s), hour(s) or minute(s). **nota bene:** last is an alias of published_timestamp''', require=False, validate=validators.Match("last", r"^[0-9]+[hdm]$")) date = Option( doc=''' **Syntax:** **date=***The user set event date field - any of valid time related filters"* **Description:**starting date. **eventid**, **last** and **date** are mutually exclusive''', require=False) # Other params category = Option( doc=''' **Syntax:** **category=***CSV string* **Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''', require=False) endpoint = Option( doc=''' **Syntax:** **endpoint=***<events|attributes>* **Description:**selection of MISP API restSearch endpoint. **default**: /attributes/restSearch''', require=False, validate=validators.Match("endpoint", r"(events|attributes)")) geteventtag = Option( doc=''' **Syntax:** **geteventtag=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean includeEventTags. By default only attribute tag(s) are returned.''', require=False, validate=validators.Boolean()) keep_related = Option( doc=''' **Syntax:** **keep_related=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to keep related events. default is to drop RelatedEvents to reduce volume.''', require=False, validate=validators.Boolean()) limit = Option( doc=''' **Syntax:** **limit=***<int>* **Description:**define the limit for each MISP search; default 1000. 0 = no pagination.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) not_tags = Option( doc=''' **Syntax:** **not_tags=***CSV string* **Description:**Comma(,)-separated string of tags to exclude. Wildcard is %.''', require=False) page = Option( doc=''' **Syntax:** **page=***<int>* **Description:**define the page for each MISP search; default 1.''', require=False, validate=validators.Match("page", r"^[0-9]+$")) tags = Option( doc=''' **Syntax:** **tags=***CSV string* **Description:**Comma(,)-separated string of tags to search for. Wildcard is %.''', require=False) to_ids = Option( doc=''' **Syntax:** **to_ids=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to search only attributes with the flag "to_ids" set to true.''', require=False, validate=validators.Boolean()) type = Option( doc=''' **Syntax:** **type=***CSV string* **Description:**Comma(,)-separated string of types to search for. Wildcard is %.''', require=False) warning_list = Option( doc=''' **Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to filter out well known values.''', require=False, validate=validators.Boolean()) @staticmethod def _record(serial_number, time_stamp, host, attributes, attribute_names, encoder): raw = encoder.encode(attributes) # Formulate record fields = dict() for f in attribute_names: if f in attributes: fields[f] = attributes[f] if serial_number > 0: fields['_serial'] = serial_number fields['_time'] = time_stamp fields['_raw'] = raw fields['host'] = host return fields record = OrderedDict(chain( (('_serial', serial_number), ('_time', time_stamp), ('_raw', raw), ('host', host)), map(lambda name: (name, fields.get(name, '')), attribute_names))) return record def generate(self): # Phase 1: Preparation misp_instance = self.misp_instance storage = self.service.storage_passwords my_args = prepare_config(self, 'misp42splunk', misp_instance, storage) if my_args is None: raise Exception("Sorry, no configuration for misp_instance={}".format(misp_instance)) my_args['host'] = my_args['misp_url'].replace('https://', '') # check that ONE of mandatory fields is present mandatory_arg = 0 if self.json_request is not None: mandatory_arg = mandatory_arg + 1 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: raise Exception('Missing "json_request", "eventid", "last" or "date" argument') elif mandatory_arg > 1: raise Exception('Options "json_request", "eventid", "last" and "date" are mutually exclusive') body_dict = dict() # Only ONE combination was provided if self.json_request is not None: body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') elif self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set with %s', json.dumps(body_dict['eventid'])) elif self.last: body_dict['last'] = self.last logging.info('Option "last" set with %s', str(body_dict['last'])) else: body_dict['date'] = self.date.split() logging.info('Option "date" set with %s', json.dumps(body_dict['date'])) # Force some values on JSON request body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False body_dict['deleted'] = False body_dict['includeEventUuid'] = True # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True if self.limit is not None: limit = int(self.limit) elif 'limit' in body_dict: limit = int(body_dict['limit']) else: limit = 1000 if limit == 0: pagination = False if self.page is not None: page = int(self.page) elif 'page' in body_dict: page = body_dict['page'] else: page = 1 # Search parameters: boolean and filter # manage to_ids and enforceWarninglist # to avoid FP enforceWarninglist is set to True if # to_ids is set to True (search criterion) if self.category is not None: if "," in self.category: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria else: body_dict['category'] = self.category if self.endpoint == 'events': my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch' else: my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' if self.geteventtag is True: body_dict['includeEventTags'] = True if self.keep_related is True: keep_related = True else: keep_related = False if self.to_ids is True: body_dict['to_ids'] = True body_dict['enforceWarninglist'] = True # protection elif self.to_ids is False: body_dict['to_ids'] = False if self.type is not None: if "," in self.type: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria else: body_dict['type'] = self.type if self.warning_list is True: body_dict['enforceWarninglist'] = True elif self.warning_list is False: body_dict['enforceWarninglist'] = False if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.debug('mispgetioc request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is if r.status_code in (200, 201, 204): logging.info( "[CO301] INFO mispcollect successful. " "url={}, HTTP status={}".format(my_args['misp_url'], r.status_code) ) else: logging.error( "[CO302] ERROR mispcollect failed. " "url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], body, r.status_code, r.text) ) raise Exception( "[CO302] ERROR mispcollect failed. " "url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], body, r.status_code, r.text) ) # response is 200 by this point or we would have thrown an exception response = r.json() encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':')) if self.endpoint == "events": if 'response' in response: for r_item in response['response']: if 'Event' in r_item: attribute_names = [] serial_number = 0 for e in list(r_item.values()): if keep_related is False: e.pop('RelatedEvent', None) if serial_number == 0: for k in list(e.keys()): attribute_names.append(k) yield MispCollectCommand._record( serial_number, e['timestamp'], my_args['host'], e, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush else: if 'response' in response: if 'Attribute' in response['response']: attribute_names = [] serial_number = 0 for a in response['response']['Attribute']: if serial_number == 0: for k in list(a.keys()): attribute_names.append(k) yield MispCollectCommand._record( serial_number, a['timestamp'], my_args['host'], a, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush
class mispgetioc(ReportingCommand): mispsrv = Option(require=False, validate=validators.Match( "mispsrv", r"^https?:\/\/[0-9a-zA-Z\.]+(?:\:\d+)?$")) mispkey = Option(require=False, validate=validators.Match("mispkey", r"^[0-9a-zA-Z]{40}$")) sslcheck = Option(require=False, validate=validators.Match("sslcheck", r"^[yYnN01]$")) eventid = Option(require=False, validate=validators.Match("eventid", r"^[0-9]+$")) last = Option(require=False, validate=validators.Match("last", r"^[0-9]+[hdwm]$")) onlyids = Option(require=False, validate=validators.Match("onlyids", r"^[yYnN01]+$")) getuuid = Option(require=False, validate=validators.Match("getuuid", r"^[yYnN01]+$")) getorg = Option(require=False, validate=validators.Match("getorg", r"^[yYnN01]+$")) category = Option(require=False) type = Option(require=False) tags = Option(require=False) not_tags = Option(require=False) @Configuration() def map(self, records): self.logger.debug('mispgetioc.map') yield {} return def reduce(self, records): self.logger.debug('mispgetioc.reduce') if self.sslcheck == None: self.sslcheck = 'n' _SPLUNK_PATH = os.environ['SPLUNK_HOME'] # open misp.conf config_file = _SPLUNK_PATH + '/etc/apps/misp42splunk/local/misp.conf' mispconf = ConfigParser.RawConfigParser() mispconf.read(config_file) # Generate args my_args = {} #MISP instance parameters if self.mispsrv: my_args['mispsrv'] = self.mispsrv else: my_args['mispsrv'] = mispconf.get('mispsetup', 'mispsrv') if self.mispkey: my_args['mispkey'] = self.mispkey else: my_args['mispkey'] = mispconf.get('mispsetup', 'mispkey') if self.sslcheck: if self.sslcheck == 'Y' or self.sslcheck == 'y' or self.sslcheck == '1': my_args['sslcheck'] = True else: my_args['sslcheck'] = False else: my_args['sslcheck'] = mispconf.getboolean('mispsetup', 'sslcheck') #Search parameters: boolean and filter if self.onlyids == 'Y' or self.onlyids == 'y' or self.onlyids == '1': my_args['onlyids'] = True else: my_args['onlyids'] = False if self.getuuid == 'Y' or self.getuuid == 'y' or self.getuuid == '1': my_args['getuuid'] = True else: my_args['getuuid'] = False if self.getorg == 'Y' or self.getorg == 'y' or self.getorg == '1': my_args['getorg'] = True else: my_args['getorg'] = False if self.category != None: my_args['category'] = self.category else: my_args['category'] = None if self.type != None: my_args['type'] = self.type else: my_args['type'] = None if self.tags != None: my_args['tags'] = self.tags else: my_args['tags'] = None if self.not_tags != None: my_args['not_tags'] = self.not_tags else: my_args['not_tags'] = None #check that ONE of mandatory fields is present if self.eventid and self.last: print('DEBUG Options "eventid" and "last" are mutually exclusive') exit(2) elif self.eventid: my_args['eventid'] = self.eventid elif self.last: my_args['last'] = self.last else: print('DEBUG Missing "eventid" or "last" argument') exit(1) #path to main components either use default values or set ones if mispconf.has_option('mispsetup', 'P3_PATH'): _NEW_PYTHON_PATH = mispconf.get('mispsetup', 'P3_PATH') else: _NEW_PYTHON_PATH = '/usr/bin/python3' if mispconf.has_option('mispsetup', 'TMP_PATH'): _TMP_PATH = mispconf.get('mispsetup', 'TMP_PATH') else: _TMP_PATH = '/tmp' _SPLUNK_PYTHON_PATH = os.environ['PYTHONPATH'] os.environ['PYTHONPATH'] = _NEW_PYTHON_PATH my_process = _SPLUNK_PATH + '/etc/apps/misp42splunk/bin/pymisp_getioc.py' # Remove LD_LIBRARY_PATH from the environment (otherwise, we will face some SSL issues env = dict(os.environ) del env['LD_LIBRARY_PATH'] FNULL = open(os.devnull, 'w') #use pickle swap_file = _TMP_PATH + '/mispgetioc_config' pickle.dump(my_args, open(swap_file, "wb"), protocol=2) env_file = _TMP_PATH + '/mispgetioc_env' pickle.dump(env, open(env_file, "wb"), protocol=2) p = subprocess.Popen([_NEW_PYTHON_PATH, my_process, swap_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) stdout, stderr = p.communicate() # if stderr: # print('DEBUG error in pymisp_getioc.py') # exit(1) output = {} output = pickle.load(open(swap_file, "rb")) if output: for v in output: yield v
class ipinfo(GeneratingCommand): url = Option(require=False, validate=validators.Match('https url', '^https:\/\/')) headers = Option(require=False) method = Option(require=False, default='POST') ip = Option(require=True) def generate(self): url = "https://ipinfo.io/batch?token=" headers = self.parseHeaders("{'Content-type': 'application/json'}") method = "post" data = self.parseData(self.ip) record = {} storage_passwords = self.service.storage_passwords local_conf = splunk_lib_util.make_splunkhome_path( ["etc", "apps", "ipinfo_app", "local", "ip_info_setup.conf"]) default_conf = splunk_lib_util.make_splunkhome_path( ["etc", "apps", "ipinfo_app", "default", "ip_info_setup.conf"]) config = ConfigParser() config.read([default_conf, local_conf]) url = config.get("ip_info_configuration", "api_url") token = config.get("ip_info_configuration", "api_token") enable = config.get("ip_info_configuration", "proxy_enable") proxy_url = config.get("ip_info_configuration", "proxy_url") disable_ssl = config.get("ip_info_configuration", "disable_ssl") cert_path = splunk_lib_util.make_splunkhome_path([ "etc", "apps", "ipinfo_app", "appserver", "static", "ipinfo.cert" ]) if (os.path.exists(cert_path)): cert_exists = True else: cert_exists = False if (disable_ssl != ""): disable_ssl_request = False else: disable_ssl_request = True if (disable_ssl_request == True and cert_exists == True): disable_ssl_request = cert_path response = "" param = {"token": token} try: if enable == "No": response = requests.request("post", url + "batch?token=" + token, headers=headers, verify=disable_ssl_request, data=data) else: proxies = {'https': proxy_url} response = requests.request("post", url + "batch?token=" + token, headers=headers, verify=disable_ssl_request, data=data, proxies=proxies) except Exception as e: logger.info(e) #url = url+token #response = requests.request(method, url, headers=headers , data=data ) records = response.json() for key, value in records.items(): record = value yield record def parseHeaders(self, headers): # Replace single quotes with double quotes for valid json return json.loads(headers.replace('\'', '"')) def parseData(self, data): data = "[\"" + data + "\"]" return (data.replace(',', '","'))
class MispSearchCommand(StreamingCommand): """ search in MISP for attributes matching the value of field. ##Syntax code-block:: mispsearch field=<field> onlyids=y|n ##Description body = { "returnFormat": "mandatory", "page": "optional", "limit": "optional", "value": "optional", "type": "optional", "category": "optional", "org": "optional", "tags": "optional", "from": "optional", "to": "optional", "last": "optional", "eventid": "optional", "withAttachments": "optional", "uuid": "optional", "publish_timestamp": "optional", "timestamp": "optional", "enforceWarninglist": "optional", "to_ids": "optional", "deleted": "optional", "includeEventUuid": "optional", "includeEventTags": "optional", "event_timestamp": "optional", "threat_level_id": "optional", "eventinfo": "optional" } ##Example Search in MISP for value of fieldname r_ip (remote IP in proxy logs). code-block:: * | mispsearch field=r_ip """ field = Option(doc=''' **Syntax:** **field=***<fieldname>* **Description:**Name of the field containing the value to search for.''', require=True, validate=validators.Fieldname()) onlyids = Option(doc=''' **Syntax:** **onlyids=***<y|n>* **Description:** Boolean to search only attributes with to_ids set''', require=False, validate=validators.Boolean()) gettag = Option(doc=''' **Syntax:** **gettag=***<y|n>* **Description:** Boolean to return attribute tags''', require=False, validate=validators.Boolean()) # Superseede MISP instance for this search misp_instance = Option(doc=''' **Syntax:** **misp_instance=instance_name* **Description:**MISP instance parameters as decibed in lookup/misp_instances.csv.''', require=False) misp_url = Option(doc=''' **Syntax:** **misp_url=***<MISP URL>* **Description:**URL of MISP instance.''', require=False, validate=validators.Match( "misp_url", r"^https?:\/\/[0-9a-zA-Z\-\.]+(?:\:\d+)?$")) misp_key = Option(doc=''' **Syntax:** **misp_key=***<AUTH_KEY>* **Description:**MISP API AUTH KEY.''', require=False, validate=validators.Match("misp_key", r"^[0-9a-zA-Z]{40}$")) misp_verifycert = Option(doc=''' **Syntax:** **misp_verifycert=***<y|n>* **Description:**Verify or not MISP certificate.''', require=False, validate=validators.Boolean()) def stream(self, records): # Generate args my_args = prepare_config(self) my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' fieldname = str(self.field) if self.onlyids is True: to_ids = True else: to_ids = False if self.gettag is True: get_tag = True else: get_tag = False for record in records: if fieldname in record: value = record.get(fieldname, None) if value is not None: body_dict = {"returnFormat": "json"} body_dict['value'] = str(value) body_dict['withAttachments'] = "false" if to_ids: body_dict['to_ids'] = "True" body = json.dumps(body_dict) misp_category = [] misp_event_id = [] misp_to_ids = [] misp_tag = [] misp_type = [] misp_value = [] misp_uuid = [] # search logging.info('INFO MISP REST API REQUEST: %s', body) r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception # print >> sys.stderr, "DEBUG MISP REST API response: %s" % response.json() response = r.json() if 'response' in response: if 'Attribute' in response['response']: for a in response['response']['Attribute']: if str(a['type']) not in misp_type: misp_type.append(str(a['type'])) if str(a['value']) not in misp_value: misp_value.append(str(a['value'])) if str(a['to_ids']) not in misp_to_ids: misp_to_ids.append(str(a['to_ids'])) if str(a['category']) not in misp_category: misp_category.append(str(a['category'])) if str(a['uuid']) not in misp_uuid: misp_uuid.append(str(a['uuid'])) if str(a['event_id']) not in misp_event_id: misp_event_id.append(str(a['event_id'])) if get_tag and 'Tag' in a: for tag in a['Tag']: if str(tag['name']) not in misp_tag: misp_tag.append(str(tag['name'])) record['misp_type'] = misp_type record['misp_value'] = misp_value record['misp_to_ids'] = misp_to_ids record['misp_category'] = misp_category record['misp_attribute_uuid'] = misp_uuid record['misp_event_id'] = misp_event_id if get_tag: record['misp_tag'] = misp_tag yield record
class mispgetevent(ReportingCommand): """ get the attributes from a MISP instance. ##Syntax .. code-block:: | mispgetevent misp_instance=<input> last=<int>(d|h|m) | mispgetevent misp_instance=<input> event=<id1>(,<id2>,...) | mispgetevent misp_instance=<input> date=<<YYYY-MM-DD> (date_to=<YYYY-MM-DD>) ##Description { "returnFormat": "mandatory", "page": "optional", "limit": "optional", "value": "optional", "type": "optional", "category": "optional", "org": "optional", "tag": "optional", "tags": "optional", "searchall": "optional", "date": "optional", "last": "optional", "eventid": "optional", "withAttachments": "optional", "metadata": "optional", "uuid": "optional", "published": "optional", "publish_timestamp": "optional", "timestamp": "optional", "enforceWarninglist": "optional", "sgReferenceOnly": "optional", "eventinfo": "optional", "excludeLocalTags": "optional" } # status "tag": "optional", "searchall": "optional", "metadata": "optional", "published": "optional", "sgReferenceOnly": "optional", "eventinfo": "optional", "excludeLocalTags": "optional" "returnFormat": forced to json, "page": param, "limit": param, "value": not managed, "type": param, CSV string, "category": param, CSV string, "org": not managed, "tags": param, see also not_tags "date": param, "last": param, "eventid": param, "withAttachments": forced to false, "uuid": not managed, "publish_timestamp": managed via param last "timestamp": not managed, "enforceWarninglist": not managed, } """ # MANDATORY MISP instance for this search misp_instance = Option(doc=''' **Syntax:** **misp_instance=instance_name* **Description:**MISP instance parameters as described in local/inputs.conf.''', require=True) # MANDATORY: json_request XOR eventid XOR last XOR date json_request = Option(doc=''' **Syntax:** **json_request=***valid JSON request* **Description:**Valid JSON request''', require=False) eventid = Option(doc=''' **Syntax:** **eventid=***id1(,id2,...)* **Description:**list of event ID(s) or event UUID(s).''', require=False, validate=validators.Match("eventid", r"^[0-9a-f,\-]+$")) last = Option(doc=''' **Syntax:** **last=***<int>d|h|m* **Description:** publication duration in day(s), hour(s) or minute(s). **nota bene:** last is an alias of published_timestamp''', require=False, validate=validators.Match("last", r"^[0-9]+[hdm]$")) date = Option(doc=''' **Syntax:** **date=***The user set event date field - any of valid time related filters"* **Description:**starting date. **eventid**, **last** and **date** are mutually exclusive''', require=False) # Other params page = Option(doc=''' **Syntax:** **page=***<int>* **Description:**define the page for each MISP search; default 1.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) limit = Option(doc=''' **Syntax:** **limit=***<int>* **Description:**define the limit for each MISP search; default 1000. 0 = no pagination.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) type = Option(doc=''' **Syntax:** **type=***CSV string* **Description:**Comma(,)-separated string of types to search for. Wildcard is %.''', require=False) category = Option(doc=''' **Syntax:** **category=***CSV string* **Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''', require=False) tags = Option(doc=''' **Syntax:** **tags=***CSV string* **Description:**Comma(,)-separated string of tags to search for. Wildcard is %.''', require=False) not_tags = Option(doc=''' **Syntax:** **not_tags=***CSV string* **Description:**Comma(,)-separated string of tags to exclude. Wildcard is %.''', require=False) published = Option(doc=''' **Syntax:** **published=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**select only published events (for option from to) .''', require=False, validate=validators.Boolean()) getioc = Option(doc=''' **Syntax:** **getioc=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to return the list of attributes together with the event.''', require=False, validate=validators.Boolean()) pipesplit = Option(doc=''' **Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to split multivalue attributes.''', require=False, validate=validators.Boolean()) @Configuration() def map(self, records): # self.logger.debug('mispevent.map') return records def reduce(self, records): # Phase 1: Preparation my_args = prepare_config(self) my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch' # check that ONE of mandatory fields is present mandatory_arg = 0 if self.json_request is not None: mandatory_arg = mandatory_arg + 1 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: logging.error('Missing "json_request", eventid", \ "last" or "date" argument') raise Exception('Missing "json_request", "eventid", \ "last" or "date" argument') elif mandatory_arg > 1: logging.error('Options "json_request", eventid", "last" \ and "date" are mutually exclusive') raise Exception('Options "json_request", "eventid", "last" \ and "date" are mutually exclusive') body_dict = dict() # Only ONE combination was provided if self.json_request is not None: body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') elif self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set with %s', json.dumps(body_dict['eventid'])) elif self.last: body_dict['last'] = self.last logging.info('Option "last" set with %s', str(body_dict['last'])) else: body_dict['date'] = self.date.split() logging.info('Option "date" set with %s', json.dumps(body_dict['date'])) # Force some values on JSON request body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True if self.limit is not None: limit = int(self.limit) elif 'limit' in body_dict: limit = int(body_dict['limit']) else: limit = 1000 if limit == 0: pagination = False if self.page is not None: page = int(self.page) elif 'page' in body_dict: page = body_dict['page'] else: page = 1 if self.published is True: body_dict['published'] = True elif self.published is False: body_dict['published'] = False if self.category is not None: if "," in self.category: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria else: body_dict['category'] = self.category if self.type is not None: if "," in self.type: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria else: body_dict['type'] = self.type if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria # output filter parameters if self.getioc is True: my_args['getioc'] = True else: my_args['getioc'] = False if self.pipesplit is True: my_args['pipe'] = True else: my_args['pipe'] = False results = [] # add colums for each type in results typelist = [] if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.error('mispgetevent request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() if 'response' in response: for r_item in response['response']: if 'Event' in r_item: for a in list(r_item.values()): v = {} v['misp_event_id'] = str(a['id']) v['misp_orgc_id'] = str(a['orgc_id']) v['misp_event_date'] = str(a['date']) v['threat_level_id'] = str(a['threat_level_id']) v['misp_event_info'] = a['info'] v['misp_event_published'] = str(a['published']) v['misp_event_uuid'] = str(a['uuid']) v['misp_attribute_count'] = str(a['attribute_count']) v['misp_analysis'] = str(a['analysis']) v['misp_timestamp'] = str(a['timestamp']) v['misp_distribution'] = str(a['distribution']) v['misp_publish_timestamp'] = \ str(a['publish_timestamp']) v['misp_sharing_group_id'] = str(a['sharing_group_id']) v['misp_extends_uuid'] = str(a['extends_uuid']) if 'Orgc' in a: v['misp_orgc_name'] = str(a['Orgc']['name']) v['misp_orgc_uuid'] = str(a['Orgc']['uuid']) tag_list = [] if 'Tag' in a: for tag in a['Tag']: try: tag_list.append(str(tag['name'])) except Exception: pass v['misp_tag'] = tag_list if my_args['getioc'] is True: v['Attribute'] = list() v['misp_attribute_count'] = 0 if 'Attribute' in a: v['misp_attribute_count'] = \ v['misp_attribute_count'] + len(a['Attribute']) if my_args['getioc'] is True: for attribute in a['Attribute']: # combined: not part of an object AND # multivalue attribute AND to be split if int(attribute['object_id']) == 0 \ and '|' in attribute['type'] \ and my_args['pipe'] is True: mv_type_list = \ attribute['type'].split('|') mv_value_list = \ str(attribute['value']).split('|') left_a = attribute.copy() left_a['type'] = mv_type_list.pop() left_a['value'] = mv_value_list.pop() v['Attribute'].append( getioc(left_a, typelist, my_args['pipe'], left_a['object_id'])) right_a = attribute.copy() right_a['type'] = mv_type_list.pop() right_a['value'] = mv_value_list.pop() v['Attribute'].append( getioc(right_a, typelist, my_args['pipe'], right_a['object_id'])) else: v['Attribute'].append( getioc(attribute, typelist, my_args['pipe'], attribute['object_id'])) if 'Object' in a: for misp_o in a['Object']: if 'Attribute' in misp_o: v['misp_attribute_count'] = \ v['misp_attribute_count'] \ + len(misp_o['Attribute']) if my_args['getioc'] is True: object_id = misp_o['id'] object_name = misp_o['name'] object_comment = misp_o['comment'] for attribute in misp_o['Attribute']: v['Attribute'].append( getioc(attribute, typelist, my_args['pipe'], object_id, object_name, object_comment)) logging.debug('event is %s', json.dumps(v)) results.append(v) logging.info('typelist is %s', json.dumps(typelist)) # relevant_cat = ['Artifacts dropped', 'Financial fraud', # 'Network activity','Payload delivery','Payload installation'] logging.debug('results is %s', json.dumps(results)) if my_args['getioc'] is False: for e in results: yield e else: output_dict = {} for e in results: if 'Attribute' in e: for r in e['Attribute']: if int(r['misp_object_id']) == 0: # not an object key = str(e['misp_event_id']) + '_' \ + r['misp_attribute_id'] is_object_member = False else: # this is a MISP object key = str(e['misp_event_id']) + \ '_object_' + str(r['misp_object_id']) is_object_member = True if key not in output_dict: v = init_misp_output(e, r) for t in typelist: misp_t = 'misp_' \ + t.replace('-', '_').replace('|', '_p_') if t == r['misp_type']: v[misp_t] = r['misp_value'] else: v[misp_t] = '' to_ids = [] to_ids.append(r['misp_to_ids']) v['misp_to_ids'] = to_ids category = [] category.append(r['misp_category']) v['misp_category'] = category attribute_uuid = [] attribute_uuid.append(r['misp_attribute_uuid']) v['misp_attribute_uuid'] = attribute_uuid if is_object_member is True: v['misp_type'] = v['misp_object_name'] v['misp_value'] = v['misp_object_id'] output_dict[key] = dict(v) else: v = dict(output_dict[key]) misp_t = 'misp_' + r['misp_type'].replace('-', '_') v[misp_t] = r['misp_value'] # set value for type to_ids = v['misp_to_ids'] if r['misp_to_ids'] not in to_ids: to_ids.append(r['misp_to_ids']) v['misp_to_ids'] = to_ids category = v['misp_category'] # append if r['misp_category'] not in category: category.append(r['misp_category']) v['misp_category'] = category attribute_uuid = v['misp_attribute_uuid'] if r['misp_attribute_uuid'] not in attribute_uuid: attribute_uuid.append(r['misp_attribute_uuid']) v['misp_attribute_uuid'] = attribute_uuid if is_object_member is False: misp_type = r['misp_type'] \ + '|' + v['misp_type'] v['misp_type'] = misp_type misp_value = r['misp_value'] + \ '|' + v['misp_value'] v['misp_value'] = misp_value output_dict[key] = dict(v) for k, v in list(output_dict.items()): yield v
class GenerateTextCommand(GeneratingCommand): account = Option( doc=''' **Syntax:** **account=**** **Description:** JIRA account to be used, if unspecified the first account configured will be taken into account.''', require=False, default=None) method = Option( doc=''' **Syntax:** **method=**** **Description:** method to use for API target. DELETE GET POST PUT are supported.''', require=False, validate=validators.Match("method", r"^(DELETE|GET|POST|PUT)$")) json_request = Option( doc=''' **Syntax:** **json_request=***JSON request* **Description:** JSON-formatted json_request.''', require=False, validate=validators.Match("json_request", r"^{.+}$")) target = Option(require=True) def generate(self): storage_passwords = self.service.storage_passwords # global configuration conf_file = "ta_service_desk_simple_addon_settings" confs = self.service.confs[str(conf_file)] jira_passthrough_mode = None proxy_enabled = "0" proxy_url = None proxy_dict = None proxy_username = None for stanza in confs: if stanza.name == "advanced_configuration": for key, value in stanza.content.items(): if key == "jira_passthrough_mode": jira_passthrough_mode = value if stanza.name == "proxy": for key, value in stanza.content.items(): if key == "proxy_enabled": proxy_enabled = value if key == "proxy_port": proxy_port = value if key == "proxy_rdns": proxy_rdns = value if key == "proxy_type": proxy_type = value if key == "proxy_url": proxy_url = value if key == "proxy_username": proxy_username = value if proxy_enabled == "1": # get proxy password if proxy_username: proxy_password = None # get proxy password, if any credential_realm = '__REST_CREDENTIAL__#TA-jira-service-desk-simple-addon#configs/conf-ta_service_desk_simple_addon_settings' for credential in storage_passwords: if credential.content.get('realm') == str(credential_realm) \ and credential.content.get('clear_password').find('proxy_password') > 0: proxy_password = json.loads(credential.content.get('clear_password')).get('proxy_password') break if proxy_type == 'http': proxy_dict= { "http" : "http://" + proxy_username + ":" + proxy_password + "@" + proxy_url + ":" + proxy_port, "https" : "https://" + proxy_username + ":" + proxy_password + "@" + proxy_url + ":" + proxy_port } else: proxy_dict= { "http" : str(proxy_type) + "://" + proxy_username + ":" + proxy_password + "@" + proxy_url + ":" + proxy_port, "https" : str(proxy_type) + "://" + proxy_username + ":" + proxy_password + "@" + proxy_url + ":" + proxy_port } else: proxy_dict= { "http" : proxy_url + ":" + proxy_port, "https" : proxy_url + ":" + proxy_port } # get all acounts accounts = [] conf_file = "ta_service_desk_simple_addon_account" confs = self.service.confs[str(conf_file)] for stanza in confs: # get all accounts for name in stanza.name: accounts.append(stanza.name) break # define the account target if not self.account or self.account == '_any': account = str(accounts[0]) else: account = str(self.account) # account configuration isfound = False jira_ssl_certificate_validation = None jira_ssl_certificate_path = None username = None password = None conf_file = "ta_service_desk_simple_addon_account" confs = self.service.confs[str(conf_file)] for stanza in confs: if stanza.name == str(account): isfound = True for key, value in stanza.content.items(): if key == "jira_url": jira_url = value if key == "jira_ssl_certificate_validation": jira_ssl_certificate_validation = value if key == "jira_ssl_certificate_path": jira_ssl_certificate_path = value if key == 'auth_type': auth_type = value if key == 'jira_auth_mode': jira_auth_mode = value if key == 'username': username = value # end of get configuration # Stop here if we cannot find the submitted account if not isfound: self.logger.fatal('This acount has not been configured on this instance, cannot proceed!: %s', self) # else get the password else: credential_username = str(account) + '``splunk_cred_sep``1' credential_realm = '__REST_CREDENTIAL__#TA-jira-service-desk-simple-addon#configs/conf-ta_service_desk_simple_addon_account' for credential in storage_passwords: if credential.content.get('username') == str(credential_username) \ and credential.content.get('realm') == str(credential_realm) \ and credential.content.get('clear_password').find('password') > 0: password = json.loads(credential.content.get('clear_password')).get('password') break # Build the authentication header for JIRA if str(jira_auth_mode) == 'basic': authorization = username + ':' + password b64_auth = base64.b64encode(authorization.encode()).decode() jira_headers = { 'Authorization': 'Basic %s' % b64_auth, 'Content-Type': 'application/json', } elif str(jira_auth_mode) == 'pat': jira_headers = { 'Authorization': 'Bearer %s' % str(password), 'Content-Type': 'application/json', } # verify the url if not jira_url.startswith("https://"): jira_url = "https://" + str(jira_url) # handle SSL verification and bundle if jira_ssl_certificate_validation: if jira_ssl_certificate_validation == '0': ssl_verify = False elif jira_ssl_certificate_validation == '1' and jira_ssl_certificate_path and os.path.isfile(jira_ssl_certificate_path): ssl_verify = str(jira_ssl_certificate_path) elif jira_ssl_certificate_validation == '1': ssl_verify = True # verify the method if self.method: jira_method = self.method else: jira_method = "GET" if self.json_request: body_dict = json.loads(self.json_request) else: if jira_method == "POST" or jira_method == "PUT": raise Exception("jirarest: method {} requires a valid json_request. It is empty".format(jira_method)) if self.target: # set proper headers if jira_method == "GET": jira_fields_response = requests.get( url=str(jira_url) + '/' + str(self.target), headers=jira_headers, verify=ssl_verify, proxies=proxy_dict ) elif jira_method == "DELETE": jira_fields_response = requests.delete( url=str(jira_url) + '/' + str(self.target), headers=jira_headers, verify=ssl_verify, proxies=proxy_dict ) elif jira_method == "POST": jira_fields_response = requests.post( url=str(jira_url) + '/' + str(self.target), data=json.dumps(body_dict).encode('utf-8'), headers=jira_headers, verify=ssl_verify, proxies=proxy_dict ) elif jira_method == "PUT": jira_fields_response = requests.put( url=str(jira_url) + '/' + str(self.target), data=json.dumps(body_dict).encode('utf-8'), headers=jira_headers, verify=ssl_verify, proxies=proxy_dict ) # Attenpt to get a JSON response, and render in Splunk try: json_response = jira_fields_response.json() data = {'_time': time.time(), '_raw': json.dumps(json_response)} yield data except Exception as e: # Build a custom response for Splunk dynamically # Create an action field, convenient to quickly understanding when things go wrong if jira_fields_response.status_code in (200, 201, 204): response_action = "success" else: response_action = "failure" # render if jira_fields_response.text: json_response = "{\"action\": \"" + str(response_action) + "\", \"status_code\": \"" + str(jira_fields_response.status_code) + "\", \"text\": \"" + str(jira_fields_response.text) + "\"}" else: json_response = "{\"action\": \"" + str(response_action) + "\", \"status_code\": \"" + str(jira_fields_response.status_code) + "\"}" data = {'_time': time.time(), '_raw': str(json.dumps(json.loads(json_response, strict=False), indent=4))} yield data
class mispgetioc(ReportingCommand): mispsrv = Option(require=False, validate=validators.Match( "mispsrv", r"^https?:\/\/[0-9a-zA-Z\.]+(?:\:\d+)?$")) mispkey = Option(require=False, validate=validators.Match("mispkey", r"^[0-9a-zA-Z]{40}$")) sslcheck = Option(require=False, validate=validators.Match("sslcheck", r"^[yYnN01]$")) eventid = Option(require=False, validate=validators.Match("eventid", r"^[0-9]+$")) last = Option(require=False, validate=validators.Match("last", r"^[0-9]+[hdwm]$")) onlyids = Option(require=False, validate=validators.Match("onlyids", r"^[yYnN01]+$")) getuuid = Option(require=False, validate=validators.Match("getuuid", r"^[yYnN01]+$")) getorg = Option(require=False, validate=validators.Match("getuuid", r"^[yYnN01]+$")) category = Option(require=False) type = Option(require=False) @Configuration() def map(self, records): self.logger.debug('mispgetioc.map') yield {} return def reduce(self, records): self.logger.debug('mispgetioc.reduce') if self.sslcheck == None: self.sslcheck = 'n' # open misp.conf config_file = '/opt/splunk/etc/apps/misp42splunk/local/misp.conf' config = ConfigParser.RawConfigParser() config.read(config_file) # Generate args my_args = {} if self.mispsrv: my_args['mispsrv'] = self.mispsrv else: my_args['mispsrv'] = config.get('mispsetup', 'mispsrv') if self.mispkey: my_args['mispkey'] = self.mispkey else: my_args['mispkey'] = config.get('mispsetup', 'mispkey') if self.sslcheck: if self.sslcheck == 'Y' or self.sslcheck == 'y' or self.sslcheck == '1': my_args['sslcheck'] = True else: my_args['sslcheck'] = False else: my_args['sslcheck'] = config.getboolean('mispsetup', 'sslcheck') if self.onlyids == 'Y' or self.onlyids == 'y' or self.onlyids == '1': onlyids = True else: onlyids = False if self.getuuid == 'Y' or self.getuuid == 'y' or self.getuuid == '1': getuuid = True else: getuuid = False if self.getorg == 'Y' or self.getorg == 'y' or self.getorg == '1': getorg = True else: getorg = False if self.eventid and self.last: print('DEBUG Options "eventid" and "last" are mutually exclusive') exit(2) if self.eventid: my_args['eventid'] = self.eventid elif self.last: my_args['last'] = self.last else: print('DEBUG Missing "eventid" or "last" argument') exit(1) _SPLUNK_PATH = '/opt/splunk' _NEW_PYTHON_PATH = '/usr/bin/python3' _SPLUNK_PYTHON_PATH = os.environ['PYTHONPATH'] os.environ['PYTHONPATH'] = _NEW_PYTHON_PATH my_process = _SPLUNK_PATH + '/etc/apps/misp42splunk/bin/pymisp_getioc.py' # Remove LD_LIBRARY_PATH from the environment (otherwise, we will face some SSL issues env = dict(os.environ) del env['LD_LIBRARY_PATH'] FNULL = open(os.devnull, 'w') p = subprocess.Popen([_NEW_PYTHON_PATH, my_process, str(my_args)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=FNULL, env=env) output = p.communicate()[0] results = {} for v in eval(output): # Do not display deleted attributes if v['deleted'] == False: # If specified, do not display attributes with the non-ids flag set to False if onlyids == True and v['to_ids'] == False: continue if self.category != None and self.category != v['category']: continue if self.type != None and self.type != v['type']: continue if getuuid == True: results['uuid'] = v['uuid'] if getorg == True: results['orgc'] = v['orgc'] results['eventid'] = v['event_id'] results['value'] = v['value'] results['category'] = v['category'] results['type'] = v['type'] results['to_ids'] = str(v['to_ids']) yield results
class MispSearchCommand(StreamingCommand): """ search in MISP for attributes matching the value of field. ##Syntax code-block:: mispsearch field=<field> to_ids=y|n ##Description body = { "returnFormat": "mandatory", "page": "optional", "limit": "optional", "value": "optional", "type": "optional", "category": "optional", "org": "optional", "tags": "optional", "from": "optional", "to": "optional", "last": "optional", "eventid": "optional", "withAttachments": "optional", "uuid": "optional", "publish_timestamp": "optional", "timestamp": "optional", "enforceWarninglist": "optional", "to_ids": "optional", "deleted": "optional", "includeEventUuid": "optional", "includeEventTags": "optional", "event_timestamp": "optional", "threat_level_id": "optional", "eventinfo": "optional" } ##Example Search in MISP for value of fieldname r_ip (remote IP in proxy logs). code-block:: * | mispsearch field=r_ip """ misp_instance = Option(doc=''' **Syntax:** **misp_instance=instance_name* **Description:**MISP instance parameters as \ described in local/misp42splunk_instances.conf''', require=True) field = Option(doc=''' **Syntax:** **field=***<fieldname>* **Description:**Name of the field containing \ the value to search for.''', require=True, validate=validators.Fieldname()) to_ids = Option(doc=''' **Syntax:** **to_ids=***<y|n>* **Description:** Boolean to search only attributes with to_ids set''', require=False, validate=validators.Boolean()) includeEventUuid = Option(doc=''' **Syntax:** **includeEventUuid=***y|Y|1|true|True|n|N|0|false|False* **Description:**Boolean to include event UUID(s) to results.''', require=False, validate=validators.Boolean()) includeEventTags = Option(doc=''' **Syntax:** **includeEventTags=***y|Y|1|true|True|n|N|0|false|False* **Description:**Boolean to include Event Tags to results.''', require=False, validate=validators.Boolean()) last = Option(doc=''' **Syntax:** **last=***<int>d|h|m* **Description:**Publication duration in day(s), hour(s) or minute(s) to limit search scope only to published events in last X timerange.''', require=False, validate=validators.Match("last", r"^[0-9]+[hdm]$")) limit = Option(doc=''' **Syntax:** **limit=***<int>* **Description:**define the limit for each MISP search; \ default 1000. 0 = no pagination.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) page = Option(doc=''' **Syntax:** **page=***<int>* **Description:**define the page for each MISP search; default 1.''', require=False, validate=validators.Match("page", r"^[0-9]+$")) json_request = Option(doc=''' **Syntax:** **json_request=***valid JSON request* **Description:**Valid JSON request''', require=False) def log_error(self, msg): logging.error(msg) def log_info(self, msg): logging.info(msg) def log_debug(self, msg): logging.debug(msg) def log_warn(self, msg): logging.warning(msg) def set_log_level(self): logging.root loglevel = logging_level('misp42splunk') logging.root.setLevel(loglevel) logging.error('[SE-101] logging level is set to %s', loglevel) logging.error('[SE-102] PYTHON VERSION: ' + sys.version) def stream(self, records): # loggging self.set_log_level() # Phase 1: Preparation misp_instance = self.misp_instance storage = self.service.storage_passwords my_args = prepare_config(self, 'misp42splunk', misp_instance, storage) if my_args is None: raise Exception( "Sorry, no configuration for misp_instance={}".format( misp_instance)) my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' fieldname = str(self.field) pagination = True if self.limit is not None: if int(self.limit) == 0: pagination = False else: limit = int(self.limit) else: limit = 1000 if self.page is not None: page = int(self.page) else: page = 1 if self.json_request is not None: body_dict = json.loads(self.json_request) self.log_info('Option "json_request" set') body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False if 'limit' in body_dict: limit = int(body_dict['limit']) if limit == 0: pagination = False if 'page' in body_dict: page = body_dict['page'] pagination = False else: # build search JSON object body_dict = {"returnFormat": "json", "withAttachments": False} if self.to_ids is True: body_dict['to_ids'] = "True" if self.includeEventUuid is not None: body_dict['includeEventUuid'] = self.includeEventUuid if self.includeEventTags is not None: body_dict['includeEventTags'] = self.includeEventTags if self.last is not None: body_dict['last'] = self.last for record in records: if fieldname in record: value = record.get(fieldname, None) if value is not None: body_dict['value'] = str(value) misp_category = [] misp_event_id = [] misp_event_uuid = [] misp_orgc_id = [] misp_to_ids = [] misp_comment = [] misp_tag = [] misp_type = [] misp_value = [] misp_uuid = [] # search if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is # check if status is anything other than 200; # throw an exception if it is if r.status_code in (200, 201, 204): self.log_info( "[SE301] INFO mispsearch successful. url={}, HTTP status={}" .format(my_args['misp_url'], r.status_code)) else: self.log_error( "[SE302] ERROR mispsearch failed. url={}, data={}, HTTP Error={}, content={}" .format(my_args['misp_url'], body, r.status_code, r.text)) raise Exception( "[SE302] ERROR mispsearch failed for url={} with HTTP Error={}. Check search.log for details" .format(my_args['misp_url'], r.status_code)) # response is 200 by this point or we would have thrown an exception response = r.json() if 'response' in response: if 'Attribute' in response['response']: for a in response['response']['Attribute']: if str(a['type']) not in misp_type: misp_type.append(str(a['type'])) if str(a['value']) not in misp_value: misp_value.append(str(a['value'])) if str(a['to_ids']) not in misp_to_ids: misp_to_ids.append(str(a['to_ids'])) if str(a['comment']) not in misp_comment: misp_comment.append(str(a['comment'])) if str(a['category']) not in misp_category: misp_category.append(str(a['category'])) if str(a['uuid']) not in misp_uuid: misp_uuid.append(str(a['uuid'])) if str(a['event_id']) not in misp_event_id: misp_event_id.append(str(a['event_id'])) if 'Tag' in a: for tag in a['Tag']: if str(tag['name']) not in misp_tag: misp_tag.append(str(tag['name'])) if 'Event' in a: if a['Event']['uuid'] \ not in misp_event_uuid: misp_event_uuid.append( str(a['Event']['uuid'])) if a['Event']['orgc_id'] \ not in misp_orgc_id: misp_orgc_id.append( str(a['Event']['orgc_id'])) record['misp_type'] = misp_type record['misp_value'] = misp_value record['misp_to_ids'] = misp_to_ids record['misp_comment'] = misp_comment record['misp_category'] = misp_category record['misp_attribute_uuid'] = misp_uuid record['misp_event_id'] = misp_event_id record['misp_event_uuid'] = misp_event_uuid record['misp_orgc_id'] = misp_orgc_id record['misp_tag'] = misp_tag yield record
class mispgetioc(ReportingCommand): """ get the attributes from a MISP instance. ##Syntax .. code-block:: | mispgetioc misp_instance=<input> last=<int>(d|h|m) | mispgetioc misp_instance=<input> event=<id1>(,<id2>,...) | mispgetioc misp_instance=<input> date_from=<<YYYY-MM-DD> (date_to=<YYYY-MM-DD>) ##Description { "returnFormat": "mandatory", "page": "optional", "limit": "optional", "value": "optional", "type": "optional", "category": "optional", "org": "optional", "tags": "optional", "from": "optional", "to": "optional", "last": "optional", "eventid": "optional", "withAttachments": "optional", "uuid": "optional", "publish_timestamp": "optional", "timestamp": "optional", "enforceWarninglist": "optional", "to_ids": "optional", "deleted": "optional", "includeEventUuid": "optional", "includeEventTags": "optional", "event_timestamp": "optional", "threat_level_id": "optional", "eventinfo": "optional", "includeProposals": "optional" } # status "returnFormat": forced to json, "page": not managed, "limit": param, "value": not managed, "type": param, CSV string, "category": param, CSV string, "org": not managed, "tags": param, "from": param, "to": param, "last": param, "eventid": param, "withAttachments": forced to false, "uuid": not managed, "publish_timestamp": not managed, "timestamp": not managed, "enforceWarninglist": param, "to_ids": param, "deleted": forced to False, "includeEventUuid": param, "includeEventTags": param, "event_timestamp": not managed, "threat_level_id": param, "eventinfo": not managed, "includeProposals": not managed } """ # Superseede MISP instance for this search misp_instance = Option( doc=''' **Syntax:** **misp_instance=instance_name* **Description:**MISP instance parameters as described in local/inputs.conf.''', require=True) # MANDATORY: eventid XOR last XOR date_from eventid = Option( doc=''' **Syntax:** **eventid=***id1(,id2,...)* **Description:**list of event ID(s). **eventid**, **last** and **date_from** are mutually exclusive''', require=False, validate=validators.Match("eventid", r"^[0-9a-f,\-]+$")) last = Option( doc=''' **Syntax:** **last=***<int>d|h|m* **Description:**publication duration in day(s), hour(s) or minute(s). **eventid**, **last** and **date_from** are mutually exclusive''', require=False, validate=validators.Match("last", r"^[0-9]+[hdm]$")) date_from = Option( doc=''' **Syntax:** **date_from=***date_string"* **Description:**starting date. **eventid**, **last** and **date_from** are mutually exclusive''', require=False) date_to = Option( doc=''' **Syntax:** **date_to=***date_string"* **Description:**(optional)ending date in searches with date_from. if not set default is now''', require=False) to_ids = Option( doc=''' **Syntax:** **to_ids=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to search only attributes with the flag "to_ids" set to true.''', require=False, validate=validators.Boolean()) published = Option( doc=''' **Syntax:** **published=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**select only published events (for option from to) .''', require=False, validate=validators.Boolean()) category = Option( doc=''' **Syntax:** **category=***CSV string* **Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''', require=False) type = Option( doc=''' **Syntax:** **type=***CSV string* **Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''', require=False) tags = Option( doc=''' **Syntax:** **tags=***CSV string* **Description:**Comma(,)-separated string of tags to search for. Wildcard is %.''', require=False) not_tags = Option( doc=''' **Syntax:** **not_tags=***CSV string* **Description:**Comma(,)-separated string of tags to exclude from results. Wildcard is %.''', require=False) threat_level_id = Option( doc=''' **Syntax:** **threat_level=***<int>* **Description:**define the threat_level_id''', require=False, validate=validators.Match("threat_level_id", r"^[1-4]$")) limit = Option( doc=''' **Syntax:** **limit=***<int>* **Description:**define the limit for each MISP search; default 10000. 0 = no pagination.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) getuuid = Option( doc=''' **Syntax:** **getuuid=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to return attribute UUID.''', require=False, validate=validators.Boolean()) getorg = Option( doc=''' **Syntax:** **getorg=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to return the ID of the organisation that created the event.''', require=False, validate=validators.Boolean()) geteventtag = Option( doc=''' **Syntax:** **geteventtag=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to return also event tag(s). By default only attribute tag(s) are returned.''', require=False, validate=validators.Boolean()) pipesplit = Option( doc=''' **Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to split multivalue attributes into 2 attributes.''', require=False, validate=validators.Boolean()) add_description = Option( doc=''' **Syntax:** **add_description=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to return misp_description.''', require=False, validate=validators.Boolean()) warning_list = Option( doc=''' **Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to filter out well known values.''', require=False, validate=validators.Boolean()) @Configuration() def map(self, records): # self.logger.debug('mispgetioc.map') return records def reduce(self, records): # Phase 1: Preparation my_args = prepare_config(self) my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' # build search JSON object body_dict = {"returnFormat": "json", "withAttachments": False, "deleted": False } # check that ONE of mandatory fields is present mandatory_arg = 0 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date_from: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: logging.error('Missing "eventid", "last" or "date_from" argument') raise Exception('Missing "eventid", "last" or "date_from" argument') elif mandatory_arg > 1: logging.error('Options "eventid", "last" and "date_from" are mutually exclusive') raise Exception('Options "eventid", "last" and "date_from" are mutually exclusive') # Only ONE combination was provided if self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set') elif self.last: body_dict['last'] = self.last logging.info('Option "last" set with %s', body_dict['last']) else: body_dict['from'] = self.date_from logging.info('Option "date_from" set with %s', body_dict['from']) if self.date_to: body_dict['to'] = self.date_to logging.info('Option "date_to" set with %s', body_dict['to']) else: logging.info('Option "date_to" will be set to now().') # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True other_page = True page = 1 page_length = 0 if self.limit is not None: if int(self.limit) == 0: pagination = False else: limit = int(self.limit) else: limit = 10000 # Search parameters: boolean and filter if self.to_ids is True: body_dict['to_ids'] = True body_dict['enforceWarninglist'] = True elif self.to_ids is False: body_dict['to_ids'] = False if self.warning_list is True: body_dict['enforceWarninglist'] = True elif self.warning_list is False: body_dict['enforceWarninglist'] = False if self.published is True: body_dict['published'] = True elif self.published is False: body_dict['published'] = False if self.geteventtag is True: body_dict['includeEventTags'] = True if self.category is not None: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria if self.type is not None: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria if self.threat_level_id is not None: body_dict['threat_level_id'] = int(self.threat_level_id) # output filter parameters if self.getuuid is True: my_args['getuuid'] = True else: my_args['getuuid'] = False if self.getorg is True: my_args['getorg'] = True else: my_args['getorg'] = False if self.pipesplit is True: my_args['pipe'] = True else: my_args['pipe'] = False if self.add_description is True: my_args['add_desc'] = True else: my_args['add_desc'] = False results = [] # add colums for each type in results typelist = [] while other_page: if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.debug('mispgetioc request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() if 'response' in response: if 'Attribute' in response['response']: page_lenth = len(response['response']['Attribute']) for a in response['response']['Attribute']: v = {} v['misp_category'] = str(a['category']) v['misp_attribute_id'] = str(a['id']) v['misp_event_id'] = str(a['event_id']) v['misp_timestamp'] = str(a['timestamp']) v['misp_to_ids'] = str(a['to_ids']) tag_list = [] if 'Tag' in a: for tag in a['Tag']: try: tag_list.append(str(tag['name'])) except Exception: pass v['misp_tag'] = tag_list # include ID of the organisation that created the attribute if requested # in previous version this was the ORG name ==> create lookup if 'Event' in a and my_args['getorg']: v['misp_orgc_id'] = str(a['Event']['orgc_id']) # include attribute UUID if requested if my_args['getuuid']: v['misp_attribute_uuid'] = str(a['uuid']) # handle object and multivalue attributes v['misp_object_id'] = str(a['object_id']) if my_args['add_desc'] is True: if int(a['object_id']) == 0: v['misp_description'] = 'MISP e' + str(a['event_id']) + ' attribute ' \ + str(a['uuid']) + ' of type "' + str(a['type']) \ + '" in category "' + str(a['category']) \ + '" (to_ids:' + str(a['to_ids']) + ')' else: v['misp_description'] = 'MISP e' + str(a['event_id']) \ + ' attribute ' + str(a['uuid']) + ' of type "' \ + str(a['type']) + '" in category "' \ + str(a['category']) \ + '" (to_ids:' + str(a['to_ids']) \ + ' - o' + str(a['object_id']) + ' )' current_type = str(a['type']) # combined: not part of an object AND multivalue attribute QND to be split if int(a['object_id']) == 0 and '|' in current_type and my_args['pipe'] is True: mv_type_list = current_type.split('|') mv_value_list = str(a['value']).split('|') left_v = v.copy() left_v['misp_type'] = mv_type_list.pop() left_v['misp_value'] = mv_value_list.pop() results.append(left_v) if left_v['misp_type'] not in typelist: typelist.append(left_v['misp_type']) right_v = v.copy() right_v['misp_type'] = mv_type_list.pop() right_v['misp_value'] = mv_value_list.pop() results.append(right_v) if right_v['misp_type'] not in typelist: typelist.append(right_v['misp_type']) else: v['misp_type'] = current_type v['misp_value'] = str(a['value']) results.append(v) if current_type not in typelist: typelist.append(current_type) if pagination is True: if page_length < limit: other_page = False else: page = page + 1 else: other_page = False logging.info(json.dumps(typelist)) output_dict = {} # relevant_cat = ['Artifacts dropped', 'Financial fraud', 'Network activity','Payload delivery','Payload installation'] for r in results: if int(r['misp_object_id']) == 0: # not an object key = str(r['misp_event_id']) + '_' + r['misp_attribute_id'] is_object_member = False else: # this is a MISP object key = str(r['misp_event_id']) + '_object_' + str(r['misp_object_id']) is_object_member = True if key not in output_dict: v = dict(r) for t in typelist: misp_t = 'misp_' + t.replace('-', '_').replace('|', '_p_') if t == r['misp_type']: v[misp_t] = r['misp_value'] else: v[misp_t] = '' to_ids = [] to_ids.append(r['misp_to_ids']) v['misp_to_ids'] = to_ids category = [] category.append(r['misp_category']) v['misp_category'] = category if my_args['add_desc'] is True: description = [] description.append(r['misp_description']) v['misp_description'] = description if my_args['getuuid'] is True: attribute_uuid = [] attribute_uuid.append(r['misp_attribute_uuid']) v['misp_attribute_uuid'] = attribute_uuid if is_object_member is True: v['misp_type'] = 'misp_object' v['misp_value'] = r['misp_object_id'] output_dict[key] = dict(v) else: v = dict(output_dict[key]) misp_t = 'misp_' + r['misp_type'].replace('-', '_') v[misp_t] = r['misp_value'] # set value for relevant type to_ids = v['misp_to_ids'] if r['misp_to_ids'] not in to_ids: to_ids.append(r['misp_to_ids']) v['misp_to_ids'] = to_ids category = v['misp_category'] if r['misp_category'] not in category: # append category category.append(r['misp_category']) v['misp_category'] = category if my_args['add_desc'] is True: description = v['misp_description'] if r['misp_description'] not in description: description.append(r['misp_description']) v['misp_description'] = description if my_args['getuuid'] is True: attribute_uuid = v['misp_attribute_uuid'] if r['misp_attribute_uuid'] not in attribute_uuid: attribute_uuid.append(r['misp_attribute_uuid']) v['misp_attribute_uuid'] = attribute_uuid if is_object_member is False: misp_type = r['misp_type'] + '|' + v['misp_type'] v['misp_type'] = misp_type misp_value = r['misp_value'] + '|' + v['misp_value'] v['misp_value'] = misp_value output_dict[key] = dict(v) for k, v in output_dict.items(): yield v
class getmispioc(ReportingCommand): ''' Extract IOC's from MISP ''' server = Option(doc="", require=False, validate=validators.Match( "server", r"^https?:\/\/[0-9a-zA-Z\.]+(?:\:\d+)?$")) authkey = Option(doc="", require=False, validate=validators.Match("authkey", r"^[0-9a-zA-Z]{40}$")) sslcheck = Option(doc="", require=False, validate=validators.Match("sslcheck", r"^[yYnN01]$")) eventid = Option(doc="", require=False, validate=validators.Match("eventid", r"^[0-9]+$")) last = Option(doc="", require=False, validate=validators.Match("last", r"^[0-9]+[hdwm]$")) onlyids = Option(doc="", require=False, validate=validators.Match("onlyids", r"^[yYnN01]+$")) category = Option(doc="", require=False) type = Option(doc="", require=False) @Configuration() def map(self, records): self.logger.debug('getmispioc.map') yield {} return def reduce(self, records): self.logger.debug('getmispioc.reduce') if self.sslcheck == None: self.sslcheck = 'n' # Generate args my_args = {} if self.server: my_args['server'] = self.server if self.authkey: my_args['authkey'] = self.authkey if self.sslcheck: my_args['sslcheck'] = self.sslcheck if self.onlyids == 'Y' or self.onlyids == 'y' or self.onlyids == '1': onlyids = True else: onlyids = False if self.eventid and self.last: print('Options "eventid" and "last" are mutually exclusive') exit(1) if self.eventid: my_args['eventid'] = self.eventid elif self.last: my_args['last'] = self.last else: print('Missing "eventid" or "last" argument') exit(1) _NEW_PYTHON_PATH = '/bin/python3' _SPLUNK_PYTHON_PATH = os.environ['PYTHONPATH'] os.environ['PYTHONPATH'] = _NEW_PYTHON_PATH my_process = '/usr/local/bin/get-misp-ioc.py' # Remove LD_LIBRARY_PATH from the environment (otherwise, we will face some SSL issues env = dict(os.environ) del env['LD_LIBRARY_PATH'] FNULL = open(os.devnull, 'w') p = subprocess.Popen( [os.environ['PYTHONPATH'], my_process, str(my_args)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=FNULL, env=env) output = p.communicate()[0] results = {} for v in eval(output): # Do not display deleted attributes if v['deleted'] == False: # If wpecified, do not display attributed with the non-ids flag set to False if onlyids == True and v['to_ids'] == False: continue if self.category != None and self.category != v['category']: continue if self.type != None and self.type != v['type']: continue results['value'] = v['value'] results['category'] = v['category'] results['type'] = v['type'] results['to_ids'] = str(v['to_ids']) yield results
class HiveCollectCommand(GeneratingCommand): """ get the attributes from a TheHive instance. ##Syntax .. code-block:: | mispgetioc hive_instance=<input> last=<int>(d|h|m) | mispgetioc hive_instance=<input> event=<id1>(,<id2>,...) | mispgetioc hive_instance=<input> date=<<YYYY-MM-DD> (date_to=<YYYY-MM-DD>) """ # MANDATORY TheHive instance for this search hive_instance = Option(doc=''' **Syntax:** **hive_instance=instance_name* **Description:** TheHive instance parameters as described in lookup/thehive_instance_list.csv.''', require=True) # MANDATORY: json_request XOR alertid XOR last XOR date # json_request = Option( # doc=''' # **Syntax:** **json_request=***valid JSON request* # **Description:**Valid JSON request''', # require=False) objectid = Option(doc=''' **Syntax:** **objectid=***id1(,id2,...)* **Description:**ID.''', require=False, validate=validators.Match("objectid", r"^([0-9a-f]|\w{20})+$")) endpoint = Option(doc=''' **Syntax:** **endpoint=***alert|case* **Description:**endpoint of TheHive API''', require=False, validate=validators.Match("endpoint", r"^(alert|case)$")) range = Option(doc=''' **Syntax:** **range=***val|start_number-end_number* **Description:**valid range to limit number of alerts returned. for example range=all or range=10-100''', require=False, validate=validators.Match("range", r"^(all|\d+\-\d+)$")) def log_error(self, msg): logging.error(msg) def log_info(self, msg): logging.info(msg) def log_debug(self, msg): logging.debug(msg) def log_warn(self, msg): logging.warning(msg) def set_log_level(self): logging.root loglevel = logging_level('TA_thehive_ce') logging.root.setLevel(loglevel) logging.error('[CO-101] logging level is set to %s', loglevel) logging.error('[CO-102] PYTHON VERSION: ' + sys.version) @staticmethod def _record(serial_number, time_stamp, host, attributes, attribute_names, encoder): raw = encoder.encode(attributes) # Formulate record fields = dict() for f in attribute_names: if f in attributes: fields[f] = attributes[f] if serial_number > 0: fields['_serial'] = serial_number fields['_time'] = time_stamp fields['_raw'] = raw fields['host'] = host return fields record = OrderedDict( chain((('_serial', serial_number), ('_time', time_stamp), ('_raw', raw), ('host', host)), map(lambda name: (name, fields.get(name, '')), attribute_names))) return record def displayResponse(results, action, host): encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':')) if action == 'list_alert' or action == 'list_case': attribute_names = list() serial_number = 0 for a in results: if serial_number == 0: for k in list(a.keys()): attribute_names.append(k) if action == 'list_alert': timestamp = int(a['date'] / 1000) elif action == 'list_case': timestamp = int(a['startDate'] / 1000) yield HiveCollectCommand._record(serial_number, timestamp, host, a, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush elif action == 'get_an_alert' or action == 'get_a_case': attribute_names = list() serial_number = 0 a = results if serial_number == 0: for k in list(a.keys()): attribute_names.append(k) if action == 'get_an_alert': timestamp = int(a['date'] / 1000) elif action == 'get_a_case': timestamp = int(a['startDate'] / 1000) yield HiveCollectCommand._record(serial_number, timestamp, host, a, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush def generate(self): # Phase 1: Preparation logging.root loglevel = logging_level(self, 'TA_thehive_ce') logging.root.setLevel(loglevel) self.log_error('[CO-101] logging level is set to {}'.format(loglevel)) storage = self.service.storage_passwords my_args = prepare_config(self, 'TA_thehive_ce', self.hive_instance, storage) if my_args is None: raise Exception( "Sorry, no configuration for hive_instance={}".format( self.hive_instance)) my_args['host'] = my_args['thehive_url'].replace('https://', '') api_action = '' if self.endpoint == 'case': my_args['thehive_url'] = my_args['thehive_url'] + '/api/case' api_action = 'list_case' else: self.endpoint = 'alert' my_args['thehive_url'] = my_args['thehive_url'] + '/api/alert' api_action = 'list_alert' if self.objectid is not None: my_args['thehive_url'] = my_args['thehive_url'] + '/' + str( self.objectid) if api_action == 'list_case': api_action = 'get_a_case' elif api_action == 'list_alert': api_action = 'get_an_alert' if self.range is not None: my_args['range'] = str(self.range) else: my_args['range'] = "0-10" headers = {"Authorization": "Bearer {}".format(my_args['thehive_key'])} params = {"range": my_args['range']} # # Search pagination # pagination = True # if self.limit is not None: # limit = int(self.limit) # elif 'limit' in body_dict: # limit = int(body_dict['limit']) # else: # limit = 1000 # if limit == 0: # pagination = False # if self.page is not None: # page = int(self.page) # elif 'page' in body_dict: # page = body_dict['page'] # else: # page = 1 # body = json.dumps(body_dict) # self.log_debug('mispgetioc request body: %s', body) # search r = requests.get(my_args['thehive_url'], headers=headers, params=params, verify=my_args['thehive_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() # HiveCollectCommand.displayResponse(response, api_action, my_args['host']) encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':')) if api_action == 'list_alert' or api_action == 'list_case': attribute_names = list() serial_number = 0 for a in response: if serial_number == 0: for k in list(a.keys()): attribute_names.append(k) if api_action == 'list_alert': timestamp = int(a['date'] / 1000) elif api_action == 'list_case': timestamp = int(a['startDate'] / 1000) yield HiveCollectCommand._record(serial_number, timestamp, my_args['host'], a, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush elif api_action == 'get_an_alert' or api_action == 'get_a_case': attribute_names = list() a = response for k in list(a.keys()): attribute_names.append(k) if api_action == 'get_an_alert': timestamp = int(a['date'] / 1000) elif api_action == 'get_a_case': timestamp = int(a['startDate'] / 1000) yield HiveCollectCommand._record(0, timestamp, my_args['host'], a, attribute_names, encoder) GeneratingCommand.flush
class ToSFXCommand(EventingCommand): """ ## Syntax <command> | tosfx ## Description One or more datapoints are generated for each input event's field(s) of the form `gauge_*`, `counter_*` or `cumulative_counter_*`. The metric name in SignalFx will be the `*` part of the field name. Any additional fields on the event will be attached as dimensions to the generated datapoints. """ access_token = None debug = Option(validate=validators.Boolean(), default=False) dry_run = Option(validate=validators.Boolean(), default=False) ingest_url = Option( validate=validators.Match("https://.*", r"^https://.*")) dp_endpoint = Option(default="/v2/datapoint") SPLUNK_PASSWORD_REALM = "realm" SPLUNK_PASSWORD_USER_NAME = "username" SPLUNK_PASSWORD_CLEAR_PASSWORD = "******" SPLUNK_KV_STORE_SFX_CONFIG_COLLECTION_NAME = "sfx_ingest_config" SPLUNK_SFX_CONFIG_INGEST_URL_KEY = "ingest_url" SPLUNK_PASSWORDS_STORAGE_SFX_ACCESS_TOKEN_REALM = "sfx_ingest_command" SPLUNK_PASSWORDS_STORAGE_SFX_ACCESS_TOKEN_USER_NAME = "access_token" def ensure_default_config(self): if not self.ingest_url: self.ingest_url = self.get_sfx_ingest_url() self.logger.error("getting access token") self.access_token = self.get_access_token() def get_access_token(self): try: for credential in self.service.storage_passwords: if (credential.content.get(self.SPLUNK_PASSWORD_REALM, None) == self.SPLUNK_PASSWORDS_STORAGE_SFX_ACCESS_TOKEN_REALM and credential.content.get( self.SPLUNK_PASSWORD_USER_NAME, None) == self. SPLUNK_PASSWORDS_STORAGE_SFX_ACCESS_TOKEN_USER_NAME): return credential.content.get( self.SPLUNK_PASSWORD_CLEAR_PASSWORD, None) except Exception as e: # pylint:disable=broad-except self.logger.error( "status=error, action=get_sfx_access_token, error_msg=%s", e, exc_info=True) return None def get_sfx_ingest_url(self): try: sfx_api_config_collection = self.service.kvstore.get( self.SPLUNK_KV_STORE_SFX_CONFIG_COLLECTION_NAME, None) if sfx_api_config_collection is not None: collection = self.service.kvstore[ self.SPLUNK_KV_STORE_SFX_CONFIG_COLLECTION_NAME] # will return a list of settings, we should only have one. # We'll grab the 'last' / most recent one by default sfx_ingest_config_records = collection.data.query() if sfx_ingest_config_records: sfx_ingest_config_latest = sfx_ingest_config_records[-1] return sfx_ingest_config_latest.get( self.SPLUNK_SFX_CONFIG_INGEST_URL_KEY, None) except Exception as e: # pylint:disable=broad-except self.logger.error( "status=error, action=get_sfx_ingest_url, error_msg=%s", str(e), exc_info=True) return None def transform(self, records): self.ensure_default_config() out = [] payload = OrderedDict() for event in records: add_event_to_payload(event=event, payload=payload) if self.debug: event["endpoint"] = self.ingest_url + self.dp_endpoint out.append(event) if not self.dry_run: resp = send_payload( payload=payload, target_url=compose_ingest_url(self.ingest_url, self.dp_endpoint), token=self.access_token, ) for event in out: event["status"] = resp.status_code if resp.status_code != 200: event["response_error"] = resp.content for event in out: yield event
class MispRestCommand(GeneratingCommand): """ get the attributes from a MISP instance. ##Syntax .. code-block:: | mispgetioc misp_instance=<input> last=<int>(d|h|m) | mispgetioc misp_instance=<input> event=<id1>(,<id2>,...) | mispgetioc misp_instance=<input> date=<<YYYY-MM-DD> (date_to=<YYYY-MM-DD>) ##Description { "returnFormat": "mandatory", "page": "optional", "limit": "optional", "value": "optional", "type": "optional", "category": "optional", "org": "optional", "tags": "optional", "date": "optional", "last": "optional", "eventid": "optional", "withAttachments": "optional", "uuid": "optional", "publish_timestamp": "optional", "timestamp": "optional", "enforceWarninglist": "optional", "to_ids": "optional", "deleted": "optional", "includeEventUuid": "optional", "includeEventTags": "optional", "event_timestamp": "optional", "threat_level_id": "optional", "eventinfo": "optional", "includeProposals": "optional", "includeDecayScore": "optional", "includeFullModel": "optional", "decayingModel": "optional", "excludeDecayed": "optional", "score": "optional" } # status "returnFormat": forced to json, "page": param, "limit": param, "value": not managed, "type": param, CSV string, "category": param, CSV string, "org": not managed, "tags": param, see also not_tags "date": param, "last": param, "eventid": param, "withAttachments": forced to false, "uuid": not managed, "publish_timestamp": managed via param last "timestamp": not managed, "enforceWarninglist": param, "to_ids": param, "deleted": forced to False, "includeEventUuid": set to True, "includeEventTags": param, "event_timestamp": not managed, "threat_level_id": not managed, "eventinfo": not managed, "includeProposals": not managed "includeDecayScore": not managed, "includeFullModel": not managed, "decayingModel": not managed, "excludeDecayed": not managed, "score": not managed } """ # MANDATORY MISP instance for this search misp_instance = Option(doc=''' **Syntax:** **misp_instance=instance_name* **Description:** MISP instance parameters as described in local/misp42splunk_instances.conf.''', require=True) method = Option(doc=''' **Syntax:** **method=**** **Description:** method to use for API target DELETE GET PATCH POST.''', require=True, validate=validators.Match("method", r"^(DELETE|GET|POST)$")) json_request = Option(doc=''' **Syntax:** **json_request=***JSON request* **Description:** JSON-formatted json_request.''', require=False, validate=validators.Match("json_request", r"^{.+}$")) limit = Option(doc=''' **Syntax:** **limit=***<int>* **Description:**define the limit for each MISP search; default 1000. 0 = no pagination.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) page = Option(doc=''' **Syntax:** **page=***<int>* **Description:**define the page for each MISP search; default 1.''', require=False, validate=validators.Match("page", r"^[0-9]+$")) target = Option( doc=''' **Syntax:** **target=api_target**** **Description:**target of MISP API.''', require=True, validate=validators.Match( "target", r"^/(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$" )) def generate(self): # Phase 1: Preparation misp_instance = self.misp_instance storage = self.service.storage_passwords my_args = prepare_config(self, 'misp42splunk', misp_instance, storage) if my_args is None: raise Exception( "Sorry, no configuration for misp_instance={}".format( misp_instance)) my_args['host'] = my_args['misp_url'].replace('https://', '') if self.target not in [None, '']: my_args['misp_url'] = my_args['misp_url'] + self.target if self.json_request not in [None, '']: body_dict = json.loads(self.json_request) logging.debug('[MR-201] body_dict is {}'.format(body_dict)) else: body_dict = {} # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' if self.method == "GET": r = requests.get(my_args['misp_url'], headers=headers, params=body_dict, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) elif self.method == "POST": r = requests.post(my_args['misp_url'], headers=headers, data=json.dumps(body_dict), verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) elif self.method == "DELETE": r = requests.delete(my_args['misp_url'], headers=headers, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) else: raise Exception( "Sorry, no valid method provided (GET/POST//DELETE)." " it was {}.".format(self.method)) # check if status is anything other than 200; # throw an exception if it is if r.status_code in (200, 201, 204): logging.info("[RE301] INFO mispcollect successful. " "url={}, HTTP status={}".format( my_args['misp_url'], r.status_code)) else: logging.error("[RE302] ERROR mispcollect failed. " "url={}, data={}, HTTP Error={}, content={}".format( my_args['misp_url'], body_dict, r.status_code, r.text)) raise Exception( "[RE302] ERROR mispcollect failed. " "url={}, data={}, HTTP Error={}, content={}".format( my_args['misp_url'], body_dict, r.status_code, r.text)) # response is 200 by this point or we would have thrown an exception data = {'_time': time.time(), '_raw': json.dumps(r.json())} yield data
class MispSearchCommand(StreamingCommand): """ search in MISP for attributes matching the value of field. ##Syntax code-block:: mispsearch field=<field> onlyids=y|n ##Description body = { "returnFormat": "mandatory", "page": "optional", "limit": "optional", "value": "optional", "type": "optional", "category": "optional", "org": "optional", "tags": "optional", "from": "optional", "to": "optional", "last": "optional", "eventid": "optional", "withAttachments": "optional", "uuid": "optional", "publish_timestamp": "optional", "timestamp": "optional", "enforceWarninglist": "optional", "to_ids": "optional", "deleted": "optional", "includeEventUuid": "optional", "includeEventTags": "optional", "event_timestamp": "optional", "threat_level_id": "optional", "eventinfo": "optional" } ##Example Search in MISP for value of fieldname r_ip (remote IP in proxy logs). code-block:: * | mispsearch field=r_ip """ misp_instance = Option(doc=''' **Syntax:** **misp_instance=instance_name* **Description:**MISP instance parameters as described in local/inputs.conf''', require=True) field = Option(doc=''' **Syntax:** **field=***<fieldname>* **Description:**Name of the field containing the value to search for.''', require=True, validate=validators.Fieldname()) onlyids = Option(doc=''' **Syntax:** **onlyids=***<y|n>* **Description:** Boolean to search only attributes with to_ids set''', require=False, validate=validators.Boolean()) gettag = Option(doc=''' **Syntax:** **gettag=***<y|n>* **Description:** Boolean to return attribute tags''', require=False, validate=validators.Boolean()) includeEventUuid = Option(doc=''' **Syntax:** **includeEventUuid=***y|Y|1|true|True|n|N|0|false|False* **Description:**Boolean to include event UUID(s) to results.''', require=False, validate=validators.Boolean()) includeEventTags = Option(doc=''' **Syntax:** **includeEventTags=***y|Y|1|true|True|n|N|0|false|False* **Description:**Boolean to include event UUID(s) to results.''', require=False, validate=validators.Boolean()) last = Option(doc=''' **Syntax:** **last=***<int>d|h|m* **Description:**publication duration in day(s), hour(s) or minute(s). **eventid**, **last** and **date_from** are mutually exclusive''', require=False, validate=validators.Match("last", r"^[0-9]+[hdm]$")) limit = Option(doc=''' **Syntax:** **limit=***<int>* **Description:**define the limit for each MISP search; default 1000. 0 = no pagination.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) page = Option(doc=''' **Syntax:** **page=***<int>* **Description:**define the page for each MISP search; default 1.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) json_request = Option(doc=''' **Syntax:** **json_request=***valid JSON request* **Description:**Valid JSON request''', require=False) def stream(self, records): # Generate args my_args = prepare_config(self) my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' fieldname = str(self.field) if self.gettag is True: get_tag = True else: get_tag = False pagination = True if self.limit is not None: if int(self.limit) == 0: pagination = False else: limit = int(self.limit) else: limit = 1000 if self.page is not None: page = int(self.page) else: page = 1 if self.json_request is not None: body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False if 'limit' in body_dict: limit = int(body_dict['limit']) if limit == 0: pagination = False if 'page' in body_dict: page = body_dict['page'] pagination = False else: # build search JSON object body_dict = {"returnFormat": "json", "withAttachments": False} if self.onlyids is True: body_dict['to_ids'] = "True" if self.includeEventUuid is not None: body_dict['includeEventUuid'] = self.includeEventUuid if self.includeEventTags is not None: body_dict['includeEventTags'] = self.includeEventTags if self.last is not None: body_dict['last'] = self.last for record in records: if fieldname in record: value = record.get(fieldname, None) if value is not None: body_dict['value'] = str(value) misp_category = [] misp_event_id = [] misp_event_uuid = [] misp_orgc_id = [] misp_to_ids = [] misp_tag = [] misp_type = [] misp_value = [] misp_uuid = [] # search if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.debug('mispsearch request body: %s', body) r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception # print >> sys.stderr, "DEBUG MISP REST API response: %s" % response.json() response = r.json() if 'response' in response: if 'Attribute' in response['response']: for a in response['response']['Attribute']: if str(a['type']) not in misp_type: misp_type.append(str(a['type'])) if str(a['value']) not in misp_value: misp_value.append(str(a['value'])) if str(a['to_ids']) not in misp_to_ids: misp_to_ids.append(str(a['to_ids'])) if str(a['category']) not in misp_category: misp_category.append(str(a['category'])) if str(a['uuid']) not in misp_uuid: misp_uuid.append(str(a['uuid'])) if str(a['event_id']) not in misp_event_id: misp_event_id.append(str(a['event_id'])) if 'Tag' in a: for tag in a['Tag']: if str(tag['name']) not in misp_tag: misp_tag.append(str(tag['name'])) if 'Event' in a: if a['Event'][ 'uuid'] not in misp_event_uuid: misp_event_uuid.append( str(a['Event']['uuid'])) if a['Event'][ 'orgc_id'] not in misp_orgc_id: misp_orgc_id.append( str(a['Event']['orgc_id'])) record['misp_type'] = misp_type record['misp_value'] = misp_value record['misp_to_ids'] = misp_to_ids record['misp_category'] = misp_category record['misp_attribute_uuid'] = misp_uuid record['misp_event_id'] = misp_event_id record['misp_event_uuid'] = misp_event_uuid record['misp_orgc_id'] = misp_orgc_id record['misp_tag'] = misp_tag yield record
class MispGetEventCommand(GeneratingCommand): """ get the attributes from a MISP instance. ##Syntax .. code-block:: | MispGetEventCommand misp_instance=<input> last=<int>(d|h|m) | MispGetEventCommand misp_instance=<input> event=<id1>(,<id2>,...) | MispGetEventCommand misp_instance=<input> date=<<YYYY-MM-DD> (date_to=<YYYY-MM-DD>) ##Description { "returnFormat": "mandatory", "page": "optional", "limit": "optional", "value": "optional", "type": "optional", "category": "optional", "org": "optional", "tag": "optional", "tags": "optional", "searchall": "optional", "date": "optional", "last": "optional", "eventid": "optional", "withAttachments": "optional", "metadata": "optional", "uuid": "optional", "published": "optional", "publish_timestamp": "optional", "timestamp": "optional", "enforceWarninglist": "optional", "sgReferenceOnly": "optional", "eventinfo": "optional", "excludeLocalTags": "optional" } # status "tag": "optional", "searchall": "optional", "metadata": "optional", "published": "optional", "sgReferenceOnly": "optional", "eventinfo": "optional", "excludeLocalTags": "optional" "returnFormat": forced to json, "page": param, "limit": param, "value": not managed, "type": param, CSV string, "category": param, CSV string, "org": not managed, "tags": param, see also not_tags "date": param, "last": param, "eventid": param, "withAttachments": forced to false, "uuid": not managed, "publish_timestamp": managed via param last "timestamp": not managed, "enforceWarninglist": not managed, } """ # MANDATORY MISP instance for this search misp_instance = Option(doc=''' **Syntax:** **misp_instance=instance_name* **Description:**MISP instance parameters as described in local/inputs.conf.''', require=True) # MANDATORY: json_request XOR eventid XOR last XOR date json_request = Option(doc=''' **Syntax:** **json_request=***valid JSON request* **Description:**Valid JSON request''', require=False) eventid = Option(doc=''' **Syntax:** **eventid=***id1(,id2,...)* **Description:**list of event ID(s) or event UUID(s).''', require=False, validate=validators.Match("eventid", r"^[0-9a-f,\-]+$")) last = Option(doc=''' **Syntax:** **last=***<int>d|h|m* **Description:** publication duration in day(s), hour(s) or minute(s). **nota bene:** last is an alias of published_timestamp''', require=False, validate=validators.Match("last", r"^[0-9]+[hdm]$")) date = Option(doc=''' **Syntax:** **date=***The user set event date field - any of valid time related filters"* **Description:**starting date. **eventid**, **last** and **date** are mutually exclusive''', require=False) # Other params category = Option(doc=''' **Syntax:** **category=***CSV string* **Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''', require=False) getioc = Option(doc=''' **Syntax:** **getioc=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to return the list of attributes together with the event.''', require=False, validate=validators.Boolean()) limit = Option(doc=''' **Syntax:** **limit=***<int>* **Description:**define the limit for each MISP search; default 1000. 0 = no pagination.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) not_tags = Option(doc=''' **Syntax:** **not_tags=***CSV string* **Description:**Comma(,)-separated string of tags to exclude. Wildcard is %.''', require=False) output = Option(doc=''' **Syntax:** **output=***<default|rawy>* **Description:**selection between a tabular or JSON output.''', require=False, validate=validators.Match("output", r"(default|raw)")) page = Option(doc=''' **Syntax:** **page=***<int>* **Description:**define the page for each MISP search; default 1.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) pipesplit = Option(doc=''' **Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to split multivalue attributes.''', require=False, validate=validators.Boolean()) published = Option(doc=''' **Syntax:** **published=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**select only published events (for option from to) .''', require=False, validate=validators.Boolean()) tags = Option(doc=''' **Syntax:** **tags=***CSV string* **Description:**Comma(,)-separated string of tags to search for. Wildcard is %.''', require=False) type = Option(doc=''' **Syntax:** **type=***CSV string* **Description:**Comma(,)-separated string of types to search for. Wildcard is %.''', require=False) warning_list = Option(doc=''' **Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to filter out well known values.''', require=False, validate=validators.Boolean()) @staticmethod def _record(serial_number, time_stamp, host, attributes, attribute_names, encoder, condensed=False): if condensed is False: raw = encoder.encode(attributes) # Formulate record fields = dict() for f in attribute_names: if f in attributes: fields[f] = attributes[f] if serial_number > 0: fields['_serial'] = serial_number fields['_time'] = time_stamp if condensed is False: fields['_raw'] = raw fields['host'] = host return fields if condensed is False: record = OrderedDict( chain((('_serial', serial_number), ('_time', time_stamp), ('_raw', raw), ('host', host)), map(lambda name: (name, fields.get(name, '')), attribute_names))) else: record = OrderedDict( chain((('_serial', serial_number), ('_time', time_stamp), ('host', host)), map(lambda name: (name, fields.get(name, '')), attribute_names))) return record def generate(self): # Phase 1: Preparation my_args = prepare_config(self, 'misp42splunk') my_args['host'] = my_args['misp_url'].replace('https://', '') my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch' # check that ONE of mandatory fields is present mandatory_arg = 0 if self.json_request is not None: mandatory_arg = mandatory_arg + 1 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: logging.error('Missing "json_request", eventid", \ "last" or "date" argument') raise Exception('Missing "json_request", "eventid", \ "last" or "date" argument') elif mandatory_arg > 1: logging.error('Options "json_request", eventid", "last" \ and "date" are mutually exclusive') raise Exception('Options "json_request", "eventid", "last" \ and "date" are mutually exclusive') body_dict = dict() # Only ONE combination was provided if self.json_request is not None: body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') elif self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set with %s', json.dumps(body_dict['eventid'])) elif self.last: body_dict['last'] = self.last logging.info('Option "last" set with %s', str(body_dict['last'])) else: body_dict['date'] = self.date.split() logging.info('Option "date" set with %s', json.dumps(body_dict['date'])) # Force some values on JSON request body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True if self.limit is not None: limit = int(self.limit) elif 'limit' in body_dict: limit = int(body_dict['limit']) else: limit = 1000 if limit == 0: pagination = False if self.page is not None: page = int(self.page) elif 'page' in body_dict: page = body_dict['page'] else: page = 1 if self.published is True: body_dict['published'] = True elif self.published is False: body_dict['published'] = False # Search parameters: boolean and filter # manage enforceWarninglist if self.warning_list is True: body_dict['enforceWarninglist'] = True elif self.warning_list is False: body_dict['enforceWarninglist'] = False if self.category is not None: if "," in self.category: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria else: body_dict['category'] = self.category if self.type is not None: if "," in self.type: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria else: body_dict['type'] = self.type if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria # output filter parameters if self.getioc is True: getioc = True else: getioc = False if self.pipesplit is True: pipesplit = True else: pipesplit = False if self.output is not None: output = self.output else: output = "default" if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.error('MispGetEventCommand request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':')) if output == "raw": if 'response' in response: attribute_names = list() serial_number = 0 for r_item in response['response']: if 'Event' in r_item: for e in r_item.values(): yield MispGetEventCommand._record( serial_number, e['timestamp'], my_args['host'], e, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush else: # build output table and list of types events = [] typelist = [] column_list = format_output_table(response, events, typelist, getioc, pipesplit) logging.info('typelist containss %s values', str(len(typelist))) logging.debug('typelist is %s', json.dumps(typelist)) logging.info('results contains %s records', str(len(events))) if getioc is False: attribute_names = list() init_attribute_names = True serial_number = 0 for e in events: # logging.debug('event is %s', json.dumps(e)) if init_attribute_names is True: for key in e.keys(): if key not in attribute_names: attribute_names.append(key) attribute_names.sort() init_attribute_names = False logging.debug(json.dumps(attribute_names)) yield MispGetEventCommand._record(serial_number, e['misp_timestamp'], my_args['host'], e, attribute_names, encoder, True) serial_number += 1 GeneratingCommand.flush else: output_dict = {} for e in events: if 'Attribute' in e: for a in e['Attribute']: if int(a['misp_object_id']) == 0: # not an object key = str(e['misp_event_id']) + '_' \ + str(a['misp_attribute_id']) is_object_member = False else: # this is a MISP object key = str(e['misp_event_id']) + \ '_object_' + str(a['misp_object_id']) is_object_member = True if key not in output_dict: v = init_misp_output(e, a, column_list) for t in typelist: misp_t = 'misp_' \ + t.replace('-', '_')\ .replace('|', '_p_') v[misp_t] = [] if t == a['misp_type']: v[misp_t].append(a['misp_value']) if is_object_member is True: v['misp_type'] = v['misp_object_name'] v['misp_value'] = v['misp_object_id'] output_dict[key] = dict(v) else: v = dict(output_dict[key]) misp_t = 'misp_' + a['misp_type']\ .replace('-', '_').replace('|', '_p_') v[misp_t].append(a['misp_value']) if a['misp_to_ids'] not in v['misp_to_ids']: v['misp_to_ids'].append(a['misp_to_ids']) if a['misp_category'] not in v[ 'misp_category']: v['misp_category'].append( a['misp_category']) v['misp_attribute_uuid']\ .append(a['misp_attribute_uuid']) v['misp_attribute_id']\ .append(a['misp_attribute_id']) if a['misp_attribute_tag'] is not None: a_tag = v['misp_attribute_tag'] for t in a['misp_attribute_tag']: if t not in a_tag: a_tag.append(t) v['misp_attribute_tag'] = a_tag if a['misp_comment'] not in v['misp_comment']: v['misp_comment'].append(a['misp_comment']) if is_object_member is False: misp_type = a['misp_type'] \ + '|' + v['misp_type'] v['misp_type'] = misp_type misp_value = a['misp_value'] + \ '|' + v['misp_value'] v['misp_value'] = misp_value output_dict[key] = dict(v) if output_dict is not None: attribute_names = list() init_attribute_names = True serial_number = 0 for v in output_dict.values(): if init_attribute_names is True: for key in v.keys(): if key not in attribute_names: attribute_names.append(key) attribute_names.sort() init_attribute_names = False logging.debug(json.dumps(attribute_names)) yield MispGetEventCommand._record( serial_number, v['misp_timestamp'], my_args['host'], v, attribute_names, encoder, True) serial_number += 1 GeneratingCommand.flush
class mispsight(StreamingCommand): """ search in MISP for attributes matching the value of field. ##Syntax code-block:: mispsearch field=<field> onlyids=y|n ##Description search_body = {"returnFormat": "json", "value": "optional", "type": "optional", "category": "optional", "org": "optional", "tags": "optional", "from": "optional", "to": "optional", "last": "optional", "eventid": "optional", "withAttachments": "optional", "uuid": "optional", "publish_timestamp": "optional", "timestamp": "optional", "enforceWarninglist": "optional", "to_ids": "optional", "deleted": "optional", "includeEventUuid": "optional", "event_timestamp": "optional", "threat_level_id": "optional" } ##Example Search in MISP for value of fieldname r_ip (remote IP in proxy logs). code-block:: * | mispsearch fieldname=r_ip """ field = Option(doc=''' **Syntax:** **field=***<fieldname>* **Description:**Name of the field containing the value to search for.''', require=True, validate=validators.Fieldname()) # Superseede MISP instance for this search misp_instance = Option(doc=''' **Syntax:** **misp_instance=instance_name* **Description:**MISP instance parameters as decibed in lookup/misp_instances.csv.''', require=False) misp_url = Option(doc=''' **Syntax:** **misp_url=***<MISP URL>* **Description:**URL of MISP instance.''', require=False, validate=validators.Match( "misp_url", r"^https?:\/\/[0-9a-zA-Z\-\.]+(?:\:\d+)?$")) misp_key = Option(doc=''' **Syntax:** **misp_key=***<AUTH_KEY>* **Description:**MISP API AUTH KEY.''', require=False, validate=validators.Match("misp_key", r"^[0-9a-zA-Z]{40}$")) misp_verifycert = Option(doc=''' **Syntax:** **misp_verifycert=***<y|n>* **Description:**Verify or not MISP certificate.''', require=False, validate=validators.Boolean()) def stream(self, records): # self.logger.debug('mispgetioc.reduce') # Generate args my_args = prepare_config(self) # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' fieldname = str(self.field) for record in records: if fieldname in record: value = record.get(fieldname, None) if value is not None: search_url = my_args['misp_url'] + '/attributes/restSearch' search_dict = {"returnFormat": "json"} search_dict['value'] = str(value) search_dict['withAttachments'] = "false", search_body = json.dumps(search_dict) sight_url = my_args[ 'misp_url'] + '/sightings/restSearch/attribute' sight_dict = {"returnFormat": "json"} misp_value = '' misp_fp = False misp_fp_timestamp = 0 misp_fp_event_id = '' misp_sight_seen = False misp_sight = { 'count': 0, 'first': 0, 'first_event_id': 0, 'last': 0, 'last_event_id': 0 } # search r = requests.post(search_url, headers=headers, data=search_body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() logging.info("INFO MISP REST API %s has response: %s" % (search_url, r.json())) if 'response' in response: if 'Attribute' in response['response']: for a in response['response']['Attribute']: if misp_value == '': misp_value = str(a['value']) if misp_fp == False: sight_dict['id'] = str(a['id']) sight_body = json.dumps(sight_dict) s = requests.post( sight_url, headers=headers, data=sight_body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; throw an exception if it is s.raise_for_status() # response is 200 by this point or we would have thrown an exception sight = s.json() logging.info( "INFO MISP REST API %s has response: %s" % (sight_url, s.json())) if 'response' in sight: for se in sight['response']: if 'Sighting' in se: if int(se['Sighting']['type'] ) == 0: #true sighting misp_sight_seen = True misp_sight[ 'count'] = misp_sight[ 'count'] + 1 if misp_sight['first'] == 0 or \ misp_sight['first'] > int(se['Sighting']['date_sighting']): misp_sight[ 'first'] = int( se['Sighting'] ['date_sighting'] ) misp_sight[ 'first_event_id'] = se[ 'Sighting'][ 'event_id'] if misp_sight['last'] < int( se['Sighting'] ['date_sighting']): misp_sight['last'] = int( se['Sighting'] ['date_sighting']) misp_sight[ 'last_event_id'] = se[ 'Sighting'][ 'event_id'] elif int( se['Sighting']['type'] ) == 1: #false positive misp_fp = True misp_fp_timestamp = int( se['Sighting'] ['date_sighting']) misp_fp_event_id = se[ 'Sighting']['event_id'] if misp_fp == True: record['misp_value'] = misp_value record['misp_fp'] = "True" record['misp_fp_timestamp'] = str( misp_fp_timestamp) record['misp_fp_event_id'] = str( misp_fp_event_id) if misp_sight_seen == True: record['misp_value'] = misp_value record['misp_sight_count'] = str( misp_sight['count']) record['misp_sight_first'] = str( misp_sight['first']) record['misp_sight_first_event_id'] = str( misp_sight['first_event_id']) record['misp_sight_last'] = str( misp_sight['last']) record['misp_sight_last_event_id'] = str( misp_sight['last_event_id']) yield record
class MispGetIocCommand(GeneratingCommand): """ get the attributes from a MISP instance. ##Syntax .. code-block:: | mispgetioc misp_instance=<input> last=<int>(d|h|m) | mispgetioc misp_instance=<input> event=<id1>(,<id2>,...) | mispgetioc misp_instance=<input> date=<<YYYY-MM-DD> (date_to=<YYYY-MM-DD>) ##Description { "returnFormat": "mandatory", "page": "optional", "limit": "optional", "value": "optional", "type": "optional", "category": "optional", "org": "optional", "tags": "optional", "date": "optional", "last": "optional", "eventid": "optional", "withAttachments": "optional", "uuid": "optional", "publish_timestamp": "optional", "timestamp": "optional", "enforceWarninglist": "optional", "to_ids": "optional", "deleted": "optional", "includeEventUuid": "optional", "includeEventTags": "optional", "event_timestamp": "optional", "threat_level_id": "optional", "eventinfo": "optional", "includeProposals": "optional", "includeDecayScore": "optional", "includeFullModel": "optional", "decayingModel": "optional", "excludeDecayed": "optional", "score": "optional" } # status "returnFormat": forced to json, "page": param, "limit": param, "value": not managed, "type": param, CSV string, "category": param, CSV string, "org": not managed, "tags": param, see also not_tags "date": param, "last": param, "eventid": param, "withAttachments": forced to false, "uuid": not managed, "publish_timestamp": managed via param last "timestamp": not managed, "enforceWarninglist": param, "to_ids": param, "deleted": forced to False, "includeEventUuid": set to True, "includeEventTags": param, "event_timestamp": not managed, "threat_level_id": not managed, "eventinfo": not managed, "includeProposals": not managed "includeDecayScore": not managed, "includeFullModel": not managed, "decayingModel": not managed, "excludeDecayed": not managed, "score": not managed } """ # MANDATORY MISP instance for this search misp_instance = Option(doc=''' **Syntax:** **misp_instance=instance_name* **Description:** MISP instance parameters as described in local/inputs.conf.''', require=True) # MANDATORY: json_request XOR eventid XOR last XOR date json_request = Option(doc=''' **Syntax:** **json_request=***valid JSON request* **Description:**Valid JSON request''', require=False) eventid = Option(doc=''' **Syntax:** **eventid=***id1(,id2,...)* **Description:**list of event ID(s) or event UUID(s).''', require=False, validate=validators.Match("eventid", r"^[0-9a-f,\-]+$")) last = Option(doc=''' **Syntax:** **last=***<int>d|h|m* **Description:** publication duration in day(s), hour(s) or minute(s). **nota bene:** last is an alias of published_timestamp''', require=False, validate=validators.Match("last", r"^[0-9]+[hdm]$")) date = Option(doc=''' **Syntax:** **date=***The user set event date field - any of valid time related filters"* **Description:**starting date. **eventid**, **last** and **date** are mutually exclusive''', require=False) # Other params add_description = Option(doc=''' **Syntax:** **add_description=***<1|y|Y|t|true|True |0|n|N|f|false|False>* **Description:**Boolean to return misp_description.''', require=False, validate=validators.Boolean()) category = Option(doc=''' **Syntax:** **category=***CSV string* **Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''', require=False) geteventtag = Option(doc=''' **Syntax:** **geteventtag=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean includeEventTags. By default only attribute tag(s) are returned.''', require=False, validate=validators.Boolean()) getorg = Option(doc=''' **Syntax:** **getorg=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to return the ID of the organisation that created the event.''', require=False, validate=validators.Boolean()) getuuid = Option(doc=''' **Syntax:** **getuuid=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to return attribute UUID.''', require=False, validate=validators.Boolean()) limit = Option(doc=''' **Syntax:** **limit=***<int>* **Description:**define the limit for each MISP search; default 1000. 0 = no pagination.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) not_tags = Option(doc=''' **Syntax:** **not_tags=***CSV string* **Description:**Comma(,)-separated string of tags to exclude. Wildcard is %.''', require=False) output = Option(doc=''' **Syntax:** **output=***<default|rawy>* **Description:**selection between the default behaviou or \ JSON output by event.''', require=False, validate=validators.Match("output", r"(default|raw)")) page = Option(doc=''' **Syntax:** **page=***<int>* **Description:**define the page for each MISP search; default 1.''', require=False, validate=validators.Match("limit", r"^[0-9]+$")) pipesplit = Option(doc=''' **Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to split multivalue attributes.''', require=False, validate=validators.Boolean()) tags = Option(doc=''' **Syntax:** **tags=***CSV string* **Description:**Comma(,)-separated string of tags to search for. Wildcard is %.''', require=False) to_ids = Option(doc=''' **Syntax:** **to_ids=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to search only attributes with the flag "to_ids" set to true.''', require=False, validate=validators.Boolean()) type = Option(doc=''' **Syntax:** **type=***CSV string* **Description:**Comma(,)-separated string of types to search for. Wildcard is %.''', require=False) warning_list = Option(doc=''' **Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>* **Description:**Boolean to filter out well known values.''', require=False, validate=validators.Boolean()) @staticmethod def _record(serial_number, time_stamp, host, attributes, attribute_names, encoder): raw = encoder.encode(attributes) # Formulate record fields = dict() for f in attribute_names: if f in attributes: fields[f] = attributes[f] if serial_number > 0: fields['_serial'] = serial_number fields['_time'] = time_stamp fields['_raw'] = raw fields['host'] = host return fields record = OrderedDict( chain((('_serial', serial_number), ('_time', time_stamp), ('_raw', raw), ('host', host)), map(lambda name: (name, fields.get(name, '')), attribute_names))) return record def generate(self): # Phase 1: Preparation my_args = prepare_config(self, 'misp42splunk') my_args['host'] = my_args['misp_url'].replace('https://', '') my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch' # check that ONE of mandatory fields is present mandatory_arg = 0 if self.json_request is not None: mandatory_arg = mandatory_arg + 1 if self.eventid: mandatory_arg = mandatory_arg + 1 if self.last: mandatory_arg = mandatory_arg + 1 if self.date: mandatory_arg = mandatory_arg + 1 if mandatory_arg == 0: logging.error('Missing "json_request", eventid", \ "last" or "date" argument') raise Exception('Missing "json_request", "eventid", \ "last" or "date" argument') elif mandatory_arg > 1: logging.error('Options "json_request", eventid", "last" \ and "date" are mutually exclusive') raise Exception('Options "json_request", "eventid", "last" \ and "date" are mutually exclusive') body_dict = dict() # Only ONE combination was provided if self.json_request is not None: body_dict = json.loads(self.json_request) logging.info('Option "json_request" set') elif self.eventid: if "," in self.eventid: event_criteria = {} event_list = self.eventid.split(",") event_criteria['OR'] = event_list body_dict['eventid'] = event_criteria else: body_dict['eventid'] = self.eventid logging.info('Option "eventid" set with %s', json.dumps(body_dict['eventid'])) elif self.last: body_dict['last'] = self.last logging.info('Option "last" set with %s', str(body_dict['last'])) else: body_dict['date'] = self.date.split() logging.info('Option "date" set with %s', json.dumps(body_dict['date'])) # Force some values on JSON request body_dict['returnFormat'] = 'json' body_dict['withAttachments'] = False body_dict['deleted'] = False body_dict['includeEventUuid'] = True # set proper headers headers = {'Content-type': 'application/json'} headers['Authorization'] = my_args['misp_key'] headers['Accept'] = 'application/json' # Search pagination pagination = True if self.limit is not None: limit = int(self.limit) elif 'limit' in body_dict: limit = int(body_dict['limit']) else: limit = 1000 if limit == 0: pagination = False if self.page is not None: page = int(self.page) elif 'page' in body_dict: page = body_dict['page'] else: page = 1 # Search parameters: boolean and filter # manage to_ids and enforceWarninglist # to avoid FP enforceWarninglist is set to True if # to_ids is set to True (search criterion) if self.to_ids is True: body_dict['to_ids'] = True body_dict['enforceWarninglist'] = True # protection elif self.to_ids is False: body_dict['to_ids'] = False if self.warning_list is True: body_dict['enforceWarninglist'] = True elif self.warning_list is False: body_dict['enforceWarninglist'] = False if self.geteventtag is True: body_dict['includeEventTags'] = True if self.category is not None: if "," in self.category: cat_criteria = {} cat_list = self.category.split(",") cat_criteria['OR'] = cat_list body_dict['category'] = cat_criteria else: body_dict['category'] = self.category if self.type is not None: if "," in self.type: type_criteria = {} type_list = self.type.split(",") type_criteria['OR'] = type_list body_dict['type'] = type_criteria else: body_dict['type'] = self.type if self.tags is not None or self.not_tags is not None: tags_criteria = {} if self.tags is not None: tags_list = self.tags.split(",") tags_criteria['OR'] = tags_list if self.not_tags is not None: tags_list = self.not_tags.split(",") tags_criteria['NOT'] = tags_list body_dict['tags'] = tags_criteria # output filter parameters if self.getuuid is True: my_args['getuuid'] = True else: my_args['getuuid'] = False if self.getorg is True: my_args['getorg'] = True else: my_args['getorg'] = False if self.pipesplit is True: my_args['pipe'] = True else: my_args['pipe'] = False if self.add_description is True: my_args['add_desc'] = True else: my_args['add_desc'] = False if self.output is not None: my_args['output'] = self.output else: my_args['output'] = "default" results = [] # add colums for each type in results typelist = [] if pagination is True: body_dict['page'] = page body_dict['limit'] = limit body = json.dumps(body_dict) logging.debug('mispgetioc request body: %s', body) # search r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies']) # check if status is anything other than 200; # throw an exception if it is r.raise_for_status() # response is 200 by this point or we would have thrown an exception response = r.json() encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':')) if my_args['output'] == "raw": if 'response' in response: if 'Attribute' in response['response']: attribute_names = [] serial_number = 0 for a in response['response']['Attribute']: yield MispGetIocCommand._record( serial_number, a['timestamp'], my_args['host'], a, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush else: if 'response' in response: if 'Attribute' in response['response']: for a in response['response']['Attribute']: v = {} v['misp_category'] = str(a['category']) v['misp_attribute_id'] = str(a['id']) v['misp_event_id'] = str(a['event_id']) v['misp_timestamp'] = str(a['timestamp']) v['misp_to_ids'] = str(a['to_ids']) v['misp_comment'] = str(a['comment']) tag_list = [] if 'Tag' in a: for tag in a['Tag']: try: tag_list.append(str(tag['name'])) except Exception: pass v['misp_tag'] = tag_list # include ID of the organisation that # created the attribute if requested if 'Event' in a: v['misp_event_uuid'] = str(a['Event']['uuid']) if my_args['getorg']: v['misp_orgc_id'] = str(a['Event']['orgc_id']) if my_args['add_desc'] is True: v['misp_event_info'] = str(a['Event']['info']) # include attribute UUID if requested if my_args['getuuid']: v['misp_attribute_uuid'] = str(a['uuid']) # handle object and multivalue attributes v['misp_object_id'] = str(a['object_id']) if my_args['add_desc'] is True: if int(a['object_id']) == 0: v['misp_description'] = 'MISP e' \ + str(a['event_id']) + ' attribute ' \ + str(a['uuid']) + ' of type "' \ + str(a['type']) \ + '" in category "' + str(a['category']) \ + '" (to_ids:' + str(a['to_ids']) + ')' else: v['misp_description'] = 'MISP e' \ + str(a['event_id']) + ' attribute ' \ + str(a['uuid']) + ' of type "' \ + str(a['type']) + '" in category "' \ + str(a['category']) \ + '" (to_ids:' + str(a['to_ids']) \ + ' - o' + str(a['object_id']) + ' )' current_type = str(a['type']) # combined: not part of an object # AND multivalue attribute AND to be split # logging.debug('misp_event: %s', json.dumps(v)) if int(a['object_id']) == 0 and '|' in current_type \ and my_args['pipe'] is True: mv_type_list = current_type.split('|') mv_value_list = str(a['value']).split('|') left_v = v.copy() left_v['misp_type'] = mv_type_list.pop() left_v['misp_value'] = mv_value_list.pop() results.append(left_v) if left_v['misp_type'] not in typelist: typelist.append(left_v['misp_type']) right_v = v.copy() right_v['misp_type'] = mv_type_list.pop() right_v['misp_value'] = mv_value_list.pop() results.append(right_v) if right_v['misp_type'] not in typelist: typelist.append(right_v['misp_type']) else: v['misp_type'] = current_type v['misp_value'] = str(a['value']) results.append(v) if current_type not in typelist: typelist.append(current_type) logging.info(json.dumps(typelist)) output_dict = {} attribute_names = [ 'misp_attribute_id', 'misp_attribute_uuid', 'misp_category', 'misp_comment', 'misp_description', 'misp_event_id', 'misp_tag', 'misp_timestamp', 'misp_to_ids', 'misp_type', 'misp_value' ] if my_args['add_desc'] is True: attribute_names.append('misp_event_info') for t in typelist: misp_t = 'misp_' + \ t.replace('-', '_').replace('|', '_p_') if misp_t not in attribute_names: attribute_names.append(misp_t) for r in results: if int(r['misp_object_id']) == 0: # not an object key = str(r['misp_event_id']) + \ '_' + r['misp_attribute_id'] is_object_member = False else: # this is a MISP object key = str(r['misp_event_id']) \ + '_object_' + str(r['misp_object_id']) is_object_member = True if key not in output_dict: v = dict(r) for t in typelist: misp_t = 'misp_' + t.replace('-', '_')\ .replace('|', '_p_') v[misp_t] = [] if t == r['misp_type']: v[misp_t].append(r['misp_value']) v['misp_to_ids'] = [] v['misp_to_ids'].append(r['misp_to_ids']) v['misp_category'] = [] v['misp_category'].append(r['misp_category']) v['misp_attribute_id'] = [] v['misp_attribute_id']\ .append(r['misp_attribute_id']) if my_args['getuuid'] is True: v['misp_attribute_uuid'] = [] v['misp_attribute_uuid']\ .append(r['misp_attribute_uuid']) if my_args['add_desc'] is True: description = [] description.append(r['misp_description']) v['misp_description'] = description if is_object_member is True: v['misp_type'] = 'misp_object' v['misp_value'] = r['misp_object_id'] output_dict[key] = dict(v) else: v = dict(output_dict[key]) misp_t = 'misp_' + r['misp_type'].replace('-', '_') v[misp_t].append(r['misp_value']) # set value for type v['misp_to_ids'].append(r['misp_to_ids']) v['misp_category'].append(r['misp_category']) tag_list = v['misp_tag'] for tag in r['misp_tag']: if tag not in tag_list: tag_list.append(tag) v['misp_tag'] = tag_list if my_args['add_desc'] is True: description = v['misp_description'] if r['misp_description'] not in description: description.append(r['misp_description']) v['misp_description'] = description v['misp_attribute_id']\ .append(r['misp_attribute_id']) if my_args['getuuid'] is True: v['misp_attribute_uuid']\ .append(r['misp_attribute_uuid']) if is_object_member is False: misp_type = r['misp_type'] + '|' + v['misp_type'] v['misp_type'] = misp_type misp_value = r['misp_value'] + '|' + v['misp_value'] v['misp_value'] = misp_value output_dict[key] = dict(v) serial_number = 0 logging.debug(json.dumps(attribute_names)) for k, v in list(output_dict.items()): yield MispGetIocCommand._record(serial_number, v['misp_timestamp'], my_args['host'], v, attribute_names, encoder) serial_number += 1 GeneratingCommand.flush
class TestSearchCommand(SearchCommand): boolean = Option( doc=''' **Syntax:** **boolean=***<value>* **Description:** A boolean value''', validate=validators.Boolean()) required_boolean = Option( doc=''' **Syntax:** **boolean=***<value>* **Description:** A boolean value''', require=True, validate=validators.Boolean()) aliased_required_boolean = Option( doc=''' **Syntax:** **boolean=***<value>* **Description:** A boolean value''', name='foo', require=True, validate=validators.Boolean()) code = Option( doc=''' **Syntax:** **code=***<value>* **Description:** A Python expression, if mode == "eval", or statement, if mode == "exec"''', validate=validators.Code()) required_code = Option( doc=''' **Syntax:** **code=***<value>* **Description:** A Python expression, if mode == "eval", or statement, if mode == "exec"''', require=True, validate=validators.Code()) duration = Option( doc=''' **Syntax:** **duration=***<value>* **Description:** A length of time''', validate=validators.Duration()) required_duration = Option( doc=''' **Syntax:** **duration=***<value>* **Description:** A length of time''', require=True, validate=validators.Duration()) fieldname = Option( doc=''' **Syntax:** **fieldname=***<value>* **Description:** Name of a field''', validate=validators.Fieldname()) required_fieldname = Option( doc=''' **Syntax:** **fieldname=***<value>* **Description:** Name of a field''', require=True, validate=validators.Fieldname()) file = Option( doc=''' **Syntax:** **file=***<value>* **Description:** Name of a file''', validate=validators.File()) required_file = Option( doc=''' **Syntax:** **file=***<value>* **Description:** Name of a file''', require=True, validate=validators.File()) integer = Option( doc=''' **Syntax:** **integer=***<value>* **Description:** An integer value''', validate=validators.Integer()) required_integer = Option( doc=''' **Syntax:** **integer=***<value>* **Description:** An integer value''', require=True, validate=validators.Integer()) map = Option( doc=''' **Syntax:** **map=***<value>* **Description:** A mapping from one value to another''', validate=validators.Map(foo=1, bar=2, test=3)) required_map = Option( doc=''' **Syntax:** **map=***<value>* **Description:** A mapping from one value to another''', require=True, validate=validators.Map(foo=1, bar=2, test=3)) match = Option( doc=''' **Syntax:** **match=***<value>* **Description:** A value that matches a regular expression pattern''', validate=validators.Match('social security number', r'\d{3}-\d{2}-\d{4}')) required_match = Option( doc=''' **Syntax:** **required_match=***<value>* **Description:** A value that matches a regular expression pattern''', require=True, validate=validators.Match('social security number', r'\d{3}-\d{2}-\d{4}')) optionname = Option( doc=''' **Syntax:** **optionname=***<value>* **Description:** The name of an option (used internally)''', validate=validators.OptionName()) required_optionname = Option( doc=''' **Syntax:** **optionname=***<value>* **Description:** The name of an option (used internally)''', require=True, validate=validators.OptionName()) regularexpression = Option( doc=''' **Syntax:** **regularexpression=***<value>* **Description:** Regular expression pattern to match''', validate=validators.RegularExpression()) required_regularexpression = Option( doc=''' **Syntax:** **regularexpression=***<value>* **Description:** Regular expression pattern to match''', require=True, validate=validators.RegularExpression()) set = Option( doc=''' **Syntax:** **set=***<value>* **Description:** A member of a set''', validate=validators.Set('foo', 'bar', 'test')) required_set = Option( doc=''' **Syntax:** **set=***<value>* **Description:** A member of a set''', require=True, validate=validators.Set('foo', 'bar', 'test')) class ConfigurationSettings(SearchCommand.ConfigurationSettings): @classmethod def fix_up(cls, command_class): pass
class GetJiraKv(GeneratingCommand): verify = Option( doc=''' **Syntax:** **verify=**** **Description:** verify the connectivity to a remote instance. True / False are supported.''', require=False, default="False", validate=validators.Match("verify", r"^(True|False)$")) def generate(self, **kwargs): if self: # Get the session key session_key = self._metadata.searchinfo.session_key # Get splunkd port entity = splunk.entity.getEntity('/server', 'settings', namespace='TA-jira-service-desk-simple-addon', sessionKey=session_key, owner='-') splunkd_port = entity['mgmtHostPort'] # Get conf conf_file = "ta_service_desk_simple_addon_settings" confs = self.service.confs[str(conf_file)] storage_passwords = self.service.storage_passwords kvstore_instance = None bearer_token = None for stanza in confs: if stanza.name == "advanced_configuration": for key, value in stanza.content.items(): if key == "jira_passthrough_mode": jira_passthrough_mode = value if key == "kvstore_instance": kvstore_instance = value if key == "kvstore_search_filters": kvstore_search_filters = value if kvstore_instance: # The bearer token is stored in the credential store # However, likely due to the number of chars, the credential.content.get SDK command is unable to return its value in a single operation # As a workaround, we concatenate the different values return to form a complete object, finally we use a regex approach to extract its clear text value credential_realm = '__REST_CREDENTIAL__#TA-jira-service-desk-simple-addon#configs/conf-ta_service_desk_simple_addon_settings' bearer_token_rawvalue = "" for credential in storage_passwords: if credential.content.get('realm') == str(credential_realm): bearer_token_rawvalue = bearer_token_rawvalue + str(credential.content.clear_password) # extract a clean json object bearer_token_rawvalue_match = re.search('\{\"bearer_token\":\s*\"(.*)\"\}', bearer_token_rawvalue) if bearer_token_rawvalue_match: bearer_token = bearer_token_rawvalue_match.group(1) else: bearer_token = None # the root search search = "| inputlookup jira_failures_replay | eval uuid=_key, mtime=if(isnull(mtime), ctime, mtime)" # If the passthrough mode is disabled, there is no distributed setup # and the instance is the localhost if (not kvstore_instance or not bearer_token) and str(jira_passthrough_mode) == '0': kvstore_instance = 'localhost:' + str(splunkd_port) header = 'Splunk ' + str(session_key) elif str(jira_passthrough_mode) == '1': # yield data = {'_time': time.time(), '_raw': "{\"response\": \"" + "INFO: Passthrough mode is currently enabled in this instance, you can safety disable the alert execution for this instance.}"} yield data sys.exit(0) elif kvstore_instance and not bearer_token: # yield data = {'_time': time.time(), '_raw': "{\"response\": \"" + "ERROR: The KVstore instance is set but not the bearer token.}"} yield data sys.exit(0) elif bearer_token and not kvstore_instance: # yield data = {'_time': time.time(), '_raw': "{\"response\": \"" + "ERROR: The bearer token is set but not the KVstore instance.}"} yield data sys.exit(0) else: header = 'Bearer ' + str(bearer_token) search = str(search) + " | search " + str(kvstore_search_filters) # Define the url url = "https://" + str(kvstore_instance) + "/services/search/jobs/export" # Get data output_mode = "csv" exec_mode = "oneshot" response = requests.post(url, headers={'Authorization': header}, verify=False, data={'search': search, 'output_mode': output_mode, 'exec_mode': exec_mode}) csv_data = response.text if response.status_code not in (200, 201, 204): response_error = 'JIRA Get remove KVstore has failed!. url={}, data={}, HTTP Error={}, content={}'.format(url, search, response.status_code, response.text) self.logger.fatal(str(response_error)) data = {'_time': time.time(), '_raw': "{\"response\": \"" + str(response_error) + "\""} yield data sys.exit(0) else: if self.verify == 'True': response_error = 'JIRA Get remove KVstore was successfull. url={}, data={}, HTTP Error={}'.format(url, search, response.status_code) data = {'_time': time.time(), '_raw': "{\"response\": \"" + str(response_error) + "\""} yield data sys.exit(0) else: # Use the CSV dict reader readCSV = csv.DictReader(csv_data.splitlines(True), delimiter=str(u','), quotechar=str(u'"')) # For row in CSV, generate the _raw for row in readCSV: yield {'_time': time.time(), 'uuid': str(row['uuid']), 'account': str(row['account']), 'data': str(row['data']), 'status': str(row['status']), 'ctime': str(row['ctime']), 'mtime': str(row['mtime']), 'no_attempts': str(row['no_attempts'])} else: # yield data = {'_time': time.time(), '_raw': "{\"response\": \"" + "Error: bad request}"} yield data