class snowIncidentCommand(GeneratingCommand): assigned = Option(require=True, validate=validators.List()) assigned_by = Option(require=False) daysAgo = Option(require=False, validate=validators.Integer(0)) active = Option(require=True, validate=validators.Boolean()) limit = Option(require=False, validate=validators.Integer(0)) env = Option(require=False) def generate(self): self.logger.debug('snowIncidentCommand: %s', self) searchinfo = self.metadata.searchinfo app = AppConf(searchinfo.splunkd_uri, searchinfo.session_key) env = self.env.lower() if self.env else 'production' conf = app.get_config('getsnow')[env] assigned_by = 'assignment_group' if self.assigned_by == 'group' else 'assigned_to' assignment = {'table': 'sys_user_group', 'field': 'name'} if self.assigned_by == 'group' else {'table': 'sys_user', 'field': 'user_name'} limit = self.limit if self.limit else 10000 snowincident = snow(conf['url'], conf['user'], conf['password']) sids = snowincident.getsysid(assignment['table'], assignment['field'], self.assigned) filters = snowincident.filterbuilder(assigned_by, sids) glide = 'sys_created_on>=javascript:gs.daysAgo({})'.format(self.daysAgo) if self.daysAgo else '' url = snowincident.reqencode(filters, table='incident', glide_system=glide, active=self.active, sysparm_limit=limit) for record in snowincident.getrecords(url): record = snowincident.updatevalue(record, sourcetype='snow:incident') record['_raw'] = json.dumps(record) record = dictexpand(record) yield record
def test_list(self): validator = validators.List() self.assertEqual(validator.__call__(''), []) self.assertEqual(validator.__call__('a,b,c'), ['a', 'b', 'c']) self.assertRaises(ValueError, validator.__call__, '"a,b,c') self.assertEqual(validator.__call__([]), []) self.assertEqual(validator.__call__(None), None) validator = validators.List(validators.Integer(1, 10)) self.assertEqual(validator.__call__(''), []) self.assertEqual(validator.__call__('1,2,3'), [1, 2, 3]) self.assertRaises(ValueError, validator.__call__, '1,2,0') self.assertEqual(validator.__call__([]), []) self.assertEqual(validator.__call__(None), None)
class GenerateCurrentWeatherCommand(GeneratingCommand): locations = Option(require=True, validate=validators.List()) def generate(self): self.logger.debug("Generating %s events" % self.locations) n = 1 for i in self.locations: Location = location(i, TomTomKey) CurrentWeather = currentweather(Location, WeatherKey) Lon = "longitude=\"" + str(Location.LocationLon) + "\" " Lat = "lattitude=\"" + str(Location.LocationLat) + "\" " Address = "address=\"" + str(Location.Address) + "\" " Temp = "temperature=\"" + str(CurrentWeather.Temperature) + "\"" Clouds = "clouds=\"" + str(CurrentWeather.Cloudiness) + "\"" Rain = "rain=\"" + str(CurrentWeather.Rainfall) + "\"" text = Lon + Lat + Address + Temp + Clouds + Rain yield {'_time': time.time(), 'event_no': n, '_raw': text} n += 1
class snowUserCommand(GeneratingCommand): user_name = Option(require=True, validate=validators.List()) daysAgo = Option(require=False, validate=validators.Integer(0)) env = Option(require=False) def generate(self): self.logger.debug('snowuser: %s', self) searchinfo = self.metadata.searchinfo app = AppConf(searchinfo.splunkd_uri, searchinfo.session_key) env = self.env.lower() if self.env else 'production' conf = app.get_config('getsnow')[env] snowuser = snow(conf['url'], conf['user'], conf['password']) filters = snowuser.filterbuilder('user_name', self.user_name) query_string = snowuser.reqencode(filters, 'sys_user') user_sid = [] for record in snowuser.getrecords(query_string): user_sid.append(record['sys_id']) record = snowuser.updatevalue(record, sourcetype='snow:user') record['_raw'] = json.dumps(record) record = dictexpand(record) yield record filters = snowuser.filterbuilder('assigned_to', user_sid) url = snowuser.reqencode(filters, table='alm_asset') for record in snowuser.getrecords(url): record = snowuser.updatevalue(record, sourcetype='snow:asset') record['_raw'] = json.dumps(record) record = dictexpand(record) yield record filters = snowuser.filterbuilder('opened_by', user_sid) glide = 'sys_created_on>=javascript:gs.daysAgo({})'.format( self.daysAgo) if self.daysAgo else '' url = snowuser.reqencode(filters, table='incident', glide_system=glide) for record in snowuser.getrecords(url): record = snowuser.updatevalue(record, sourcetype='snow:incident') record['_raw'] = json.dumps(record) record = dictexpand(record) yield record
class EMGroupEntityMatchCommand(StreamingCommand): """ Match groups and entities based on group filter and entity dimensions ##Syntax .. code-block:: emgroupentitymatch selectedGroupIds="states,aws_instances" retainInput=false ##Description This custom search command will add 'group_id' and 'group_title' to all input entity records if they are members of a group - otherwise it will be omitted from the results unless retainInput is 'true'. Options: 1. selectedGroupIds -- indicates the selected groups that you want to match against the entities 2. retainInput -- indicates if the original input records should be attached to the output records if true, those records will have 'group_id' and 'group_title' set to 'N/A' for you to distinguish them. ##Example .. code-block:: | inputlookup em_entities | emgroupentitymatch selectedGroupIds="states,aws_instances" retainInput=false | stats count by group_title """ _group_records = None selected_group_ids = Option(doc='List of selected group ids, separated by comma.', name='selectedGroupIds', default=None, require=False, validate=validators.List()) retain_input_record = Option(doc='Boolean to indicate if user wants the input ' 'record to be added to the output without modification.', name='retainInput', default=False, require=False, validate=validators.Boolean()) def stream(self, records): """ Generator function that processes and yields event records to the Splunk stream pipeline. :param records: splunk event records :return: """ self._setup_group_records() self.logger.debug('EMGroupEntityMatchCommand: %s', self) # logs command line for record in records: if self.retain_input_record: record['group_id'] = 'N/A' record['group_title'] = 'N/A' yield record if len(self._group_records) > 0: for group_record in self._group_records: if self._match_group_entity(record, group_record.group_filter): record['group_id'] = group_record.group_id record['group_title'] = group_record.group_content.get('title') yield record else: yield record def _setup_group_records(self): """ Grabs the groups from KV Store and builds out the filter objects if they have yet to be built :return: None """ if self._group_records is None: collection = self.service.kvstore[STORE_GROUPS] group_data = collection.data.query() if self.selected_group_ids: selected_group_set = set(self.selected_group_ids) group_data = filter(lambda g: g['_key'] in selected_group_set, group_data) group_records = [] for group in group_data: filter_val = group.get('filter') d = {} if filter_val: for v in filter_val.split(','): dim_name, dim_val = v.strip().split('=') d.setdefault(dim_name, set()).add(dim_val) group_records.append(GroupRecord(group_id=group['_key'], group_filter=d, group_content=group)) self._group_records = group_records def _match_group_entity(self, record, filter_dict): """ Verify whether this record matches to group filters. Support wildcard in end of string :param record: :param filter_dict: :return: """ for dim_name, dim_val_set in filter_dict.iteritems(): record_vals = record.get('dimensions.%s' % dim_name) if not record_vals: return False if not isinstance(record_vals, list): record_vals = [record_vals] matched = False for rval in record_vals: # check if record value is one of the filter values if rval in dim_val_set: matched = True break # otherwise check if record value matches any of the fuzzy match values fuzzy_matches = filter(lambda v: v.endswith('*'), dim_val_set) if len(fuzzy_matches): matched = any(rval.startswith(v[:-1]) for v in fuzzy_matches) if not matched: return False return True
class MatchupCommandRex(StreamingCommand): """ SCP v2 """ opt_outputcols = Option(doc=''' **Syntax:** **outputcols=***<CSV list of fields>* **Description:** The field(s) that should be returned from the table. **Default:** <empty> returns all fields''', name='outputcols', require=False, default='', validate=validators.List()) opt_regex = Option(doc=''' **Syntax:** **regex=***<regex field in lookup>* **Description:** Names the field in the lookup that stores the regex. **Default:** none''', name='regex', require=True) opt_field = Option(doc=''' **Syntax:** **field=***<field to match>* **Description:** Names the field in the event that is matched to the regex. **Default:** none''', name='field', require=True) opt_lookup = Option(doc=''' **Syntax:** **lookup=***<lookup_name>* **Description:** name of a valid accessible lookup **Default:** none''', name='lookup', require=True) opt_where = Option(doc=''' **Syntax:** **where=***<valid where clause>* **Description:** where clause that can be used with inputlookup to restrict the rows returned and improve performance **Default:** none''', name='where', default=None, require=False) opt_sort = Option(doc=''' **Syntax:** **sort=***<string>* **Description:** SPL style sort string, ie '- order' **Default:** empty string''', name='sort', default=None, require=False) opt_undefined_str = Option(doc=''' **Syntax:** **undefined=***<string>* **Description:** string to use if the column doesn't exist in the output **Default:** undefined''', name='undefined', default='undefined', require=False) table = None def __init__(self): super(MatchupCommandRex, self).__init__() self.perf = Perf(self.logger) def get_where_clause(self): where = self.opt_where self.logger.info("option where is (%s)" % where) if where is not None: return " WHERE " + where return "" def get_sort_clause(self): sort = "" if self.opt_sort is not None: sort = " | sort " + self.opt_sort self.logger.info("option sort is (%s)" % sort) else: self.logger.info("option sort is empty") return sort def get_table_fast_and_raw(self): if self.table is None: import requests, json self.perf.start("get_table_raw") res = requests.post( "%s/services/search/jobs/export" % (self.metadata.searchinfo.splunkd_uri), verify=False, headers={ "Authorization": "Splunk %s" % self.metadata.searchinfo.session_key, "Accept-Encoding": "gzip,deflate", "User-Agent": "Python Requests", "Content-Type": "application/json; charset=UTF-8", "Accept": "application/json" }, data={ "exec_mode": "oneshot", "output_mode": "json", "search": "| inputlookup {} {} {}".format(self.opt_lookup, self.get_where_clause(), self.get_sort_clause()) }) table = MatchTableRex(self.opt_regex, self.logger) res.raise_for_status() for line in res.text.strip().split("\n"): table.add_row(json.loads(line)["result"]) self.logger.info("Line is {}".format(line)) self.perf.end("get_table_raw", "%d Rows" % table.len()) self.table = table return self.table def stream(self, records): """ Main entry point for the command :param records: An iterable stream of events from the command pipeline. :return: `None`. """ self.logger.info("================== Stream") sz = 0 self.perf.start("full stream method") for record in records: sz += 1 perf_str = "checking row {}".format(sz) tbl = self.get_table_fast_and_raw() self.perf.start(perf_str) match = tbl.get_match(self.opt_field, record) if match is not None: for col in self.opt_outputcols: try: record[col] = match[col] except KeyError: record[col] = self.opt_undefined_str else: self.logger.warn("No match for row {}".format(record)) record["debug"] = "nothing matched this record" self.perf.end(perf_str) yield record self.logger.info("Iterated over {} records".format(sz)) self.perf.end("full stream method", "{} events processed".format(sz))
class MACFormatCommand(StreamingCommand): """ Convert a given MAC address field to specified format. ##Syntax .. code-block:: | macformat input=field-list output=field-list format=[cisco|dash|ieee|none] ## Description Convert the fields in the `input` field list to the ones in the `output` list; Both lists are optional. The `input` list defaults to `macaddress`. The`output` list is filled with fields in the `input` list it the `output` list is shorter than the `input`. The `format` option is one of [cisco|dash|ieee|none]. The default is `none`. Raises a ValueError exception if the MAC address is invalid. """ format = Option(doc=''' **Syntax:** **format=**`[cisco|dash|ieee|none]` **Description:** Format of the output MAC address. Defaults to `none`.''', require=False, validate=validators.Set('cisco', 'dash', 'ieee', 'none')) inputs = Option(doc=''' **Syntax:** **inputs=***<field-list>* **Description:** A comma-delimited list of input fields to convert. Defaults to `macaddress`.''', require=False, validate=validators.List()) outputs = Option(doc=''' **Syntax:** **outputs=***<field-list>* **Description:** A comma-delimited list of fields for the results. Defaults to `inputs`.''', require=False, validate=validators.List()) def prepare(self): """ Prepare the options. :return: :const:`None` :rtype: NoneType """ self.toform = globals()['_' + (self.format or self.def_format)] inputs = self.inputs if inputs is None: self.inputs = inputs = self.def_inputs outputs = self.outputs if outputs is None: outputs = inputs elif len(outputs) < len(inputs): outputs += inputs[len(outputs):] self.outputs = outputs self.logger.debug( 'MACFormatCommand.prepare: inputs = %s, outputs = %s', self.inputs, outputs) def stream(self, records): toform = self.toform inputs = self.inputs outputs = self.outputs if outputs is None: outputs = inputs elif len(outputs) < len(inputs): outputs += inputs[len(outputs):] for record in records: self.logger.debug('MACFormatCommand: record = %s', record) for i in range(len(inputs)): mac = record.get(inputs[i]) if mac != None: try: record[outputs[i]] = toform(mac) except Exception as err: record[outputs[i]] = mac self.logger.error('(input=%s) %s', inputs[i], err.message) yield record def __init__(self): StreamingCommand.__init__(self) appdir = path.dirname(path.dirname(__file__)) defconfpath = path.join(appdir, "default", "app.conf") defconf = cli.readConfFile(defconfpath).get('macformat') or {} localconfpath = path.join(appdir, "local", "app.conf") localconf = (cli.readConfFile(localconfpath).get('macformat') or {}) if path.exists(localconfpath) else {} self.def_format = localconf.get('format') or defconf.get( 'format') or 'none' inputs = localconf.get('inputs') or defconf.get('inputs') self.def_inputs = re.split('[\s,]', inputs) if inputs else ['macaddress']
class LdapFetchCommand(StreamingCommand): """ Filters and augments events with information from Active Directory. This command follows a search or similar command in the pipeline so that you can feed it events: .. code-block:: text | ldapsearch domain=splunk.com search="(objectClass=groups)" | ldapfetch domain=splunk.com dn=memberOf attributes="cn,description" """ # region Command options attrs = Option( doc=''' Specifies a comma separated list of attributes to return as fields. **Default:** '*', specifying that all attributes should be returned as fields. ''', default=[ldap3.ALL_ATTRIBUTES], validate=validators.List()) debug = Option( doc=''' True, if the logging_level should be set to DEBUG; otherwise False. **Default:** The current value of logging_level. ''', default=False, validate=validators.Boolean()) decode = Option( doc=''' True, if Active Directory formatting rules should be applied to attribute types. **Default:** The value of decode as specified in the configuration stanza for domain. ''', default=True, validate=validators.Boolean()) dn = Option( doc=''' Specifies the name of the field holding the distinguished name to fetch. ''', default='distinguishedName') domain = Option( doc=''' Specifies the Active Directory domain to search. ''', default='default') # endregion # region Command implementation def stream(self, records): """ :param records: An iterable stream of events from the command pipeline. :return: `None`. """ configuration = app.Configuration(self, is_expanded=True) expanded_domain = app.ExpandedString(self.domain) search_scope = ldap3.SEARCH_SCOPE_BASE_OBJECT search_filter = '(objectClass=*)' try: with configuration.open_connection_pool(self.attrs) as connection_pool: attribute_names = connection_pool.attributes for record in records: dn = record.get(self.dn) if not dn: # got a falsey value yield record continue domain = expanded_domain.get_value(record) if domain is None: yield record continue connection = connection_pool.select(domain) if not connection: self.logger.warning('dn="%s": domain="%s" is not configured', self.dn, domain) yield record continue for search_base in dn if isinstance(dn, list) else (dn,): if search_base: try: connection.search(search_base, search_filter, search_scope, attributes=attribute_names) except ldap3.LDAPNoSuchObjectResult: self.logger.warning( 'dn="%s" domain="%s": distinguishedName="%s" does not exist', self.dn, domain, search_base) else: response = connection.response[0] attributes = app.get_attributes(self, connection.response[0]) if attributes: self._augment_record(record, response['dn'], attributes, attribute_names) yield record pass except ldap3.LDAPException as error: self.error_exit(error, app.get_ldap_error_message(error, configuration)) return def _augment_record(self, record, dn, attributes, attribute_names): """ :param record: :param dn: :param attributes: :return: """ record[self.dn] = dn for name in attribute_names: record[name] = attributes.get(name, '') return
class MatchupCommand(StreamingCommand): """ SCP v2 """ opt_outputcols = Option(doc=''' **Syntax:** **outputcols=***<CSV list of fields>* **Description:** The field(s) that should be returned from the table. **Default:** <empty> returns all fields''', name='outputcols', require=False, default='', validate=validators.List()) opt_inputcols = Option(doc=''' **Syntax:** **inputcols=***<CSV list of key fields to match>* **Description:** Each field in keys is matched with the lookup using the rules defined for matchup, this does not work like lookup, read the docs. **Default:** none''', name='inputcols', require=True, validate=validators.List()) opt_lookup = Option(doc=''' **Syntax:** **lookup=***<lookup_name>* **Description:** name of a valid accessible lookup **Default:** none''', name='lookup', require=True) opt_where = Option(doc=''' **Syntax:** **where=***<valid where clause>* **Description:** where clause that can be used with inputlookup to restrict the rows returned and improve performance **Default:** none''', name='where', default=None, require=False) opt_undefined_str = Option(doc=''' **Syntax:** **undefined=***<string>* **Description:** string to use if the column doesn't exist in the output **Default:** undefined''', name='undefined', default='undefined', require=False) opt_debug = Option(doc=''' **Syntax:** **debug=***<boolean>* **Description:** Wether the command should populate a error messege in a dubug field. **Default:** none''', name='debug', default='False', require=False) table = None def __init__(self): super(MatchupCommand, self).__init__() self.perf = Perf(self.logger) self.barocstringformat = BarocStringFormat() def normalize_debug(self, debug): return debug.lower() in ['true', '1', 't', 'y', 'yes'] def get_where_clause(self): where = self.opt_where self.logger.info("option where is (%s)" % where) if where is not None: return " WHERE " + where return "" def get_table_slow_with_splunklib(self, force=False): """ Here we'll add the code to fetch the lookup, once only ... Not pretending this isan awesome super scalable solution so for now we just read the whole file. Would be nice if we could use some static memory to store it as well, need FDSE/Eng input for that job though """ if self.table == None: self.perf.start("load table") self.logger.info("================== Live call starting") search = "| inputlookup {} {} ".format(self.opt_lookup, self.get_where_clause()) self.logger.info("Lookup search is : {}".format(search)) self.perf.start("rest call") self.logger.info("FARK : {}".format(self._metadata)) res = self.service.jobs.oneshot(search, **{"count": 0}) self.perf.end("rest call") # Get the results and display them using the ResultsReader reader = results.ResultsReader(res) self.table = MatchTable(self.opt_inputcols) self.perf.end("rest call", "part 2") rows = [] i = 0 for row in reader: #rows[i] = row #i = i+1 pass self.perf.end("load table", "table size: " + str(self.table.len())) return self.table def get_table_fast_and_raw(self): if self.table is None: import requests, json self.perf.start("get_table_raw") res = requests.post( "%s/services/search/jobs/export" % (self.metadata.searchinfo.splunkd_uri), verify=False, headers={ "Authorization": "Splunk %s" % self.metadata.searchinfo.session_key, "Accept-Encoding": "gzip,deflate", "User-Agent": "Python Requests", "Content-Type": "application/json; charset=UTF-8", "Accept": "application/json" }, data={ "exec_mode": "oneshot", "output_mode": "json", "search": "| inputlookup {} {} ".format(self.opt_lookup, self.get_where_clause()) }) table = MatchTable(self.opt_inputcols) res.raise_for_status() for line in res.text.strip().split("\n"): table.add_row(json.loads(line)["result"]) self.perf.end("get_table_raw", "%d Rows" % table.len()) self.table = table return self.table def map_to_output_colums(self, match, record, debug='False'): if self.opt_debug: debug = self.normalize_debug(self.opt_debug) if match is not None: for col in self.opt_outputcols: try: record[col] = self.barocstringformat.string_template( match[col], record) if debug: record['debug'] = self.barocstringformat.debug_msg except KeyError: record[col] = self.opt_undefined_str else: self.logger.warn("No match for row {}".format(record)) record["debug"] = "nothing matched this record" return record def stream(self, records): """ :param records: An iterable stream of events from the command pipeline. :return: `None`. """ self.logger.info("================== Stream") sz = 0 self.perf.start("full stream method") for record in records: sz += 1 perf_str = "checking row {}".format(sz) tbl = self.get_table_fast_and_raw() self.perf.start(perf_str) match = tbl.match_table(record) record = self.map_to_output_colums(match, record) self.perf.end(perf_str) yield record self.logger.info("Iterated over {} records".format(sz)) self.perf.end("full stream method", "{} events processed".format(sz))
class LdapFilterCommand(StreamingCommand): """ Filters and augments events with information from Active Directory. This command follows a search or similar command in the pipeline so that you can feed it events: .. code-block:: text eventtype=msad-user-logons | ldapfilter domain=$dest_nt_domain$ search="(objectClass=$src_user$)" attrs="telephoneNumber,displayName" """ # region Command options search = Option( doc=''' Specifies an RFC 2254 compliant search string. ''', require=True) domain = Option( doc=''' Specifies the Active Directory domain to search. ''', default='default') attrs = Option( doc=''' Specifies a comma separated list of attributes to return as fields. **Default:** '*', specifying that all attributes should be returned as fields. ''', default=[ldap3.ALL_ATTRIBUTES], validate=validators.List()) basedn = Option( doc=''' Specifies the starting point for the search. Default: The value of `basedn` as specified in the configuration stanza for `domain`. ''') scope = Option( doc=''' Specifies the scope of the search to be one of `base`, `one`, or `sub`. **Default:** sub. ''', default='sub', validate=validators.Map( base=ldap3.SEARCH_SCOPE_BASE_OBJECT, one=ldap3.SEARCH_SCOPE_SINGLE_LEVEL, sub=ldap3.SEARCH_SCOPE_WHOLE_SUBTREE )) decode = Option( doc=''' True, if Active Directory formatting rules should be applied to attribute types. **Default:** The value of decode as specified in the configuration stanza for domain. ''', default=True, validate=validators.Boolean()) limit = Option( doc=''' Specifies an upper bound on the number of matching entries returned by the search. **Default:** 0, specifying that there is no upper bound on the number of entries returned by the search. ''', default=0, validate=validators.Integer(minimum=0)) debug = Option( doc=''' True, if the logging_level should be set to DEBUG; otherwise False. **Default:** The current value of logging_level. ''', default=False, validate=validators.Boolean()) # endregion # region Command implementation def stream(self, records): """ :param records: An iterable stream of events from the command pipeline. :return: `None`. """ configuration = app.Configuration(self, is_expanded=True) expanded_domain = app.ExpandedString(self.domain) expanded_search_filter = app.ExpandedString(self.search, converter=app.escape_assertion_value) try: with configuration.open_connection_pool(self.attrs) as connection_pool: for record in records: domain = expanded_domain.get_value(record) if domain is None: continue search_filter = expanded_search_filter.get_value(record) if len(search_filter) == 0: continue connection = connection_pool.select(domain) if not connection: self.logger.warning('search="%s": domain="%s" is not configured', search_filter, domain) continue search_base = app.ExpandedString(self.basedn).get_value(record) # must be instantiated here entry_generator = connection.extend.standard.paged_search( search_base=search_base, search_filter=search_filter, search_scope=self.scope, attributes=connection_pool.attributes, paged_size=configuration.paged_size) for entry in entry_generator: attributes = app.get_attributes(self, entry) if not attributes: continue for name in connection_pool.attributes: record[name] = attributes.get(name, '') yield record pass except ldap3.LDAPException as error: self.error_exit(error, app.get_ldap_error_message(error, configuration)) return
class epbox(ReportingCommand): doc = ''' **Syntax:** search | epbox target=<target alias> outputfile=<output path/filename> outputformat=[json|raw|kv|csv|tsv|pipe] fields="field1, field2, field3" compress=[true|false] **Description** Export Splunk events to Box in any format. ''' # Define Parameters target = Option(doc=''' **Syntax:** **target=***<target alias>* **Description:** Reference to a target Box app within the configuration **Default:** The target configured as "Default" within the setup page (if any)''', require=False) outputfile = Option(doc=''' **Syntax:** **outputfile=***<file path/file name>* **Description:** The name of the file to be written to Box **Default:** The name of the user plus the timestamp and the output format, e.g. admin_1588000000.log json=.json, csv=.csv, tsv=.tsv, pipe=.log, kv=.log, raw=.log''', require=False) outputformat = Option(doc=''' **Syntax:** **outputformat=***[json|raw|kv|csv|tsv|pipe]* **Description:** The format written for the output events/search results **Default:** *csv*''', require=False) fields = Option(doc=''' **Syntax:** **fields=***"field1, field2, field3"* **Description:** Limit the fields to be written to the file **Default:** All (Unspecified)''', require=False, validate=validators.List()) compress = Option(doc=''' **Syntax:** **compress=***[true|false]* **Description:** Option to compress the output file into .gz format before uploading **Default:** The setting from the target configuration, or True if .gz is in the filename ''', require=False, validate=validators.Boolean()) # Validators found @ https://github.com/splunk/splunk-sdk-python/blob/master/splunklib/searchcommands/validators.py def __getitem__(self, key): return getattr(self, key) def map(self, events): for e in events: yield (e) #define main function def reduce(self, events): try: app_config = cli.getConfStanza('ep_general', 'settings') cmd_config = cli.getConfStanzas('ep_box') except BaseException as e: raise Exception("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(app_config["log_level"], 'export_everything.log', facility) except BaseException as e: raise Exception("Could not create logger: " + repr(e)) logger.info('Box Export search command initiated') logger.debug('search_ep_box command: %s', self) # logs command line # Enumerate proxy settings http_proxy = os.environ.get('HTTP_PROXY') https_proxy = os.environ.get('HTTPS_PROXY') proxy_exceptions = os.environ.get('NO_PROXY') if http_proxy is not None: logger.debug("HTTP proxy: %s" % http_proxy) if https_proxy is not None: logger.debug("HTTPS proxy: %s" % https_proxy) if proxy_exceptions is not None: logger.debug("Proxy Exceptions: %s" % proxy_exceptions) # Enumerate settings app = self._metadata.searchinfo.app user = self._metadata.searchinfo.username dispatch = self._metadata.searchinfo.dispatch_dir session_key = self._metadata.searchinfo.session_key if self.target is None and 'target=' in str(self): recover_parameters(self) # Replace all tokenized parameter strings replace_object_tokens(self) try: target_config = get_config_from_alias(session_key, cmd_config, self.target) if target_config is None: exit_error( logger, "Unable to find target configuration (%s)." % self.target, 100937) logger.debug("Target configuration: " + str(target_config)) except BaseException as e: exit_error(logger, "Error reading target server configuration: " + repr(e), 124812) file_extensions = { 'raw': '.log', 'kv': '.log', 'pipe': '.log', 'csv': '.csv', 'tsv': '.tsv', 'json': '.json' } if self.outputformat is None: self.outputformat = 'csv' # Create the default filename now = str(int(time.time())) default_filename = ('export_' + user + '___now__' + file_extensions[self.outputformat]).strip("'") # Split the output into folder and filename if self.outputfile is not None: folder_list = self.outputfile.split('/') if len(folder_list) == 1: # No folder specified, use the default use_default_folder = True filename = folder_list[0] elif folder_list[0] == '': # Length > 1, outputfile points to the root folder (leading /) use_default_folder = False else: # Length > 1 and outputfile points to a relative path (no leading /) use_default_folder = True if len(folder_list) > 1 and folder_list[-1] == '': # No filename provided, trailing / filename = default_filename folder_list.pop() elif len(folder_list) > 1 and len(folder_list[-1]) > 0: filename = folder_list[-1] folder_list.pop() else: use_default_folder = True filename = default_filename folder_list = [] if use_default_folder: if 'default_folder' in list(target_config.keys()): # Use the configured default folder folder_list = target_config['default_folder'].strip('/').split( '/') + folder_list else: # Use the root folder folder_list = [''] # Replace keywords from output filename and folder folder = replace_keywords('/'.join(folder_list)) filename = replace_keywords(filename) logger.debug("Folder = " + folder) logger.debug("Filename = " + filename) if self.compress is not None: logger.debug('Compression: %s', self.compress) else: try: self.compress = target_config.get('compress') except: self.compress = False # Use the random number to support running multiple outputs in a single search random_number = str(random.randint(10000, 100000)) staging_filename = 'export_everything_staging_' + random_number + '.txt' local_output_file = os.path.join(dispatch, staging_filename) if self.compress: local_output_file = local_output_file + '.gz' logger.debug("Staging file: %s" % local_output_file) # Append .gz to the output file if compress=true if not self.compress and len(filename) > 3: if filename[-3:] == '.gz': # We have a .gz extension when compression was not specified. Enable compression. self.compress = True elif self.compress and len(filename) > 3: if filename[-3:] != '.gz': filename = filename + '.gz' #if auth is not None: # Use the credential to connect to Box try: client = get_box_connection(target_config) except BaseException as e: exit_error(logger, "Could not connect to box: " + repr(e)) subfolders = folder.strip('/').split('/') if '' in subfolders: subfolders.remove('') logger.debug("Folders: %s" % str(subfolders)) # Prepend the list with the root element box_folder_object = client.root_folder().get() # Walk the folder path until we find the target directory for subfolder_name in subfolders: logger.debug("Looking for folder: %s" % subfolder_name) # Get the folder ID for the string specified from the list of child subfolders # folder object is from the previous iteration folder_contents = box_folder_object.get_items() folder_found = False for item in folder_contents: if item.type == 'folder': #logger.debug('{0} {1} is named "{2}"'.format(item.type.capitalize(), item.id, item.name)) if subfolder_name == item.name: logger.debug("Found a target folder ID: %s" % str(item.id)) box_folder_object = client.folder(folder_id=item.id) folder_found = True if not folder_found: # Create the required subfolder box_folder_object = box_folder_object.create_subfolder( subfolder_name) try: event_counter = 0 # Write the output file to disk in the dispatch folder logger.debug( "Writing events to dispatch file. file=\"%s\" format=%s compress=%s fields=%s", local_output_file, self.outputformat, self.compress, self.fields) for event in event_file.write_events_to_file( events, self.fields, local_output_file, self.outputformat, self.compress): yield event event_counter += 1 except BoxAPIException as be: exit_error(logger, be.message, 833928) except BaseException as e: exit_error(logger, "Error writing file to upload", 398372) try: new_file = box_folder_object.upload(local_output_file, file_name=filename) message = "Box Export Status: Success. File name: %s, File ID: %s" % ( new_file.name, new_file.id) eprint(message) logger.info(message) except BaseException as e: exit_error(logger, "Error uploading file to Box: " + repr(e), 109693)
class epsmb(ReportingCommand): doc = ''' **Syntax:** search | epsmb target=<target host alias> outputfile=<output path/filename> outputformat=[json|raw|kv|csv|tsv|pipe] fields="field1, field2, field3" compress=[true|false] **Description** Export Splunk events to an SMB server share in any format. ''' # Define Parameters target = Option(doc=''' **Syntax:** **target=***<target_host_alias>* **Description:** Reference to a target SMB share within the configuration **Default:** The target configured as "Default" within the setup page (if any)''', require=False) outputfile = Option(doc=''' **Syntax:** **outputfile=***<file path/file name>* **Description:** The name of the file to be written remotely **Default:** The name of the user plus the timestamp and the output format, e.g. admin_1588000000.log json=.json, csv=.csv, tsv=.tsv, pipe=.log, kv=.log, raw=.log''', require=False) outputformat = Option(doc=''' **Syntax:** **outputformat=***[json|raw|kv|csv|tsv|pipe]* **Description:** The format written for the output events/search results **Default:** *csv*''', require=False) fields = Option(doc=''' **Syntax:** **fields=***"field1, field2, field3"* **Description:** Limit the fields to be written to the file **Default:** All (Unspecified)''', require=False, validate=validators.List()) compress = Option(doc=''' **Syntax:** **compress=***[true|false]* **Description:** Option to compress the output file into .gz format before uploading **Default:** The setting from the target configuration, or True if .gz is in the filename ''', require=False, validate=validators.Boolean()) # Validators found @ https://github.com/splunk/splunk-sdk-python/blob/master/splunklib/searchcommands/validators.py def __getitem__(self, key): return getattr(self, key) def map(self, events): for e in events: yield (e) #define main function def reduce(self, events): try: app_config = cli.getConfStanza('ep_general', 'settings') cmd_config = cli.getConfStanzas('ep_smb') except BaseException as e: raise Exception("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(app_config["log_level"], 'export_everything.log', facility) except BaseException as e: raise Exception("Could not create logger: " + repr(e)) logger.info('SMB Export search command initiated') logger.debug('search_ep_smb command: %s', self) # logs command line # Enumerate proxy settings http_proxy = os.environ.get('HTTP_PROXY') https_proxy = os.environ.get('HTTPS_PROXY') proxy_exceptions = os.environ.get('NO_PROXY') if http_proxy is not None: logger.debug("HTTP proxy: %s" % http_proxy) if https_proxy is not None: logger.debug("HTTPS proxy: %s" % https_proxy) if proxy_exceptions is not None: logger.debug("Proxy Exceptions: %s" % proxy_exceptions) # Enumerate settings app = self._metadata.searchinfo.app user = self._metadata.searchinfo.username dispatch = self._metadata.searchinfo.dispatch_dir session_key = self._metadata.searchinfo.session_key if self.target is None and 'target=' in str(self): recover_parameters(self) # Replace all tokenized parameter strings replace_object_tokens(self) # Use the random number to support running multiple outputs in a single search random_number = str(random.randint(10000, 100000)) try: target_config = get_config_from_alias(session_key, cmd_config, self.target) if target_config is None: exit_error( logger, "Unable to find target configuration (%s)." % self.target, 100937) except BaseException as e: exit_error(logger, "Error reading target server configuration: " + repr(e), 124812) # Get the local client hostname client_name = socket.gethostname() # Delete any domain from the client hostname string if '.' in client_name: client_name = client_name[0:client_name.index('.')] # Check to see if we have credentials valid_settings = [] for l in list(target_config.keys()): if len(target_config[l]) > 0: valid_settings.append(l) if 'host' in valid_settings: # A target has been configured. Check for credentials. try: if 'credential_username' in valid_settings and 'credential_password' in valid_settings and 'share_name' in valid_settings: domain = target_config[ 'credential_realm'] if 'credential_realm' in list( target_config.keys()) else target_config['host'] try: # Try port 445 first conn = SMBConnection( target_config['credential_username'], target_config['credential_password'], client_name, target_config['host'], domain=domain, use_ntlm_v2=True, sign_options=SMBConnection.SIGN_WHEN_SUPPORTED, is_direct_tcp=True) connected = conn.connect(target_config['host'], 445, timeout=5) if target_config['share_name'] not in ( s.name for s in conn.listShares(timeout=10)): exit_error( logger, "Unable to find the specified share name on the server", 553952) ''' p445_error = repr(e445) try: # Try port 139 if that didn't work conn = SMBConnection(target_config['credential_username'], target_config['credential_password'], client_name, target_config['host'], domain=domain, use_ntlm_v2=True, sign_options = SMBConnection.SIGN_WHEN_SUPPORTED) connected = conn.connect(target_config['host'], 139, timeout=5) except BaseException as e139: p139_error = repr(e139) raise Exception("Errors connecting to host: \\nPort 139: %s\\nPort 445: %s" % (p139_error, p445_error)) conn = SMBConnection(target_config['credential_username'], target_config['credential_password'], client_name, target_config['host'], domain=domain, use_ntlm_v2=True, sign_options = SMBConnection.SIGN_WHEN_SUPPORTED) connected = conn.connect(target_config['host'], 139) shares = share_exists = False for i in range(len(shares)): if shares[i].name == target_config['share_name']: share_exists = True break ''' except BaseException as e: exit_error( logger, "Unable to setup SMB connection: " + repr(e), 921982) else: exit_error(logger, "Required settings not found", 101926) except BaseException as e: exit_error(logger, "Error reading the configuration: " + repr(e), 230494) else: exit_error(logger, "Could not find required configuration settings", 2823874) file_extensions = { 'raw': '.log', 'kv': '.log', 'pipe': '.log', 'csv': '.csv', 'tsv': '.tsv', 'json': '.json' } if self.outputformat is None: self.outputformat = 'csv' # Create the default filename default_filename = ('export_' + user + '___now__' + file_extensions[self.outputformat]).strip("'") folder, filename = event_file.parse_outputfile(self.outputfile, default_filename, target_config) if self.compress is not None: logger.debug('Compression: %s', self.compress) else: try: self.compress = target_config.get('compress') except: self.compress = False staging_filename = 'export_everything_staging_' + random_number + '.txt' local_output_file = os.path.join(dispatch, staging_filename) if self.compress: local_output_file = local_output_file + '.gz' # Append .gz to the output file if compress=true if not self.compress and len(filename) > 3: if filename[-3:] == '.gz': # We have a .gz extension when compression was not specified. Enable compression. self.compress = True elif self.compress and len(filename) > 3: if filename[-3:] != '.gz': filename = filename + '.gz' if conn is not None: # Use the credential to connect to the SFTP server try: # Check to see if the folder exists folder_attrs = conn.getAttributes(target_config['share_name'], folder, timeout=10) except BaseException: # Remote directory could not be loaded. It must not exist. Create it. # Create the folders required to store the file subfolders = ['/'] + folder.strip('/').split('/') if '' in subfolders: subfolders.remove('') logger.debug("Folders list for dir creation: %s" % str(subfolders)) current_folder = '' folder_depth = len(subfolders) - 1 for i, subfolder_name in enumerate(subfolders): current_folder = (current_folder + '/' + subfolder_name).replace('//', '/') logger.debug("Current folder = " + current_folder) try: conn.getAttributes(target_config['share_name'], current_folder, timeout=10) except: conn.createDirectory(target_config['share_name'], current_folder, timeout=10) try: folder_attrs = conn.getAttributes( target_config['share_name'], folder, timeout=10) except BaseException as e: exit_error( logger, "Could not load or create remote directory: " + repr(e), 377890) # This should always be true if folder_attrs is not None: if folder_attrs.isReadOnly or not folder_attrs.isDirectory: exit_error( logger, "Could not access the remote directory: " + repr(e), 184772) else: try: event_counter = 0 # Write the output file to disk in the dispatch folder logger.debug( "Writing events to dispatch file. file=\"%s\" format=%s compress=%s fields=%s", local_output_file, self.outputformat, self.compress, self.fields) for event in event_file.write_events_to_file( events, self.fields, local_output_file, self.outputformat, self.compress): yield event event_counter += 1 except BaseException as e: exit_error(logger, "Error writing file to upload: " + repr(e), 296733) # Write the file to the remote location try: with open(local_output_file, 'rb', buffering=0) as local_file: bytes_uploaded = conn.storeFile( target_config['share_name'], folder + '/' + filename, local_file) except BaseException as e: exit_error( logger, "Error uploading file to SMB server: " + repr(e), 109693) if bytes_uploaded > 0: message = "SMB Export Status: Success. File name: %s" % ( folder + '/' + filename) eprint(message) logger.info(message) else: exit_error(logger, "Zero bytes uploaded", 771293) else: exit_error(logger, "Could not connect to server.", 159528)
class epawss3(ReportingCommand): doc = ''' **Syntax:** search | epawss3 target=<target alias> bucket=<bucket> outputfile=<output path/filename> outputformat=[json|raw|kv|csv|tsv|pipe] **Description** Export Splunk events to AWS S3 (or compatible) over JSON or raw text. ''' #Define Parameters target = Option(doc=''' **Syntax:** **target=***<target alias>* **Description:** The name of the AWS target alias provided on the configuration dashboard **Default:** The target configured as "Default" within the AWS S3 Setup page (if any)''', require=False) bucket = Option(doc=''' **Syntax:** **bucket=***<bucket name>* **Description:** The name of the destination S3 bucket **Default:** The bucket name from the AWS S3 Setup page (if any)''', require=False) outputfile = Option(doc=''' **Syntax:** **outputfile=***<output path/filename>* **Description:** The path and filename to be written to the S3 bucket **Default:** The name of the user plus the timestamp and the output format, e.g. admin_1588000000.log json=.json, csv=.csv, tsv=.tsv, pipe=.log, kv=.log, raw=.log''', require=False) outputformat = Option(doc=''' **Syntax:** **outputformat=***[json|raw|kv|csv|tsv|pipe]* **Description:** The format written for the output events/search results **Default:** *csv*''', require=False) fields = Option(doc=''' **Syntax:** **fields=***"field1, field2, field3"* **Description:** Limit the fields to be written to the S3 file **Default:** All (Unspecified)''', require=False, validate=validators.List()) compress = Option(doc=''' **Syntax:** **compress=***[true|false]* **Description:** Option to compress the output file into .gz format before uploading **Default:** The setting from the target configuration, or True if .gz is in the filename ''', require=False, validate=validators.Boolean()) # Validators found @ https://github.com/splunk/splunk-sdk-python/blob/master/splunklib/searchcommands/validators.py def __getitem__(self, key): return getattr(self, key) def map(self, events): for e in events: yield (e) #define main function def reduce(self, events): try: app_config = cli.getConfStanza('ep_general', 'settings') cmd_config = cli.getConfStanzas('ep_aws_s3') except BaseException as e: raise Exception("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(app_config["log_level"], 'export_everything.log', facility) except BaseException as e: raise Exception("Could not create logger: " + repr(e)) logger.info('AWS S3 Export search command initiated') logger.debug("Configuration: " + str(cmd_config)) logger.debug('search_ep_awss3 command: %s', self) # logs command line # Enumerate settings app = self._metadata.searchinfo.app user = self._metadata.searchinfo.username dispatch = self._metadata.searchinfo.dispatch_dir session_key = self._metadata.searchinfo.session_key if self.target is None and 'target=' in str(self): recover_parameters(self) # Replace all tokenized parameter strings replace_object_tokens(self) # Build the configuration try: aws_config = get_config_from_alias(session_key, cmd_config, self.target) if aws_config is None: exit_error( logger, "Unable to find target configuration (%s)." % self.target, 100937) logger.debug("Target configuration: " + str(aws_config)) except BaseException as e: exit_error(logger, "Error reading target server configuration: " + repr(e), 124812) if self.bucket is None: if 'default_s3_bucket' in list(aws_config.keys()): t = aws_config['default_s3_bucket'] if t is not None and len(t) > 0: self.bucket = t else: exit_error(logger, "No bucket specified", 4) else: exit_error(logger, "No bucket specified", 5) file_extensions = { 'raw': '.log', 'kv': '.log', 'pipe': '.log', 'csv': '.csv', 'tsv': '.tsv', 'json': '.json' } if self.outputformat is None: self.outputformat = 'csv' if self.outputfile is None: # Boto is special. We need repr to give it the encoding it expects to match the hashing. self.outputfile = repr('export_' + user + '___now__' + file_extensions[self.outputformat]).strip( "'") # Replace keywords from output filename self.outputfile = replace_keywords(self.outputfile) if self.compress is not None: logger.debug('Compression: %s', self.compress) else: try: self.compress = str2bool(aws_config['compress']) except: self.compress = False # Use the random number to support running multiple outputs in a single search random_number = str(random.randint(10000, 100000)) staging_filename = 'export_everything_staging_' + random_number + '.txt' local_output_file = os.path.join(dispatch, staging_filename) # Append .gz to the output file if compress=true if not self.compress and len(self.outputfile) > 3: # We have a .gz extension when compression was not specified. Enable compression. if self.outputfile[-3:] == '.gz': self.compress = True elif self.compress and len(self.outputfile) > 3: if self.outputfile[-3:] != '.gz': self.outputfile = self.outputfile + '.gz' if self.compress: local_output_file = local_output_file + '.gz' logger.debug("Staging file: %s" % local_output_file) try: s3 = get_aws_connection(aws_config) except BaseException as e: exit_error(logger, "Could not connect to AWS: " + repr(e), 741423) event_counter = 0 # Write the output file to disk in the dispatch folder logger.debug( "Writing events to file %s in %s format. Compress=%s\n\tfields=%s", local_output_file, self.outputformat, self.compress, self.fields) for event in event_file.write_events_to_file(events, self.fields, local_output_file, self.outputformat, self.compress): yield event event_counter += 1 # Upload file to s3 try: with open(local_output_file, "rb") as f: s3.upload_fileobj(f, self.bucket, self.outputfile) s3 = None sts_client = None logger.info( "Successfully exported events to s3. app=%s count=%s bucket=%s file=%s user=%s" % (app, event_counter, self.bucket, self.outputfile, user)) os.remove(local_output_file) except s3.exceptions.NoSuchBucket as e: exit_error(logger, "Error: No such bucket", 123833) except BaseException as e: exit_error(logger, "Could not upload file to S3: " + repr(e), 9)
class LdapSearchCommand(GeneratingCommand): """ Retrieves results from the specified search in a configured domain and generates events. This command must be placed at the beginning of a search pipeline: .. code-block:: text | ldapsearch domain=splunk.com search="(objectCategory=User)" attrs="distinguishedName" """ search = Option(doc=''' Specifies an RFC 2254 compliant search string. ''', require=True) attrs = Option( doc= ''' Specifies a comma separated list of attributes to be returned as fields. **Default:** '*', specifying that all attributes should be returned as fields. ''', default=[ldap3.ALL_ATTRIBUTES], validate=validators.List()) basedn = Option(doc=''' Specifies the starting point for the search. Default: The value of basedn as specified in the configuration stanza for domain. ''') domain = Option( doc= ''' Specifies the LDAP or Active Directory domain directory to search. ''', default='default') scope = Option( doc=''' Specifies the scope of the search to be one of base, one, or sub. **Default:** sub. ''', default='sub', validate=validators.Map(base=ldap3.SEARCH_SCOPE_BASE_OBJECT, one=ldap3.SEARCH_SCOPE_SINGLE_LEVEL, sub=ldap3.SEARCH_SCOPE_WHOLE_SUBTREE)) debug = Option( doc= ''' True, if the logging_level should be set to DEBUG; otherwise False. **Default:** The current value of logging_level. ''', default=False, validate=validators.Boolean()) decode = Option( doc= ''' True, if Active Directory formatting rules should be applied to attribute types. **Default:** The value of decode as specified in the configuration stanza for domain. ''', default=True, validate=validators.Boolean()) limit = Option( doc= ''' Specifies an upper bound on the number of matching entries returned by the search. **Default:** 0, specifying that there is no upper bound on the number of entries returned by the search. ''', default=0, validate=validators.Integer(minimum=0)) def generate(self): """ :return: `None`. """ configuration = app.Configuration(self) try: with ldap3.Connection( configuration.server, read_only=True, raise_exceptions=True, user=configuration.credentials.username, password=configuration.credentials.password) as connection: attribute_names = app.get_normalized_attribute_names( self.attrs, connection, configuration) entry_generator = connection.extend.standard.paged_search( search_base=self.basedn, search_filter=self.search, search_scope=self.scope, attributes=self.attrs, paged_size=configuration.paged_size) encoder = JSONEncoder(ensure_ascii=False, separators=(',', ':')) time_stamp = time() serial_number = 0 for entry in entry_generator: attributes = app.get_attributes(self, entry) if attributes: dn = entry['dn'] yield LdapSearchCommand._record( serial_number, time_stamp, connection.server.host, dn, attributes, attribute_names, encoder) serial_number += 1 if self.limit and serial_number == self.limit: break pass pass except ldap3.LDAPException as error: self.error_exit(error, app.get_ldap_error_message(error, configuration)) return @staticmethod def _record(serial_number, time_stamp, host, dn, attributes, attribute_names, encoder): # Base-64 encode binary values (they're stored as str values--byte strings--not unicode values) for name, value in attributes.iteritems(): if isinstance(value, str): attributes[name] = b64encode(value) elif isinstance(value, list): for i in range(len(value)): if isinstance(value[i], str): value[i] = b64encode(value[i]) raw = encoder.encode(attributes) # Formulate record if serial_number > 0: attributes['_serial'] = serial_number attributes['_time'] = time_stamp attributes['_raw'] = raw attributes['host'] = host attributes['dn'] = dn return attributes record = OrderedDict( chain((('_serial', serial_number), ('_time', time_stamp), ('_raw', raw), ('host', host), ('dn', dn)), imap(lambda name: (name, attributes.get(name, '')), attribute_names))) return record