def handleEdit(self, confInfo): facility = config_file + '_edit' logger = setup_logger(app_config["log_level"], setup_log, facility) logger.debug("Setup edit handler started") config_id = self.callerArgs.id config = self.callerArgs.data logger.debug("Config: %s/%s" % (config_id, config)) new_config = {} for k, v in list(config.items()): try: if isinstance(v, list) and len(v) == 1: v = v[0] logger.debug(v) if k == 'stanza': logger.debug("Setting stanza to %s" % v) config_id = v else: if v is None: logger.debug('%s Setting %s to blank', facility, k) new_config[k] = '' else: new_config[k] = v except BaseException as e: logger.exception("Error parsing config value \"%s\": %s" % (v, repr(e))) logger.debug("%s Writing new config for %s: %s", facility, config_id, str(new_config)) try: # Write the config stanza self.writeConf(config_file, config_id, new_config) except BaseException as e: logger.critical("%s Error writing config: %s", facility, repr(e)) exit(1)
def delete_key_from_event(self, delete_event): try: cfg = cli.getConfStanza('kvstore_tools', 'settings') except BaseException as e: eprint("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(cfg["log_level"], 'kvstore_tools.log', facility) except BaseException as e: eprint("Could not create logger: " + repr(e)) print("Could not create logger: " + repr(e)) exit(1) url_tmpl_delete = '%(server_uri)s/servicesNS/%(owner)s/%(app)s/storage/collections/data/%(collection)s/%(id)s?output_mode=json' headers = { 'Authorization': 'Splunk %s' % self.session_key, 'Content-Type': 'application/json' } for key, value in list(delete_event.items()): delete_event[key] = value if key == '_key' and len(value) > 0: logger.debug("Found %s (%s) in event" % (key, value)) try: delete_url = url_tmpl_delete % dict( server_uri=self.splunkd_uri, owner='nobody', app=self.app, collection=self.collection, id=urllib.parse.quote(value, safe='')) logger.debug("Delete url: " + delete_url) try: response, response_code = request( 'DELETE', delete_url, '', headers) logger.debug('Server response: %s' % response) except BaseException as e: logger.error('ERROR Failed to delete key: %s', repr(e)) if response_code == 200: logger.debug("Successfully deleted " + key) delete_event['delete_status'] = "success" return delete_event else: logger.error("Error %d deleting %s: %s" % (response_code, key, response)) delete_event['delete_status'] = "error" return delete_event except BaseException as e: logger.error("Error deleting %s: %s" % (key, repr(e))) delete_event['delete_status'] = "error" return delete_event
def handleList(self, confInfo): facility = config_file + '_list' logger = setup_logger(app_config["log_level"], setup_log, facility) confDict = self.readConf(config_file) credentials = {} logger.info(config_file + " list handler started") try: session_key = self.getSessionKey() entity = en.getEntity('/server', 'settings', namespace='-', sessionKey=session_key, owner='-') splunkd_port = entity["mgmtHostPort"] service = client.connect(token=session_key, port=splunkd_port) # Get all credentials for this app storage_passwords = service.storage_passwords for credential in storage_passwords: if credential.access.app == app: credentials[credential._state.title] = { 'username': credential.content.get('username'), 'password': credential.content.get('clear_password'), 'realm': credential.content.get('realm') } except BaseException as e: logger.exception('Could not connect to service: %s' % e) raise (e) if None != confDict: for stanza, settings in list(confDict.items()): for k, v in list(settings.items()): if stanza != 'default': logger.debug("%s stanza: %s, key: %s, value: %s", facility, stanza, k, v) if k.lower( ) in password_options and v is not None and len( v) > 0 and not '$7$' in v: v = encrypt_new(splunk_secret, v) if 'credential' in k: if v in list(credentials.keys()): confInfo[stanza].append( k + '_username', credentials[v]['username']) confInfo[stanza].append( k + '_realm', credentials[v]['realm']) confInfo[stanza].append( k + '_password', credentials[v]['password']) confInfo[stanza].append(k, v)
def setup(self): facility = config_file + '_setup' logger = setup_logger(app_config["log_level"], setup_log, facility) logger.debug("Setup script started") try: if self.requestedAction == admin.ACTION_EDIT: # ACTION_EDIT == 4 for arg in options: self.supportedArgs.addOptArg(arg) except BaseException as e: logger.exception(e)
def handleList(self, confInfo): self.capabilityRead = 'read_kvst_config' try: cfg = cli.getConfStanza('kvstore_tools','settings') except BaseException as e: raise Exception("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(cfg["log_level"], 'kvstore_tools.log', facility) except BaseException as e: raise Exception("Could not create logger: " + repr(e)) logger.debug('KV Store Tools Settings handler started (List)') # Check for permissions to read the configuration session_key = self.getSessionKey() content = rest.simpleRequest('/services/authentication/current-context?output_mode=json', sessionKey=session_key, method='GET')[1] content = json.loads(content) current_user = content['entry'][0]['content']['username'] current_user_capabilities = content['entry'][0]['content']['capabilities'] if self.capabilityRead in current_user_capabilities: logger.debug("User %s is authorized" % current_user) confDict = self.readConf("kvstore_tools") if None != confDict: for stanza, settings in list(confDict.items()): for key, val in list(settings.items()): logger.debug("key: {0}, value: {1}".format(key, val)) if key in ['compression']: if str2bool(val): val = '1' else: val = '0' ''' if key in ['default_path'] and val in [None, '', 'unset']: val = os.path.join('$SPLUNK_HOME', 'etc', 'apps', 'kvstore_tools', 'backups') # Windows wildcard support (works with $ but this is more clear). if '\\' in val: val = val.replace('$SPLUNK_HOME', '%SPLUNK_HOME%') if key in ['backup_batch_size'] and val in [None, '']: val = '50000' if key in ['retention_days'] and val in [None, '']: val = '0' if key in ['retention_size'] and val in [None, '']: val = '0' ''' confInfo[stanza].append(key, val) else: raise Exception("User %s is unauthorized. Has the read_kvst_config capability been granted?" % current_user)
def handleList(self, confInfo): facility = config_file + '_list' logger = setup_logger(app_config["log_level"], setup_log, facility) logger.info("Setup list handler started") confDict = self.readConf(config_file) if None != confDict: for stanza, settings in list(confDict.items()): for k, v in list(settings.items()): logger.debug("%s stanza: %s, key: %s, value: %s", facility, stanza, k, v) # Set blank value for each setting if one does not exist if v is None: v = '' confInfo[stanza].append(k, v)
def handleRemove(self, confInfo): facility = config_file + '_delete' logger = setup_logger(app_config["log_level"], setup_log, facility) logger.debug(config_file + " delete handler started") config_id = self.callerArgs.id logger.debug("Config: %s" % config_id) try: en.deleteEntity('/configs/conf-' + config_file, self.callerArgs.id, namespace=self.appName, owner=self.userName, sessionKey=self.getSessionKey()) except BaseException as e: logger.exception(e)
def handleEdit(self, confInfo): facility = config_file + '_edit' logger = setup_logger(app_config["log_level"], setup_log, facility) logger.debug(config_file + " edit handler started") config_id = self.callerArgs.id config = self.callerArgs.data logger.debug("Config: %s/%s" % (config_id, config)) new_config = {} for k, v in list(config.items()): try: if isinstance(v, list) and len(v) == 1: v = v[0] # Dynamic stanza name - GUIDs only guid_pattern = r'^([0-9A-Fa-f]{8}[-][0-9A-Fa-f]{4}[-][0-9A-Fa-f]{4}[-][0-9A-Fa-f]{4}[-][0-9A-Fa-f]{12})$' if k == 'stanza' and re.match(guid_pattern, str(v)): config_id = v logger.debug("Setting stanza to %s" % v) else: if v is None: logger.debug('%s Setting %s to blank', facility, k) new_config[k] = '' else: #logger.debug('%s Setting %s to %s', facility, k, v) if k.lower() in password_options and not '$7$' in v: logger.debug( '%s Value has an unencrypted password. Encrypting.', facility) try: v = encrypt_new(splunk_secret, v) except BaseException as e: logger.error( "%s Error saving encrypted password for %s: %s", facility, v, repr(e)) continue new_config[k] = v except BaseException as e: logger.exception("Error parsing config value \"%s\": %s" % (v, repr(e))) logger.debug("%s Writing new config for %s: %s", facility, config_id, str(new_config)) try: # Write the config stanza self.writeConf(config_file, config_id, new_config) except BaseException as e: logger.exception("%s Error writing config: %s", facility, e)
def generate(self): try: cfg = cli.getConfStanza('kvstore_tools', 'settings') except BaseException as e: eprint("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(cfg["log_level"], 'kvstore_tools.log', facility) except BaseException as e: eprint("Could not create logger: " + repr(e)) exit(1) logger.info('Script started by %s' % self._metadata.searchinfo.username) session_key = self._metadata.searchinfo.session_key splunkd_uri = self._metadata.searchinfo.splunkd_uri # Check for permissions to run the command content = rest.simpleRequest( '/services/authentication/current-context?output_mode=json', sessionKey=session_key, method='GET')[1] content = json.loads(content) current_user = self._metadata.searchinfo.username current_user_capabilities = content['entry'][0]['content'][ 'capabilities'] if 'run_kvstore_restore' in current_user_capabilities or 'run_kvst_all' in current_user_capabilities: logger.debug("User %s is authorized." % current_user) else: logger.error( "User %s is unauthorized. Has the run_kvstore_restore capability been granted?" % current_user) yield ({ 'Error': 'User %s is unauthorized. Has the run_kvstore_restore capability been granted?' % current_user }) sys.exit(3) # Sanitize input if self.filename: logger.debug('Restore filename: %s' % self.filename) list_only = False else: self.filename = "*#*#*.json*" list_only = True if self.append: logger.debug('Appending to existing collection') else: self.append = False logger.debug('Append to existing collection: %s' % str(self.append)) backup_file_list = [] # Get the default path from the configuration default_path_dirlist = cfg.get('default_path').split('/') default_path = os.path.abspath( os.path.join(os.sep, *default_path_dirlist)) # Replace environment variables default_path = os.path.expandvars(default_path) default_path = default_path.replace('//', '/') if '*' in self.filename: # Expand the wildcard to include all matching files from the filesystem for name in glob.glob(self.filename): backup_file_list.append(name) if len(backup_file_list) == 0: self.filename = os.path.join(default_path, self.filename) for name in glob.glob(self.filename): backup_file_list.append(name) if len(backup_file_list) == 0: logger.critical("No matching files: %s" % self.filename) sys.exit(1) else: logger.debug('No wildcard string found in %s' % self.filename) if os.path.isfile(self.filename): backup_file_list.append(self.filename) elif os.path.isfile(os.path.join(default_path, self.filename)): backup_file_list.append( os.path.join(default_path, self.filename)) else: logger.critical("File does not exist: %s" % self.filename) sys.exit(1) deleted_collections = [] # f is now an array of filenames for name in backup_file_list: logger.debug('Parsing filename: %s' % name) try: # Isolate the filename from the path matches = re.search(r'(.*)(?:\/|\\)([^\/\\]+)', name) #path = matches.group(1) file_param = matches.group(2) name_split = file_param.split('#') except BaseException as e: logger.critical('Invalid filename: %s\n\t%s' % (name, repr(e))) yield ({'Error': 'Invalid filename: %s' % name}) sys.exit(1) # Open the file if it's a supported format if (name.endswith('.json') or name.endswith('.json.gz')) and len(name_split) == 3: # Extract the app name and collection name from the file name file_app = name_split[0] file_collection = name_split[1] if list_only: yield { 'filename': name, 'app': file_app, 'collection': file_collection, 'status': 'ready' } else: if not self.append: # Delete the collection contents using the KV Store REST API try: collection_id = file_app + "/" + file_collection # Make sure we aren't trying to delete the same collection twice if not collection_id in deleted_collections: kv.delete_collection(logger, splunkd_uri, session_key, file_app, file_collection) deleted_collections.append(collection_id) except BaseException as e: logger.critical(repr(e), exc_info=True) yield ({ 'Error': 'Failed to delete collection %s/%s: %s' % (file_app, file_collection, repr(e)) }) sys.exit(4) # Upload the collection to the KV Store REST API try: result, message, record_count = kv.upload_collection( logger, splunkd_uri, session_key, file_app, file_collection, name) yield ({ 'result': result, 'message': message, 'records': record_count }) except BaseException as e: logger.error("Error restoring collection: %s" % repr(e), exc_info=True) yield ({ 'result': 'error', 'message': 'Failed to delete collection: %s' % repr(e), 'records': 0 }) elif name.endswith('.tar.gz') or name.endswith('.tgz'): logger.info('Skipping filename (unsupported format): %s' % name) yield { '_time': time.time(), 'source': name, 'app': '', 'collection': '', 'records': 0, 'result': 'error' } continue else: # Skip this file logger.info( 'Skipping filename (does not meet naming convention): %s' % name) yield { '_time': time.time(), 'source': name, 'app': '', 'collection': '', 'records': 0, 'result': 'error' } continue
def reduce(self, events): try: app_config = cli.getConfStanza('ep_general', 'settings') cmd_config = cli.getConfStanzas('ep_box') except BaseException as e: raise Exception("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(app_config["log_level"], 'export_everything.log', facility) except BaseException as e: raise Exception("Could not create logger: " + repr(e)) logger.info('Box Export search command initiated') logger.debug('search_ep_box command: %s', self) # logs command line # Enumerate proxy settings http_proxy = os.environ.get('HTTP_PROXY') https_proxy = os.environ.get('HTTPS_PROXY') proxy_exceptions = os.environ.get('NO_PROXY') if http_proxy is not None: logger.debug("HTTP proxy: %s" % http_proxy) if https_proxy is not None: logger.debug("HTTPS proxy: %s" % https_proxy) if proxy_exceptions is not None: logger.debug("Proxy Exceptions: %s" % proxy_exceptions) # Enumerate settings app = self._metadata.searchinfo.app user = self._metadata.searchinfo.username dispatch = self._metadata.searchinfo.dispatch_dir session_key = self._metadata.searchinfo.session_key if self.target is None and 'target=' in str(self): recover_parameters(self) # Replace all tokenized parameter strings replace_object_tokens(self) try: target_config = get_config_from_alias(session_key, cmd_config, self.target) if target_config is None: exit_error( logger, "Unable to find target configuration (%s)." % self.target, 100937) logger.debug("Target configuration: " + str(target_config)) except BaseException as e: exit_error(logger, "Error reading target server configuration: " + repr(e), 124812) file_extensions = { 'raw': '.log', 'kv': '.log', 'pipe': '.log', 'csv': '.csv', 'tsv': '.tsv', 'json': '.json' } if self.outputformat is None: self.outputformat = 'csv' # Create the default filename now = str(int(time.time())) default_filename = ('export_' + user + '___now__' + file_extensions[self.outputformat]).strip("'") # Split the output into folder and filename if self.outputfile is not None: folder_list = self.outputfile.split('/') if len(folder_list) == 1: # No folder specified, use the default use_default_folder = True filename = folder_list[0] elif folder_list[0] == '': # Length > 1, outputfile points to the root folder (leading /) use_default_folder = False else: # Length > 1 and outputfile points to a relative path (no leading /) use_default_folder = True if len(folder_list) > 1 and folder_list[-1] == '': # No filename provided, trailing / filename = default_filename folder_list.pop() elif len(folder_list) > 1 and len(folder_list[-1]) > 0: filename = folder_list[-1] folder_list.pop() else: use_default_folder = True filename = default_filename folder_list = [] if use_default_folder: if 'default_folder' in list(target_config.keys()): # Use the configured default folder folder_list = target_config['default_folder'].strip('/').split( '/') + folder_list else: # Use the root folder folder_list = [''] # Replace keywords from output filename and folder folder = replace_keywords('/'.join(folder_list)) filename = replace_keywords(filename) logger.debug("Folder = " + folder) logger.debug("Filename = " + filename) if self.compress is not None: logger.debug('Compression: %s', self.compress) else: try: self.compress = target_config.get('compress') except: self.compress = False # Use the random number to support running multiple outputs in a single search random_number = str(random.randint(10000, 100000)) staging_filename = 'export_everything_staging_' + random_number + '.txt' local_output_file = os.path.join(dispatch, staging_filename) if self.compress: local_output_file = local_output_file + '.gz' logger.debug("Staging file: %s" % local_output_file) # Append .gz to the output file if compress=true if not self.compress and len(filename) > 3: if filename[-3:] == '.gz': # We have a .gz extension when compression was not specified. Enable compression. self.compress = True elif self.compress and len(filename) > 3: if filename[-3:] != '.gz': filename = filename + '.gz' #if auth is not None: # Use the credential to connect to Box try: client = get_box_connection(target_config) except BaseException as e: exit_error(logger, "Could not connect to box: " + repr(e)) subfolders = folder.strip('/').split('/') if '' in subfolders: subfolders.remove('') logger.debug("Folders: %s" % str(subfolders)) # Prepend the list with the root element box_folder_object = client.root_folder().get() # Walk the folder path until we find the target directory for subfolder_name in subfolders: logger.debug("Looking for folder: %s" % subfolder_name) # Get the folder ID for the string specified from the list of child subfolders # folder object is from the previous iteration folder_contents = box_folder_object.get_items() folder_found = False for item in folder_contents: if item.type == 'folder': #logger.debug('{0} {1} is named "{2}"'.format(item.type.capitalize(), item.id, item.name)) if subfolder_name == item.name: logger.debug("Found a target folder ID: %s" % str(item.id)) box_folder_object = client.folder(folder_id=item.id) folder_found = True if not folder_found: # Create the required subfolder box_folder_object = box_folder_object.create_subfolder( subfolder_name) try: event_counter = 0 # Write the output file to disk in the dispatch folder logger.debug( "Writing events to dispatch file. file=\"%s\" format=%s compress=%s fields=%s", local_output_file, self.outputformat, self.compress, self.fields) for event in event_file.write_events_to_file( events, self.fields, local_output_file, self.outputformat, self.compress): yield event event_counter += 1 except BoxAPIException as be: exit_error(logger, be.message, 833928) except BaseException as e: exit_error(logger, "Error writing file to upload", 398372) try: new_file = box_folder_object.upload(local_output_file, file_name=filename) message = "Box Export Status: Success. File name: %s, File ID: %s" % ( new_file.name, new_file.id) eprint(message) logger.info(message) except BaseException as e: exit_error(logger, "Error uploading file to Box: " + repr(e), 109693)
def stream(self, events): try: app_config = cli.getConfStanza('ep_general', 'settings') cmd_config = cli.getConfStanzas('ep_hec') except BaseException as e: raise Exception("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(app_config["log_level"], 'export_everything.log', facility) except BaseException as e: raise Exception("Could not create logger: " + repr(e)) logger.info('HEC Export search command initiated') logger.debug('search_ep_hec command: %s', self) # logs command line # Set defaults if self.host is None: self.host = "$host$" # Get the default values used for data originating from this machine inputs_host = cli.getConfStanza('inputs', 'splunktcp')["host"] if self.source is None: self.source = "$source$" if self.sourcetype is None: self.sourcetype = "$sourcetype$" if self.index is None: self.index = "$index$" # Enumerate proxy settings http_proxy = os.environ.get('HTTP_PROXY') https_proxy = os.environ.get('HTTPS_PROXY') proxy_exceptions = os.environ.get('NO_PROXY') if http_proxy is not None: logger.debug("HTTP proxy: %s" % http_proxy) if https_proxy is not None: logger.debug("HTTPS proxy: %s" % https_proxy) if proxy_exceptions is not None: logger.debug("Proxy Exceptions: %s" % proxy_exceptions) # Enumerate settings searchinfo = self._metadata.searchinfo app = searchinfo.app user = searchinfo.username session_key = self._metadata.searchinfo.session_key if self.target is None and 'target=' in str(self): recover_parameters(self) # Replace all tokenized parameter strings replace_object_tokens(self) try: target_config = get_config_from_alias(session_key, cmd_config, self.target) if target_config is None: exit_error( logger, "Unable to find target configuration (%s)." % self.target, 100937) logger.debug("Target configuration: " + str(target_config)) hec_token = target_config['token'] hec_host = target_config['host'] hec_port = target_config['port'] hec_ssl = str2bool(target_config['ssl']) except BaseException as e: exit_error(logger, "Error reading target server configuration: " + repr(e), 124812) if len(hec_host) == 0: exit_error(logger, "No host specified", 119371) # Create HEC object hec = http_event_collector(hec_token, hec_host, http_event_port=hec_port, http_event_server_ssl=hec_ssl) if port_is_open(hec_host, hec_port): logger.debug("Port connectivity check passed") if hec.check_connectivity(): # Special event key fields that can be specified/overridden in the alert action meta_keys = ['source', 'sourcetype', 'host', 'index'] event_count = 0 for event in events: # Get the fields list for the event event_keys = list(event.keys()) payload = {} payload_event_src = {} # Copy event to new event, so we can change it for f in event_keys: payload_event_src[f] = event[f] if '_time' in event_keys: payload.update({"time": payload_event_src['_time']}) del (payload_event_src['_time']) else: payload.update({"time": time.time()}) for k in meta_keys: # Loop through the metadata keys: host/source/sourcetype/index if getattr(self, k)[0] == getattr( self, k)[-1] and getattr(self, k)[-1] == "$": if k in event_keys: # If the key field is in the event and its output argument is set to a variable payload.update({ k: payload_event_src[getattr(self, k)[1:-1]] }) # Delete it from the payload event source so it's not included when we dump the rest of the fields later. del (payload_event_src[getattr(self, k)[1:-1]]) elif k == "host" and self.host == "$host$": # "host" field not found in event, but has the default value. Use the one from inputs.conf. payload.update({k: inputs_host}) else: # Plaintext entry payload.update({k: self[k]}) # Only send _raw (no other fields) if the _raw field was included in the search result. # (Don't include other fields/values) if '_raw' in list(payload_event_src.keys()): #logger.debug("Using _raw from search result") payload.update({"event": payload_event_src['_raw']}) else: payload.update({"event": payload_event_src}) event_count += 1 logger.debug("Payload = " + str(payload)) hec.batchEvent(payload) yield (event) hec.flushBatch() logger.info( "Successfully exported events. count=%s target=%s app=%s user=%s" % (event_count, hec_host, app, user)) else: # Connectivity check failed exit_error( logger, "HEC endpoint port open but connection test failed.", 104893) else: if str2bool(hec_ssl): protocol = 'https' else: protocol = 'http' exit_error( logger, "Unable to connect to the HEC endpoint: %s" % protocol + '://' + hec_host + ':' + hec_port, 100253)
def handleEdit(self, confInfo): self.capabilityWrite = 'write_kvst_config' try: cfg = cli.getConfStanza('kvstore_tools','settings') except BaseException as e: raise Exception("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(cfg["log_level"], 'kvstore_tools.log', facility) except BaseException as e: raise Exception("Could not create logger: " + repr(e)) logger.debug('KV Store Tools Settings handler started (Edit)') # Check for permissions to read the configuration session_key = self.getSessionKey() content = rest.simpleRequest('/services/authentication/current-context?output_mode=json', sessionKey=session_key, method='GET')[1] content = json.loads(content) current_user = content['entry'][0]['content']['username'] current_user_capabilities = content['entry'][0]['content']['capabilities'] if self.capabilityWrite in current_user_capabilities: logger.debug("User %s is authorized" % current_user) # Read the splunk.secret file with open(os.path.join(os.getenv('SPLUNK_HOME'), 'etc', 'auth', 'splunk.secret'), 'r') as ssfh: splunk_secret = ssfh.readline() config = self.callerArgs.data new_config = {} for k, v in list(config.items()): if isinstance(v, list) and len(v) == 1: v = v[0] if v is None: logger.debug('Setting %s to blank' % k) new_config[k] = '' else: logger.debug('Setting %s to %s' % (k, v)) if k[:10] == 'credential' and not '$7$' in v: logger.debug('Value has an unencrypted password. Encrypting.') # Split the value into alias/username/password hostname, username, password = v.split(':') try: v = hostname + ":" + username + ":" + encrypt_new(splunk_secret, password) except BaseException as e: logger.error("Error saving encrypted password for %s: %s" % (hostname, repr(e))) continue logger.debug('Encrypted') new_config[k] = v logger.debug('applied to configuration dict') try: if 'compression' in list(new_config.keys()): if str2bool(config['compression'][0]): new_config['compression'][0] = '1' else: new_config['compression'][0] = '0' if 'default_path' in list(new_config.keys()): if config['default_path'][0] in [None, '']: new_config['default_path'][0] = None if 'backup_batch_size' in list(new_config.keys()): if config['backup_batch_size'][0] in [None, '']: new_config['backup_batch_size'][0] = None logger.debug("Writing configuration") except BaseException as e: logger.critical("Error parsing configuration: %s" % repr(e)) # Write the config stanza self.writeConf('kvstore_tools', 'settings', new_config) else: raise Exception("User %s is unauthorized. Has the write_kvst_config capability been granted?" % current_user)
def generate(self): try: cfg = cli.getConfStanza('kvstore_tools', 'settings') except BaseException as e: eprint("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(cfg["log_level"], 'kvstore_tools.log', facility) except BaseException as e: eprint("Could not create logger: " + repr(e)) print("Could not create logger: " + repr(e)) exit(1) logger.info('Script started by %s' % self._metadata.searchinfo.username) session_key = self._metadata.searchinfo.session_key splunkd_uri = self._metadata.searchinfo.splunkd_uri if self.app: logger.debug('App: %s' % self.app) else: self.app = self._metadata.searchinfo.app if self.collection: logger.debug('Collection: %s' % self.collection) else: logger.critical("No collection specified. Exiting.") print("Error: No collection specified.") exit(1) if self.key: logger.debug('Key ID: %s' % self.collection) else: logger.critical("No key value specified. Exiting.") print("Error: No key value specified.") exit(1) headers = { 'Authorization': 'Splunk %s' % session_key, 'Content-Type': 'application/json' } #url_tmpl_app = '%(server_uri)s/servicesNS/%(owner)s/%(app)s/storage/collections/config?output_mode=json&count=0' # Enumerate all apps app_list = kv.get_server_apps(splunkd_uri, session_key, self.app) collection_list = kv.get_app_collections(splunkd_uri, session_key, self.collection, self.app, app_list, True) logger.debug('Collections present: %s', str(collection_list)) try: # Create an object for the collection collection_present = False for c in collection_list: # Extract the app and collection name from the array # c[0] = app, c[1] = collection name collection_app = c[0] collection_name = c[1] if (collection_name == self.collection): if self.app is None or self.app == collection_app: self.app = collection_app collection_present = True elif self.app != collection_app: pass logger.debug("Collection found: {0} in app {1}".format( self.collection, self.app)) if not collection_present: logger.critical( "KVStore collection %s not found within app %s" % (self.collection, self.app)) exit(1) except BaseException as e: logger.critical('Error enumerating collections: ' + str(e)) exit(1) url_tmpl_delete = '%(server_uri)s/servicesNS/%(owner)s/%(app)s/storage/collections/data/%(collection)s/%(id)s?output_mode=json' try: delete_url = url_tmpl_delete % dict(server_uri=splunkd_uri, owner='nobody', app=self.app, collection=self.collection, id=urllib.parse.quote(self.key, safe='')) logger.debug("Delete url: " + delete_url) try: response, response_code = request('DELETE', delete_url, '', headers) logger.debug('Server response: %s', response) except BaseException as e: logger.error( 'Failed to delete key %s from collection %s/%s: %s' % (self.key, self.app, self.collection, repr(e))) if response_code == 200: logger.debug( "Successfully deleted key %s from collection %s/%s" % (self.key, self.app, self.collection)) result = "success" else: logger.error( "Error deleting key %s from collection %s/%s: %s" % (self.key, self.app, self.collection, response)) result = "error" except BaseException as e: logger.error("Error deleting key %s from collection %s/%s: %s" % (self.key, self.app, self.collection, repr(e))) result = "error" # Entry deleted yield { '_time': time.time(), 'app': self.app, 'collection': self.collection, 'key': self.key, 'result': result }
def reduce(self, events): try: app_config = cli.getConfStanza('ep_general', 'settings') cmd_config = cli.getConfStanzas('ep_smb') except BaseException as e: raise Exception("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(app_config["log_level"], 'export_everything.log', facility) except BaseException as e: raise Exception("Could not create logger: " + repr(e)) logger.info('SMB Export search command initiated') logger.debug('search_ep_smb command: %s', self) # logs command line # Enumerate proxy settings http_proxy = os.environ.get('HTTP_PROXY') https_proxy = os.environ.get('HTTPS_PROXY') proxy_exceptions = os.environ.get('NO_PROXY') if http_proxy is not None: logger.debug("HTTP proxy: %s" % http_proxy) if https_proxy is not None: logger.debug("HTTPS proxy: %s" % https_proxy) if proxy_exceptions is not None: logger.debug("Proxy Exceptions: %s" % proxy_exceptions) # Enumerate settings app = self._metadata.searchinfo.app user = self._metadata.searchinfo.username dispatch = self._metadata.searchinfo.dispatch_dir session_key = self._metadata.searchinfo.session_key if self.target is None and 'target=' in str(self): recover_parameters(self) # Replace all tokenized parameter strings replace_object_tokens(self) # Use the random number to support running multiple outputs in a single search random_number = str(random.randint(10000, 100000)) try: target_config = get_config_from_alias(session_key, cmd_config, self.target) if target_config is None: exit_error( logger, "Unable to find target configuration (%s)." % self.target, 100937) except BaseException as e: exit_error(logger, "Error reading target server configuration: " + repr(e), 124812) # Get the local client hostname client_name = socket.gethostname() # Delete any domain from the client hostname string if '.' in client_name: client_name = client_name[0:client_name.index('.')] # Check to see if we have credentials valid_settings = [] for l in list(target_config.keys()): if len(target_config[l]) > 0: valid_settings.append(l) if 'host' in valid_settings: # A target has been configured. Check for credentials. try: if 'credential_username' in valid_settings and 'credential_password' in valid_settings and 'share_name' in valid_settings: domain = target_config[ 'credential_realm'] if 'credential_realm' in list( target_config.keys()) else target_config['host'] try: # Try port 445 first conn = SMBConnection( target_config['credential_username'], target_config['credential_password'], client_name, target_config['host'], domain=domain, use_ntlm_v2=True, sign_options=SMBConnection.SIGN_WHEN_SUPPORTED, is_direct_tcp=True) connected = conn.connect(target_config['host'], 445, timeout=5) if target_config['share_name'] not in ( s.name for s in conn.listShares(timeout=10)): exit_error( logger, "Unable to find the specified share name on the server", 553952) ''' p445_error = repr(e445) try: # Try port 139 if that didn't work conn = SMBConnection(target_config['credential_username'], target_config['credential_password'], client_name, target_config['host'], domain=domain, use_ntlm_v2=True, sign_options = SMBConnection.SIGN_WHEN_SUPPORTED) connected = conn.connect(target_config['host'], 139, timeout=5) except BaseException as e139: p139_error = repr(e139) raise Exception("Errors connecting to host: \\nPort 139: %s\\nPort 445: %s" % (p139_error, p445_error)) conn = SMBConnection(target_config['credential_username'], target_config['credential_password'], client_name, target_config['host'], domain=domain, use_ntlm_v2=True, sign_options = SMBConnection.SIGN_WHEN_SUPPORTED) connected = conn.connect(target_config['host'], 139) shares = share_exists = False for i in range(len(shares)): if shares[i].name == target_config['share_name']: share_exists = True break ''' except BaseException as e: exit_error( logger, "Unable to setup SMB connection: " + repr(e), 921982) else: exit_error(logger, "Required settings not found", 101926) except BaseException as e: exit_error(logger, "Error reading the configuration: " + repr(e), 230494) else: exit_error(logger, "Could not find required configuration settings", 2823874) file_extensions = { 'raw': '.log', 'kv': '.log', 'pipe': '.log', 'csv': '.csv', 'tsv': '.tsv', 'json': '.json' } if self.outputformat is None: self.outputformat = 'csv' # Create the default filename default_filename = ('export_' + user + '___now__' + file_extensions[self.outputformat]).strip("'") folder, filename = event_file.parse_outputfile(self.outputfile, default_filename, target_config) if self.compress is not None: logger.debug('Compression: %s', self.compress) else: try: self.compress = target_config.get('compress') except: self.compress = False staging_filename = 'export_everything_staging_' + random_number + '.txt' local_output_file = os.path.join(dispatch, staging_filename) if self.compress: local_output_file = local_output_file + '.gz' # Append .gz to the output file if compress=true if not self.compress and len(filename) > 3: if filename[-3:] == '.gz': # We have a .gz extension when compression was not specified. Enable compression. self.compress = True elif self.compress and len(filename) > 3: if filename[-3:] != '.gz': filename = filename + '.gz' if conn is not None: # Use the credential to connect to the SFTP server try: # Check to see if the folder exists folder_attrs = conn.getAttributes(target_config['share_name'], folder, timeout=10) except BaseException: # Remote directory could not be loaded. It must not exist. Create it. # Create the folders required to store the file subfolders = ['/'] + folder.strip('/').split('/') if '' in subfolders: subfolders.remove('') logger.debug("Folders list for dir creation: %s" % str(subfolders)) current_folder = '' folder_depth = len(subfolders) - 1 for i, subfolder_name in enumerate(subfolders): current_folder = (current_folder + '/' + subfolder_name).replace('//', '/') logger.debug("Current folder = " + current_folder) try: conn.getAttributes(target_config['share_name'], current_folder, timeout=10) except: conn.createDirectory(target_config['share_name'], current_folder, timeout=10) try: folder_attrs = conn.getAttributes( target_config['share_name'], folder, timeout=10) except BaseException as e: exit_error( logger, "Could not load or create remote directory: " + repr(e), 377890) # This should always be true if folder_attrs is not None: if folder_attrs.isReadOnly or not folder_attrs.isDirectory: exit_error( logger, "Could not access the remote directory: " + repr(e), 184772) else: try: event_counter = 0 # Write the output file to disk in the dispatch folder logger.debug( "Writing events to dispatch file. file=\"%s\" format=%s compress=%s fields=%s", local_output_file, self.outputformat, self.compress, self.fields) for event in event_file.write_events_to_file( events, self.fields, local_output_file, self.outputformat, self.compress): yield event event_counter += 1 except BaseException as e: exit_error(logger, "Error writing file to upload: " + repr(e), 296733) # Write the file to the remote location try: with open(local_output_file, 'rb', buffering=0) as local_file: bytes_uploaded = conn.storeFile( target_config['share_name'], folder + '/' + filename, local_file) except BaseException as e: exit_error( logger, "Error uploading file to SMB server: " + repr(e), 109693) if bytes_uploaded > 0: message = "SMB Export Status: Success. File name: %s" % ( folder + '/' + filename) eprint(message) logger.info(message) else: exit_error(logger, "Zero bytes uploaded", 771293) else: exit_error(logger, "Could not connect to server.", 159528)
# pyright: reportMissingImports=false from splunk.clilib import cli_common as cli import splunk.entity as en import splunklib.client as client from splunk.persistconn.application import PersistentServerConnectionApplication from splunk.appserver.mrsparkle.lib.util import make_splunkhome_path import splunk.rest from deductiv_helpers import setup_logger from ep_helpers import get_config_from_alias, get_aws_s3_directory, get_box_directory, get_sftp_directory, get_smb_directory config = cli.getConfStanza('ep_general', 'settings') # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] logger = setup_logger(config["log_level"], 'export_everything.log', facility) temp_dir = os.path.join(os.environ.get('SPLUNK_HOME'), 'etc', 'users', 'splunk-system-user', '.eptemp') os.makedirs(temp_dir, exist_ok=True) os.chdir(temp_dir) app = 'export_everything' def return_error(error_text): error_text = re.sub(r'Exception\(|\\|\'|"', '', error_text) error_text = re.sub(r'\(+', '(', error_text) error_text = re.sub(r'\)+', ')', error_text) return {'error': error_text, 'payload': error_text, 'status': 500}
def generate(self): try: cfg = cli.getConfStanza('kvstore_tools', 'settings') except BaseException as e: eprint("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(cfg["log_level"], 'kvstore_tools.log', facility) except BaseException as e: eprint("Could not create logger: " + repr(e)) print("Could not create logger: " + repr(e)) exit(1) logger.info('Script started by %s' % self._metadata.searchinfo.username) batch_size = int(cfg.get('backup_batch_size')) logger.debug("Batch size: %d rows" % batch_size) local_session_key = self._metadata.searchinfo.session_key splunkd_uri = self._metadata.searchinfo.splunkd_uri # Check for permissions to run the command content = rest.simpleRequest( '/services/authentication/current-context?output_mode=json', sessionKey=local_session_key, method='GET')[1] content = json.loads(content) current_user = self._metadata.searchinfo.username current_user_capabilities = content['entry'][0]['content'][ 'capabilities'] if 'run_kvstore_pull' in current_user_capabilities or 'run_kvst_all' in current_user_capabilities: logger.debug("User %s is authorized." % current_user) else: logger.error( "User %s is unauthorized. Has the run_kvstore_pull capability been granted?" % current_user) yield ({ 'Error': 'User %s is unauthorized. Has the run_kvstore_pull capability been granted?' % current_user }) sys.exit(3) # Sanitize input if self.app: logger.debug('App Context: %s' % self.app) else: self.app = None if self.collection: logger.debug('Collection: %s' % self.collection) else: self.collection = None if self.global_scope: logger.debug('Global Scope: %s' % self.global_scope) else: self.global_scope = False if self.append: logger.debug('Appending to existing collection') else: self.append = False logger.debug('Append to existing collection: %s' % str(self.append)) if self.targetport: logger.debug('Port for remote connect: %s' % self.targetport) else: self.targetport = '8089' # Get credentials try: # Use the credential where the realm matches the target hostname # Otherwise, use the last entry in the list credentials = kv.parse_custom_credentials(logger, cfg) try: credential = credentials[self.target] except: try: hostname = self.target.split('.')[0] credential = credentials[hostname] except: logger.critical("Could not get password for %s: %s" % (self.target, repr(e))) print("Could not get password for %s: %s" % (self.target, repr(e))) exit(1593) remote_user = credential['username'] remote_password = credential['password'] except BaseException as e: logger.critical( 'Failed to get credentials for remote Splunk instance: %s' % repr(e), exc_info=True) yield ({ 'Error': 'Failed to get credentials for remote Splunk instance: %s' % repr(e) }) exit(7372) # Login to the remote host and get the session key try: remote_host = self.target remote_port = self.targetport remote_uri = 'https://%s:%s' % (self.target, self.targetport) remote_service = client.connect(host=remote_host, port=remote_port, username=remote_user, password=remote_password) remote_service.login() remote_session_key = remote_service.token.replace('Splunk ', '') logger.debug('Remote Session_key: %s' % remote_session_key) except (urllib.error.HTTPError, BaseException) as e: logger.exception('Failed to login on remote Splunk instance: %s' % repr(e)) yield ({ 'Error': 'Failed to login on remote Splunk instance: %s' % repr(e) }) sys.exit(4424) # Get the list of remote apps and collections remote_app_list = kv.get_server_apps(remote_uri, remote_session_key, self.app) remote_collection_list = kv.get_app_collections( remote_uri, remote_session_key, self.collection, self.app, remote_app_list, self.global_scope) logger.debug('Collections to pull: %s' % str(remote_collection_list)) for remote_collection in remote_collection_list: # Extract the app and collection name from the array collection_app = remote_collection[0] collection_name = remote_collection[1] try: yield (kv.copy_collection(logger, remote_session_key, remote_uri, local_session_key, splunkd_uri, collection_app, collection_name, self.append)) except BaseException as e: logger.critical( 'Failed to copy collections from %s to local KV store: %s' % (self.target, repr(e)), exc_info=True) yield ({ 'Error': 'Failed to copy collections from %s to local KV store: %s' % (self.target, repr(e)) }) sys.exit(11)
def stream(self, events): try: cfg = cli.getConfStanza('kvstore_tools', 'settings') except BaseException as e: eprint("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(cfg["log_level"], 'kvstore_tools.log', facility) except BaseException as e: eprint("Could not create logger: " + repr(e)) print("Could not create logger: " + repr(e)) exit(1) logger.info('Script started by %s' % self._metadata.searchinfo.username) if self.app: logger.debug('App: %s' % self.app) else: self.app = self._metadata.searchinfo.app if self.collection: logger.debug('Collection: %s' % self.collection) else: logger.critical("No collection specified. Exiting.") print("Error: No collection specified.") exit(1) self.session_key = self._metadata.searchinfo.session_key self.splunkd_uri = self._metadata.searchinfo.splunkd_uri # Enumerate all app_list app_list = kv.get_server_apps(self.splunkd_uri, self.session_key, self.app) collection_list = kv.get_app_collections(self.splunkd_uri, self.session_key, self.collection, self.app, app_list, True) logger.debug('Collections present: %s', str(collection_list)) try: # Create an object for the collection collection_present = False for c in collection_list: # Extract the app and collection name from the array # c[0] = app, c[1] = collection name collection_app = c[0] collection_name = c[1] if (collection_name == self.collection): if self.app is None or self.app == collection_app: self.app = collection_app collection_present = True elif self.app != collection_app: pass logger.debug("Collection {0} found in app {1}".format( self.collection, self.app)) if not collection_present: logger.critical("KVStore collection %s/%s not found" % (self.app, self.collection)) exit(1) except BaseException as e: logger.critical('Error enumerating collections: %s' % repr(e)) exit(1) # Make a Pool of workers pool = ThreadPool(4) try: results = pool.map(self.delete_key_from_event, events) except BaseException as e: logger.error("%s" % repr(e), exc_info=True) results = {} for result in results: yield result
def generate(self): try: cfg = cli.getConfStanza('kvstore_tools','settings') except BaseException as e: eprint("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(cfg["log_level"], 'kvstore_tools.log', facility) except BaseException as e: eprint("Could not create logger: " + repr(e)) exit(1) logger.info('Script started by %s' % self._metadata.searchinfo.username) batch_size = int(cfg.get('backup_batch_size')) logger.debug("Batch size: %d rows" % batch_size) session_key = self._metadata.searchinfo.session_key splunkd_uri = self._metadata.searchinfo.splunkd_uri # Check for permissions to run the command content = rest.simpleRequest('/services/authentication/current-context?output_mode=json', sessionKey=session_key, method='GET')[1] content = json.loads(content) current_user = self._metadata.searchinfo.username current_user_capabilities = content['entry'][0]['content']['capabilities'] if 'run_kvstore_backup' in current_user_capabilities or 'run_kvst_all' in current_user_capabilities: logger.debug("User %s is authorized." % current_user) else: logger.error("User %s is unauthorized. Has the run_kvstore_backup capability been granted?" % current_user) yield({'Error': 'User %s is unauthorized. Has the run_kvstore_backup capability been granted?' % current_user }) sys.exit(3) # Sanitize input if self.app: logger.debug('App Context: %s' % self.app) else: self.app = None if self.path: pass else: # Get path from configuration try: # Break path out and re-join it so it's OS independent default_path = cfg.get('default_path').split('/') self.path = os.path.abspath(os.path.join(os.sep, *default_path)) except: logger.critical("Unable to get backup path") yield({'Error': "Path not provided in search arguments and default path is not set."}) sys.exit(1) # Replace environment variables self.path = os.path.expandvars(self.path) self.path = self.path.replace('//', '/') logger.debug('Backup path: %s' % self.path) if not os.path.isdir(self.path): logger.critical("Path does not exist: {0}".format(self.path)) yield({'Error': "Path does not exist: {0}".format(self.path)}) sys.exit(1) if self.collection: logger.debug('Collection: %s' % self.collection) else: self.collection = None if self.global_scope: logger.debug('Global Scope: %s' % self.global_scope) else: self.global_scope = False if self.compression or self.compression == False: logger.debug('Compression: %s' % self.compression) else: try: self.compression = cfg.get('compression') except: self.compression = False app_list = kv.get_server_apps(splunkd_uri, session_key, self.app) logger.debug("Apps list: %s" % str(app_list)) collection_list = kv.get_app_collections(splunkd_uri, session_key, self.collection, self.app, app_list, self.global_scope) logger.info('Collections to backup: %s', str(collection_list)) for collection in collection_list: # Extract the app and collection name from the array entry_app = collection[0] collection_name = collection[1] ts = time.time() st = datetime.fromtimestamp(ts).strftime('%Y%m%d_%H%M%S') #maxrows = int(limits_cfg.get('max_rows_per_query')) # Set the filename and location for the output (expanding environment variables) output_filename = entry_app + "#" + collection_name + "#" + st + ".json" if self.compression: output_filename = output_filename + '.gz' output_file = os.path.join(self.path, output_filename) # Download the collection to a local file result, message, total_record_count = kv.download_collection(logger, splunkd_uri, session_key, entry_app, collection_name, output_file, self.compression) logger.debug("Retrieved {0} records from {1}".format(total_record_count, collection_name)) yield {'_time': time.time(), 'app': entry_app, 'collection': collection_name, 'result': result, 'records': total_record_count, 'message': message, 'file': output_file } # Execute retention routine max_age = 0 max_age = int(cfg.get('retention_days')) max_size = 0 max_size = int(cfg.get('retention_size')) * 1024 * 1024 if max_size > 0 or max_age > 0: # Check the size of all *.json and *.json.gz files in the directory #dir = self.path pattern = os.path.join(self.path, "*#*#*.json*") # Get a listing of the files and check the file sizes backup_file_list = glob.glob(pattern) # Sort descending based on file timestamp backup_file_list.sort(key=os.path.getmtime, reverse=True) # Count the total bytes in all of the files totalbytes = 0 logger.debug("Max age (days): %s / Max size: %s" % (max_age, max_size)) for f in backup_file_list: logger.debug("File %s", f) # Get the file size (bytes) and age (days) bytes = os.path.getsize(f) age_days = old_div((time.time() - os.stat(f)[stat.ST_MTIME]), 86400) logger.debug("Age (days): %d", age_days) # increment the total byte count totalbytes += bytes if totalbytes > max_size and max_size > 0: # Delete the files logger.debug("Total bytes ({0}) > max_size ({1})".format(totalbytes, max_size)) os.remove(f) logger.info("Deleted file due to size retention policy: %s" % f) elif age_days > max_age and max_age > 0: logger.debug("Age ({0}) > max_age ({1})".format(age_days, max_age)) os.remove(f) logger.info("Deleted file due to age retention policy: %s" % f)
import kv_common as kv from deductiv_helpers import eprint, request, str2bool, setup_logger, read_config # Examples: # http://docs.splunk.com/Documentation/Splunk/6.5.0/AdvancedDev/CustomAlertKVStoreExample print(sys.argv) if len(sys.argv) > 1: if sys.argv[1] == "--execute": payload = json.loads(sys.stdin.read()) # Build the logger object based on the config file setting for log_level config = read_config('kvstore_tools.conf') log_level = config['settings']['log_level'] facility = os.path.splitext(os.path.basename(__file__))[0] logger = setup_logger(log_level, 'kvstore_tools.log', facility) # Get the stdin payload alert_config = payload.get('configuration', dict()) # Get the app / collection name supplied by the user/search app = urllib.parse.quote( alert_config.get('app') if 'app' in alert_config else payload.get('app')) collection = alert_config.get('collection') # Build the URL for the Splunkd REST endpoint url_tmpl_batch = '%(server_uri)s/servicesNS/%(owner)s/%(app)s/storage/collections/data/%(collection)s/batch_save?output_mode=json' record_url = url_tmpl_batch % dict( server_uri=payload.get('server_uri'), owner='nobody', app=urllib.parse.quote(
def stream(self, events): try: cfg = cli.getConfStanza('kvstore_tools', 'settings') except BaseException as e: eprint("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(cfg["log_level"], 'kvstore_tools.log', facility) except BaseException as e: eprint("Could not create logger: " + repr(e)) print("Could not create logger: " + repr(e)) exit(1) logger.info('Script started by %s' % self._metadata.searchinfo.username) if self.app: logger.debug('App: %s' % self.app) else: self.app = self._metadata.searchinfo.app if self.collection: logger.debug('Collection: %s' % self.collection) else: logger.critical("No collection specified. Exiting.") print("Error: No collection specified.") exit(1) if self.outputkeyfield: logger.debug('Output Key Field: %s' % self.outputkeyfield) else: self.outputkeyfield = self.collection + "_key" if self.outputvalues: logger.debug('Output Values: %s' % self.outputvalues) else: self.outputvalues = "" if self.delimiter: logger.debug('Delimiter: %s' % self.delimiter) else: self.delimiter = "," if self.groupby: logger.debug('Group by field: %s' % self.groupby) else: self.groupby = None opts = {} opts["owner"] = "nobody" opts["token"] = self._metadata.searchinfo.session_key opts["app"] = self.app #epoch_time = int(time.time()) current_user = self._metadata.searchinfo.username lookup_output_kvpairs = [] # Static output fields are literal values that are given within the search command arguments # e.g. "lookup_field1=value1" static_output_fields = {} # variable output fields are values taken from the events and pushed into the lookup record # as events are processed # e.g. "lookup_field2=$sourcetype$" variable_output_fields = {} resolved_variables = {} # Check for lockfile from previous invocations for this search ID dispatch = self._metadata.searchinfo.dispatch_dir static_kvfields_file = os.path.join(dispatch, "kvfields_static") #dict variable_kvfields_file = os.path.join(dispatch, "kvfields_variable") #dict resolved_variables_file = os.path.join(dispatch, "resolved_variables") #dict try: if os.path.isfile(static_kvfields_file): with open(static_kvfields_file, 'r') as f: # Set static kvfields values static_output_fields = json.loads(f.read()) #dict if os.path.isfile(variable_kvfields_file): with open(variable_kvfields_file, 'r') as f: # Set variable kvfields values variable_output_fields = json.loads(f.read()) #dict # Connect to the kv store service = connect(**opts) if self.collection in service.kvstore: obj_collection = service.kvstore[self.collection] else: logger.critical("KVStore not found: %s" % self.collection) print('KVStore not found: %s' % self.collection) exit(1) # First invocation - build the lists for static and variable values if static_output_fields == {} and variable_output_fields == {}: # Split the key-value pairs argument into individual key-value pairs # Account for quoted string values and delimiters within the quoted value kvpair_split_re = r'([^=]+=(?:"[^"\\]*(?:\\.[^"\\]*)*"|[^{}]+))'.format( self.delimiter) x = re.findall(kvpair_split_re, self.outputvalues) for i in x: i = i.strip(self.delimiter).strip() lookup_output_kvpairs.append(i) for lof in lookup_output_kvpairs: k, v = lof.split("=") k = k.strip() v = v.strip().strip('"').replace('\\"', '"') logger.debug("k = %s, v = %s" % (k, v)) # Replace special values v = v.replace("$kv_current_userid$", current_user) v = v.replace("$kv_now$", str(time.time())) # Value starts and ends with $ - variable field if v[0] + v[-1] == '$$': # Add to the list of variable fields variable_output_fields[k] = v.replace("$", "") else: # Add to the list of static fields static_output_fields[k] = v logger.info( "Unpacked %d static and %d variable fields from arguments" % (len(list(static_output_fields.keys())), len(list(variable_output_fields.keys())))) # Write the static payload to the file # File doesn't exist. Open/claim it. with open(static_kvfields_file, 'w') as f: f.write( json.dumps(static_output_fields, ensure_ascii=False)) with open(variable_kvfields_file, 'w') as f: f.write( json.dumps(variable_output_fields, ensure_ascii=False)) except BaseException as e: logger.critical('Error connecting to collection: %s' % repr(e), exc_info=True) print('Error connecting to collection: %s' % repr(e)) exit(1) # Read the events, resolve the variables, store them on a per-groupby-fieldvalue basis i = 0 inserts = 0 for e in events: update = False # (Re)read the latest data if os.path.isfile(resolved_variables_file): with open(resolved_variables_file, 'r') as f: # Open in non-blocking mode fd = f.fileno() flag = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, flag | os.O_NONBLOCK) # Set static kvfields values resolved_variables = json.loads( f.read()) #dict [groupby value][field name] if self.groupby is not None: groupby_value = e[self.groupby] else: # Make this value the same for every event (no group-by) groupby_value = '____placeholder' new_kv_record = {} if groupby_value in list(resolved_variables.keys()): # Set the previously recorded key value for this group-by value within the event kvstore_entry_key = resolved_variables[groupby_value]["_key"] # We've already resolved the variables for this groupby, but see if any are not populated for lookup_field, event_field in list( variable_output_fields.items()): if lookup_field not in list( resolved_variables[groupby_value].keys()): if event_field in list(e.keys()): if e[event_field] is not None and e[ event_field] != '': resolved_variables[groupby_value][ lookup_field] = e[event_field] new_kv_record[lookup_field] = e[event_field] update = True if update: # Update the collection new_kv_record.update(static_output_fields) response = obj_collection.data.update( kvstore_entry_key, json.dumps(new_kv_record)) # Write the data to disk immediately so other threads can benefit with open(resolved_variables_file, 'w') as f: f.write( json.dumps(resolved_variables, ensure_ascii=False)) else: # First time we're seeing this groupby value. Resolve variables and write the KV store record. # Define the dictionary resolved_variables[groupby_value] = {} # Update the static values new_kv_record = static_output_fields.copy() # Resolve the variables for lookup_field, event_field in list( variable_output_fields.items()): if event_field in list(e.keys()): if e[event_field] is not None: resolved_variables[groupby_value][ lookup_field] = e[event_field] new_kv_record[lookup_field] = e[event_field] # Write the new kvstore record and get the ID (_key) response = obj_collection.data.insert( json.dumps(new_kv_record)) kvstore_entry_key = response["_key"] resolved_variables[groupby_value]["_key"] = kvstore_entry_key # Write the data to disk immediately so other threads can benefit with open(resolved_variables_file, 'w') as f: f.write(json.dumps(resolved_variables, ensure_ascii=False)) inserts += 1 # Write the KV store record's _key value to the event e[self.outputkeyfield] = kvstore_entry_key yield e i += 1 logger.info("Modified %d events and inserted %s new records into %s" % (i, inserts, self.collection))
def reduce(self, events): try: app_config = cli.getConfStanza('ep_general', 'settings') cmd_config = cli.getConfStanzas('ep_aws_s3') except BaseException as e: raise Exception("Could not read configuration: " + repr(e)) # Facility info - prepended to log lines facility = os.path.basename(__file__) facility = os.path.splitext(facility)[0] try: logger = setup_logger(app_config["log_level"], 'export_everything.log', facility) except BaseException as e: raise Exception("Could not create logger: " + repr(e)) logger.info('AWS S3 Export search command initiated') logger.debug("Configuration: " + str(cmd_config)) logger.debug('search_ep_awss3 command: %s', self) # logs command line # Enumerate settings app = self._metadata.searchinfo.app user = self._metadata.searchinfo.username dispatch = self._metadata.searchinfo.dispatch_dir session_key = self._metadata.searchinfo.session_key if self.target is None and 'target=' in str(self): recover_parameters(self) # Replace all tokenized parameter strings replace_object_tokens(self) # Build the configuration try: aws_config = get_config_from_alias(session_key, cmd_config, self.target) if aws_config is None: exit_error( logger, "Unable to find target configuration (%s)." % self.target, 100937) logger.debug("Target configuration: " + str(aws_config)) except BaseException as e: exit_error(logger, "Error reading target server configuration: " + repr(e), 124812) if self.bucket is None: if 'default_s3_bucket' in list(aws_config.keys()): t = aws_config['default_s3_bucket'] if t is not None and len(t) > 0: self.bucket = t else: exit_error(logger, "No bucket specified", 4) else: exit_error(logger, "No bucket specified", 5) file_extensions = { 'raw': '.log', 'kv': '.log', 'pipe': '.log', 'csv': '.csv', 'tsv': '.tsv', 'json': '.json' } if self.outputformat is None: self.outputformat = 'csv' if self.outputfile is None: # Boto is special. We need repr to give it the encoding it expects to match the hashing. self.outputfile = repr('export_' + user + '___now__' + file_extensions[self.outputformat]).strip( "'") # Replace keywords from output filename self.outputfile = replace_keywords(self.outputfile) if self.compress is not None: logger.debug('Compression: %s', self.compress) else: try: self.compress = str2bool(aws_config['compress']) except: self.compress = False # Use the random number to support running multiple outputs in a single search random_number = str(random.randint(10000, 100000)) staging_filename = 'export_everything_staging_' + random_number + '.txt' local_output_file = os.path.join(dispatch, staging_filename) # Append .gz to the output file if compress=true if not self.compress and len(self.outputfile) > 3: # We have a .gz extension when compression was not specified. Enable compression. if self.outputfile[-3:] == '.gz': self.compress = True elif self.compress and len(self.outputfile) > 3: if self.outputfile[-3:] != '.gz': self.outputfile = self.outputfile + '.gz' if self.compress: local_output_file = local_output_file + '.gz' logger.debug("Staging file: %s" % local_output_file) try: s3 = get_aws_connection(aws_config) except BaseException as e: exit_error(logger, "Could not connect to AWS: " + repr(e), 741423) event_counter = 0 # Write the output file to disk in the dispatch folder logger.debug( "Writing events to file %s in %s format. Compress=%s\n\tfields=%s", local_output_file, self.outputformat, self.compress, self.fields) for event in event_file.write_events_to_file(events, self.fields, local_output_file, self.outputformat, self.compress): yield event event_counter += 1 # Upload file to s3 try: with open(local_output_file, "rb") as f: s3.upload_fileobj(f, self.bucket, self.outputfile) s3 = None sts_client = None logger.info( "Successfully exported events to s3. app=%s count=%s bucket=%s file=%s user=%s" % (app, event_counter, self.bucket, self.outputfile, user)) os.remove(local_output_file) except s3.exceptions.NoSuchBucket as e: exit_error(logger, "Error: No such bucket", 123833) except BaseException as e: exit_error(logger, "Could not upload file to S3: " + repr(e), 9)