def generate(self):
try:
cfg = cli.getConfStanza('kvstore_tools', 'settings')
except BaseException as e:
eprint("Could not read configuration: " + repr(e))
# Facility info - prepended to log lines
facility = os.path.basename(__file__)
facility = os.path.splitext(facility)[0]
try:
logger = setup_logger(cfg["log_level"], 'kvstore_tools.log',
facility)
except BaseException as e:
eprint("Could not create logger: " + repr(e))
print("Could not create logger: " + repr(e))
exit(1)
logger.info('Script started by %s' %
self._metadata.searchinfo.username)
session_key = self._metadata.searchinfo.session_key
splunkd_uri = self._metadata.searchinfo.splunkd_uri
if self.app:
logger.debug('App: %s' % self.app)
else:
self.app = self._metadata.searchinfo.app
if self.collection:
logger.debug('Collection: %s' % self.collection)
else:
logger.critical("No collection specified. Exiting.")
print("Error: No collection specified.")
exit(1)
if self.key:
logger.debug('Key ID: %s' % self.collection)
else:
logger.critical("No key value specified. Exiting.")
print("Error: No key value specified.")
exit(1)
headers = {
'Authorization': 'Splunk %s' % session_key,
'Content-Type': 'application/json'
}
#url_tmpl_app = '%(server_uri)s/servicesNS/%(owner)s/%(app)s/storage/collections/config?output_mode=json&count=0'
# Enumerate all apps
app_list = kv.get_server_apps(splunkd_uri, session_key, self.app)
collection_list = kv.get_app_collections(splunkd_uri, session_key,
self.collection, self.app,
app_list, True)
logger.debug('Collections present: %s', str(collection_list))
try:
# Create an object for the collection
collection_present = False
for c in collection_list:
# Extract the app and collection name from the array
# c[0] = app, c[1] = collection name
collection_app = c[0]
collection_name = c[1]
if (collection_name == self.collection):
if self.app is None or self.app == collection_app:
self.app = collection_app
collection_present = True
elif self.app != collection_app:
pass
logger.debug("Collection found: {0} in app {1}".format(
self.collection, self.app))
if not collection_present:
logger.critical(
"KVStore collection %s not found within app %s" %
(self.collection, self.app))
exit(1)
except BaseException as e:
logger.critical('Error enumerating collections: ' + str(e))
exit(1)
url_tmpl_delete = '%(server_uri)s/servicesNS/%(owner)s/%(app)s/storage/collections/data/%(collection)s/%(id)s?output_mode=json'
try:
delete_url = url_tmpl_delete % dict(server_uri=splunkd_uri,
owner='nobody',
app=self.app,
collection=self.collection,
id=urllib.parse.quote(self.key,
safe=''))
logger.debug("Delete url: " + delete_url)
try:
response, response_code = request('DELETE', delete_url, '',
headers)
logger.debug('Server response: %s', response)
except BaseException as e:
logger.error(
'Failed to delete key %s from collection %s/%s: %s' %
(self.key, self.app, self.collection, repr(e)))
if response_code == 200:
logger.debug(
"Successfully deleted key %s from collection %s/%s" %
(self.key, self.app, self.collection))
result = "success"
else:
logger.error(
"Error deleting key %s from collection %s/%s: %s" %
(self.key, self.app, self.collection, response))
result = "error"
except BaseException as e:
logger.error("Error deleting key %s from collection %s/%s: %s" %
(self.key, self.app, self.collection, repr(e)))
result = "error"
# Entry deleted
yield {
'_time': time.time(),
'app': self.app,
'collection': self.collection,
'key': self.key,
'result': result
}
def stream(self, events):
try:
cfg = cli.getConfStanza('kvstore_tools', 'settings')
except BaseException as e:
eprint("Could not read configuration: " + repr(e))
# Facility info - prepended to log lines
facility = os.path.basename(__file__)
facility = os.path.splitext(facility)[0]
try:
logger = setup_logger(cfg["log_level"], 'kvstore_tools.log',
facility)
except BaseException as e:
eprint("Could not create logger: " + repr(e))
print("Could not create logger: " + repr(e))
exit(1)
logger.info('Script started by %s' %
self._metadata.searchinfo.username)
if self.app:
logger.debug('App: %s' % self.app)
else:
self.app = self._metadata.searchinfo.app
if self.collection:
logger.debug('Collection: %s' % self.collection)
else:
logger.critical("No collection specified. Exiting.")
print("Error: No collection specified.")
exit(1)
self.session_key = self._metadata.searchinfo.session_key
self.splunkd_uri = self._metadata.searchinfo.splunkd_uri
# Enumerate all app_list
app_list = kv.get_server_apps(self.splunkd_uri, self.session_key,
self.app)
collection_list = kv.get_app_collections(self.splunkd_uri,
self.session_key,
self.collection, self.app,
app_list, True)
logger.debug('Collections present: %s', str(collection_list))
try:
# Create an object for the collection
collection_present = False
for c in collection_list:
# Extract the app and collection name from the array
# c[0] = app, c[1] = collection name
collection_app = c[0]
collection_name = c[1]
if (collection_name == self.collection):
if self.app is None or self.app == collection_app:
self.app = collection_app
collection_present = True
elif self.app != collection_app:
pass
logger.debug("Collection {0} found in app {1}".format(
self.collection, self.app))
if not collection_present:
logger.critical("KVStore collection %s/%s not found" %
(self.app, self.collection))
exit(1)
except BaseException as e:
logger.critical('Error enumerating collections: %s' % repr(e))
exit(1)
# Make a Pool of workers
pool = ThreadPool(4)
try:
results = pool.map(self.delete_key_from_event, events)
except BaseException as e:
logger.error("%s" % repr(e), exc_info=True)
results = {}
for result in results:
yield result
def generate(self):
try:
cfg = cli.getConfStanza('kvstore_tools','settings')
except BaseException as e:
eprint("Could not read configuration: " + repr(e))
# Facility info - prepended to log lines
facility = os.path.basename(__file__)
facility = os.path.splitext(facility)[0]
try:
logger = setup_logger(cfg["log_level"], 'kvstore_tools.log', facility)
except BaseException as e:
eprint("Could not create logger: " + repr(e))
exit(1)
logger.info('Script started by %s' % self._metadata.searchinfo.username)
batch_size = int(cfg.get('backup_batch_size'))
logger.debug("Batch size: %d rows" % batch_size)
session_key = self._metadata.searchinfo.session_key
splunkd_uri = self._metadata.searchinfo.splunkd_uri
# Check for permissions to run the command
content = rest.simpleRequest('/services/authentication/current-context?output_mode=json', sessionKey=session_key, method='GET')[1]
content = json.loads(content)
current_user = self._metadata.searchinfo.username
current_user_capabilities = content['entry'][0]['content']['capabilities']
if 'run_kvstore_backup' in current_user_capabilities or 'run_kvst_all' in current_user_capabilities:
logger.debug("User %s is authorized." % current_user)
else:
logger.error("User %s is unauthorized. Has the run_kvstore_backup capability been granted?" % current_user)
yield({'Error': 'User %s is unauthorized. Has the run_kvstore_backup capability been granted?' % current_user })
sys.exit(3)
# Sanitize input
if self.app:
logger.debug('App Context: %s' % self.app)
else:
self.app = None
if self.path:
pass
else:
# Get path from configuration
try:
# Break path out and re-join it so it's OS independent
default_path = cfg.get('default_path').split('/')
self.path = os.path.abspath(os.path.join(os.sep, *default_path))
except:
logger.critical("Unable to get backup path")
yield({'Error': "Path not provided in search arguments and default path is not set."})
sys.exit(1)
# Replace environment variables
self.path = os.path.expandvars(self.path)
self.path = self.path.replace('//', '/')
logger.debug('Backup path: %s' % self.path)
if not os.path.isdir(self.path):
logger.critical("Path does not exist: {0}".format(self.path))
yield({'Error': "Path does not exist: {0}".format(self.path)})
sys.exit(1)
if self.collection:
logger.debug('Collection: %s' % self.collection)
else:
self.collection = None
if self.global_scope:
logger.debug('Global Scope: %s' % self.global_scope)
else:
self.global_scope = False
if self.compression or self.compression == False:
logger.debug('Compression: %s' % self.compression)
else:
try:
self.compression = cfg.get('compression')
except:
self.compression = False
app_list = kv.get_server_apps(splunkd_uri, session_key, self.app)
logger.debug("Apps list: %s" % str(app_list))
collection_list = kv.get_app_collections(splunkd_uri, session_key, self.collection, self.app, app_list, self.global_scope)
logger.info('Collections to backup: %s', str(collection_list))
for collection in collection_list:
# Extract the app and collection name from the array
entry_app = collection[0]
collection_name = collection[1]
ts = time.time()
st = datetime.fromtimestamp(ts).strftime('%Y%m%d_%H%M%S')
#maxrows = int(limits_cfg.get('max_rows_per_query'))
# Set the filename and location for the output (expanding environment variables)
output_filename = entry_app + "#" + collection_name + "#" + st + ".json"
if self.compression:
output_filename = output_filename + '.gz'
output_file = os.path.join(self.path, output_filename)
# Download the collection to a local file
result, message, total_record_count = kv.download_collection(logger, splunkd_uri, session_key, entry_app, collection_name, output_file, self.compression)
logger.debug("Retrieved {0} records from {1}".format(total_record_count, collection_name))
yield {'_time': time.time(), 'app': entry_app, 'collection': collection_name, 'result': result, 'records': total_record_count, 'message': message, 'file': output_file }
# Execute retention routine
max_age = 0
max_age = int(cfg.get('retention_days'))
max_size = 0
max_size = int(cfg.get('retention_size')) * 1024 * 1024
if max_size > 0 or max_age > 0:
# Check the size of all *.json and *.json.gz files in the directory
#dir = self.path
pattern = os.path.join(self.path, "*#*#*.json*")
# Get a listing of the files and check the file sizes
backup_file_list = glob.glob(pattern)
# Sort descending based on file timestamp
backup_file_list.sort(key=os.path.getmtime, reverse=True)
# Count the total bytes in all of the files
totalbytes = 0
logger.debug("Max age (days): %s / Max size: %s" % (max_age, max_size))
for f in backup_file_list:
logger.debug("File %s", f)
# Get the file size (bytes) and age (days)
bytes = os.path.getsize(f)
age_days = old_div((time.time() - os.stat(f)[stat.ST_MTIME]), 86400)
logger.debug("Age (days): %d", age_days)
# increment the total byte count
totalbytes += bytes
if totalbytes > max_size and max_size > 0:
# Delete the files
logger.debug("Total bytes ({0}) > max_size ({1})".format(totalbytes, max_size))
os.remove(f)
logger.info("Deleted file due to size retention policy: %s" % f)
elif age_days > max_age and max_age > 0:
logger.debug("Age ({0}) > max_age ({1})".format(age_days, max_age))
os.remove(f)
logger.info("Deleted file due to age retention policy: %s" % f)
def generate(self):
try:
cfg = cli.getConfStanza('kvstore_tools', 'settings')
except BaseException as e:
eprint("Could not read configuration: " + repr(e))
# Facility info - prepended to log lines
facility = os.path.basename(__file__)
facility = os.path.splitext(facility)[0]
try:
logger = setup_logger(cfg["log_level"], 'kvstore_tools.log',
facility)
except BaseException as e:
eprint("Could not create logger: " + repr(e))
print("Could not create logger: " + repr(e))
exit(1)
logger.info('Script started by %s' %
self._metadata.searchinfo.username)
batch_size = int(cfg.get('backup_batch_size'))
logger.debug("Batch size: %d rows" % batch_size)
local_session_key = self._metadata.searchinfo.session_key
splunkd_uri = self._metadata.searchinfo.splunkd_uri
# Check for permissions to run the command
content = rest.simpleRequest(
'/services/authentication/current-context?output_mode=json',
sessionKey=local_session_key,
method='GET')[1]
content = json.loads(content)
current_user = self._metadata.searchinfo.username
current_user_capabilities = content['entry'][0]['content'][
'capabilities']
if 'run_kvstore_pull' in current_user_capabilities or 'run_kvst_all' in current_user_capabilities:
logger.debug("User %s is authorized." % current_user)
else:
logger.error(
"User %s is unauthorized. Has the run_kvstore_pull capability been granted?"
% current_user)
yield ({
'Error':
'User %s is unauthorized. Has the run_kvstore_pull capability been granted?'
% current_user
})
sys.exit(3)
# Sanitize input
if self.app:
logger.debug('App Context: %s' % self.app)
else:
self.app = None
if self.collection:
logger.debug('Collection: %s' % self.collection)
else:
self.collection = None
if self.global_scope:
logger.debug('Global Scope: %s' % self.global_scope)
else:
self.global_scope = False
if self.append:
logger.debug('Appending to existing collection')
else:
self.append = False
logger.debug('Append to existing collection: %s' %
str(self.append))
if self.targetport:
logger.debug('Port for remote connect: %s' % self.targetport)
else:
self.targetport = '8089'
# Get credentials
try:
# Use the credential where the realm matches the target hostname
# Otherwise, use the last entry in the list
credentials = kv.parse_custom_credentials(logger, cfg)
try:
credential = credentials[self.target]
except:
try:
hostname = self.target.split('.')[0]
credential = credentials[hostname]
except:
logger.critical("Could not get password for %s: %s" %
(self.target, repr(e)))
print("Could not get password for %s: %s" %
(self.target, repr(e)))
exit(1593)
remote_user = credential['username']
remote_password = credential['password']
except BaseException as e:
logger.critical(
'Failed to get credentials for remote Splunk instance: %s' %
repr(e),
exc_info=True)
yield ({
'Error':
'Failed to get credentials for remote Splunk instance: %s' %
repr(e)
})
exit(7372)
# Login to the remote host and get the session key
try:
remote_host = self.target
remote_port = self.targetport
remote_uri = 'https://%s:%s' % (self.target, self.targetport)
remote_service = client.connect(host=remote_host,
port=remote_port,
username=remote_user,
password=remote_password)
remote_service.login()
remote_session_key = remote_service.token.replace('Splunk ', '')
logger.debug('Remote Session_key: %s' % remote_session_key)
except (urllib.error.HTTPError, BaseException) as e:
logger.exception('Failed to login on remote Splunk instance: %s' %
repr(e))
yield ({
'Error':
'Failed to login on remote Splunk instance: %s' % repr(e)
})
sys.exit(4424)
# Get the list of remote apps and collections
remote_app_list = kv.get_server_apps(remote_uri, remote_session_key,
self.app)
remote_collection_list = kv.get_app_collections(
remote_uri, remote_session_key, self.collection, self.app,
remote_app_list, self.global_scope)
logger.debug('Collections to pull: %s' % str(remote_collection_list))
for remote_collection in remote_collection_list:
# Extract the app and collection name from the array
collection_app = remote_collection[0]
collection_name = remote_collection[1]
try:
yield (kv.copy_collection(logger, remote_session_key,
remote_uri, local_session_key,
splunkd_uri, collection_app,
collection_name, self.append))
except BaseException as e:
logger.critical(
'Failed to copy collections from %s to local KV store: %s'
% (self.target, repr(e)),
exc_info=True)
yield ({
'Error':
'Failed to copy collections from %s to local KV store: %s'
% (self.target, repr(e))
})
sys.exit(11)