class LookupBackupsHandler(rest_handler.RESTHandler):
"""
This is a REST handler that supports backing up lookup files.
This is broken out as a separate handler so that this handler can be replayed on other search
heads via the allowRestReplay setting in restmap.conf.
"""
def __init__(self, command_line, command_arg):
super(LookupBackupsHandler, self).__init__(command_line, command_arg,
logger)
self.lookup_editor = LookupEditor(logger)
def post_backup(self,
request_info,
lookup_file=None,
namespace="lookup_editor",
owner=None,
file_time=None,
**kwargs):
"""
Make a backup of the given lookup file.
"""
try:
# Determine the final path of the file
resolved_file_path = self.lookup_editor.resolve_lookup_filename(
lookup_file,
namespace,
owner,
session_key=request_info.session_key,
throw_not_found=True)
# Backup the file passing the time so that the REST handler will use the same time for
# all lookups even when the REST call is being replayed in an SHC cluster
file_path = self.lookup_editor.backup_lookup_file(
request_info.session_key, lookup_file, namespace,
resolved_file_path, owner, file_time)
self.logger.info("Created a backup of a lookup file, file_path=%s",
file_path)
# Everything worked, return accordingly
return {
'payload': str(file_path), # Payload of the request.
'status': 200 # HTTP status code
}
except ResourceNotFound:
self.logger.warn("Unable to find the lookup to backup")
return self.render_error_json(
"Unable to find the lookup to backup", 404)
except:
self.logger.exception(
"Exception generated when attempting to backup a lookup file")
return self.render_error_json("Unable to backup the lookup")
class TestLookupEditor(LookupEditorTestCase):
"""
This tests the class which manages lookup backups.
"""
def setUp(self):
self.lookup_editor = LookupEditor(logger=logger)
@skipIfCantAuthenticate
def test_resolve_lookup_filename(self):
"""
Test resolve_lookup_filename() to resolve a lookup file name.
"""
file_path = self.lookup_editor.resolve_lookup_filename('test.csv', 'search', 'nobody',
False, None,
session_key=self.get_session_key())
self.assertEquals(self.strip_splunk_path(file_path),
'/etc/apps/lookup_editor/lookups/test.csv')
@skipIfCantAuthenticate
def test_resolve_lookup_filename_version(self):
"""
Test resolve_lookup_filename() to resolve a lookup file name to the backed up version.
"""
file_path = self.lookup_editor.resolve_lookup_filename('test.csv', 'search', 'nobody',
False, '1234',
session_key=self.get_session_key())
self.assertEquals(self.strip_splunk_path(file_path),
'/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234')
@skipIfCantAuthenticate
def test_resolve_lookup_filename_version_no_user(self):
"""
Test resolve_lookup_filename() to resolve a lookup file name to the backed up version
without providing a user.
"""
file_path = self.lookup_editor.resolve_lookup_filename('test.csv', 'search', None, False,
'1234', session_key=self.get_session_key())
self.assertEquals(self.strip_splunk_path(file_path),
'/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234')
@skipIfLookupTestNotInstalled
def test_get_kv_fields_from_transform(self):
"""
Test getting the fields of a lookup from a transform.
"""
fields = self.lookup_editor.get_kv_fields_from_transform(self.get_session_key(), 'test_kv_store_transform_fields', 'lookup_test', None)
self.assertEquals(len(fields), 4)
class LookupBackupsHandler(rest_handler.RESTHandler):
"""
This is a REST handler that supports backing up lookup files.
This is broken out as a separate handler so that this handler can be replayed on other search
heads via the allowRestReplay setting in restmap.conf.
"""
def __init__(self, command_line, command_arg):
super(LookupBackupsHandler, self).__init__(command_line, command_arg, logger)
self.lookup_editor = LookupEditor(logger)
def post_backup(self, request_info, lookup_file=None, namespace="lookup_editor",
owner=None, file_time=None, **kwargs):
"""
Make a backup of the given lookup file.
"""
try:
# Determine the final path of the file
resolved_file_path = self.lookup_editor.resolve_lookup_filename(lookup_file,
namespace,
owner,
session_key=request_info.session_key,
throw_not_found=True)
# Backup the file passing the time so that the REST handler will use the same time for
# all lookups even when the REST call is being replayed in an SHC cluster
file_path = self.lookup_editor.backup_lookup_file(request_info.session_key,
lookup_file, namespace,
resolved_file_path, owner,
file_time)
self.logger.info("Created a backup of a lookup file, file_path=%s", file_path)
# Everything worked, return accordingly
return {
'payload': str(file_path), # Payload of the request.
'status': 200 # HTTP status code
}
except ResourceNotFound:
self.logger.warn("Unable to find the lookup to backup")
return self.render_error_json("Unable to find the lookup to backup", 404)
except:
self.logger.exception("Exception generated when attempting to backup a lookup file")
return self.render_error_json("Unable to backup the lookup")
def test_get_backup_directory_with_resolved(self):
"""
Ensure that get_backup_directory() returns the correct directory when given a value for resolved_lookup_path.
"""
self.lookup_editor = LookupEditor(logger=logger)
resolved_lookup_path = self.lookup_editor.resolve_lookup_filename(
'test.csv',
'search',
'nobody',
False,
None,
session_key=self.get_session_key())
dir = self.lookup_backups.get_backup_directory(
self.get_session_key(),
"test.csv",
namespace="search",
owner="nobody",
resolved_lookup_path=resolved_lookup_path)
self.assertEquals(
self.strip_splunk_path(dir),
"/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv"
)
class TestLookupBackups(LookupEditorTestCase):
"""
This tests the class which manages lookup backups.
"""
def setUp(self):
self.lookup_backups = lookup_backups.LookupBackups(logger=logger)
@skipIfCantAuthenticate
def test_get_backup_directory(self):
"""
Ensure that get_backup_directory() returns the correct directory.
"""
dir = self.lookup_backups.get_backup_directory(self.get_session_key(),
"test.csv",
namespace="search",
owner="nobody")
self.assertEquals(
self.strip_splunk_path(dir),
"/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv"
)
@skipIfCantAuthenticate
def test_get_backup_directory_with_resolved(self):
"""
Ensure that get_backup_directory() returns the correct directory when given a value for resolved_lookup_path.
"""
self.lookup_editor = LookupEditor(logger=logger)
resolved_lookup_path = self.lookup_editor.resolve_lookup_filename(
'test.csv',
'search',
'nobody',
False,
None,
session_key=self.get_session_key())
dir = self.lookup_backups.get_backup_directory(
self.get_session_key(),
"test.csv",
namespace="search",
owner="nobody",
resolved_lookup_path=resolved_lookup_path)
self.assertEquals(
self.strip_splunk_path(dir),
"/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv"
)
def test_get_backup_directory_with_resolved(self):
"""
Ensure that get_backup_directory() returns the correct directory when given a value for resolved_lookup_path.
"""
self.lookup_editor = LookupEditor(logger=logger)
resolved_lookup_path = self.lookup_editor.resolve_lookup_filename('test.csv',
'search',
'nobody',
False,
None,
session_key=self.get_session_key())
dir = self.lookup_backups.get_backup_directory(self.get_session_key(), "test.csv",
namespace="search", owner="nobody",
resolved_lookup_path=resolved_lookup_path)
self.assertEquals(self.strip_splunk_path(dir),
"/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv")
class TestLookupBackups(LookupEditorTestCase):
"""
This tests the class which manages lookup backups.
"""
def setUp(self):
self.lookup_backups = lookup_backups.LookupBackups(logger=logger)
@skipIfCantAuthenticate
def test_get_backup_directory(self):
"""
Ensure that get_backup_directory() returns the correct directory.
"""
dir = self.lookup_backups.get_backup_directory(self.get_session_key(),
"test.csv", namespace="search",
owner="nobody")
self.assertEquals(self.strip_splunk_path(dir),
"/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv")
@skipIfCantAuthenticate
def test_get_backup_directory_with_resolved(self):
"""
Ensure that get_backup_directory() returns the correct directory when given a value for resolved_lookup_path.
"""
self.lookup_editor = LookupEditor(logger=logger)
resolved_lookup_path = self.lookup_editor.resolve_lookup_filename('test.csv',
'search',
'nobody',
False,
None,
session_key=self.get_session_key())
dir = self.lookup_backups.get_backup_directory(self.get_session_key(), "test.csv",
namespace="search", owner="nobody",
resolved_lookup_path=resolved_lookup_path)
self.assertEquals(self.strip_splunk_path(dir),
"/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv")
def __init__(self, command_line, command_arg):
super(LookupEditorHandler, self).__init__(command_line, command_arg, logger)
self.lookup_editor = LookupEditor(logger)
class LookupEditorHandler(rest_handler.RESTHandler):
"""
This is a REST handler that supports editing lookup files. All calls from the user-interface
should pass through this handler.
"""
def __init__(self, command_line, command_arg):
super(LookupEditorHandler, self).__init__(command_line, command_arg, logger)
self.lookup_editor = LookupEditor(logger)
def get_lookup_info(self, request_info, lookup_file=None, namespace="lookup_editor", **kwargs):
"""
Get information about a lookup file (owner, size, etc.)
"""
return {
'payload': str(lookup_file), # Payload of the request.
'status': 200 # HTTP status code
}
def get_lookup_backups(self, request_info, lookup_file=None, namespace=None, owner=None, **kwargs):
"""
Get a list of the lookup file backups rendered as JSON.
"""
backups = self.lookup_editor.get_backup_files(request_info.session_key, lookup_file, namespace, owner)
# Make the response
backups_meta = []
for backup in backups:
try:
backups_meta.append(
{
'time': backup,
'time_readable' : datetime.datetime.fromtimestamp(float(backup)).strftime('%Y-%m-%d %H:%M:%S')
}
)
except ValueError:
self.logger.warning("Backup file name is invalid, file_name=%s", backup)
# Sort the list
backups_meta = sorted(backups_meta, key=lambda x: float(x['time']), reverse=True)
return self.render_json(backups_meta)
def get_lookup_contents(self, request_info, lookup_file=None, namespace="lookup_editor",
owner=None, header_only=False, version=None, lookup_type=None,
**kwargs):
"""
Provides the contents of a lookup file as JSON.
"""
self.logger.info("Retrieving lookup contents, namespace=%s, lookup=%s, type=%s, owner=%s,"
" version=%s", namespace, lookup_file, lookup_type, owner, version)
if lookup_type is None or len(lookup_type) == 0:
lookup_type = "csv"
self.logger.warning("No type for the lookup provided when attempting to load a lookup" +
" file, it will default to CSV")
if header_only in ["1", "true", 1, True]:
header_only = True
else:
header_only = False
try:
# Load the KV store lookup
if lookup_type == "kv":
return self.render_json(self.lookup_editor.get_kv_lookup(lookup_file, namespace, owner))
# Load the CSV lookup
elif lookup_type == "csv":
with self.lookup_editor.get_lookup(request_info.session_key, lookup_file, namespace,
owner, version=version,
throw_exception_if_too_big=True) as csv_file:
csv_reader = csv.reader(csv_file)
# Convert the content to JSON
lookup_contents = []
for row in csv_reader:
lookup_contents.append(row)
# If we are only loading the header, then stop here
if header_only:
break
return self.render_json(lookup_contents)
else:
self.logger.warning('Lookup file type is not recognized,' +
' lookup_type=' + lookup_type)
return self.render_error_json('Lookup file type is not recognized', 421)
except IOError:
self.logger.warning("Unable to find the requested lookup")
return self.render_error_json("Unable to find the lookup", 404)
except (AuthorizationFailed, PermissionDeniedException) as e:
self.logger.warning("Access to lookup denied")
return self.render_error_json(str(e), 403)
except LookupFileTooBigException as e:
self.logger.warning("Lookup file is too large to load")
data = {
'message': 'Lookup file is too large to load' +
'(file-size must be less than 10 MB to be edited)',
'file_size' : e.file_size
}
return {
'payload': json.dumps(data),
'status': 420
}
except:
self.logger.exception('Lookup file could not be loaded')
return self.render_error_json('Lookup file could not be loaded', 500)
return {
'payload': 'Response',
'status': 500
}
def get_lookup_as_file(self, request_info, lookup_file=None, namespace="lookup_editor", owner=None, lookup_type='csv', **kwargs):
"""
Provides the lookup file in a way to be downloaded by the browser
"""
logger.info("Exporting lookup, namespace=%s, lookup=%s, type=%s, owner=%s", namespace,
lookup_file, lookup_type, owner)
try:
# If we are getting the CSV, then just pipe the file to the user
if lookup_type == "csv":
with self.lookup_editor.get_lookup(request_info.session_key, lookup_file, namespace, owner) as csv_file_handle:
csv_data = csv_file_handle.read()
# If we are getting a KV store lookup, then convert it to a CSV file
else:
rows = self.lookup_editor.get_kv_lookup(lookup_file, namespace, owner)
csv_data = shortcuts.convert_array_to_csv(rows)
# Tell the browser to download this as a file
if lookup_file.endswith(".csv"):
filename = 'attachment; filename="%s"' % lookup_file
else:
filename = 'attachment; filename="%s"' % (lookup_file + ".csv")
return {
'payload': json.dumps(csv_data),
'status': 200,
'headers': {
'Content-Type': 'text/csv',
'Content-Disposition': filename
},
}
except IOError:
return self.render_json([], 404)
except PermissionDeniedException as exception:
return self.render_error_json(str(exception), 403)
return {
'payload': str(lookup_file), # Payload of the request.
'status': 200 # HTTP status code
}
def post_lookup_contents(self, request_info, lookup_file=None, namespace="lookup_editor",
owner=None, header_only=False, version=None, lookup_type=None,
**kwargs):
"""
Save the JSON contents to the lookup file
"""
return {
'payload': str(lookup_file), # Payload of the request.
'status': 200 # HTTP status code
}
def setUp(self):
self.lookup_editor = LookupEditor(logger=logger)
def __init__(self, command_line, command_arg):
super(LookupEditorHandler, self).__init__(command_line, command_arg,
logger)
self.lookup_editor = LookupEditor(logger)
class LookupEditorHandler(rest_handler.RESTHandler):
"""
This is a REST handler that supports editing lookup files. All calls from the user-interface
should pass through this handler.
"""
def __init__(self, command_line, command_arg):
super(LookupEditorHandler, self).__init__(command_line, command_arg,
logger)
self.lookup_editor = LookupEditor(logger)
def get_lookup_info(self,
request_info,
lookup_file=None,
namespace="lookup_editor",
**kwargs):
"""
Get information about a lookup file (owner, size, etc.)
"""
return {
'payload': str(lookup_file), # Payload of the request.
'status': 200 # HTTP status code
}
def get_lookup_backups(self,
request_info,
lookup_file=None,
namespace=None,
owner=None,
**kwargs):
"""
Get a list of the lookup file backups rendered as JSON.
"""
backups = self.lookup_editor.get_backup_files(request_info.session_key,
lookup_file, namespace,
owner)
# Make the response
backups_meta = []
for backup in backups:
try:
backups_meta.append({
'time':
backup,
'time_readable':
datetime.datetime.fromtimestamp(
float(backup)).strftime('%Y-%m-%d %H:%M:%S')
})
except ValueError:
self.logger.warning(
"Backup file name is invalid, file_name=%s", backup)
# Sort the list
backups_meta = sorted(backups_meta,
key=lambda x: float(x['time']),
reverse=True)
return self.render_json(backups_meta)
def get_lookup_contents(self,
request_info,
lookup_file=None,
namespace="lookup_editor",
owner=None,
header_only=False,
version=None,
lookup_type=None,
**kwargs):
"""
Provides the contents of a lookup file as JSON.
"""
self.logger.info(
"Retrieving lookup contents, namespace=%s, lookup=%s, type=%s, owner=%s,"
" version=%s", namespace, lookup_file, lookup_type, owner, version)
if lookup_type is None or len(lookup_type) == 0:
lookup_type = "csv"
self.logger.warning(
"No type for the lookup provided when attempting to load a lookup"
+ " file, it will default to CSV")
if header_only in ["1", "true", 1, True]:
header_only = True
else:
header_only = False
try:
# Load the KV store lookup
if lookup_type == "kv":
return self.render_json(
self.lookup_editor.get_kv_lookup(request_info.session_key,
lookup_file, namespace,
owner))
# Load the CSV lookup
elif lookup_type == "csv":
with self.lookup_editor.get_lookup(
request_info.session_key,
lookup_file,
namespace,
owner,
version=version,
throw_exception_if_too_big=True) as csv_file:
csv_reader = csv.reader(csv_file)
# Convert the content to JSON
lookup_contents = []
for row in csv_reader:
lookup_contents.append(row)
# If we are only loading the header, then stop here
if header_only:
break
return self.render_json(lookup_contents)
else:
self.logger.warning('Lookup file type is not recognized,' +
' lookup_type=' + lookup_type)
return self.render_error_json(
'Lookup file type is not recognized', 421)
except IOError:
self.logger.warning("Unable to find the requested lookup")
return self.render_error_json("Unable to find the lookup", 404)
except (AuthorizationFailed, PermissionDeniedException) as e:
self.logger.warning("Access to lookup denied")
return self.render_error_json(str(e), 403)
except LookupFileTooBigException as e:
self.logger.warning("Lookup file is too large to load")
data = {
'message': 'Lookup file is too large to load' +
'(file-size must be less than 10 MB to be edited)',
'file_size': e.file_size
}
return {'payload': json.dumps(data), 'status': 420}
except:
self.logger.exception('Lookup file could not be loaded')
return self.render_error_json('Lookup file could not be loaded',
500)
return {'payload': 'Response', 'status': 500}
def get_lookup_as_file(self,
request_info,
lookup_file=None,
namespace="lookup_editor",
owner=None,
lookup_type='csv',
**kwargs):
"""
Provides the lookup file in a way to be downloaded by the browser
"""
self.logger.info(
"Exporting lookup, namespace=%s, lookup=%s, type=%s, owner=%s",
namespace, lookup_file, lookup_type, owner)
try:
# If we are getting the CSV, then just pipe the file to the user
if lookup_type == "csv":
with self.lookup_editor.get_lookup(request_info.session_key,
lookup_file, namespace,
owner) as csv_file_handle:
csv_data = csv_file_handle.read()
# If we are getting a KV store lookup, then convert it to a CSV file
else:
rows = self.lookup_editor.get_kv_lookup(
request_info.session_key, lookup_file, namespace, owner)
csv_data = shortcuts.convert_array_to_csv(rows)
# Tell the browser to download this as a file
if lookup_file.endswith(".csv"):
filename = 'attachment; filename="%s"' % lookup_file
else:
filename = 'attachment; filename="%s"' % (lookup_file + ".csv")
return {
'payload': csv_data,
'status': 200,
'headers': {
'Content-Type': 'text/csv',
'Content-Disposition': filename
},
}
except IOError:
return self.render_json([], 404)
except PermissionDeniedException as exception:
return self.render_error_json(str(exception), 403)
return {
'payload': str(lookup_file), # Payload of the request.
'status': 200 # HTTP status code
}
def post_lookup_contents(self,
request_info,
contents=None,
lookup_file=None,
namespace="lookup_editor",
owner=None,
**kwargs):
"""
Save the JSON contents to the lookup file.
"""
self.logger.info("Saving lookup contents...")
try:
# Backup the lookup file
data = {
'lookup_file': lookup_file,
'namespace': namespace,
'owner': owner
}
try:
_, _ = simpleRequest('/services/data/lookup_backup/backup',
sessionKey=request_info.session_key,
method='POST',
postargs=data)
except ResourceNotFound:
self.logger.info(
"Existing lookup could not be found for backup")
file_name = self.lookup_editor.update(contents, lookup_file,
namespace, owner,
request_info.session_key,
request_info.user)
# Everything worked, return accordingly
return {
'payload': str(file_name), # Payload of the request.
'status': 200 # HTTP status code
}
except LookupNameInvalidException:
return self.render_error_json("Lookup name is invalid", 400)
except:
self.logger.exception("Unable to save the lookup")
return self.render_error_json("Unable to save the lookup")
def setUp(self):
self.lookup_editor = LookupEditor(logger=logger)
class TestLookupEditor(LookupEditorTestCase):
"""
This tests the class which manages lookup backups.
"""
def setUp(self):
self.lookup_editor = LookupEditor(logger=logger)
@skipIfCantAuthenticate
def test_resolve_lookup_filename(self):
"""
Test resolve_lookup_filename() to resolve a lookup file name.
"""
file_path = self.lookup_editor.resolve_lookup_filename(
'test.csv',
'search',
'nobody',
False,
None,
session_key=self.get_session_key())
self.assertEquals(self.strip_splunk_path(file_path),
'/etc/apps/lookup_editor/lookups/test.csv')
@skipIfCantAuthenticate
def test_resolve_lookup_filename_version(self):
"""
Test resolve_lookup_filename() to resolve a lookup file name to the backed up version.
"""
file_path = self.lookup_editor.resolve_lookup_filename(
'test.csv',
'search',
'nobody',
False,
'1234',
session_key=self.get_session_key())
self.assertEquals(
self.strip_splunk_path(file_path),
'/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234'
)
@skipIfCantAuthenticate
def test_resolve_lookup_filename_version_no_user(self):
"""
Test resolve_lookup_filename() to resolve a lookup file name to the backed up version
without providing a user.
"""
file_path = self.lookup_editor.resolve_lookup_filename(
'test.csv',
'search',
None,
False,
'1234',
session_key=self.get_session_key())
self.assertEquals(
self.strip_splunk_path(file_path),
'/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234'
)
@skipIfLookupTestNotInstalled
def test_get_kv_fields_from_transform(self):
"""
Test getting the fields of a lookup from a transform.
"""
fields = self.lookup_editor.get_kv_fields_from_transform(
self.get_session_key(), 'test_kv_store_transform_fields',
'lookup_test', None)
self.assertEquals(len(fields), 4)
class TestLookupEditor(LookupEditorTestCase):
"""
This tests the class which manages lookup backups.
"""
def setUp(self):
self.lookup_editor = LookupEditor(logger=logger)
@skipIfCantAuthenticate
def test_resolve_lookup_filename(self):
"""
Test resolve_lookup_filename() to resolve a lookup file name.
"""
file_path = self.lookup_editor.resolve_lookup_filename(
'test.csv',
'search',
'nobody',
False,
None,
session_key=self.get_session_key())
self.assertEquals(self.strip_splunk_path(file_path),
'/etc/apps/lookup_editor/lookups/test.csv')
@skipIfCantAuthenticate
def test_resolve_lookup_filename_version(self):
"""
Test resolve_lookup_filename() to resolve a lookup file name to the backed up version.
"""
file_path = self.lookup_editor.resolve_lookup_filename(
'test.csv',
'search',
'nobody',
False,
'1234',
session_key=self.get_session_key())
self.assertEquals(
self.strip_splunk_path(file_path),
'/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234'
)
@skipIfCantAuthenticate
def test_resolve_lookup_filename_version_no_user(self):
"""
Test resolve_lookup_filename() to resolve a lookup file name to the backed up version
without providing a user.
"""
file_path = self.lookup_editor.resolve_lookup_filename(
'test.csv',
'search',
None,
False,
'1234',
session_key=self.get_session_key())
self.assertEquals(
self.strip_splunk_path(file_path),
'/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234'
)