class LookupBackupsHandler(rest_handler.RESTHandler): """ This is a REST handler that supports backing up lookup files. This is broken out as a separate handler so that this handler can be replayed on other search heads via the allowRestReplay setting in restmap.conf. """ def __init__(self, command_line, command_arg): super(LookupBackupsHandler, self).__init__(command_line, command_arg, logger) self.lookup_editor = LookupEditor(logger) def post_backup(self, request_info, lookup_file=None, namespace="lookup_editor", owner=None, file_time=None, **kwargs): """ Make a backup of the given lookup file. """ try: # Determine the final path of the file resolved_file_path = self.lookup_editor.resolve_lookup_filename( lookup_file, namespace, owner, session_key=request_info.session_key, throw_not_found=True) # Backup the file passing the time so that the REST handler will use the same time for # all lookups even when the REST call is being replayed in an SHC cluster file_path = self.lookup_editor.backup_lookup_file( request_info.session_key, lookup_file, namespace, resolved_file_path, owner, file_time) self.logger.info("Created a backup of a lookup file, file_path=%s", file_path) # Everything worked, return accordingly return { 'payload': str(file_path), # Payload of the request. 'status': 200 # HTTP status code } except ResourceNotFound: self.logger.warn("Unable to find the lookup to backup") return self.render_error_json( "Unable to find the lookup to backup", 404) except: self.logger.exception( "Exception generated when attempting to backup a lookup file") return self.render_error_json("Unable to backup the lookup")
class TestLookupEditor(LookupEditorTestCase): """ This tests the class which manages lookup backups. """ def setUp(self): self.lookup_editor = LookupEditor(logger=logger) @skipIfCantAuthenticate def test_resolve_lookup_filename(self): """ Test resolve_lookup_filename() to resolve a lookup file name. """ file_path = self.lookup_editor.resolve_lookup_filename('test.csv', 'search', 'nobody', False, None, session_key=self.get_session_key()) self.assertEquals(self.strip_splunk_path(file_path), '/etc/apps/lookup_editor/lookups/test.csv') @skipIfCantAuthenticate def test_resolve_lookup_filename_version(self): """ Test resolve_lookup_filename() to resolve a lookup file name to the backed up version. """ file_path = self.lookup_editor.resolve_lookup_filename('test.csv', 'search', 'nobody', False, '1234', session_key=self.get_session_key()) self.assertEquals(self.strip_splunk_path(file_path), '/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234') @skipIfCantAuthenticate def test_resolve_lookup_filename_version_no_user(self): """ Test resolve_lookup_filename() to resolve a lookup file name to the backed up version without providing a user. """ file_path = self.lookup_editor.resolve_lookup_filename('test.csv', 'search', None, False, '1234', session_key=self.get_session_key()) self.assertEquals(self.strip_splunk_path(file_path), '/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234') @skipIfLookupTestNotInstalled def test_get_kv_fields_from_transform(self): """ Test getting the fields of a lookup from a transform. """ fields = self.lookup_editor.get_kv_fields_from_transform(self.get_session_key(), 'test_kv_store_transform_fields', 'lookup_test', None) self.assertEquals(len(fields), 4)
class LookupBackupsHandler(rest_handler.RESTHandler): """ This is a REST handler that supports backing up lookup files. This is broken out as a separate handler so that this handler can be replayed on other search heads via the allowRestReplay setting in restmap.conf. """ def __init__(self, command_line, command_arg): super(LookupBackupsHandler, self).__init__(command_line, command_arg, logger) self.lookup_editor = LookupEditor(logger) def post_backup(self, request_info, lookup_file=None, namespace="lookup_editor", owner=None, file_time=None, **kwargs): """ Make a backup of the given lookup file. """ try: # Determine the final path of the file resolved_file_path = self.lookup_editor.resolve_lookup_filename(lookup_file, namespace, owner, session_key=request_info.session_key, throw_not_found=True) # Backup the file passing the time so that the REST handler will use the same time for # all lookups even when the REST call is being replayed in an SHC cluster file_path = self.lookup_editor.backup_lookup_file(request_info.session_key, lookup_file, namespace, resolved_file_path, owner, file_time) self.logger.info("Created a backup of a lookup file, file_path=%s", file_path) # Everything worked, return accordingly return { 'payload': str(file_path), # Payload of the request. 'status': 200 # HTTP status code } except ResourceNotFound: self.logger.warn("Unable to find the lookup to backup") return self.render_error_json("Unable to find the lookup to backup", 404) except: self.logger.exception("Exception generated when attempting to backup a lookup file") return self.render_error_json("Unable to backup the lookup")
def test_get_backup_directory_with_resolved(self): """ Ensure that get_backup_directory() returns the correct directory when given a value for resolved_lookup_path. """ self.lookup_editor = LookupEditor(logger=logger) resolved_lookup_path = self.lookup_editor.resolve_lookup_filename( 'test.csv', 'search', 'nobody', False, None, session_key=self.get_session_key()) dir = self.lookup_backups.get_backup_directory( self.get_session_key(), "test.csv", namespace="search", owner="nobody", resolved_lookup_path=resolved_lookup_path) self.assertEquals( self.strip_splunk_path(dir), "/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv" )
class TestLookupBackups(LookupEditorTestCase): """ This tests the class which manages lookup backups. """ def setUp(self): self.lookup_backups = lookup_backups.LookupBackups(logger=logger) @skipIfCantAuthenticate def test_get_backup_directory(self): """ Ensure that get_backup_directory() returns the correct directory. """ dir = self.lookup_backups.get_backup_directory(self.get_session_key(), "test.csv", namespace="search", owner="nobody") self.assertEquals( self.strip_splunk_path(dir), "/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv" ) @skipIfCantAuthenticate def test_get_backup_directory_with_resolved(self): """ Ensure that get_backup_directory() returns the correct directory when given a value for resolved_lookup_path. """ self.lookup_editor = LookupEditor(logger=logger) resolved_lookup_path = self.lookup_editor.resolve_lookup_filename( 'test.csv', 'search', 'nobody', False, None, session_key=self.get_session_key()) dir = self.lookup_backups.get_backup_directory( self.get_session_key(), "test.csv", namespace="search", owner="nobody", resolved_lookup_path=resolved_lookup_path) self.assertEquals( self.strip_splunk_path(dir), "/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv" )
def test_get_backup_directory_with_resolved(self): """ Ensure that get_backup_directory() returns the correct directory when given a value for resolved_lookup_path. """ self.lookup_editor = LookupEditor(logger=logger) resolved_lookup_path = self.lookup_editor.resolve_lookup_filename('test.csv', 'search', 'nobody', False, None, session_key=self.get_session_key()) dir = self.lookup_backups.get_backup_directory(self.get_session_key(), "test.csv", namespace="search", owner="nobody", resolved_lookup_path=resolved_lookup_path) self.assertEquals(self.strip_splunk_path(dir), "/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv")
class TestLookupBackups(LookupEditorTestCase): """ This tests the class which manages lookup backups. """ def setUp(self): self.lookup_backups = lookup_backups.LookupBackups(logger=logger) @skipIfCantAuthenticate def test_get_backup_directory(self): """ Ensure that get_backup_directory() returns the correct directory. """ dir = self.lookup_backups.get_backup_directory(self.get_session_key(), "test.csv", namespace="search", owner="nobody") self.assertEquals(self.strip_splunk_path(dir), "/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv") @skipIfCantAuthenticate def test_get_backup_directory_with_resolved(self): """ Ensure that get_backup_directory() returns the correct directory when given a value for resolved_lookup_path. """ self.lookup_editor = LookupEditor(logger=logger) resolved_lookup_path = self.lookup_editor.resolve_lookup_filename('test.csv', 'search', 'nobody', False, None, session_key=self.get_session_key()) dir = self.lookup_backups.get_backup_directory(self.get_session_key(), "test.csv", namespace="search", owner="nobody", resolved_lookup_path=resolved_lookup_path) self.assertEquals(self.strip_splunk_path(dir), "/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv")
def __init__(self, command_line, command_arg): super(LookupEditorHandler, self).__init__(command_line, command_arg, logger) self.lookup_editor = LookupEditor(logger)
class LookupEditorHandler(rest_handler.RESTHandler): """ This is a REST handler that supports editing lookup files. All calls from the user-interface should pass through this handler. """ def __init__(self, command_line, command_arg): super(LookupEditorHandler, self).__init__(command_line, command_arg, logger) self.lookup_editor = LookupEditor(logger) def get_lookup_info(self, request_info, lookup_file=None, namespace="lookup_editor", **kwargs): """ Get information about a lookup file (owner, size, etc.) """ return { 'payload': str(lookup_file), # Payload of the request. 'status': 200 # HTTP status code } def get_lookup_backups(self, request_info, lookup_file=None, namespace=None, owner=None, **kwargs): """ Get a list of the lookup file backups rendered as JSON. """ backups = self.lookup_editor.get_backup_files(request_info.session_key, lookup_file, namespace, owner) # Make the response backups_meta = [] for backup in backups: try: backups_meta.append( { 'time': backup, 'time_readable' : datetime.datetime.fromtimestamp(float(backup)).strftime('%Y-%m-%d %H:%M:%S') } ) except ValueError: self.logger.warning("Backup file name is invalid, file_name=%s", backup) # Sort the list backups_meta = sorted(backups_meta, key=lambda x: float(x['time']), reverse=True) return self.render_json(backups_meta) def get_lookup_contents(self, request_info, lookup_file=None, namespace="lookup_editor", owner=None, header_only=False, version=None, lookup_type=None, **kwargs): """ Provides the contents of a lookup file as JSON. """ self.logger.info("Retrieving lookup contents, namespace=%s, lookup=%s, type=%s, owner=%s," " version=%s", namespace, lookup_file, lookup_type, owner, version) if lookup_type is None or len(lookup_type) == 0: lookup_type = "csv" self.logger.warning("No type for the lookup provided when attempting to load a lookup" + " file, it will default to CSV") if header_only in ["1", "true", 1, True]: header_only = True else: header_only = False try: # Load the KV store lookup if lookup_type == "kv": return self.render_json(self.lookup_editor.get_kv_lookup(lookup_file, namespace, owner)) # Load the CSV lookup elif lookup_type == "csv": with self.lookup_editor.get_lookup(request_info.session_key, lookup_file, namespace, owner, version=version, throw_exception_if_too_big=True) as csv_file: csv_reader = csv.reader(csv_file) # Convert the content to JSON lookup_contents = [] for row in csv_reader: lookup_contents.append(row) # If we are only loading the header, then stop here if header_only: break return self.render_json(lookup_contents) else: self.logger.warning('Lookup file type is not recognized,' + ' lookup_type=' + lookup_type) return self.render_error_json('Lookup file type is not recognized', 421) except IOError: self.logger.warning("Unable to find the requested lookup") return self.render_error_json("Unable to find the lookup", 404) except (AuthorizationFailed, PermissionDeniedException) as e: self.logger.warning("Access to lookup denied") return self.render_error_json(str(e), 403) except LookupFileTooBigException as e: self.logger.warning("Lookup file is too large to load") data = { 'message': 'Lookup file is too large to load' + '(file-size must be less than 10 MB to be edited)', 'file_size' : e.file_size } return { 'payload': json.dumps(data), 'status': 420 } except: self.logger.exception('Lookup file could not be loaded') return self.render_error_json('Lookup file could not be loaded', 500) return { 'payload': 'Response', 'status': 500 } def get_lookup_as_file(self, request_info, lookup_file=None, namespace="lookup_editor", owner=None, lookup_type='csv', **kwargs): """ Provides the lookup file in a way to be downloaded by the browser """ logger.info("Exporting lookup, namespace=%s, lookup=%s, type=%s, owner=%s", namespace, lookup_file, lookup_type, owner) try: # If we are getting the CSV, then just pipe the file to the user if lookup_type == "csv": with self.lookup_editor.get_lookup(request_info.session_key, lookup_file, namespace, owner) as csv_file_handle: csv_data = csv_file_handle.read() # If we are getting a KV store lookup, then convert it to a CSV file else: rows = self.lookup_editor.get_kv_lookup(lookup_file, namespace, owner) csv_data = shortcuts.convert_array_to_csv(rows) # Tell the browser to download this as a file if lookup_file.endswith(".csv"): filename = 'attachment; filename="%s"' % lookup_file else: filename = 'attachment; filename="%s"' % (lookup_file + ".csv") return { 'payload': json.dumps(csv_data), 'status': 200, 'headers': { 'Content-Type': 'text/csv', 'Content-Disposition': filename }, } except IOError: return self.render_json([], 404) except PermissionDeniedException as exception: return self.render_error_json(str(exception), 403) return { 'payload': str(lookup_file), # Payload of the request. 'status': 200 # HTTP status code } def post_lookup_contents(self, request_info, lookup_file=None, namespace="lookup_editor", owner=None, header_only=False, version=None, lookup_type=None, **kwargs): """ Save the JSON contents to the lookup file """ return { 'payload': str(lookup_file), # Payload of the request. 'status': 200 # HTTP status code }
def setUp(self): self.lookup_editor = LookupEditor(logger=logger)
class LookupEditorHandler(rest_handler.RESTHandler): """ This is a REST handler that supports editing lookup files. All calls from the user-interface should pass through this handler. """ def __init__(self, command_line, command_arg): super(LookupEditorHandler, self).__init__(command_line, command_arg, logger) self.lookup_editor = LookupEditor(logger) def get_lookup_info(self, request_info, lookup_file=None, namespace="lookup_editor", **kwargs): """ Get information about a lookup file (owner, size, etc.) """ return { 'payload': str(lookup_file), # Payload of the request. 'status': 200 # HTTP status code } def get_lookup_backups(self, request_info, lookup_file=None, namespace=None, owner=None, **kwargs): """ Get a list of the lookup file backups rendered as JSON. """ backups = self.lookup_editor.get_backup_files(request_info.session_key, lookup_file, namespace, owner) # Make the response backups_meta = [] for backup in backups: try: backups_meta.append({ 'time': backup, 'time_readable': datetime.datetime.fromtimestamp( float(backup)).strftime('%Y-%m-%d %H:%M:%S') }) except ValueError: self.logger.warning( "Backup file name is invalid, file_name=%s", backup) # Sort the list backups_meta = sorted(backups_meta, key=lambda x: float(x['time']), reverse=True) return self.render_json(backups_meta) def get_lookup_contents(self, request_info, lookup_file=None, namespace="lookup_editor", owner=None, header_only=False, version=None, lookup_type=None, **kwargs): """ Provides the contents of a lookup file as JSON. """ self.logger.info( "Retrieving lookup contents, namespace=%s, lookup=%s, type=%s, owner=%s," " version=%s", namespace, lookup_file, lookup_type, owner, version) if lookup_type is None or len(lookup_type) == 0: lookup_type = "csv" self.logger.warning( "No type for the lookup provided when attempting to load a lookup" + " file, it will default to CSV") if header_only in ["1", "true", 1, True]: header_only = True else: header_only = False try: # Load the KV store lookup if lookup_type == "kv": return self.render_json( self.lookup_editor.get_kv_lookup(request_info.session_key, lookup_file, namespace, owner)) # Load the CSV lookup elif lookup_type == "csv": with self.lookup_editor.get_lookup( request_info.session_key, lookup_file, namespace, owner, version=version, throw_exception_if_too_big=True) as csv_file: csv_reader = csv.reader(csv_file) # Convert the content to JSON lookup_contents = [] for row in csv_reader: lookup_contents.append(row) # If we are only loading the header, then stop here if header_only: break return self.render_json(lookup_contents) else: self.logger.warning('Lookup file type is not recognized,' + ' lookup_type=' + lookup_type) return self.render_error_json( 'Lookup file type is not recognized', 421) except IOError: self.logger.warning("Unable to find the requested lookup") return self.render_error_json("Unable to find the lookup", 404) except (AuthorizationFailed, PermissionDeniedException) as e: self.logger.warning("Access to lookup denied") return self.render_error_json(str(e), 403) except LookupFileTooBigException as e: self.logger.warning("Lookup file is too large to load") data = { 'message': 'Lookup file is too large to load' + '(file-size must be less than 10 MB to be edited)', 'file_size': e.file_size } return {'payload': json.dumps(data), 'status': 420} except: self.logger.exception('Lookup file could not be loaded') return self.render_error_json('Lookup file could not be loaded', 500) return {'payload': 'Response', 'status': 500} def get_lookup_as_file(self, request_info, lookup_file=None, namespace="lookup_editor", owner=None, lookup_type='csv', **kwargs): """ Provides the lookup file in a way to be downloaded by the browser """ self.logger.info( "Exporting lookup, namespace=%s, lookup=%s, type=%s, owner=%s", namespace, lookup_file, lookup_type, owner) try: # If we are getting the CSV, then just pipe the file to the user if lookup_type == "csv": with self.lookup_editor.get_lookup(request_info.session_key, lookup_file, namespace, owner) as csv_file_handle: csv_data = csv_file_handle.read() # If we are getting a KV store lookup, then convert it to a CSV file else: rows = self.lookup_editor.get_kv_lookup( request_info.session_key, lookup_file, namespace, owner) csv_data = shortcuts.convert_array_to_csv(rows) # Tell the browser to download this as a file if lookup_file.endswith(".csv"): filename = 'attachment; filename="%s"' % lookup_file else: filename = 'attachment; filename="%s"' % (lookup_file + ".csv") return { 'payload': csv_data, 'status': 200, 'headers': { 'Content-Type': 'text/csv', 'Content-Disposition': filename }, } except IOError: return self.render_json([], 404) except PermissionDeniedException as exception: return self.render_error_json(str(exception), 403) return { 'payload': str(lookup_file), # Payload of the request. 'status': 200 # HTTP status code } def post_lookup_contents(self, request_info, contents=None, lookup_file=None, namespace="lookup_editor", owner=None, **kwargs): """ Save the JSON contents to the lookup file. """ self.logger.info("Saving lookup contents...") try: # Backup the lookup file data = { 'lookup_file': lookup_file, 'namespace': namespace, 'owner': owner } try: _, _ = simpleRequest('/services/data/lookup_backup/backup', sessionKey=request_info.session_key, method='POST', postargs=data) except ResourceNotFound: self.logger.info( "Existing lookup could not be found for backup") file_name = self.lookup_editor.update(contents, lookup_file, namespace, owner, request_info.session_key, request_info.user) # Everything worked, return accordingly return { 'payload': str(file_name), # Payload of the request. 'status': 200 # HTTP status code } except LookupNameInvalidException: return self.render_error_json("Lookup name is invalid", 400) except: self.logger.exception("Unable to save the lookup") return self.render_error_json("Unable to save the lookup")
class TestLookupEditor(LookupEditorTestCase): """ This tests the class which manages lookup backups. """ def setUp(self): self.lookup_editor = LookupEditor(logger=logger) @skipIfCantAuthenticate def test_resolve_lookup_filename(self): """ Test resolve_lookup_filename() to resolve a lookup file name. """ file_path = self.lookup_editor.resolve_lookup_filename( 'test.csv', 'search', 'nobody', False, None, session_key=self.get_session_key()) self.assertEquals(self.strip_splunk_path(file_path), '/etc/apps/lookup_editor/lookups/test.csv') @skipIfCantAuthenticate def test_resolve_lookup_filename_version(self): """ Test resolve_lookup_filename() to resolve a lookup file name to the backed up version. """ file_path = self.lookup_editor.resolve_lookup_filename( 'test.csv', 'search', 'nobody', False, '1234', session_key=self.get_session_key()) self.assertEquals( self.strip_splunk_path(file_path), '/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234' ) @skipIfCantAuthenticate def test_resolve_lookup_filename_version_no_user(self): """ Test resolve_lookup_filename() to resolve a lookup file name to the backed up version without providing a user. """ file_path = self.lookup_editor.resolve_lookup_filename( 'test.csv', 'search', None, False, '1234', session_key=self.get_session_key()) self.assertEquals( self.strip_splunk_path(file_path), '/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234' ) @skipIfLookupTestNotInstalled def test_get_kv_fields_from_transform(self): """ Test getting the fields of a lookup from a transform. """ fields = self.lookup_editor.get_kv_fields_from_transform( self.get_session_key(), 'test_kv_store_transform_fields', 'lookup_test', None) self.assertEquals(len(fields), 4)
class TestLookupEditor(LookupEditorTestCase): """ This tests the class which manages lookup backups. """ def setUp(self): self.lookup_editor = LookupEditor(logger=logger) @skipIfCantAuthenticate def test_resolve_lookup_filename(self): """ Test resolve_lookup_filename() to resolve a lookup file name. """ file_path = self.lookup_editor.resolve_lookup_filename( 'test.csv', 'search', 'nobody', False, None, session_key=self.get_session_key()) self.assertEquals(self.strip_splunk_path(file_path), '/etc/apps/lookup_editor/lookups/test.csv') @skipIfCantAuthenticate def test_resolve_lookup_filename_version(self): """ Test resolve_lookup_filename() to resolve a lookup file name to the backed up version. """ file_path = self.lookup_editor.resolve_lookup_filename( 'test.csv', 'search', 'nobody', False, '1234', session_key=self.get_session_key()) self.assertEquals( self.strip_splunk_path(file_path), '/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234' ) @skipIfCantAuthenticate def test_resolve_lookup_filename_version_no_user(self): """ Test resolve_lookup_filename() to resolve a lookup file name to the backed up version without providing a user. """ file_path = self.lookup_editor.resolve_lookup_filename( 'test.csv', 'search', None, False, '1234', session_key=self.get_session_key()) self.assertEquals( self.strip_splunk_path(file_path), '/etc/apps/lookup_editor/lookups/lookup_file_backups/search/nobody/test.csv/1234' )