def create_cache(self):
"""Create a cache from all namespaces when the napp setup."""
backend = FileSystem()
for namespace in backend.list_namespaces():
if namespace not in self.metadata_cache:
self.metadata_cache[namespace] = []
for box_id in backend.list(namespace):
box = backend.retrieve(namespace, box_id)
cache = metadata_from_box(box)
self.metadata_cache[namespace].append(cache)
class Main(KytosNApp):
"""Main class of kytos/storehouse NApp.
This class is the entry point for this napp.
"""
metadata_cache = {}
def setup(self):
"""Replace the '__init__' method for the KytosNApp subclass.
Execute right after the NApp is loaded.
"""
# pylint: disable=import-outside-toplevel
if settings.BACKEND == "etcd":
from napps.kytos.storehouse.backends.etcd import Etcd
log.info("Loading 'etcd' backend...")
self.backend = Etcd()
else:
from napps.kytos.storehouse.backends.fs import FileSystem
log.info("Loading 'filesystem' backend...")
self.backend = FileSystem()
self.metadata_cache = {}
self.create_cache()
log.info("Storehouse NApp started.")
def execute(self):
"""Execute after the setup method."""
def create_cache(self):
"""Create a cache from all namespaces when the napp setup."""
log.debug('Creating storehouse cache...')
for namespace in self.backend.list_namespaces():
if namespace not in self.metadata_cache:
self.metadata_cache[namespace] = []
for box_id in self.backend.list(namespace):
box = self.backend.retrieve(namespace, box_id)
log.debug("Loading box '%s'...", box)
cache = metadata_from_box(box)
self.metadata_cache[namespace].append(cache)
def delete_metadata_from_cache(self, namespace, box_id=None):
"""Delete a metadata from cache.
Args:
namespace(str): A namespace, when the box will be deleted
box_id(str): Box identifier
"""
for cache in self.metadata_cache.get(namespace, []):
if (box_id and box_id in cache["box_id"]):
self.metadata_cache.get(namespace, []).remove(cache)
def add_metadata_to_cache(self, box):
"""Add a box cache into the namespace cache."""
cache = metadata_from_box(box)
if box.namespace not in self.metadata_cache:
self.metadata_cache[box.namespace] = []
self.metadata_cache[box.namespace].append(cache)
def search_metadata_by(self, namespace, filter_option="box_id", query=""):
"""Search for all metadata with specific pattern.
Args:
namespace(str): namespace where the box is stored
filter_option(str): metadata option
query(str): query to be searched
Returns:
list: list of metadata box filtered
"""
namespace_cache = self.metadata_cache.get(namespace, [])
results = []
for metadata in namespace_cache:
field_value = metadata.get(filter_option, "")
if re.match(f".*{query}.*", field_value):
results.append(metadata)
return results
@staticmethod
def _execute_callback(event, data, error):
"""Run the callback function for event calls to the NApp."""
try:
event.content['callback'](event, data, error)
except KeyError:
log.error(f'Event {event!r} without callback function!')
except TypeError as exception:
log.error(f"Bad callback function {event.content['callback']}!")
log.error(exception)
@rest('v1/<namespace>', methods=['POST'])
def rest_create(self, namespace):
"""Create a box in a namespace based on JSON input."""
data = request.get_json(silent=True)
if not data:
return jsonify({"response": "Invalid Request"}), 400
box = Box(data, namespace)
self.backend.create(box)
self.add_metadata_to_cache(box)
result = {"response": "Box created.", "id": box.box_id}
return jsonify(result), 201
@rest('v2/<namespace>', methods=['POST'])
@rest('v2/<namespace>/<box_id>', methods=['POST'])
def rest_create_v2(self, namespace, box_id=None):
"""Create a box in a namespace based on JSON input."""
data = request.get_json(silent=True)
if not data:
return jsonify({"response": "Invalid Request"}), 400
box = Box(data, namespace, box_id=box_id)
self.backend.create(box)
self.add_metadata_to_cache(box)
result = {"response": "Box created.", "id": box.box_id}
return jsonify(result), 201
@rest('v1/<namespace>', methods=['GET'])
def rest_list(self, namespace):
"""List all boxes in a namespace."""
result = self.backend.list(namespace)
return jsonify(result), 200
@rest('v1/<namespace>/<box_id>', methods=['PUT', 'PATCH'])
def rest_update(self, namespace, box_id):
"""Update a box_id from namespace."""
data = request.get_json(silent=True)
if not data:
return jsonify({"response": "Invalid request: empty data"}), 400
box = self.backend.retrieve(namespace, box_id)
if not box:
return jsonify({"response": "Not Found"}), 404
if request.method == "PUT":
box.data = data
else:
box.data.update(data)
self.backend.update(namespace, box)
return jsonify(box.data), 200
@rest('v1/<namespace>/<box_id>', methods=['GET'])
def rest_retrieve(self, namespace, box_id):
"""Retrieve and return a box from a namespace."""
box = self.backend.retrieve(namespace, box_id)
if not box:
return jsonify({"response": "Not Found"}), 404
return jsonify(box.data), 200
@rest('v1/<namespace>/<box_id>', methods=['DELETE'])
def rest_delete(self, namespace, box_id):
"""Delete a box from a namespace."""
result = self.backend.delete(namespace, box_id)
if result:
self.delete_metadata_from_cache(namespace, box_id)
return jsonify({"response": "Box deleted"}), 200
# or 204 - No Content
return jsonify({"response": "Box not found"}), 404
@rest("v1/<namespace>/search_by/<filter_option>/<query>", methods=['GET'])
def rest_search_by(self, namespace, filter_option="name", query=""):
"""Filter the boxes with specific pattern.
Args:
namespace(str): namespace where the box is stored
filter_option(str): metadata option
query(str): query to be searched
Returns:
list: list of metadata box filtered
"""
results = self.search_metadata_by(namespace, filter_option, query)
if not results:
return jsonify({"response": f"{filter_option} not found"}), 404
return jsonify(results), 200
@listen_to('kytos.storehouse.create')
def event_create(self, event):
"""Create a box in a namespace based on an event."""
error = None
box_id = event.content.get('box_id')
try:
data = event.content['data']
namespace = event.content['namespace']
if self.search_metadata_by(namespace, query=box_id):
raise KeyError("Box id already exists.")
except KeyError as exc:
box = None
error = exc
else:
box = Box(data, namespace, box_id=box_id)
self.backend.create(box)
self.add_metadata_to_cache(box)
self._execute_callback(event, box, error)
@listen_to('kytos.storehouse.retrieve')
def event_retrieve(self, event):
"""Retrieve a box from a namespace based on an event."""
error = None
try:
box = self.backend.retrieve(event.content['namespace'],
event.content['box_id'])
except KeyError as exc:
box = None
error = exc
self._execute_callback(event, box, error)
@listen_to('kytos.storehouse.update')
def event_update(self, event):
"""Update a box_id from namespace.
This method receive an KytosEvent with the content bellow.
namespace: namespace where the box is stored
box_id: the box identify
method: 'PUT' or 'PATCH', the default update method is 'PATCH'
data: a python dict with the data
"""
error = False
try:
namespace = event.content['namespace']
box_id = event.content['box_id']
box = self.backend.retrieve(namespace, box_id)
if not box:
raise KeyError("Box id does not exist.")
except KeyError as exc:
box = None
error = exc
else:
method = event.content.get('method', 'PATCH')
data = event.content.get('data', {})
if method == 'PUT':
box.data = data
elif method == 'PATCH':
box.data.update(data)
self.backend.update(namespace, box)
self._execute_callback(event, box, error)
@listen_to('kytos.storehouse.delete')
def event_delete(self, event):
"""Delete a box from a namespace based on an event."""
error = None
try:
namespace = event.content['namespace']
box_id = event.content['box_id']
except KeyError as exc:
result = None
error = exc
else:
result = self.backend.delete(namespace, box_id)
self.delete_metadata_from_cache(namespace, box_id)
self._execute_callback(event, result, error)
@listen_to('kytos.storehouse.list')
def event_list(self, event):
"""List all boxes in a namespace based on an event."""
error = None
try:
result = self.backend.list(event.content['namespace'])
except KeyError as exc:
result = None
error = exc
self._execute_callback(event, result, error)
@rest("v1/backup/<namespace>/", methods=['GET'])
@rest("v1/backup/<namespace>/<box_id>", methods=['GET'])
def rest_backup(self, namespace, box_id=None):
"""Backup an entire namespace or an object based on its id."""
try:
return jsonify(self.backend.backup(namespace, box_id)), 200
except ValueError:
return jsonify({"response": "Not Found"}), 404
def shutdown(self):
"""Execute before the NApp is unloaded."""
log.info("Storehouse NApp is shutting down.")
class TestFileSystem(TestCase):
"""Tests for the FileSystem class."""
# pylint: disable=arguments-differ
@patch('napps.kytos.storehouse.backends.fs.FileSystem._parse_settings')
def setUp(self, mock_parse_settings):
"""Execute steps before each tests."""
mock_parse_settings.return_value = MagicMock()
self.file_system = FileSystem()
@staticmethod
@patch('napps.kytos.storehouse.backends.fs.Path')
def test_create_dirs(mock_path):
"""Test _create_dirs method."""
_create_dirs('destination')
mock_path.assert_called_with('destination')
mock_path.return_value.mkdir.assert_called()
@patch('napps.kytos.storehouse.backends.fs._create_dirs')
@patch('os.environ.get', return_value='/')
def test_parse_settings(self, *args):
"""Test _parse_settings method."""
(_, mock_create_dirs) = args
self.file_system._parse_settings()
calls = [
call(self.file_system.destination_path),
call(self.file_system.lock_path)
]
mock_create_dirs.assert_has_calls(calls)
@patch('napps.kytos.storehouse.backends.fs.Path')
def test_get_destination(self, mock_path):
"""Test _get_destination method."""
destination = self.file_system._get_destination('namespace')
mock_path.assert_called_with(self.file_system.destination_path,
'namespace')
self.assertEqual(destination, mock_path.return_value)
@patch('pickle.dump')
@patch('builtins.open')
@patch('napps.kytos.storehouse.backends.fs.FileLock')
def test_write_to_file(self, *args):
"""Test _write_to_file method."""
(_, mock_open, mock_dump) = args
save_file = MagicMock()
mock_open.return_value = save_file
box = MagicMock()
self.file_system._write_to_file('filename', box)
mock_dump.assert_called_with(box, save_file.__enter__())
@patch('pickle.load')
@patch('builtins.open')
@patch('napps.kytos.storehouse.backends.fs.FileLock')
def test_load_from_file(self, *args):
"""Test _load_from_file method."""
(_, mock_open, mock_load) = args
load_file = MagicMock()
mock_open.return_value = load_file
mock_load.return_value = 'data'
data = self.file_system._load_from_file('filename')
mock_load.assert_called_with(load_file.__enter__())
self.assertEqual(data, 'data')
def test_delete_file(self):
"""Test _delete_file method to success and failure cases."""
path = MagicMock()
path.exists.side_effect = [True, False]
path.unlink.return_value = 'unlink'
return_1 = self.file_system._delete_file(path)
return_2 = self.file_system._delete_file(path)
self.assertTrue(return_1)
self.assertFalse(return_2)
@patch('napps.kytos.storehouse.backends.fs.Path')
def test_list_namespace_success_case(self, mock_path):
"""Test _list_namespace method to success case."""
content = []
for is_dir in [False, False, True]:
obj = MagicMock()
obj.name = 'name'
obj.is_dir.return_value = is_dir
content.append(obj)
path = MagicMock()
path.exists.return_value = True
path.iterdir.return_value = content
mock_path.return_value = path
list_namespace = self.file_system._list_namespace('namespace')
self.assertEqual(list_namespace, [content[0].name, content[1].name])
@patch('napps.kytos.storehouse.backends.fs.Path')
def test_list_namespace_failure_case(self, mock_path):
"""Test _list_namespace method to failure case."""
path = MagicMock()
path.exists.return_value = False
mock_path.return_value = path
list_namespace = self.file_system._list_namespace('namespace')
self.assertEqual(list_namespace, [])
@patch('napps.kytos.storehouse.backends.fs.Path')
@patch('napps.kytos.storehouse.backends.fs._create_dirs')
@patch('napps.kytos.storehouse.backends.fs.FileSystem._write_to_file')
def test_create(self, *args):
"""Test create method."""
(mock_write_to_file, mock_create_dirs, mock_path) = args
destination = MagicMock()
mock_path.return_value = destination
box = Box('any', 'namespace', box_id='123')
self.file_system.create(box)
mock_path.assert_called_with(self.file_system.destination_path,
'namespace')
mock_create_dirs.assert_called_with(destination)
mock_write_to_file.assert_called_with(destination.joinpath('123'), box)
@patch('napps.kytos.storehouse.backends.fs.Path')
@patch('napps.kytos.storehouse.backends.fs.FileSystem._load_from_file',
return_value='data')
def test_retrieve_success_case(self, *args):
"""Test retrieve method to success case."""
(mock_load_from_file, mock_path) = args
path = MagicMock()
destination = MagicMock()
destination.is_file.return_value = True
mock_path.return_value = path
path.joinpath.return_value = destination
box = Box('any', 'namespace', box_id='123')
retrieve = self.file_system.retrieve(box.namespace, box.box_id)
mock_path.assert_called_with(self.file_system.destination_path,
'namespace')
mock_load_from_file.assert_called_with(destination)
self.assertEqual(retrieve, 'data')
@patch('napps.kytos.storehouse.backends.fs.Path')
def test_retrieve_failure_case(self, mock_path):
"""Test retrieve method to failure case."""
path = MagicMock()
destination = MagicMock()
destination.is_file.return_value = False
mock_path.return_value = path
path.joinpath.return_value = destination
box = Box('any', 'namespace', box_id='123')
retrieve = self.file_system.retrieve(box.namespace, box.box_id)
self.assertFalse(retrieve)
@patch('napps.kytos.storehouse.backends.fs.Path')
@patch('napps.kytos.storehouse.backends.fs.FileSystem._write_to_file')
def test_update(self, *args):
"""Test update method."""
(mock_write, mock_path) = args
destination = MagicMock()
mock_path.return_value = destination
box = Box('any', 'namespace', box_id='123')
self.file_system.update(box.namespace, box)
mock_path.assert_called_with(self.file_system.destination_path,
'namespace')
destination.joinpath.assert_called_with('123')
mock_write.assert_called_with(destination.joinpath.return_value, box)
@patch('napps.kytos.storehouse.backends.fs.Path')
@patch('napps.kytos.storehouse.backends.fs.FileSystem._delete_file')
def test_delete(self, *args):
"""Test delete method."""
(mock_delete_file, mock_path) = args
path = MagicMock()
destination = MagicMock()
mock_path.return_value = path
path.joinpath.return_value = destination
box = Box('any', 'namespace', box_id='123')
self.file_system.delete(box.namespace, box.box_id)
mock_path.assert_called_with(self.file_system.destination_path,
'namespace')
path.joinpath.assert_called_with('123')
mock_delete_file.assert_called_with(destination)
@patch('napps.kytos.storehouse.backends.fs.FileSystem._list_namespace')
def test_list(self, mock_list_namespace):
"""Test list method."""
list_namespace = self.file_system.list('namespace')
mock_list_namespace.assert_called_with('namespace')
self.assertEqual(list_namespace, mock_list_namespace.return_value)
@patch('napps.kytos.storehouse.backends.fs.Path')
def test_list_namespaces_success_case(self, mock_path):
"""Test list_namespaces method to success case."""
content = []
for is_dir in [False, False, True]:
obj = MagicMock()
obj.name = 'name'
obj.is_dir.return_value = is_dir
content.append(obj)
path = MagicMock()
path.exists.return_value = True
path.iterdir.return_value = content
mock_path.return_value = path
list_namespace = self.file_system.list_namespaces()
mock_path.assert_called_with(self.file_system.destination_path, '.')
self.assertEqual(list_namespace, [content[2].name])
@patch('napps.kytos.storehouse.backends.fs.Path')
def test_list_namespaces_failure_case(self, mock_path):
"""Test list_namespaces method to failure case."""
path = MagicMock()
path.exists.return_value = False
mock_path.return_value = path
list_namespace = self.file_system.list_namespaces()
mock_path.assert_called_with(self.file_system.destination_path, '.')
self.assertEqual(list_namespace, [])
@patch('napps.kytos.storehouse.backends.fs.FileSystem.retrieve')
@patch('napps.kytos.storehouse.backends.fs.FileSystem.list')
@patch('napps.kytos.storehouse.backends.fs.FileSystem.list_namespaces',
return_value=['namespace'])
def test_backup_without_box_id(self, *args):
"""Test backup method without box_id parameter."""
(_, mock_list, mock_retrieve) = args
mock_list.return_value = ['456']
retrieve = MagicMock()
retrieve.to_json.return_value = {}
mock_retrieve.return_value = retrieve
box = Box('any', 'namespace', box_id='123')
boxes_dict_1 = self.file_system.backup(box.namespace, box.box_id)
boxes_dict_2 = self.file_system.backup(box.namespace)
self.assertEqual(boxes_dict_1, {'123': {}})
self.assertEqual(boxes_dict_2, {'456': {}})