def fetch(backend_class, backend_args, category):
"""Fetch items using the given backend.
Generator to get items using the given backend class.
The parameters needed to initialize the `backend` class and
get the items are given using `backend_args` dict parameter.
:param backend_class: backend class to fetch items
:param backend_args: dict of arguments needed to fetch the items
:param category: category of the items to retrieve
:returns: a generator of items
"""
init_args = find_signature_parameters(backend_class.__init__, backend_args)
init_args['archive'] = None
backend = backend_class(**init_args)
if category:
backend_args['category'] = category
fetch_args = find_signature_parameters(backend.fetch, backend_args)
items = backend.fetch(**fetch_args)
try:
for item in items:
yield item
except Exception as e:
raise e
def transfer(backend_class, backend_args):
"""Transfer items from a data storage to another one.
:param backend_class: backend class to transfer items
:param backend_args: dict of arguments needed to init the backend
"""
init_args = find_signature_parameters(backend_class.__init__, backend_args)
backend = backend_class(**init_args)
transfer_args = find_signature_parameters(backend.transfer, backend_args)
backend.transfer(**transfer_args)
def __init__(self,
backend_class,
backend_args,
category,
filter_classified=False,
manager=None,
fetch_archive=False,
archived_after=None):
init_args = find_signature_parameters(backend_class.__init__,
backend_args)
if not fetch_archive:
archive = manager.create_archive() if manager else None
init_args['archive'] = archive
self.backend = backend_class(**init_args)
items = self.__fetch(backend_args,
category,
filter_classified=filter_classified,
manager=manager)
else:
self.backend = backend_class(**init_args)
items = self.__fetch_from_archive(category, manager,
archived_after)
self.items = items
def fetch_from_archive(backend_class, backend_args, manager, category,
archived_after):
"""Fetch items from an archive manager.
Generator to get the items of a category (previously fetched
by the given backend class) from an archive manager. Only those
items archived after the given date will be returned.
The parameters needed to initialize `backend` and get the
items are given using `backend_args` dict parameter.
:param backend_class: backend class to retrive items
:param backend_args: dict of arguments needed to retrieve the items
:param manager: archive manager where the items will be retrieved
:param category: category of the items to retrieve
:param archived_after: return items archived after this date
:returns: a generator of archived items
"""
init_args = find_signature_parameters(backend_class.__init__, backend_args)
backend = backend_class(**init_args)
filepaths = manager.search(backend.origin, backend.__class__.__name__,
category, archived_after)
for filepath in filepaths:
backend.archive = Archive(filepath)
items = backend.fetch_from_archive()
try:
for item in items:
yield item
except ArchiveError as e:
logger.warning("Ignoring %s archive due to: %s", filepath, str(e))
def fetch(backend_class,
backend_args,
category,
filter_classified=False,
manager=None):
"""Fetch items using the given backend.
Generator to get items using the given backend class. When
an archive manager is given, this function will store
the fetched items in an `Archive`. If an exception is raised,
this archive will be removed to avoid corrupted archives.
The parameters needed to initialize the `backend` class and
get the items are given using `backend_args` dict parameter.
:param backend_class: backend class to fetch items
:param backend_args: dict of arguments needed to fetch the items
:param category: category of the items to retrieve.
If None, it will use the default backend category
:param filter_classified: remove classified fields from the resulting items
:param manager: archive manager needed to store the items
:returns: a generator of items
"""
init_args = find_signature_parameters(backend_class.__init__, backend_args)
archive = manager.create_archive() if manager else None
init_args['archive'] = archive
backend = backend_class(**init_args)
if category:
backend_args['category'] = category
if filter_classified:
backend_args['filter_classified'] = filter_classified
fetch_args = find_signature_parameters(backend.fetch, backend_args)
items = backend.fetch(**fetch_args)
try:
for item in items:
yield item
except Exception as e:
if manager:
archive_path = archive.archive_path
manager.remove_archive(archive_path)
raise e
def test_attribute_error(self):
"""Test if it raises an exception for not found parameters."""
with self.assertRaises(AttributeError) as e:
params = {'a': 1, 'd': 3}
_ = find_signature_parameters(FakeCallable.test, params)
self.assertEqual(e.exception.args[1], 'b')
def test_find_excluding_parameters(self):
"""Test if a list of parameters is generated excluding some."""
expected = {'b': 2, 'c': 3}
params = {'a': 1, 'b': 2, 'c': 3}
excluded = ('self', 'a')
found = find_signature_parameters(FakeCallable.test, params,
excluded=excluded)
self.assertDictEqual(found, expected)
def test_find_parameters(self):
"""Test if a list of parameters is generated."""
expected = {'a': 1, 'b': 2, 'c': 3}
params = {'a': 1, 'b': 2, 'c': 3}
found = find_signature_parameters(FakeCallable.test, params)
self.assertDictEqual(found, expected)
expected = {'a': 1, 'b': 2, 'c': 3}
params = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
found = find_signature_parameters(FakeCallable.test, params)
self.assertDictEqual(found, expected)
expected = {'a': 1, 'b': 2}
params = {'a': 1, 'b': 2, 'd': 3}
found = find_signature_parameters(FakeCallable.test, params)
self.assertDictEqual(found, expected)
expected = {'a': 1, 'b': 2}
params = {'a': 1, 'b': 2}
found = find_signature_parameters(FakeCallable.test_args, params)
self.assertDictEqual(found, expected)
