def fetch(backend_class, backend_args, manager=None):
"""Fetch items using the given backend.
Generator to get items using the given backend class. When
an archive manager is given, this function will store
the fetched items in an `Archive`. If an exception is raised,
this archive will be removed to avoid corrupted archives.
The parameters needed to initialize the `backend` class and
get the items are given using `backend_args` dict parameter.
:param backend_class: backend class to fetch items
:param backend_args: dict of arguments needed to fetch the items
:param manager: archive manager needed to store the items
:returns: a generator of items
"""
init_args = find_signature_parameters(backend_class.__init__, backend_args)
archive = manager.create_archive() if manager else None
init_args['archive'] = archive
backend = backend_class(**init_args)
fetch_args = find_signature_parameters(backend.fetch, backend_args)
items = backend.fetch(**fetch_args)
try:
for item in items:
yield item
except Exception as e:
if manager:
archive_path = archive.archive_path
manager.remove_archive(archive_path)
raise e
def fetch(backend_class, backend_args, category):
"""Fetch items using the given backend.
Generator to get items using the given backend class.
The parameters needed to initialize the `backend` class and
get the items are given using `backend_args` dict parameter.
:param backend_class: backend class to fetch items
:param backend_args: dict of arguments needed to fetch the items
:param category: category of the items to retrieve
:returns: a generator of items
"""
init_args = find_signature_parameters(backend_class.__init__,
backend_args)
init_args['archive'] = None
backend = backend_class(**init_args)
if category:
backend_args['category'] = category
fetch_args = find_signature_parameters(backend.fetch,
backend_args)
items = backend.fetch(**fetch_args)
try:
for item in items:
yield item
except Exception as e:
raise e
def test_find_parameters(self):
"""Test if a list of parameters is generated."""
expected = {'a': 1, 'b': 2, 'c': 3}
params = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
found = find_signature_parameters(FakeCallable.test, params)
self.assertDictEqual(found, expected)
expected = {'a': 1, 'b': 2}
params = {'a': 1, 'b': 2, 'd': 3}
found = find_signature_parameters(FakeCallable.test, params)
self.assertDictEqual(found, expected)
def run(self):
"""Fetch and write items.
This method runs the backend to fetch the items from the given
origin. Items are converted to JSON objects and written to the
defined output.
If `fetch-cache` parameter was given as an argument during
the inizialization of the instance, the items will be retrieved
from the cache.
"""
if self.backend.cache and self.parsed_args.fetch_cache:
fetch = self.backend.fetch_from_cache
else:
fetch = self.backend.fetch
parsed_args = vars(self.parsed_args)
kw = find_signature_parameters(fetch, parsed_args)
items = fetch(**kw)
try:
for item in items:
obj = json.dumps(item, indent=4, sort_keys=True)
self.outfile.write(obj)
self.outfile.write('\n')
except IOError as e:
raise RuntimeError(str(e))
except Exception as e:
if self.backend.cache:
self.backend.cache.recover()
raise RuntimeError(str(e))
def fetch_from_archive(backend_class, backend_args, manager, category,
archived_after):
"""Fetch items from an archive manager.
Generator to get the items of a category (previously fetched
by the given backend class) from an archive manager. Only those
items archived after the given date will be returned.
The parameters needed to initialize `backend` and get the
items are given using `backend_args` dict parameter.
:param backend_class: backend class to retrive items
:param backend_args: dict of arguments needed to retrieve the items
:param manager: archive manager where the items will be retrieved
:param category: category of the items to retrieve
:param archived_after: return items archived after this date
:returns: a generator of archived items
"""
init_args = find_signature_parameters(backend_class.__init__, backend_args)
backend = backend_class(**init_args)
filepaths = manager.search(backend.origin, backend.__class__.__name__,
category, archived_after)
for filepath in filepaths:
backend.archive = Archive(filepath)
items = backend.fetch_from_archive()
try:
for item in items:
yield item
except ArchiveError as e:
logger.warning("Ignoring %s archive due to: %s", filepath, str(e))
def test_attribute_error(self):
"""Test if it raises an exception for not found parameters."""
with self.assertRaises(AttributeError) as e:
params = {'a': 1, 'd': 3}
_ = find_signature_parameters(FakeCallable.test, params)
self.assertEqual(e.exception.args[1], 'b')
def test_find_excluding_parameters(self):
"""Test if a list of parameters is generated excluding some."""
expected = {'b': 2, 'c': 3}
params = {'a': 1, 'b': 2, 'c': 3}
excluded = ('self', 'a')
found = find_signature_parameters(FakeCallable.test,
params,
excluded=excluded)
self.assertDictEqual(found, expected)
def __init__(self, *args):
parser = self.setup_cmd_parser()
self.parsed_args = parser.parse(*args)
self._pre_init()
parsed_args = vars(self.parsed_args)
kw = find_signature_parameters(self.BACKEND.__init__, parsed_args)
self.backend = self.BACKEND(**kw)
self.backend.cache = self._initialize_cache()
self._post_init()
self.outfile = self.parsed_args.outfile
