def fetch(backend_class, backend_args, category): """Fetch items using the given backend. Generator to get items using the given backend class. The parameters needed to initialize the `backend` class and get the items are given using `backend_args` dict parameter. :param backend_class: backend class to fetch items :param backend_args: dict of arguments needed to fetch the items :param category: category of the items to retrieve :returns: a generator of items """ init_args = find_signature_parameters(backend_class.__init__, backend_args) init_args['archive'] = None backend = backend_class(**init_args) if category: backend_args['category'] = category fetch_args = find_signature_parameters(backend.fetch, backend_args) items = backend.fetch(**fetch_args) try: for item in items: yield item except Exception as e: raise e
def transfer(backend_class, backend_args): """Transfer items from a data storage to another one. :param backend_class: backend class to transfer items :param backend_args: dict of arguments needed to init the backend """ init_args = find_signature_parameters(backend_class.__init__, backend_args) backend = backend_class(**init_args) transfer_args = find_signature_parameters(backend.transfer, backend_args) backend.transfer(**transfer_args)
def __init__(self, backend_class, backend_args, category, filter_classified=False, manager=None, fetch_archive=False, archived_after=None): init_args = find_signature_parameters(backend_class.__init__, backend_args) if not fetch_archive: archive = manager.create_archive() if manager else None init_args['archive'] = archive self.backend = backend_class(**init_args) items = self.__fetch(backend_args, category, filter_classified=filter_classified, manager=manager) else: self.backend = backend_class(**init_args) items = self.__fetch_from_archive(category, manager, archived_after) self.items = items
def fetch_from_archive(backend_class, backend_args, manager, category, archived_after): """Fetch items from an archive manager. Generator to get the items of a category (previously fetched by the given backend class) from an archive manager. Only those items archived after the given date will be returned. The parameters needed to initialize `backend` and get the items are given using `backend_args` dict parameter. :param backend_class: backend class to retrive items :param backend_args: dict of arguments needed to retrieve the items :param manager: archive manager where the items will be retrieved :param category: category of the items to retrieve :param archived_after: return items archived after this date :returns: a generator of archived items """ init_args = find_signature_parameters(backend_class.__init__, backend_args) backend = backend_class(**init_args) filepaths = manager.search(backend.origin, backend.__class__.__name__, category, archived_after) for filepath in filepaths: backend.archive = Archive(filepath) items = backend.fetch_from_archive() try: for item in items: yield item except ArchiveError as e: logger.warning("Ignoring %s archive due to: %s", filepath, str(e))
def fetch(backend_class, backend_args, category, filter_classified=False, manager=None): """Fetch items using the given backend. Generator to get items using the given backend class. When an archive manager is given, this function will store the fetched items in an `Archive`. If an exception is raised, this archive will be removed to avoid corrupted archives. The parameters needed to initialize the `backend` class and get the items are given using `backend_args` dict parameter. :param backend_class: backend class to fetch items :param backend_args: dict of arguments needed to fetch the items :param category: category of the items to retrieve. If None, it will use the default backend category :param filter_classified: remove classified fields from the resulting items :param manager: archive manager needed to store the items :returns: a generator of items """ init_args = find_signature_parameters(backend_class.__init__, backend_args) archive = manager.create_archive() if manager else None init_args['archive'] = archive backend = backend_class(**init_args) if category: backend_args['category'] = category if filter_classified: backend_args['filter_classified'] = filter_classified fetch_args = find_signature_parameters(backend.fetch, backend_args) items = backend.fetch(**fetch_args) try: for item in items: yield item except Exception as e: if manager: archive_path = archive.archive_path manager.remove_archive(archive_path) raise e
def test_attribute_error(self): """Test if it raises an exception for not found parameters.""" with self.assertRaises(AttributeError) as e: params = {'a': 1, 'd': 3} _ = find_signature_parameters(FakeCallable.test, params) self.assertEqual(e.exception.args[1], 'b')
def test_find_excluding_parameters(self): """Test if a list of parameters is generated excluding some.""" expected = {'b': 2, 'c': 3} params = {'a': 1, 'b': 2, 'c': 3} excluded = ('self', 'a') found = find_signature_parameters(FakeCallable.test, params, excluded=excluded) self.assertDictEqual(found, expected)
def test_find_parameters(self): """Test if a list of parameters is generated.""" expected = {'a': 1, 'b': 2, 'c': 3} params = {'a': 1, 'b': 2, 'c': 3} found = find_signature_parameters(FakeCallable.test, params) self.assertDictEqual(found, expected) expected = {'a': 1, 'b': 2, 'c': 3} params = {'a': 1, 'b': 2, 'c': 3, 'd': 4} found = find_signature_parameters(FakeCallable.test, params) self.assertDictEqual(found, expected) expected = {'a': 1, 'b': 2} params = {'a': 1, 'b': 2, 'd': 3} found = find_signature_parameters(FakeCallable.test, params) self.assertDictEqual(found, expected) expected = {'a': 1, 'b': 2} params = {'a': 1, 'b': 2} found = find_signature_parameters(FakeCallable.test_args, params) self.assertDictEqual(found, expected)