def fetch(backend_class, backend_args, manager=None): """Fetch items using the given backend. Generator to get items using the given backend class. When an archive manager is given, this function will store the fetched items in an `Archive`. If an exception is raised, this archive will be removed to avoid corrupted archives. The parameters needed to initialize the `backend` class and get the items are given using `backend_args` dict parameter. :param backend_class: backend class to fetch items :param backend_args: dict of arguments needed to fetch the items :param manager: archive manager needed to store the items :returns: a generator of items """ init_args = find_signature_parameters(backend_class.__init__, backend_args) archive = manager.create_archive() if manager else None init_args['archive'] = archive backend = backend_class(**init_args) fetch_args = find_signature_parameters(backend.fetch, backend_args) items = backend.fetch(**fetch_args) try: for item in items: yield item except Exception as e: if manager: archive_path = archive.archive_path manager.remove_archive(archive_path) raise e
def fetch(backend_class, backend_args, category): """Fetch items using the given backend. Generator to get items using the given backend class. The parameters needed to initialize the `backend` class and get the items are given using `backend_args` dict parameter. :param backend_class: backend class to fetch items :param backend_args: dict of arguments needed to fetch the items :param category: category of the items to retrieve :returns: a generator of items """ init_args = find_signature_parameters(backend_class.__init__, backend_args) init_args['archive'] = None backend = backend_class(**init_args) if category: backend_args['category'] = category fetch_args = find_signature_parameters(backend.fetch, backend_args) items = backend.fetch(**fetch_args) try: for item in items: yield item except Exception as e: raise e
def test_find_parameters(self): """Test if a list of parameters is generated.""" expected = {'a': 1, 'b': 2, 'c': 3} params = {'a': 1, 'b': 2, 'c': 3, 'd': 4} found = find_signature_parameters(FakeCallable.test, params) self.assertDictEqual(found, expected) expected = {'a': 1, 'b': 2} params = {'a': 1, 'b': 2, 'd': 3} found = find_signature_parameters(FakeCallable.test, params) self.assertDictEqual(found, expected)
def run(self): """Fetch and write items. This method runs the backend to fetch the items from the given origin. Items are converted to JSON objects and written to the defined output. If `fetch-cache` parameter was given as an argument during the inizialization of the instance, the items will be retrieved from the cache. """ if self.backend.cache and self.parsed_args.fetch_cache: fetch = self.backend.fetch_from_cache else: fetch = self.backend.fetch parsed_args = vars(self.parsed_args) kw = find_signature_parameters(fetch, parsed_args) items = fetch(**kw) try: for item in items: obj = json.dumps(item, indent=4, sort_keys=True) self.outfile.write(obj) self.outfile.write('\n') except IOError as e: raise RuntimeError(str(e)) except Exception as e: if self.backend.cache: self.backend.cache.recover() raise RuntimeError(str(e))
def fetch_from_archive(backend_class, backend_args, manager, category, archived_after): """Fetch items from an archive manager. Generator to get the items of a category (previously fetched by the given backend class) from an archive manager. Only those items archived after the given date will be returned. The parameters needed to initialize `backend` and get the items are given using `backend_args` dict parameter. :param backend_class: backend class to retrive items :param backend_args: dict of arguments needed to retrieve the items :param manager: archive manager where the items will be retrieved :param category: category of the items to retrieve :param archived_after: return items archived after this date :returns: a generator of archived items """ init_args = find_signature_parameters(backend_class.__init__, backend_args) backend = backend_class(**init_args) filepaths = manager.search(backend.origin, backend.__class__.__name__, category, archived_after) for filepath in filepaths: backend.archive = Archive(filepath) items = backend.fetch_from_archive() try: for item in items: yield item except ArchiveError as e: logger.warning("Ignoring %s archive due to: %s", filepath, str(e))
def test_attribute_error(self): """Test if it raises an exception for not found parameters.""" with self.assertRaises(AttributeError) as e: params = {'a': 1, 'd': 3} _ = find_signature_parameters(FakeCallable.test, params) self.assertEqual(e.exception.args[1], 'b')
def test_find_excluding_parameters(self): """Test if a list of parameters is generated excluding some.""" expected = {'b': 2, 'c': 3} params = {'a': 1, 'b': 2, 'c': 3} excluded = ('self', 'a') found = find_signature_parameters(FakeCallable.test, params, excluded=excluded) self.assertDictEqual(found, expected)
def __init__(self, *args): parser = self.setup_cmd_parser() self.parsed_args = parser.parse(*args) self._pre_init() parsed_args = vars(self.parsed_args) kw = find_signature_parameters(self.BACKEND.__init__, parsed_args) self.backend = self.BACKEND(**kw) self.backend.cache = self._initialize_cache() self._post_init() self.outfile = self.parsed_args.outfile