def __init__(self, identifiers, workers=20, max_requests=10): """Makes a generator for an list of `(index, item)` where `item` is an instance of `Item` containing metadata, and index is the index, for each id in `identifiers`. Note: this does not return the items in the same order as given in the identifiers list :type identifiers: list :param identifiers: a list of identifiers to get the metadata of :type workers: int :param workers: the number of concurrent workers to have fecthing the metadata :type max_requests: int or None :param max_requests: the number of times to try fetching the metadata, in case there is something wrong with requesting it :rtype: Mine """ self.skips = [] self.queue = queue self.workers = workers self.identifiers = identifiers self.item_count = len(identifiers) self.max_requests = max_requests self.queued_count = 0 self.got_count = 0 self.input_queue = self.queue.JoinableQueue(1000) self.json_queue = self.queue.Queue(1000) # Use the same session for each item fetch. self.session = get_session()
def test_session(): s = get_session() s.set_file_logger(0, 'test.log') os.remove('test.log')