Exemple #1
0
    def __init__(self, identifiers, workers=20, max_requests=10):
        """Makes a generator for an list of `(index, item)` where `item`
        is an instance of `Item` containing metadata, and index is the index,
        for each id in `identifiers`. Note: this does not return the
        items in the same order as given in the identifiers list

        :type identifiers: list
        :param identifiers: a list of identifiers to get the metadata of
        :type workers: int
        :param workers: the number of concurrent workers to have fecthing the metadata
        :type max_requests: int or None
        :param max_requests: the number of times to try fetching the metadata,
        in case there is something wrong with requesting it

        :rtype: Mine

        """
        self.skips = []
        self.queue = queue
        self.workers = workers
        self.identifiers = identifiers
        self.item_count = len(identifiers)
        self.max_requests = max_requests
        self.queued_count = 0
        self.got_count = 0
        self.input_queue = self.queue.JoinableQueue(1000)
        self.json_queue = self.queue.Queue(1000)

        # Use the same session for each item fetch.
        self.session = get_session()
Exemple #2
0
    def __init__(self, identifiers, workers=20, max_requests=10):
        """Makes a generator for an list of `(index, item)` where `item`
        is an instance of `Item` containing metadata, and index is the index,
        for each id in `identifiers`. Note: this does not return the
        items in the same order as given in the identifiers list

        :type identifiers: list
        :param identifiers: a list of identifiers to get the metadata of
        :type workers: int
        :param workers: the number of concurrent workers to have fecthing the metadata
        :type max_requests: int or None
        :param max_requests: the number of times to try fetching the metadata,
        in case there is something wrong with requesting it

        :rtype: Mine

        """
        self.skips = []
        self.queue = queue
        self.workers = workers
        self.identifiers = identifiers
        self.item_count = len(identifiers)
        self.max_requests = max_requests
        self.queued_count = 0
        self.got_count = 0
        self.input_queue = self.queue.JoinableQueue(1000)
        self.json_queue = self.queue.Queue(1000)

        # Use the same session for each item fetch.
        self.session = get_session()
Exemple #3
0
def test_session():
    s = get_session()
    s.set_file_logger(0, 'test.log')
    os.remove('test.log')