Example #1
0
class DownloaderPool(threading.Thread):
    def __init__(self, num_threads, server, port, ssl, username, password):
        super(DownloaderPool, self).__init__()

        self._num_threads = num_threads
        self._server = server
        self._port = port
	self._ssl = ssl
        self._username = username
        self._password = password

        self._queue = Queue()
        self._body_tasks = weakref.WeakValueDictionary() # message_id -> BodyTask

        # TODO: Make disk cache configurable
        #self._segment_cache = CacheChain((MemCache(5 * 1024 * 1024), DiskCache('/mnt/cache')))
        self._segment_cache = MemCache(20 * 1024 * 1024)
        self._lock = threading.Lock()

	self._threads = []

    def run(self):
        for i in range(self._num_threads):
            thread = DownloaderThread(self._queue, i, self, self._server, self._port, self._ssl, self._username, self._password)
	    thread.start()
	    self._threads.append(thread)

    def queue(self, part, line_handler_factory):
        with self._lock:
            cached_result = self._segment_cache.get(part.message_id)
            if (part.begin is None or part.bytes is None) or cached_result is None:
                if part.message_id in self._body_tasks:
                    task = self._body_tasks[part.message_id]
                else:
                    line_handler = line_handler_factory()
                    task = BodyTask(line_handler)
		    def _update_cache_cb():
			with self._lock:
			    self._segment_cache.set(part.message_id, line_handler.get_data())
		    task.callbacks.append(_update_cache_cb)
                    self._queue.put(task)
                    self._body_tasks[part.message_id] = task
            else:
                return cached_result

        return task

    def queue_stat(self, file):
        task = StatTask(file)
        self._queue.put(task)
        return task

    def status(self):
	s = []
	for thread in self._threads:
	    if thread.cur_task is None:
		s.append('None')
	    else:
		s.append(thread.status())
	logging.info("Currently processing: " + " ".join(s))
Example #2
0
class DownloaderPool(threading.Thread):
    def __init__(self, num_threads, server, port, ssl, username, password):
        super(DownloaderPool, self).__init__()

        self._num_threads = num_threads
        self._server = server
        self._port = port
        self._ssl = ssl
        self._username = username
        self._password = password

        # A queue of available tasks to process
        self._task_queue = Queue()
        # message_id -> BodyTask
        self._body_tasks = weakref.WeakValueDictionary()

        self._segment_cache = MemCache(20 * 1024 * 1024)
        self._lock = threading.Lock()

        self._threads = []

    def run(self):
        for i in range(self._num_threads):
            thread = DownloaderThread(
                self._task_queue, i, self, self._server, self._port, self._ssl,
                self._username, self._password)
            thread.start()
            self._threads.append(thread)

    def queue(self, part, line_handler_factory):
        """Queue a task to fetch a particular part.

        Returns a queue where updates about ecoded data will be fed. Each tuple
        returned from the queue is of the form: (finished, size, error)

        finished: True/False, if the part is finished downloading/decoding.
        data: The currently decoded data.
        error: Optional exception.
        """
        with self._lock:
            update_queue = Queue()
            cached_data = self._segment_cache.get(part.message_id)

            if cached_data:
                line_handler = line_handler_factory()
                line_handler.set_data(cached_data)
                update_queue.put((True, len(cached_data), None))
                return update_queue, line_handler

            if part.message_id in self._body_tasks:
                task = self._body_tasks[part.message_id]
                line_handler = task.line_handler
                if task.state is not None:
                    update_queue.put(task.state)
            else:
                line_handler = line_handler_factory()
                task = BodyTask(line_handler)

                def _update_cache_cb():
                    with self._lock:
                        self._segment_cache.set(
                            part.message_id, line_handler.get_data())
                task.callbacks.append(_update_cache_cb)

                self._task_queue.put(task)
                self._body_tasks[part.message_id] = task

            task.update_queues.append(update_queue)
            return update_queue, line_handler

    def status(self):
        s = []
        for thread in self._threads:
            if thread.cur_task is None:
                s.append('None')
            else:
                s.append(thread.status())
        log.info("Currently processing: " + " ".join(s))
Example #3
0
class DownloaderPool(threading.Thread):
    def __init__(self, num_threads, server, port, ssl, username, password):
        super(DownloaderPool, self).__init__()

        self._num_threads = num_threads
        self._server = server
        self._port = port
        self._ssl = ssl
        self._username = username
        self._password = password

        # A queue of available tasks to process
        self._task_queue = Queue()
        # message_id -> BodyTask
        self._body_tasks = weakref.WeakValueDictionary()

        self._segment_cache = MemCache(20 * 1024 * 1024)
        self._lock = threading.Lock()

        self._threads = []

    def run(self):
        for i in range(self._num_threads):
            thread = DownloaderThread(self._task_queue, i, self, self._server,
                                      self._port, self._ssl, self._username,
                                      self._password)
            thread.start()
            self._threads.append(thread)

    def queue(self, part, line_handler_factory):
        """Queue a task to fetch a particular part.

        Returns a queue where updates about ecoded data will be fed. Each tuple
        returned from the queue is of the form: (finished, size, error)

        finished: True/False, if the part is finished downloading/decoding.
        data: The currently decoded data.
        error: Optional exception.
        """
        with self._lock:
            update_queue = Queue()
            cached_data = self._segment_cache.get(part.message_id)

            if cached_data:
                line_handler = line_handler_factory()
                line_handler.set_data(cached_data)
                update_queue.put((True, len(cached_data), None))
                return update_queue, line_handler

            if part.message_id in self._body_tasks:
                task = self._body_tasks[part.message_id]
                line_handler = task.line_handler
                if task.state is not None:
                    update_queue.put(task.state)
            else:
                line_handler = line_handler_factory()
                task = BodyTask(line_handler)

                def _update_cache_cb():
                    with self._lock:
                        self._segment_cache.set(part.message_id,
                                                line_handler.get_data())

                task.callbacks.append(_update_cache_cb)

                self._task_queue.put(task)
                self._body_tasks[part.message_id] = task

            task.update_queues.append(update_queue)
            return update_queue, line_handler

    def status(self):
        s = []
        for thread in self._threads:
            if thread.cur_task is None:
                s.append('None')
            else:
                s.append(thread.status())
        log.info("Currently processing: " + " ".join(s))