class DownloaderPool(threading.Thread):
def __init__(self, num_threads, server, port, ssl, username, password):
super(DownloaderPool, self).__init__()
self._num_threads = num_threads
self._server = server
self._port = port
self._ssl = ssl
self._username = username
self._password = password
self._queue = Queue()
self._body_tasks = weakref.WeakValueDictionary() # message_id -> BodyTask
# TODO: Make disk cache configurable
#self._segment_cache = CacheChain((MemCache(5 * 1024 * 1024), DiskCache('/mnt/cache')))
self._segment_cache = MemCache(20 * 1024 * 1024)
self._lock = threading.Lock()
self._threads = []
def run(self):
for i in range(self._num_threads):
thread = DownloaderThread(self._queue, i, self, self._server, self._port, self._ssl, self._username, self._password)
thread.start()
self._threads.append(thread)
def queue(self, part, line_handler_factory):
with self._lock:
cached_result = self._segment_cache.get(part.message_id)
if (part.begin is None or part.bytes is None) or cached_result is None:
if part.message_id in self._body_tasks:
task = self._body_tasks[part.message_id]
else:
line_handler = line_handler_factory()
task = BodyTask(line_handler)
def _update_cache_cb():
with self._lock:
self._segment_cache.set(part.message_id, line_handler.get_data())
task.callbacks.append(_update_cache_cb)
self._queue.put(task)
self._body_tasks[part.message_id] = task
else:
return cached_result
return task
def queue_stat(self, file):
task = StatTask(file)
self._queue.put(task)
return task
def status(self):
s = []
for thread in self._threads:
if thread.cur_task is None:
s.append('None')
else:
s.append(thread.status())
logging.info("Currently processing: " + " ".join(s))
def __init__(self, num_threads, server, port, ssl, username, password):
super(DownloaderPool, self).__init__()
self._num_threads = num_threads
self._server = server
self._port = port
self._ssl = ssl
self._username = username
self._password = password
# A queue of available tasks to process
self._task_queue = Queue()
# message_id -> BodyTask
self._body_tasks = weakref.WeakValueDictionary()
self._segment_cache = MemCache(20 * 1024 * 1024)
self._lock = threading.Lock()
self._threads = []
def __init__(self, num_threads, server, port, ssl, username, password):
super(DownloaderPool, self).__init__()
self._num_threads = num_threads
self._server = server
self._port = port
self._ssl = ssl
self._username = username
self._password = password
self._queue = Queue()
self._body_tasks = weakref.WeakValueDictionary() # message_id -> BodyTask
# TODO: Make disk cache configurable
#self._segment_cache = CacheChain((MemCache(5 * 1024 * 1024), DiskCache('/mnt/cache')))
self._segment_cache = MemCache(20 * 1024 * 1024)
self._lock = threading.Lock()
self._threads = []
def __init__(self, num_threads, server, port, ssl, username, password):
super(DownloaderPool, self).__init__()
self._num_threads = num_threads
self._server = server
self._port = port
self._ssl = ssl
self._username = username
self._password = password
# A queue of available tasks to process
self._task_queue = Queue()
# message_id -> BodyTask
self._body_tasks = weakref.WeakValueDictionary()
self._segment_cache = MemCache(20 * 1024 * 1024)
self._lock = threading.Lock()
self._threads = []
class DownloaderPool(threading.Thread):
def __init__(self, num_threads, server, port, ssl, username, password):
super(DownloaderPool, self).__init__()
self._num_threads = num_threads
self._server = server
self._port = port
self._ssl = ssl
self._username = username
self._password = password
# A queue of available tasks to process
self._task_queue = Queue()
# message_id -> BodyTask
self._body_tasks = weakref.WeakValueDictionary()
self._segment_cache = MemCache(20 * 1024 * 1024)
self._lock = threading.Lock()
self._threads = []
def run(self):
for i in range(self._num_threads):
thread = DownloaderThread(
self._task_queue, i, self, self._server, self._port, self._ssl,
self._username, self._password)
thread.start()
self._threads.append(thread)
def queue(self, part, line_handler_factory):
"""Queue a task to fetch a particular part.
Returns a queue where updates about ecoded data will be fed. Each tuple
returned from the queue is of the form: (finished, size, error)
finished: True/False, if the part is finished downloading/decoding.
data: The currently decoded data.
error: Optional exception.
"""
with self._lock:
update_queue = Queue()
cached_data = self._segment_cache.get(part.message_id)
if cached_data:
line_handler = line_handler_factory()
line_handler.set_data(cached_data)
update_queue.put((True, len(cached_data), None))
return update_queue, line_handler
if part.message_id in self._body_tasks:
task = self._body_tasks[part.message_id]
line_handler = task.line_handler
if task.state is not None:
update_queue.put(task.state)
else:
line_handler = line_handler_factory()
task = BodyTask(line_handler)
def _update_cache_cb():
with self._lock:
self._segment_cache.set(
part.message_id, line_handler.get_data())
task.callbacks.append(_update_cache_cb)
self._task_queue.put(task)
self._body_tasks[part.message_id] = task
task.update_queues.append(update_queue)
return update_queue, line_handler
def status(self):
s = []
for thread in self._threads:
if thread.cur_task is None:
s.append('None')
else:
s.append(thread.status())
log.info("Currently processing: " + " ".join(s))
class DownloaderPool(threading.Thread):
def __init__(self, num_threads, server, port, ssl, username, password):
super(DownloaderPool, self).__init__()
self._num_threads = num_threads
self._server = server
self._port = port
self._ssl = ssl
self._username = username
self._password = password
# A queue of available tasks to process
self._task_queue = Queue()
# message_id -> BodyTask
self._body_tasks = weakref.WeakValueDictionary()
self._segment_cache = MemCache(20 * 1024 * 1024)
self._lock = threading.Lock()
self._threads = []
def run(self):
for i in range(self._num_threads):
thread = DownloaderThread(self._task_queue, i, self, self._server,
self._port, self._ssl, self._username,
self._password)
thread.start()
self._threads.append(thread)
def queue(self, part, line_handler_factory):
"""Queue a task to fetch a particular part.
Returns a queue where updates about ecoded data will be fed. Each tuple
returned from the queue is of the form: (finished, size, error)
finished: True/False, if the part is finished downloading/decoding.
data: The currently decoded data.
error: Optional exception.
"""
with self._lock:
update_queue = Queue()
cached_data = self._segment_cache.get(part.message_id)
if cached_data:
line_handler = line_handler_factory()
line_handler.set_data(cached_data)
update_queue.put((True, len(cached_data), None))
return update_queue, line_handler
if part.message_id in self._body_tasks:
task = self._body_tasks[part.message_id]
line_handler = task.line_handler
if task.state is not None:
update_queue.put(task.state)
else:
line_handler = line_handler_factory()
task = BodyTask(line_handler)
def _update_cache_cb():
with self._lock:
self._segment_cache.set(part.message_id,
line_handler.get_data())
task.callbacks.append(_update_cache_cb)
self._task_queue.put(task)
self._body_tasks[part.message_id] = task
task.update_queues.append(update_queue)
return update_queue, line_handler
def status(self):
s = []
for thread in self._threads:
if thread.cur_task is None:
s.append('None')
else:
s.append(thread.status())
log.info("Currently processing: " + " ".join(s))
