def _fetch_list_locked(self, url, query_string, query_params, headers):
url_full = "%s?%s" % (url, query_string)
jobs = [(self._fetch, url, query_string, query_params, headers)]
q = pool_apply(self.store.finder.worker_pool(), jobs)
log.debug('RemoteReader:: Storing FetchInProgress for %s' % url_full)
return FetchInProgress(_Results(q))
def _fetch_list_locked(self, url, query_string, query_params, headers):
url_full = "%s?%s" % (url, query_string)
jobs = [(self._fetch, url, query_string, query_params, headers)]
q = pool_apply(self.store.finder.worker_pool(), jobs)
log.debug('RemoteReader:: Storing FetchInProgress for %s' % url_full)
return FetchInProgress(_Results(q))
def test_named(self):
default = pool.get_pool()
p = pool.get_pool(name='test')
self.assertNotEqual(default, p)
q = pool.pool_apply(p, [(lambda v: v, 'a')])
self.assertEqual(q.get(True, 1), 'a')
def test_named(self):
default = pool.get_pool()
p = pool.get_pool(name='test')
self.assertNotEqual(default, p)
q = pool.pool_apply(p, [(lambda v: v, 'a')])
self.assertEqual(q.get(True, 1), 'a')
def test_named_no_worker_pool(self):
default = pool.get_pool()
p = pool.get_pool(name='test')
self.assertIsNone(p)
self.assertIsNone(default)
q = pool.pool_apply(p, [(lambda v: v, 'a')])
self.assertEqual(q.get_nowait(), 'a')
def test_named_no_worker_pool(self):
default = pool.get_pool()
p = pool.get_pool(name='test')
self.assertIsNone(p)
self.assertIsNone(default)
q = pool.pool_apply(p, [(lambda v: v, 'a')])
self.assertEqual(q.get_nowait(), 'a')
def find_all(self, query):
start = time.time()
jobs = []
# Start local searches
for finder in self.finders:
# Support legacy finders by defaulting to 'local = True'
is_local = not hasattr(finder, 'local') or finder.local
if query.local and not is_local:
continue
if getattr(finder, 'disabled', False):
continue
jobs.append((finder.find_nodes, query))
result_queue = pool_apply(get_pool(), jobs)
# Group matching nodes by their path
nodes_by_path = defaultdict(list)
timeout = settings.REMOTE_FIND_TIMEOUT
deadline = start + timeout
done = 0
total = len(jobs)
while done < total:
wait_time = deadline - time.time()
nodes = []
try:
nodes = result_queue.get(True, wait_time)
# ValueError could happen if due to really unlucky timing wait_time
# is negative
except (Queue.Empty, ValueError):
if time.time() > deadline:
log.debug("Timed out in find_nodes after %fs" % timeout)
break
else:
continue
log.debug("Got a find result after %fs" % (time.time() - start))
done += 1
for node in nodes or []:
nodes_by_path[node.path].append(node)
log.debug("Got all find results in %fs" % (time.time() - start))
return self._list_nodes(query, nodes_by_path)
def find_all(self, query):
start = time.time()
jobs = []
# Start local searches
for finder in self.finders:
# Support legacy finders by defaulting to 'local = True'
is_local = not hasattr(finder, 'local') or finder.local
if query.local and not is_local:
continue
jobs.append((finder.find_nodes, query))
result_queue = pool_apply(get_pool(), jobs)
# Group matching nodes by their path
nodes_by_path = defaultdict(list)
timeout = settings.REMOTE_FIND_TIMEOUT
deadline = start + timeout
done = 0
total = len(jobs)
while done < total:
wait_time = deadline - time.time()
nodes = []
try:
nodes = result_queue.get(True, wait_time)
# ValueError could happen if due to really unlucky timing wait_time
# is negative
except (Queue.Empty, ValueError):
if time.time() > deadline:
log.debug("Timed out in find_nodes after %fs" % timeout)
break
else:
continue
log.debug("Got a find result after %fs" % (time.time() - start))
done += 1
for node in nodes or []:
nodes_by_path[node.path].append(node)
log.debug("Got all find results in %fs" % (time.time() - start))
return self._list_nodes(query, nodes_by_path)
def find_nodes(self, query):
start = time.time()
jobs = []
random.shuffle(self.remote_stores)
for store in self.remote_stores:
if store.available:
jobs.append((store.find, query))
queue = pool_apply(self.worker_pool(), jobs)
timeout = settings.REMOTE_FIND_TIMEOUT
deadline = start + timeout
done = 0
total = len(jobs)
while done < total:
wait_time = deadline - time.time()
nodes = []
try:
nodes = queue.get(True, wait_time)
# ValueError could happen if due to really unlucky timing wait_time
# is negative.
except (Queue.Empty, ValueError):
if time.time() > deadline:
log.debug("Timed out in find_nodes after %fs" % timeout)
break
else:
continue
log.debug("Got a remote find result after %fs" %
(time.time() - start))
done += 1
for node in nodes or []:
yield node
log.debug("Got all remote find results in %fs" % (time.time() - start))
def find_nodes(self, query):
start = time.time()
jobs = []
random.shuffle(self.remote_stores)
for store in self.remote_stores:
if store.available:
jobs.append((store.find, query))
queue = pool_apply(self.worker_pool(), jobs)
timeout = settings.REMOTE_FIND_TIMEOUT
deadline = start + timeout
done = 0
total = len(jobs)
while done < total:
wait_time = deadline - time.time()
nodes = []
try:
nodes = queue.get(True, wait_time)
# ValueError could happen if due to really unlucky timing wait_time
# is negative.
except (Queue.Empty, ValueError):
if time.time() > deadline:
log.debug("Timed out in find_nodes after %fs" % timeout)
break
else:
continue
log.debug("Got a remote find result after %fs" % (time.time() - start))
done += 1
for node in nodes or []:
yield node
log.debug("Got all remote find results in %fs" % (time.time() - start))
def test_basic(self):
p = pool.get_pool()
pool.pool_apply(p, [])
def test_basic(self):
p = pool.get_pool()
pool.pool_apply(p, [])