def startFindOptional(self):
time.sleep(0.01) # Wait for more file requests
optional_tasks = [task for task in self.tasks if task["optional_hash_id"]]
optional_hash_ids = set([task["optional_hash_id"] for task in optional_tasks])
self.log.debug("Finding peers for optional files: %s" % optional_hash_ids)
found = self.findOptionalTasks(optional_tasks)
if found:
found_peers = set([peer for peers in found.values() for peer in peers])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids):
self.log.debug("No local result for optional files: %s" % (optional_hash_ids - set(found)))
# Query hashfield from connected peers
threads = []
peers = self.site.getConnectedPeers()
if not peers:
peers = self.site.getConnectablePeers()
for peer in peers:
if not peer.time_hashfield:
threads.append(gevent.spawn(peer.updateHashfield))
gevent.joinall(threads, timeout=5)
found = self.findOptionalTasks(optional_tasks)
self.log.debug("Found optional files after query hashtable connected peers: %s/%s" % (len(found), len(optional_hash_ids)))
if found:
found_peers = set([peer for hash_id_peers in found.values() for peer in hash_id_peers])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids):
self.log.debug("No connected hashtable result for optional files: %s" % (optional_hash_ids - set(found)))
# Try to query connected peers
threads = []
peers = self.site.getConnectedPeers()
if not peers:
peers = self.site.getConnectablePeers()
for peer in peers:
threads.append(gevent.spawn(peer.findHashIds, list(optional_hash_ids)))
gevent.joinall(threads, timeout=5)
found_ips = helper.mergeDicts([thread.value for thread in threads if thread.value])
found = self.addOptionalPeers(found_ips)
self.log.debug("Found optional files after findhash connected peers: %s/%s" % (len(found), len(optional_hash_ids)))
if found:
found_peers = set([peer for hash_id_peers in found.values() for peer in hash_id_peers])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids):
self.log.debug("No findhash result for optional files: %s" % (optional_hash_ids - set(found)))
def startFindOptional(self):
time.sleep(0.01) # Wait for more file requests
optional_tasks = [
task for task in self.tasks if task["optional_hash_id"]
]
optional_hash_ids = set(
[task["optional_hash_id"] for task in optional_tasks])
self.log.debug("Finding peers for optional files: %s" %
optional_hash_ids)
found = self.findOptionalTasks(optional_tasks)
if found:
found_peers = set(
[peer for peers in found.values() for peer in peers])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids):
self.log.debug("No local result for optional files: %s" %
(optional_hash_ids - set(found)))
# Query hashfield from connected peers
threads = []
peers = self.site.getConnectedPeers()
if not peers:
peers = self.site.getConnectablePeers()
for peer in peers:
if not peer.time_hashfield:
threads.append(gevent.spawn(peer.updateHashfield))
gevent.joinall(threads, timeout=5)
found = self.findOptionalTasks(optional_tasks)
self.log.debug(
"Found optional files after query hashtable connected peers: %s/%s"
% (len(found), len(optional_hash_ids)))
if found:
found_peers = set([
peer for hash_id_peers in found.values()
for peer in hash_id_peers
])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids):
self.log.debug(
"No connected hashtable result for optional files: %s" %
(optional_hash_ids - set(found)))
# Try to query connected peers
threads = []
peers = self.site.getConnectedPeers()
if not peers:
peers = self.site.getConnectablePeers()
for peer in peers:
threads.append(
gevent.spawn(peer.findHashIds, list(optional_hash_ids)))
gevent.joinall(threads, timeout=5)
found_ips = helper.mergeDicts(
[thread.value for thread in threads if thread.value])
found = self.addOptionalPeers(found_ips)
self.log.debug(
"Found optional files after findhash connected peers: %s/%s" %
(len(found), len(optional_hash_ids)))
if found:
found_peers = set([
peer for hash_id_peers in found.values()
for peer in hash_id_peers
])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids):
self.log.debug("No findHash result, try random peers: %s" %
(optional_hash_ids - set(found)))
# Try to query random peers
threads = []
peers = self.site.getConnectablePeers()
for peer in peers[0:5]:
threads.append(
gevent.spawn(peer.findHashIds, list(optional_hash_ids)))
gevent.joinall(threads, timeout=15)
found_ips = helper.mergeDicts(
[thread.value for thread in threads if thread.value])
found = self.addOptionalPeers(found_ips)
self.log.debug(
"Found optional files after findhash random peers: %s/%s" %
(len(found), len(optional_hash_ids)))
if found:
found_peers = set([
peer for hash_id_peers in found.values()
for peer in hash_id_peers
])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids):
self.log.debug("No findhash result for optional files: %s" %
(optional_hash_ids - set(found)))
def startFindOptional(self,
reset_task=False,
find_more=False,
high_priority=False):
# Wait for more file requests
if len(self.tasks) < 20 or high_priority:
time.sleep(0.01)
if len(self.tasks) > 90:
time.sleep(5)
else:
time.sleep(0.5)
optional_tasks = [
task for task in self.tasks if task["optional_hash_id"]
]
if not optional_tasks:
return False
optional_hash_ids = set(
[task["optional_hash_id"] for task in optional_tasks])
time_tasks = self.time_task_added
self.log.debug(
"Finding peers for optional files: %s (reset_task: %s, find_more: %s)"
% (optional_hash_ids, reset_task, find_more))
found = self.findOptionalTasks(optional_tasks, reset_task=reset_task)
if found:
found_peers = set(
[peer for peers in found.values() for peer in peers])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids) or find_more or (
high_priority
and any(len(peers) < 10 for peers in found.itervalues())):
self.log.debug("No local result for optional files: %s" %
(optional_hash_ids - set(found)))
# Query hashfield from connected peers
threads = []
peers = self.site.getConnectedPeers()
if not peers:
peers = self.site.getConnectablePeers()
for peer in peers:
if not peer.time_hashfield:
threads.append(gevent.spawn(peer.updateHashfield))
gevent.joinall(threads, timeout=5)
if time_tasks != self.time_task_added: # New task added since start
optional_tasks = [
task for task in self.tasks if task["optional_hash_id"]
]
optional_hash_ids = set(
[task["optional_hash_id"] for task in optional_tasks])
found = self.findOptionalTasks(optional_tasks)
self.log.debug(
"Found optional files after query hashtable connected peers: %s/%s"
% (len(found), len(optional_hash_ids)))
if found:
found_peers = set([
peer for hash_id_peers in found.values()
for peer in hash_id_peers
])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids) or find_more:
self.log.debug(
"No connected hashtable result for optional files: %s" %
(optional_hash_ids - set(found)))
# Try to query connected peers
threads = []
peers = [
peer for peer in self.site.getConnectedPeers()
if peer not in self.asked_peers
]
if not peers:
peers = self.site.getConnectablePeers()
for peer in peers:
threads.append(
gevent.spawn(peer.findHashIds, list(optional_hash_ids)))
self.asked_peers.append(peer)
for i in range(5):
time.sleep(1)
thread_values = [
thread.value for thread in threads if thread.value
]
if not thread_values:
continue
found_ips = helper.mergeDicts(thread_values)
found = self.addOptionalPeers(found_ips)
self.log.debug(
"Found optional files after findhash connected peers: %s/%s (asked: %s)"
% (len(found), len(optional_hash_ids), len(threads)))
if found:
found_peers = set([
peer for hash_id_peers in found.values()
for peer in hash_id_peers
])
self.startWorkers(found_peers)
if len(thread_values) == len(threads):
# Got result from all started thread
break
if len(found) < len(optional_hash_ids):
self.log.debug("No findHash result, try random peers: %s" %
(optional_hash_ids - set(found)))
# Try to query random peers
if time_tasks != self.time_task_added: # New task added since start
optional_tasks = [
task for task in self.tasks if task["optional_hash_id"]
]
optional_hash_ids = set(
[task["optional_hash_id"] for task in optional_tasks])
threads = []
peers = self.site.getConnectablePeers(ignore=self.asked_peers)
for peer in peers:
threads.append(
gevent.spawn(peer.findHashIds, list(optional_hash_ids)))
self.asked_peers.append(peer)
gevent.joinall(threads, timeout=15)
found_ips = helper.mergeDicts(
[thread.value for thread in threads if thread.value])
found = self.addOptionalPeers(found_ips)
self.log.debug(
"Found optional files after findhash random peers: %s/%s" %
(len(found), len(optional_hash_ids)))
if found:
found_peers = set([
peer for hash_id_peers in found.values()
for peer in hash_id_peers
])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids):
self.log.debug("No findhash result for optional files: %s" %
(optional_hash_ids - set(found)))
def startFindOptional(self, reset_task=False, find_more=False, high_priority=False):
# Wait for more file requests
if len(self.tasks) < 20 or high_priority:
time.sleep(0.01)
if len(self.tasks) > 90:
time.sleep(5)
else:
time.sleep(0.5)
optional_tasks = [task for task in self.tasks if task["optional_hash_id"]]
if not optional_tasks:
return False
optional_hash_ids = set([task["optional_hash_id"] for task in optional_tasks])
time_tasks = self.time_task_added
self.log.debug(
"Finding peers for optional files: %s (reset_task: %s, find_more: %s)" %
(optional_hash_ids, reset_task, find_more)
)
found = self.findOptionalTasks(optional_tasks, reset_task=reset_task)
if found:
found_peers = set([peer for peers in found.values() for peer in peers])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids) or find_more or (high_priority and any(len(peers) < 10 for peers in found.itervalues())):
self.log.debug("No local result for optional files: %s" % (optional_hash_ids - set(found)))
# Query hashfield from connected peers
threads = []
peers = self.site.getConnectedPeers()
if not peers:
peers = self.site.getConnectablePeers()
for peer in peers:
if not peer.time_hashfield:
threads.append(gevent.spawn(peer.updateHashfield))
gevent.joinall(threads, timeout=5)
if time_tasks != self.time_task_added: # New task added since start
optional_tasks = [task for task in self.tasks if task["optional_hash_id"]]
optional_hash_ids = set([task["optional_hash_id"] for task in optional_tasks])
found = self.findOptionalTasks(optional_tasks)
self.log.debug("Found optional files after query hashtable connected peers: %s/%s" % (
len(found), len(optional_hash_ids)
))
if found:
found_peers = set([peer for hash_id_peers in found.values() for peer in hash_id_peers])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids) or find_more:
self.log.debug("No connected hashtable result for optional files: %s" % (optional_hash_ids - set(found)))
# Try to query connected peers
threads = []
peers = [peer for peer in self.site.getConnectedPeers() if peer not in self.asked_peers]
if not peers:
peers = self.site.getConnectablePeers()
for peer in peers:
threads.append(gevent.spawn(peer.findHashIds, list(optional_hash_ids)))
self.asked_peers.append(peer)
for i in range(5):
time.sleep(1)
thread_values = [thread.value for thread in threads if thread.value]
if not thread_values:
continue
found_ips = helper.mergeDicts(thread_values)
found = self.addOptionalPeers(found_ips)
self.log.debug("Found optional files after findhash connected peers: %s/%s (asked: %s)" % (
len(found), len(optional_hash_ids), len(threads)
))
if found:
found_peers = set([peer for hash_id_peers in found.values() for peer in hash_id_peers])
self.startWorkers(found_peers)
if len(thread_values) == len(threads):
# Got result from all started thread
break
if len(found) < len(optional_hash_ids):
self.log.debug("No findHash result, try random peers: %s" % (optional_hash_ids - set(found)))
# Try to query random peers
if time_tasks != self.time_task_added: # New task added since start
optional_tasks = [task for task in self.tasks if task["optional_hash_id"]]
optional_hash_ids = set([task["optional_hash_id"] for task in optional_tasks])
threads = []
peers = self.site.getConnectablePeers(ignore=self.asked_peers)
for peer in peers:
threads.append(gevent.spawn(peer.findHashIds, list(optional_hash_ids)))
self.asked_peers.append(peer)
gevent.joinall(threads, timeout=15)
found_ips = helper.mergeDicts([thread.value for thread in threads if thread.value])
found = self.addOptionalPeers(found_ips)
self.log.debug("Found optional files after findhash random peers: %s/%s" % (len(found), len(optional_hash_ids)))
if found:
found_peers = set([peer for hash_id_peers in found.values() for peer in hash_id_peers])
self.startWorkers(found_peers)
if len(found) < len(optional_hash_ids):
self.log.debug("No findhash result for optional files: %s" % (optional_hash_ids - set(found)))
