def startFindOptional(self): time.sleep(0.01) # Wait for more file requests optional_tasks = [task for task in self.tasks if task["optional_hash_id"]] optional_hash_ids = set([task["optional_hash_id"] for task in optional_tasks]) self.log.debug("Finding peers for optional files: %s" % optional_hash_ids) found = self.findOptionalTasks(optional_tasks) if found: found_peers = set([peer for peers in found.values() for peer in peers]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids): self.log.debug("No local result for optional files: %s" % (optional_hash_ids - set(found))) # Query hashfield from connected peers threads = [] peers = self.site.getConnectedPeers() if not peers: peers = self.site.getConnectablePeers() for peer in peers: if not peer.time_hashfield: threads.append(gevent.spawn(peer.updateHashfield)) gevent.joinall(threads, timeout=5) found = self.findOptionalTasks(optional_tasks) self.log.debug("Found optional files after query hashtable connected peers: %s/%s" % (len(found), len(optional_hash_ids))) if found: found_peers = set([peer for hash_id_peers in found.values() for peer in hash_id_peers]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids): self.log.debug("No connected hashtable result for optional files: %s" % (optional_hash_ids - set(found))) # Try to query connected peers threads = [] peers = self.site.getConnectedPeers() if not peers: peers = self.site.getConnectablePeers() for peer in peers: threads.append(gevent.spawn(peer.findHashIds, list(optional_hash_ids))) gevent.joinall(threads, timeout=5) found_ips = helper.mergeDicts([thread.value for thread in threads if thread.value]) found = self.addOptionalPeers(found_ips) self.log.debug("Found optional files after findhash connected peers: %s/%s" % (len(found), len(optional_hash_ids))) if found: found_peers = set([peer for hash_id_peers in found.values() for peer in hash_id_peers]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids): self.log.debug("No findhash result for optional files: %s" % (optional_hash_ids - set(found)))
def startFindOptional(self): time.sleep(0.01) # Wait for more file requests optional_tasks = [ task for task in self.tasks if task["optional_hash_id"] ] optional_hash_ids = set( [task["optional_hash_id"] for task in optional_tasks]) self.log.debug("Finding peers for optional files: %s" % optional_hash_ids) found = self.findOptionalTasks(optional_tasks) if found: found_peers = set( [peer for peers in found.values() for peer in peers]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids): self.log.debug("No local result for optional files: %s" % (optional_hash_ids - set(found))) # Query hashfield from connected peers threads = [] peers = self.site.getConnectedPeers() if not peers: peers = self.site.getConnectablePeers() for peer in peers: if not peer.time_hashfield: threads.append(gevent.spawn(peer.updateHashfield)) gevent.joinall(threads, timeout=5) found = self.findOptionalTasks(optional_tasks) self.log.debug( "Found optional files after query hashtable connected peers: %s/%s" % (len(found), len(optional_hash_ids))) if found: found_peers = set([ peer for hash_id_peers in found.values() for peer in hash_id_peers ]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids): self.log.debug( "No connected hashtable result for optional files: %s" % (optional_hash_ids - set(found))) # Try to query connected peers threads = [] peers = self.site.getConnectedPeers() if not peers: peers = self.site.getConnectablePeers() for peer in peers: threads.append( gevent.spawn(peer.findHashIds, list(optional_hash_ids))) gevent.joinall(threads, timeout=5) found_ips = helper.mergeDicts( [thread.value for thread in threads if thread.value]) found = self.addOptionalPeers(found_ips) self.log.debug( "Found optional files after findhash connected peers: %s/%s" % (len(found), len(optional_hash_ids))) if found: found_peers = set([ peer for hash_id_peers in found.values() for peer in hash_id_peers ]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids): self.log.debug("No findHash result, try random peers: %s" % (optional_hash_ids - set(found))) # Try to query random peers threads = [] peers = self.site.getConnectablePeers() for peer in peers[0:5]: threads.append( gevent.spawn(peer.findHashIds, list(optional_hash_ids))) gevent.joinall(threads, timeout=15) found_ips = helper.mergeDicts( [thread.value for thread in threads if thread.value]) found = self.addOptionalPeers(found_ips) self.log.debug( "Found optional files after findhash random peers: %s/%s" % (len(found), len(optional_hash_ids))) if found: found_peers = set([ peer for hash_id_peers in found.values() for peer in hash_id_peers ]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids): self.log.debug("No findhash result for optional files: %s" % (optional_hash_ids - set(found)))
def startFindOptional(self, reset_task=False, find_more=False, high_priority=False): # Wait for more file requests if len(self.tasks) < 20 or high_priority: time.sleep(0.01) if len(self.tasks) > 90: time.sleep(5) else: time.sleep(0.5) optional_tasks = [ task for task in self.tasks if task["optional_hash_id"] ] if not optional_tasks: return False optional_hash_ids = set( [task["optional_hash_id"] for task in optional_tasks]) time_tasks = self.time_task_added self.log.debug( "Finding peers for optional files: %s (reset_task: %s, find_more: %s)" % (optional_hash_ids, reset_task, find_more)) found = self.findOptionalTasks(optional_tasks, reset_task=reset_task) if found: found_peers = set( [peer for peers in found.values() for peer in peers]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids) or find_more or ( high_priority and any(len(peers) < 10 for peers in found.itervalues())): self.log.debug("No local result for optional files: %s" % (optional_hash_ids - set(found))) # Query hashfield from connected peers threads = [] peers = self.site.getConnectedPeers() if not peers: peers = self.site.getConnectablePeers() for peer in peers: if not peer.time_hashfield: threads.append(gevent.spawn(peer.updateHashfield)) gevent.joinall(threads, timeout=5) if time_tasks != self.time_task_added: # New task added since start optional_tasks = [ task for task in self.tasks if task["optional_hash_id"] ] optional_hash_ids = set( [task["optional_hash_id"] for task in optional_tasks]) found = self.findOptionalTasks(optional_tasks) self.log.debug( "Found optional files after query hashtable connected peers: %s/%s" % (len(found), len(optional_hash_ids))) if found: found_peers = set([ peer for hash_id_peers in found.values() for peer in hash_id_peers ]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids) or find_more: self.log.debug( "No connected hashtable result for optional files: %s" % (optional_hash_ids - set(found))) # Try to query connected peers threads = [] peers = [ peer for peer in self.site.getConnectedPeers() if peer not in self.asked_peers ] if not peers: peers = self.site.getConnectablePeers() for peer in peers: threads.append( gevent.spawn(peer.findHashIds, list(optional_hash_ids))) self.asked_peers.append(peer) for i in range(5): time.sleep(1) thread_values = [ thread.value for thread in threads if thread.value ] if not thread_values: continue found_ips = helper.mergeDicts(thread_values) found = self.addOptionalPeers(found_ips) self.log.debug( "Found optional files after findhash connected peers: %s/%s (asked: %s)" % (len(found), len(optional_hash_ids), len(threads))) if found: found_peers = set([ peer for hash_id_peers in found.values() for peer in hash_id_peers ]) self.startWorkers(found_peers) if len(thread_values) == len(threads): # Got result from all started thread break if len(found) < len(optional_hash_ids): self.log.debug("No findHash result, try random peers: %s" % (optional_hash_ids - set(found))) # Try to query random peers if time_tasks != self.time_task_added: # New task added since start optional_tasks = [ task for task in self.tasks if task["optional_hash_id"] ] optional_hash_ids = set( [task["optional_hash_id"] for task in optional_tasks]) threads = [] peers = self.site.getConnectablePeers(ignore=self.asked_peers) for peer in peers: threads.append( gevent.spawn(peer.findHashIds, list(optional_hash_ids))) self.asked_peers.append(peer) gevent.joinall(threads, timeout=15) found_ips = helper.mergeDicts( [thread.value for thread in threads if thread.value]) found = self.addOptionalPeers(found_ips) self.log.debug( "Found optional files after findhash random peers: %s/%s" % (len(found), len(optional_hash_ids))) if found: found_peers = set([ peer for hash_id_peers in found.values() for peer in hash_id_peers ]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids): self.log.debug("No findhash result for optional files: %s" % (optional_hash_ids - set(found)))
def startFindOptional(self, reset_task=False, find_more=False, high_priority=False): # Wait for more file requests if len(self.tasks) < 20 or high_priority: time.sleep(0.01) if len(self.tasks) > 90: time.sleep(5) else: time.sleep(0.5) optional_tasks = [task for task in self.tasks if task["optional_hash_id"]] if not optional_tasks: return False optional_hash_ids = set([task["optional_hash_id"] for task in optional_tasks]) time_tasks = self.time_task_added self.log.debug( "Finding peers for optional files: %s (reset_task: %s, find_more: %s)" % (optional_hash_ids, reset_task, find_more) ) found = self.findOptionalTasks(optional_tasks, reset_task=reset_task) if found: found_peers = set([peer for peers in found.values() for peer in peers]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids) or find_more or (high_priority and any(len(peers) < 10 for peers in found.itervalues())): self.log.debug("No local result for optional files: %s" % (optional_hash_ids - set(found))) # Query hashfield from connected peers threads = [] peers = self.site.getConnectedPeers() if not peers: peers = self.site.getConnectablePeers() for peer in peers: if not peer.time_hashfield: threads.append(gevent.spawn(peer.updateHashfield)) gevent.joinall(threads, timeout=5) if time_tasks != self.time_task_added: # New task added since start optional_tasks = [task for task in self.tasks if task["optional_hash_id"]] optional_hash_ids = set([task["optional_hash_id"] for task in optional_tasks]) found = self.findOptionalTasks(optional_tasks) self.log.debug("Found optional files after query hashtable connected peers: %s/%s" % ( len(found), len(optional_hash_ids) )) if found: found_peers = set([peer for hash_id_peers in found.values() for peer in hash_id_peers]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids) or find_more: self.log.debug("No connected hashtable result for optional files: %s" % (optional_hash_ids - set(found))) # Try to query connected peers threads = [] peers = [peer for peer in self.site.getConnectedPeers() if peer not in self.asked_peers] if not peers: peers = self.site.getConnectablePeers() for peer in peers: threads.append(gevent.spawn(peer.findHashIds, list(optional_hash_ids))) self.asked_peers.append(peer) for i in range(5): time.sleep(1) thread_values = [thread.value for thread in threads if thread.value] if not thread_values: continue found_ips = helper.mergeDicts(thread_values) found = self.addOptionalPeers(found_ips) self.log.debug("Found optional files after findhash connected peers: %s/%s (asked: %s)" % ( len(found), len(optional_hash_ids), len(threads) )) if found: found_peers = set([peer for hash_id_peers in found.values() for peer in hash_id_peers]) self.startWorkers(found_peers) if len(thread_values) == len(threads): # Got result from all started thread break if len(found) < len(optional_hash_ids): self.log.debug("No findHash result, try random peers: %s" % (optional_hash_ids - set(found))) # Try to query random peers if time_tasks != self.time_task_added: # New task added since start optional_tasks = [task for task in self.tasks if task["optional_hash_id"]] optional_hash_ids = set([task["optional_hash_id"] for task in optional_tasks]) threads = [] peers = self.site.getConnectablePeers(ignore=self.asked_peers) for peer in peers: threads.append(gevent.spawn(peer.findHashIds, list(optional_hash_ids))) self.asked_peers.append(peer) gevent.joinall(threads, timeout=15) found_ips = helper.mergeDicts([thread.value for thread in threads if thread.value]) found = self.addOptionalPeers(found_ips) self.log.debug("Found optional files after findhash random peers: %s/%s" % (len(found), len(optional_hash_ids))) if found: found_peers = set([peer for hash_id_peers in found.values() for peer in hash_id_peers]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids): self.log.debug("No findhash result for optional files: %s" % (optional_hash_ids - set(found)))