def get_rtts(self):
"""Test latency to all mirrors"""
stderr.write("Testing latency to mirror(s)\n")
self.__kickoff_trips()
processed = 0
progress_msg(processed, self._num_trips)
for _ in xrange(self._num_trips):
try:
min_rtt = self._trip_queue.get(block=True)
except Empty:
pass
else:
# we can ignore empty rtt results (None) from the queue
# as in this case ConnectError was already raised
if min_rtt:
self._trip_queue.task_done()
self.urls[min_rtt[0]].update({"Latency": min_rtt[1]})
self.got["ping"] += 1
processed += 1
progress_msg(processed, self._num_trips)
stderr.write('\n')
# Mirrors without latency info are removed
self.urls = {
key: val
for key, val in self.urls.items() if "Latency" in val
}
self.ranked = sorted(self.urls, key=lambda x: self.urls[x]["Latency"])
def lookup_statuses(self, min_status):
"""Scrape statuses/info in from launchpad.net mirror pages"""
while (self.got["data"] < self.status_num) and self.ranked:
data_queue = Queue()
num_threads = self.__queue_lookups(data_queue)
if num_threads == 0:
break
# Get output of all started thread methods from queue
progress_msg(self.got["data"], self.status_num)
for _ in xrange(num_threads):
try:
# We don't care about timeouts longer than 7 seconds as
# we're only getting 16 KB
info = data_queue.get(block=True, timeout=7)
except Empty:
pass
else:
data_queue.task_done()
if info[1] and info[1]["Status"] in self._status_opts:
self.urls[info[0]].update(info[1])
self.got["data"] += 1
self.top_list.append(info[0])
progress_msg(self.got["data"], self.status_num)
# Eliminate the url from the ranked list as long as
# something is received from the queue (for selective
# iteration if another queue needs to be built)
self.ranked.remove(info[0])
if (self.got["data"] == self.status_num):
break
# Reorder by latency as queue returns vary building final list
self.top_list = sorted(self.top_list,
key=lambda x: self.urls[x]["Latency"])
data_queue.join()