def get_rtts(self):
"""Test latency to all mirrors"""
self.__kickoff_trips()
processed = 0
stderr.write("Testing %d mirror(s)\n" % self.num_trips)
progress_msg(processed, self.num_trips)
for _ in range(self.num_trips):
try:
min_rtt = self.trip_queue.get(block=True)
except Empty:
pass
else:
# we can ignore empty rtt results (None) from the queue
# as in this case ConnectError was already raised
if min_rtt:
self.trip_queue.task_done()
self.urls[min_rtt[0]].update({"Latency": min_rtt[1]})
self.got["ping"] += 1
processed += 1
progress_msg(processed, self.num_trips)
stderr.write('\n')
# Mirrors without latency info are removed
self.urls = {
key: val for key, val in self.urls.items() if "Latency" in val
}
self.ranked = sorted(
self.urls, key=lambda x: self.urls[x]["Latency"]
)
def get_rtts(self):
"""Test latency to all mirrors"""
processed = 0
stderr.write("Testing %d mirror(s)\n" % self.test_num)
progress_msg(processed, self.test_num)
for url, info in self.urls.items():
host = info["Host"]
try:
trip = _RoundTrip(host)
except gaierror as err:
stderr.write("%s: %s ignored\n" % (err, url))
else:
try:
rtt = trip.min_rtt()
except ConnectError as err:
stderr.write("\nconnection to %s: %s\n" % (host, err))
else:
self.urls[url].update({"Latency": rtt})
self.got["ping"] += 1
processed += 1
progress_msg(processed, self.test_num)
stderr.write('\n')
# Mirrors without latency info are removed
self.urls = {
key: val for key, val in self.urls.items() if "Latency" in val
}
self.ranked = sorted(self.urls, key=lambda x: self.urls[x]["Latency"])
def lookup_statuses(self, min_status, codename, hardware):
"""Scrape statuses/info in from launchpad.net mirror pages"""
while (self.got["data"] < self.status_num) and self.ranked:
data_queue = Queue()
num_threads = self.__queue_lookups(codename, hardware, data_queue)
if num_threads == 0:
break
# Get output of all started thread methods from queue
progress_msg(self.got["data"], self.status_num)
for _ in range(num_threads):
try:
# We don't care about timeouts longer than 7 seconds as
# we're only getting 16 KB
info = data_queue.get(block=True, timeout=7)
except Empty:
pass
else:
data_queue.task_done()
if info[1] and info[1]["Status"] in self.status_opts:
self.urls[info[0]].update(info[1])
self.got["data"] += 1
self.top_list.append(info[0])
progress_msg(self.got["data"], self.status_num)
else:
# Remove unqualified results from ranked list so
# queueing can use it to populate the right threads
self.ranked.remove(info[0])
if (self.got["data"] == self.status_num):
break
data_queue.join()
def get_rtts(self):
"""Test latency to all mirrors"""
self.__kickoff_trips()
processed = 0
stderr.write("Testing %d mirror(s)\n" % self.num_trips)
progress_msg(processed, self.num_trips)
for _ in range(self.num_trips):
try:
min_rtt = self.trip_queue.get(block=True)
except Empty:
pass
else:
# we can ignore empty rtt results (None) from the queue
# as in this case ConnectError was already raised
if min_rtt:
self.trip_queue.task_done()
self.urls[min_rtt[0]].update({"Latency": min_rtt[1]})
self.got["ping"] += 1
processed += 1
progress_msg(processed, self.num_trips)
stderr.write('\n')
# Mirrors without latency info are removed
self.urls = {
key: val for key, val in self.urls.items() if "Latency" in val
}
self.ranked = sorted(
self.urls, key=lambda x: self.urls[x]["Latency"]
)
def lookup_statuses(self):
"""Scrape requested number of statuses/info from Launchpad"""
total = 0
progress_msg(self.got["data"], self.status_num)
for url in (x for x in self.ranked
if "Status" not in self.urls[x]):
try:
info = self.__get_info(url)
except DataError as err:
self.ranked.remove(url)
stderr.write("\n%s\n" % err)
else:
if info and info[1] and info[1]["Status"] in self.status_opts:
self.urls[url].update(info[1])
self.got["data"] += 1
self.top_list.append(info[0])
total += 1
progress_msg(self.got["data"], self.status_num)
if ((self.got["data"] == self.status_num) or
(total == self.got["ping"])):
break
def lookup_statuses(self, min_status, codename, hardware):
"""Scrape statuses/info in from launchpad.net mirror pages"""
while (self.got["data"] < self.status_num) and self.ranked:
data_queue = Queue()
num_threads = self.__queue_lookups(codename, hardware, data_queue)
if num_threads == 0:
break
# Get output of all started thread methods from queue
progress_msg(self.got["data"], self.status_num)
for _ in range(num_threads):
try:
# We don't care about timeouts longer than 7 seconds as
# we're only getting 16 KB
info = data_queue.get(block=True, timeout=7)
except Empty:
pass
else:
data_queue.task_done()
if info[1] and info[1]["Status"] in self.status_opts:
self.urls[info[0]].update(info[1])
self.got["data"] += 1
self.top_list.append(info[0])
progress_msg(self.got["data"], self.status_num)
# Eliminate the url from the ranked list as long as
# something is received from the queue (for selective
# iteration if another queue needs to be built)
self.ranked.remove(info[0])
if (self.got["data"] == self.status_num):
break
# Reorder by latency as queue returns vary building final list
self.top_list = sorted(
self.top_list, key=lambda x: self.urls[x]["Latency"]
)
data_queue.join()