Esempio n. 1
0
    def get_rtts(self):
        """Test latency to all mirrors"""

        self.__kickoff_trips()

        processed = 0
        stderr.write("Testing %d mirror(s)\n" % self.num_trips)
        progress_msg(processed, self.num_trips)
        for _ in range(self.num_trips):
            try:
                min_rtt = self.trip_queue.get(block=True)
            except Empty:
                pass
            else:
                # we can ignore empty rtt results (None) from the queue
                # as in this case ConnectError was already raised
                if min_rtt:
                    self.trip_queue.task_done()
                    self.urls[min_rtt[0]].update({"Latency": min_rtt[1]})
                    self.got["ping"] += 1

            processed += 1
            progress_msg(processed, self.num_trips)

        stderr.write('\n')
        # Mirrors without latency info are removed
        self.urls = {
            key: val for key, val in self.urls.items() if "Latency" in val
        }

        self.ranked = sorted(
            self.urls, key=lambda x: self.urls[x]["Latency"]
        )
Esempio n. 2
0
    def get_rtts(self):
        """Test latency to all mirrors"""
        processed = 0
        stderr.write("Testing %d mirror(s)\n" % self.test_num)
        progress_msg(processed, self.test_num)
        for url, info in self.urls.items():
            host = info["Host"]
            try:
                trip = _RoundTrip(host)
            except gaierror as err:
                stderr.write("%s: %s ignored\n" % (err, url))
            else:
                try:
                    rtt = trip.min_rtt()
                except ConnectError as err:
                    stderr.write("\nconnection to %s: %s\n" % (host, err))
                else:
                    self.urls[url].update({"Latency": rtt})
                    self.got["ping"] += 1

            processed += 1
            progress_msg(processed, self.test_num)

        stderr.write('\n')
        # Mirrors without latency info are removed
        self.urls = {
            key: val for key, val in self.urls.items() if "Latency" in val
        }

        self.ranked = sorted(self.urls, key=lambda x: self.urls[x]["Latency"])
Esempio n. 3
0
    def lookup_statuses(self, min_status, codename, hardware):
        """Scrape statuses/info in from launchpad.net mirror pages"""
        while (self.got["data"] < self.status_num) and self.ranked:
            data_queue = Queue()
            num_threads = self.__queue_lookups(codename, hardware, data_queue)
            if num_threads == 0:
                break
            # Get output of all started thread methods from queue
            progress_msg(self.got["data"], self.status_num)
            for _ in range(num_threads):
                try:
                    # We don't care about timeouts longer than 7 seconds as
                    # we're only getting 16 KB
                    info = data_queue.get(block=True, timeout=7)
                except Empty:
                    pass
                else:
                    data_queue.task_done()
                    if info[1] and info[1]["Status"] in self.status_opts:
                        self.urls[info[0]].update(info[1])
                        self.got["data"] += 1
                        self.top_list.append(info[0])
                        progress_msg(self.got["data"], self.status_num)
                    else:
                        # Remove unqualified results from ranked list so
                        # queueing can use it to populate the right threads
                        self.ranked.remove(info[0])

                if (self.got["data"] == self.status_num):
                    break

            data_queue.join()
Esempio n. 4
0
    def get_rtts(self):
        """Test latency to all mirrors"""

        self.__kickoff_trips()

        processed = 0
        stderr.write("Testing %d mirror(s)\n" % self.num_trips)
        progress_msg(processed, self.num_trips)
        for _ in range(self.num_trips):
            try:
                min_rtt = self.trip_queue.get(block=True)
            except Empty:
                pass
            else:
                # we can ignore empty rtt results (None) from the queue
                # as in this case ConnectError was already raised
                if min_rtt:
                    self.trip_queue.task_done()
                    self.urls[min_rtt[0]].update({"Latency": min_rtt[1]})
                    self.got["ping"] += 1

            processed += 1
            progress_msg(processed, self.num_trips)

        stderr.write('\n')
        # Mirrors without latency info are removed
        self.urls = {
            key: val for key, val in self.urls.items() if "Latency" in val
        }

        self.ranked = sorted(
            self.urls, key=lambda x: self.urls[x]["Latency"]
        )
Esempio n. 5
0
    def lookup_statuses(self):
        """Scrape requested number of statuses/info from Launchpad"""
        total = 0
        progress_msg(self.got["data"], self.status_num)
        for url in (x for x in self.ranked
                    if "Status" not in self.urls[x]):
            try:
                info = self.__get_info(url)
            except DataError as err:
                self.ranked.remove(url)
                stderr.write("\n%s\n" % err)
            else:
                if info and info[1] and info[1]["Status"] in self.status_opts:
                    self.urls[url].update(info[1])
                    self.got["data"] += 1
                    self.top_list.append(info[0])

            total += 1
            progress_msg(self.got["data"], self.status_num)
            if ((self.got["data"] == self.status_num) or
                    (total == self.got["ping"])):
                break
Esempio n. 6
0
    def lookup_statuses(self, min_status, codename, hardware):
        """Scrape statuses/info in from launchpad.net mirror pages"""
        while (self.got["data"] < self.status_num) and self.ranked:
            data_queue = Queue()
            num_threads = self.__queue_lookups(codename, hardware, data_queue)
            if num_threads == 0:
                break
            # Get output of all started thread methods from queue
            progress_msg(self.got["data"], self.status_num)
            for _ in range(num_threads):
                try:
                    # We don't care about timeouts longer than 7 seconds as
                    # we're only getting 16 KB
                    info = data_queue.get(block=True, timeout=7)
                except Empty:
                    pass
                else:
                    data_queue.task_done()
                    if info[1] and info[1]["Status"] in self.status_opts:
                        self.urls[info[0]].update(info[1])
                        self.got["data"] += 1
                        self.top_list.append(info[0])
                        progress_msg(self.got["data"], self.status_num)

                    # Eliminate the url from the ranked list as long as
                    # something is received from the queue (for selective
                    # iteration if another queue needs to be built)
                    self.ranked.remove(info[0])

                if (self.got["data"] == self.status_num):
                    break

            # Reorder by latency as queue returns vary building final list
            self.top_list = sorted(
                self.top_list, key=lambda x: self.urls[x]["Latency"]
            )

            data_queue.join()