def fill_database(self): logging.basicConfig(level=logging.INFO) database = MirrorDatabase() helper = Helpers() database.create_database() downloader = MirrorDownloader() if not (database.conn.number_of_rows('mirrors')): mirrors = downloader.download_mirrors() ip, location = helper.get_location() for distro in mirrors: database.conn.executemany('INSERT INTO mirrors (Distro, Mirror) VALUES (?, ?)', [(distro['name'], m) for m in distro['mirrors']]) database.conn.commit() else: database.log.warn('No mirrors were added') hosts = database.resolve_mirrors() for mirror_id, addresses in hosts: for address in addresses: if not database.get_mirror_by_ip(address): database.log.info("IP %s was new", address) location = helper.get_mirror_location(address) country = location['country']['names']['en'] city = location['city']['names']['en'] if 'city' in location else "" area = ", ".join([s['names']['en'] for s in location['subdivisions']]) if 'subdivisions' in location else "" database.conn.execute("INSERT INTO mirror_ip (MirrorID, IP, Country, Area, City) VALUES (?, ?, ?, ?, ?)", [mirror_id, address, country, area, city]) else: database.log.info("IP %s was already in the database", address) database.conn.commit() database.conn.close()
def get_mirrors(self, location=None): masterlist = requests.get("http://anonscm.debian.org/viewvc/webwml/webwml/english/mirror/Mirrors.masterlist?revision=HEAD").text mirrors = collections.defaultdict(set) for mirror in deb822.Deb822.iter_paragraphs(masterlist): if not "Country" in mirror: continue country = mirror["Country"].split(None, 1)[0] site = mirror["Site"] for proto in ['http']:#, 'ftp': if "Archive-%s" % proto in mirror: mirrors[country].add("%s://%s%s" % (proto, site, mirror["Archive-%s" % proto])) if len(mirrors) == 0: raise IOError("Couldn't get mirrors for an unknown reason") sys.exit(1) if location is None: h = Helpers() location = h.get_location()[1] return mirrors[location]