def fill_database(self):
logging.basicConfig(level=logging.INFO)
database = MirrorDatabase()
helper = Helpers()
database.create_database()
downloader = MirrorDownloader()
if not (database.conn.number_of_rows('mirrors')):
mirrors = downloader.download_mirrors()
ip, location = helper.get_location()
for distro in mirrors:
database.conn.executemany('INSERT INTO mirrors (Distro, Mirror) VALUES (?, ?)', [(distro['name'], m) for m in distro['mirrors']])
database.conn.commit()
else:
database.log.warn('No mirrors were added')
hosts = database.resolve_mirrors()
for mirror_id, addresses in hosts:
for address in addresses:
if not database.get_mirror_by_ip(address):
database.log.info("IP %s was new", address)
location = helper.get_mirror_location(address)
country = location['country']['names']['en']
city = location['city']['names']['en'] if 'city' in location else ""
area = ", ".join([s['names']['en'] for s in location['subdivisions']]) if 'subdivisions' in location else ""
database.conn.execute("INSERT INTO mirror_ip (MirrorID, IP, Country, Area, City) VALUES (?, ?, ?, ?, ?)", [mirror_id, address, country, area, city])
else:
database.log.info("IP %s was already in the database", address)
database.conn.commit()
database.conn.close()
def get_mirrors(self, location=None):
masterlist = requests.get("http://anonscm.debian.org/viewvc/webwml/webwml/english/mirror/Mirrors.masterlist?revision=HEAD").text
mirrors = collections.defaultdict(set)
for mirror in deb822.Deb822.iter_paragraphs(masterlist):
if not "Country" in mirror:
continue
country = mirror["Country"].split(None, 1)[0]
site = mirror["Site"]
for proto in ['http']:#, 'ftp':
if "Archive-%s" % proto in mirror:
mirrors[country].add("%s://%s%s" % (proto, site,
mirror["Archive-%s" % proto]))
if len(mirrors) == 0:
raise IOError("Couldn't get mirrors for an unknown reason")
sys.exit(1)
if location is None:
h = Helpers()
location = h.get_location()[1]
return mirrors[location]
