def downloadGeoLiteDb(self, db_path): import urllib import gzip import shutil from util import helper self.log.info("Downloading GeoLite2 City database...") self.cmd("progress", [ "geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], 0 ]) db_urls = [ "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz", "https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz" ] for db_url in db_urls: try: # Download response = helper.httpRequest(db_url) data_size = response.getheader('content-length') data_recv = 0 data = StringIO.StringIO() while True: buff = response.read(1024 * 512) if not buff: break data.write(buff) data_recv += 1024 * 512 if data_size: progress = int(float(data_recv) / int(data_size) * 100) self.cmd("progress", [ "geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], progress ]) self.log.info( "GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell()) data.seek(0) # Unpack with gzip.GzipFile(fileobj=data) as gzip_file: shutil.copyfileobj(gzip_file, open(db_path, "wb")) self.cmd("progress", [ "geolite-info", _["GeoLite2 City database downloaded!"], 100 ]) time.sleep(2) # Wait for notify animation return True except Exception as err: self.log.error("Error downloading %s: %s" % (db_url, err)) pass self.cmd("progress", [ "geolite-info", _["GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}"] .format(err, db_urls[0]), -100 ])
def downloadTor(self): self.log.info("Downloading Tor...") # Check Tor webpage for link download_page = helper.httpRequest( "https://www.torproject.org/download/download.html").read() download_url = re.search('href="(.*?tor.*?win32.*?zip)"', download_page).group(1) if not download_url.startswith("http"): download_url = "https://www.torproject.org/download/" + download_url # Download Tor client self.log.info("Downloading %s" % download_url) data = helper.httpRequest(download_url, as_file=True) data_size = data.tell() # Handle redirect if data_size < 1024 and "The document has moved" in data.getvalue(): download_url = re.search('href="(.*?tor.*?win32.*?zip)"', data.getvalue()).group(1) data = helper.httpRequest(download_url, as_file=True) data_size = data.tell() if data_size > 1024: import zipfile zip = zipfile.ZipFile(data) self.log.info("Unpacking Tor") for inner_path in zip.namelist(): if ".." in inner_path: continue dest_path = inner_path dest_path = re.sub("^Data/Tor/", "tools/tor/data/", dest_path) dest_path = re.sub("^Data/", "tools/tor/data/", dest_path) dest_path = re.sub("^Tor/", "tools/tor/", dest_path) dest_dir = os.path.dirname(dest_path) if dest_dir and not os.path.isdir(dest_dir): os.makedirs(dest_dir) if dest_dir != dest_path.strip("/"): data = zip.read(inner_path) if not os.path.isfile(dest_path): open(dest_path, 'wb').write(data) else: self.log.error("Bad response from server: %s" % data.getvalue()) return False
def downloadTor(self): self.log.info("Downloading Tor...") # Check Tor webpage for link download_page = helper.httpRequest("https://www.torproject.org/download/download.html").read() download_url = re.search('href="(.*?tor.*?win32.*?zip)"', download_page).group(1) if not download_url.startswith("http"): download_url = "https://www.torproject.org/download/" + download_url # Download Tor client self.log.info("Downloading %s" % download_url) data = helper.httpRequest(download_url, as_file=True) data_size = data.tell() # Handle redirect if data_size < 1024 and "The document has moved" in data.getvalue(): download_url = re.search('href="(.*?tor.*?win32.*?zip)"', data.getvalue()).group(1) data = helper.httpRequest(download_url, as_file=True) data_size = data.tell() if data_size > 1024: import zipfile zip = zipfile.ZipFile(data) self.log.info("Unpacking Tor") for inner_path in zip.namelist(): if ".." in inner_path: continue dest_path = inner_path dest_path = re.sub("^Data/Tor/", "tools/tor/data/", dest_path) dest_path = re.sub("^Data/", "tools/tor/data/", dest_path) dest_path = re.sub("^Tor/", "tools/tor/", dest_path) dest_dir = os.path.dirname(dest_path) if dest_dir and not os.path.isdir(dest_dir): os.makedirs(dest_dir) if dest_dir != dest_path.strip("/"): data = zip.read(inner_path) if not os.path.isfile(dest_path): open(dest_path, "wb").write(data) else: self.log.error("Bad response from server: %s" % data.getvalue()) return False
def downloadGeoLiteDb(self, db_path): import urllib import gzip import shutil from util import helper self.log.info("Downloading GeoLite2 City database...") self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], 0]) db_urls = [ "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz", "https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz" ] for db_url in db_urls: try: # Download response = helper.httpRequest(db_url) data_size = response.getheader('content-length') data_recv = 0 data = StringIO.StringIO() while True: buff = response.read(1024 * 512) if not buff: break data.write(buff) data_recv += 1024 * 512 if data_size: progress = int(float(data_recv) / int(data_size) * 100) self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], progress]) self.log.info("GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell()) data.seek(0) # Unpack with gzip.GzipFile(fileobj=data) as gzip_file: shutil.copyfileobj(gzip_file, open(db_path, "wb")) self.cmd("progress", ["geolite-info", _["GeoLite2 City database downloaded!"], 100]) time.sleep(2) # Wait for notify animation return True except Exception as err: self.log.error("Error downloading %s: %s" % (db_url, err)) pass self.cmd("progress", [ "geolite-info", _["GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}"].format(err, db_urls[0]), -100 ])
def downloadGeoLiteDb(self, db_path): import urllib import gzip import shutil from util import helper self.log.info("Downloading GeoLite2 City database...") self.cmd("notification", [ "geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], 0 ]) db_urls = [ "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz", "https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz" ] for db_url in db_urls: try: # Download response = helper.httpRequest(db_url) data = StringIO.StringIO() while True: buff = response.read(1024 * 512) if not buff: break data.write(buff) self.log.info( "GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell()) data.seek(0) # Unpack with gzip.GzipFile(fileobj=data) as gzip_file: shutil.copyfileobj(gzip_file, open(db_path, "wb")) self.cmd("notification", [ "geolite-done", _["GeoLite2 City database downloaded!"], 5000 ]) time.sleep(2) # Wait for notify animation return True except Exception, err: self.log.error("Error downloading %s: %s" % (db_url, err)) pass
def downloadGeoLiteDb(self, db_path): import urllib import gzip import shutil from util import helper self.log.info("Downloading GeoLite2 City database...") self.cmd("notification", ["geolite-info", "Downloading GeoLite2 City database (one time only, ~15MB)...", 0]) db_urls = [ "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz", "https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz" ] for db_url in db_urls: try: # Download response = helper.httpRequest(db_url) data = StringIO.StringIO() while True: buff = response.read(1024 * 512) if not buff: break data.write(buff) self.log.info("GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell()) data.seek(0) # Unpack with gzip.GzipFile(fileobj=data) as gzip_file: shutil.copyfileobj(gzip_file, open(db_path, "wb")) self.cmd("notification", ["geolite-done", "GeoLite2 City database downloaded!", 5000]) time.sleep(2) # Wait for notify animation return True except Exception, err: self.log.error("Error downloading %s: %s" % (db_url, err)) pass