def downloadGeoLiteDb(self, db_path):
import urllib
import gzip
import shutil
from util import helper
self.log.info("Downloading GeoLite2 City database...")
self.cmd("progress", [
"geolite-info",
_["Downloading GeoLite2 City database (one time only, ~20MB)..."],
0
])
db_urls = [
"https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz",
"https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz"
]
for db_url in db_urls:
try:
# Download
response = helper.httpRequest(db_url)
data_size = response.getheader('content-length')
data_recv = 0
data = StringIO.StringIO()
while True:
buff = response.read(1024 * 512)
if not buff:
break
data.write(buff)
data_recv += 1024 * 512
if data_size:
progress = int(float(data_recv) / int(data_size) * 100)
self.cmd("progress", [
"geolite-info",
_["Downloading GeoLite2 City database (one time only, ~20MB)..."],
progress
])
self.log.info(
"GeoLite2 City database downloaded (%s bytes), unpacking..."
% data.tell())
data.seek(0)
# Unpack
with gzip.GzipFile(fileobj=data) as gzip_file:
shutil.copyfileobj(gzip_file, open(db_path, "wb"))
self.cmd("progress", [
"geolite-info", _["GeoLite2 City database downloaded!"],
100
])
time.sleep(2) # Wait for notify animation
return True
except Exception as err:
self.log.error("Error downloading %s: %s" % (db_url, err))
pass
self.cmd("progress", [
"geolite-info",
_["GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}"]
.format(err, db_urls[0]), -100
])
def downloadTor(self):
self.log.info("Downloading Tor...")
# Check Tor webpage for link
download_page = helper.httpRequest(
"https://www.torproject.org/download/download.html").read()
download_url = re.search('href="(.*?tor.*?win32.*?zip)"',
download_page).group(1)
if not download_url.startswith("http"):
download_url = "https://www.torproject.org/download/" + download_url
# Download Tor client
self.log.info("Downloading %s" % download_url)
data = helper.httpRequest(download_url, as_file=True)
data_size = data.tell()
# Handle redirect
if data_size < 1024 and "The document has moved" in data.getvalue():
download_url = re.search('href="(.*?tor.*?win32.*?zip)"',
data.getvalue()).group(1)
data = helper.httpRequest(download_url, as_file=True)
data_size = data.tell()
if data_size > 1024:
import zipfile
zip = zipfile.ZipFile(data)
self.log.info("Unpacking Tor")
for inner_path in zip.namelist():
if ".." in inner_path:
continue
dest_path = inner_path
dest_path = re.sub("^Data/Tor/", "tools/tor/data/", dest_path)
dest_path = re.sub("^Data/", "tools/tor/data/", dest_path)
dest_path = re.sub("^Tor/", "tools/tor/", dest_path)
dest_dir = os.path.dirname(dest_path)
if dest_dir and not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
if dest_dir != dest_path.strip("/"):
data = zip.read(inner_path)
if not os.path.isfile(dest_path):
open(dest_path, 'wb').write(data)
else:
self.log.error("Bad response from server: %s" % data.getvalue())
return False
def downloadTor(self):
self.log.info("Downloading Tor...")
# Check Tor webpage for link
download_page = helper.httpRequest("https://www.torproject.org/download/download.html").read()
download_url = re.search('href="(.*?tor.*?win32.*?zip)"', download_page).group(1)
if not download_url.startswith("http"):
download_url = "https://www.torproject.org/download/" + download_url
# Download Tor client
self.log.info("Downloading %s" % download_url)
data = helper.httpRequest(download_url, as_file=True)
data_size = data.tell()
# Handle redirect
if data_size < 1024 and "The document has moved" in data.getvalue():
download_url = re.search('href="(.*?tor.*?win32.*?zip)"', data.getvalue()).group(1)
data = helper.httpRequest(download_url, as_file=True)
data_size = data.tell()
if data_size > 1024:
import zipfile
zip = zipfile.ZipFile(data)
self.log.info("Unpacking Tor")
for inner_path in zip.namelist():
if ".." in inner_path:
continue
dest_path = inner_path
dest_path = re.sub("^Data/Tor/", "tools/tor/data/", dest_path)
dest_path = re.sub("^Data/", "tools/tor/data/", dest_path)
dest_path = re.sub("^Tor/", "tools/tor/", dest_path)
dest_dir = os.path.dirname(dest_path)
if dest_dir and not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
if dest_dir != dest_path.strip("/"):
data = zip.read(inner_path)
if not os.path.isfile(dest_path):
open(dest_path, "wb").write(data)
else:
self.log.error("Bad response from server: %s" % data.getvalue())
return False
def downloadGeoLiteDb(self, db_path):
import urllib
import gzip
import shutil
from util import helper
self.log.info("Downloading GeoLite2 City database...")
self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], 0])
db_urls = [
"https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz",
"https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz"
]
for db_url in db_urls:
try:
# Download
response = helper.httpRequest(db_url)
data_size = response.getheader('content-length')
data_recv = 0
data = StringIO.StringIO()
while True:
buff = response.read(1024 * 512)
if not buff:
break
data.write(buff)
data_recv += 1024 * 512
if data_size:
progress = int(float(data_recv) / int(data_size) * 100)
self.cmd("progress", ["geolite-info", _["Downloading GeoLite2 City database (one time only, ~20MB)..."], progress])
self.log.info("GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell())
data.seek(0)
# Unpack
with gzip.GzipFile(fileobj=data) as gzip_file:
shutil.copyfileobj(gzip_file, open(db_path, "wb"))
self.cmd("progress", ["geolite-info", _["GeoLite2 City database downloaded!"], 100])
time.sleep(2) # Wait for notify animation
return True
except Exception as err:
self.log.error("Error downloading %s: %s" % (db_url, err))
pass
self.cmd("progress", [
"geolite-info",
_["GeoLite2 City database download error: {}!<br>Please download manually and unpack to data dir:<br>{}"].format(err, db_urls[0]),
-100
])
def downloadGeoLiteDb(self, db_path):
import urllib
import gzip
import shutil
from util import helper
self.log.info("Downloading GeoLite2 City database...")
self.cmd("notification", [
"geolite-info",
_["Downloading GeoLite2 City database (one time only, ~20MB)..."],
0
])
db_urls = [
"https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz",
"https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz"
]
for db_url in db_urls:
try:
# Download
response = helper.httpRequest(db_url)
data = StringIO.StringIO()
while True:
buff = response.read(1024 * 512)
if not buff:
break
data.write(buff)
self.log.info(
"GeoLite2 City database downloaded (%s bytes), unpacking..."
% data.tell())
data.seek(0)
# Unpack
with gzip.GzipFile(fileobj=data) as gzip_file:
shutil.copyfileobj(gzip_file, open(db_path, "wb"))
self.cmd("notification", [
"geolite-done", _["GeoLite2 City database downloaded!"],
5000
])
time.sleep(2) # Wait for notify animation
return True
except Exception, err:
self.log.error("Error downloading %s: %s" % (db_url, err))
pass
def downloadGeoLiteDb(self, db_path):
import urllib
import gzip
import shutil
from util import helper
self.log.info("Downloading GeoLite2 City database...")
self.cmd("notification", ["geolite-info", "Downloading GeoLite2 City database (one time only, ~15MB)...", 0])
db_urls = [
"http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz",
"https://raw.githubusercontent.com/texnikru/GeoLite2-Database/master/GeoLite2-City.mmdb.gz"
]
for db_url in db_urls:
try:
# Download
response = helper.httpRequest(db_url)
data = StringIO.StringIO()
while True:
buff = response.read(1024 * 512)
if not buff:
break
data.write(buff)
self.log.info("GeoLite2 City database downloaded (%s bytes), unpacking..." % data.tell())
data.seek(0)
# Unpack
with gzip.GzipFile(fileobj=data) as gzip_file:
shutil.copyfileobj(gzip_file, open(db_path, "wb"))
self.cmd("notification", ["geolite-done", "GeoLite2 City database downloaded!", 5000])
time.sleep(2) # Wait for notify animation
return True
except Exception, err:
self.log.error("Error downloading %s: %s" % (db_url, err))
pass
