def test_read_from_cache(self):
create_cwd(cwd())
Config._cache = {}
assert cfg("operationality", "ip_api") == "http://api.ipify.org"
assert "operationality" in Config._cache
Config._cache["operationality"]["ip_api"] = "http://example.com"
assert cfg("operationality", "ip_api") == "http://example.com"
def verify(self):
"""
Test if this socks5 can be connected to and retrieve its own
IP through the configured IP api. Automatically updates the
'operational' value in the database
:returns: True if server is operational, false otherwise
:rtype: bool
"""
operational = False
ip = self.host
if not is_ipv4(ip):
ip = get_ipv4_hostname(ip)
response = get_over_socks5(
cfg("operationality", "ip_api"), self.host, self.port,
username=self.username, password=self.password,
timeout=cfg("operationality", "timeout")
)
if response:
if ip == response:
operational = True
# If a private ip is used, the api response will not match with
# the configured host or its ip. There was however a response,
# therefore we still mark it as operational
elif is_reserved_ipv4(ip) and is_ipv4(response):
operational = True
db.set_operational(self.id, operational)
return operational
def measure_connection_time(self):
"""
Measure the time it takes to connect to the specified connection
test URL in the config. Result is automatically stored in the
database
:returns: An approximate connection time in seconds
:rtype: float
"""
s = socks.socksocket()
s.set_proxy(socks.SOCKS5,
self.host,
self.port,
username=self.username,
password=self.password)
s.settimeout(cfg("connection_time", "timeout"))
start = time.time()
try:
s.connect((cfg("connection_time",
"hostname"), cfg("connection_time", "port")))
s.close()
# socket.error, socks.ProxyError
except Exception as e:
log.error("Error connecting in connection time test: %s", e)
connect_time = None
else:
connect_time = time.time() - start
db.set_connect_time(self.id, connect_time)
return connect_time
def measure_connection_time(self):
"""
Measure the time it takes to connect to the specified connection
test URL in the config. Result is automatically stored in the
database
:returns: An approximate connection time in seconds
:rtype: float
"""
s = socks.socksocket()
s.set_proxy(
socks.SOCKS5, self.host, self.port, username=self.username,
password=self.password
)
s.settimeout(cfg("connection_time", "timeout"))
start = time.time()
try:
s.connect((
cfg("connection_time", "hostname"),
cfg("connection_time", "port")
))
s.close()
except (socks.ProxyError, socket.error) as e:
log.error("Error connecting in connection time test: %s", e)
connect_time = None
else:
connect_time = time.time() - start
db.set_connect_time(self.id, connect_time)
return connect_time
def test_invalid_conf(self):
create_cwd(cwd())
Config._cache = {}
with open(cwd("conf", "socks5man.conf"), "wb") as fw:
fw.write(os.urandom(512))
with pytest.raises(Socks5ConfigError):
cfg("socks5man", "verify_interval")
def test_ip_api(self):
"""Verify that the default ip api returns an actual ip"""
create_cwd(cwd())
res = urllib2.urlopen(cfg("operationality", "ip_api"),
timeout=cfg("operationality", "timeout"))
assert res.getcode() == 200
assert re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", res.read())
def test_invalid_conf(self):
create_cwd(cwd())
Config._cache = {}
with open(cwd("conf", "socks5man.conf"), "w") as fw:
fw.write("socks5man to dominate them all")
with pytest.raises(Socks5ConfigError):
cfg("socks5man", "verify_interval")
def test_download_url(self):
"""Verify that the url used to measure an approximate bandwidth
is still available"""
create_cwd(cwd())
res = urllib2.urlopen(cfg("bandwidth", "download_url"),
timeout=cfg("bandwidth", "timeout"))
assert res.getcode() == 200
assert len(res.read()) > 0
def test_ip_api(self):
"""Verify that the default ip api returns an actual ip"""
create_cwd(cwd())
res = urllib2.urlopen(
cfg("operationality", "ip_api"),
timeout=cfg("operationality", "timeout")
)
assert res.getcode() == 200
assert re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", res.read())
def test_measure_time_host(self):
"""Verify that the default connection measurement still accepts
connections"""
create_cwd(cwd())
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(cfg("connection_time", "timeout"))
s.connect((cfg("connection_time",
"hostname"), cfg("connection_time", "port")))
s.close()
def test_measure_time_host(self):
"""Verify that the default connection measurement still accepts
connections"""
create_cwd(cwd())
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(cfg("connection_time", "timeout"))
s.connect((
cfg("connection_time", "hostname"), cfg("connection_time", "port")
))
s.close()
def test_download_url(self):
"""Verify that the url used to measure an approximate bandwidth
is still available"""
create_cwd(cwd())
res = urllib2.urlopen(
cfg("bandwidth", "download_url"),
timeout=cfg("bandwidth", "timeout")
)
assert res.getcode() == 200
assert len(res.read()) > 0
def approx_bandwidth(self):
"""
Calculate an approximate Mbit/s download speed using
the file specified in the config to download. Automatically
updated in the database
:returns: An approximate download speed in Mbit/s
:rtype: float
"""
approx_bandwidth = approximate_bandwidth(
self.host, self.port, username=self.username,
password=self.password, times=cfg("bandwidth", "times"),
timeout=cfg("bandwidth", "timeout")
)
db.set_approx_bandwidth(self.id, approx_bandwidth)
return approx_bandwidth
def approximate_bandwidth(host,
port,
username=None,
password=None,
maxfail=1,
times=2,
timeout=10):
"""Tries to determine the average download speed in Mbit/s.
Higher 'times' values will result in more accurate speeds, but will
take a longer time.
specified socks5. This value is subtracted from the total time it takes
to download a test file.
@param maxfail: The maximum amount of times the socks5 is allowed to
timeout before the measurement should stop
@param times: The amount of times the test file should be downloaded.
Optimal amount would be 3-4.
"""
total = 0
fails = 0
test_url = cfg("bandwidth", "download_url")
for t in range(times):
start = time.time()
response = get_over_socks5(test_url,
host,
port,
username=username,
password=password,
timeout=timeout)
if not response:
if fails >= maxfail:
return None
fails += 1
continue
took = time.time() - start
# Calculate the approximate Mbit/s
try:
speed = (len(response) / took) / 1000000 * 8
except ZeroDivisionError:
# Can be thrown if the download was instant. To still calculate
# a speed, use 0.001 as the time it took
speed = (len(response) / 0.001) / 1000000 * 8
# If the used file to measure is smaller than approx 1 MB,
# add a small amount as the small size might cause the TCP window
# to stay small
if len(response) < 1000000:
speed += speed * 0.1
total += speed
return total / times
def test_geoipdb_hash_url(self):
create_cwd(cwd())
res = urllib.request.urlopen(cfg("geodb", "geodb_md5_url"))
assert res.getcode() == 200
assert len(res.read()) == 32
def test_invalid_optiontype(self):
create_cwd(cwd())
Config._cache = {}
Config._conf["operationality"]["ip_api"] = int
with pytest.raises(Socks5ConfigError):
cfg("socks5man", "verify_interval")
def test_unknown_section(self):
create_cwd(cwd())
Config._cache = {}
del Config._conf["operationality"]
with pytest.raises(Socks5ConfigError):
cfg("socks5man", "verify_interval")
def test_missing_conf(self):
create_cwd(cwd())
os.remove(cwd("conf", "socks5man.conf"))
Config._cache = {}
with pytest.raises(Socks5ConfigError):
cfg("socks5man", "verify_interval")
def test_cfg_invalid(self):
create_cwd(cwd())
with pytest.raises(Socks5ConfigError):
cfg("socks5man", "nonexistingkeystuffdogestosti")
with pytest.raises(Socks5ConfigError):
cfg("doges", "wut")
def test_cfg_values(self):
create_cwd(cwd())
assert cfg("socks5man", "verify_interval") == 300
assert cfg("operationality", "ip_api") == "http://api.ipify.org"
assert not cfg("bandwidth", "enabled")
assert cfg("connection_time", "enabled")
def test_cfg_defaults(self):
create_cwd(cwd())
assert isinstance(cfg("socks5man", "verify_interval"), int)
assert isinstance(cfg("socks5man", "bandwidth_interval"), int)
assert isinstance(cfg("operationality", "ip_api"), (str, basestring))
assert isinstance(cfg("operationality", "timeout"), int)
assert isinstance(cfg("connection_time", "enabled"), bool)
assert isinstance(cfg("connection_time", "timeout"), int)
assert isinstance(cfg("connection_time", "hostname"),(str, basestring))
assert isinstance(cfg("connection_time", "port"), int)
assert isinstance(cfg("bandwidth", "enabled"), bool)
assert isinstance(cfg("bandwidth", "download_url"), (str, basestring))
assert isinstance(cfg("bandwidth", "times"), int)
assert isinstance(cfg("bandwidth", "timeout"), int)
assert isinstance(cfg("geodb", "geodb_url"), (str, basestring))
assert isinstance(cfg("geodb", "geodb_md5_url"), (str, basestring))
def test_geoipdb_hash_url(self):
create_cwd(cwd())
res = urllib2.urlopen(cfg("geodb", "geodb_md5_url"))
assert res.getcode() == 200
assert len(res.read()) == 32
def verify_all(repeated=False, operational=None, unverified=None):
last_bandwidth = None
bandwidth_checked = False
download_verified = False
if repeated:
log.info("Starting continuous verification")
while True:
socks5_list = db.list_socks5(
operational=operational, unverified=unverified
)
log.debug("Verifying %s socks5 servers", len(socks5_list))
for socks5 in socks5_list:
socks5 = Socks5(socks5)
log.info(
"Testing socks5 server: '%s:%s'", socks5.host, socks5.port
)
if socks5.verify():
log.info("Operationality check: OK")
else:
log.warning(
"Operationality check (%s:%s): FAILED",
socks5.host, socks5.port
)
continue
if cfg("connection_time", "enabled"):
con_time = socks5.measure_connection_time()
if con_time:
log.debug("Measured connection time: %s", con_time)
else:
log.warning(
"Connection time measurement failed for: '%s:%s'",
socks5.host, socks5.port
)
continue
if cfg("bandwidth", "enabled"):
print "BLABLA 2"
if last_bandwidth:
waited = time.time() - last_bandwidth
if waited < cfg("socks5man", "bandwidth_interval"):
continue
if not download_verified:
download_url = cfg("bandwidth", "download_url")
try:
urllib2.urlopen(download_url, timeout=5)
download_verified = True
except (socket.error, urllib2.URLError) as e:
log.error(
"Failed to download speed test file: '%s'. Please"
" verify the configured file is still online!"
" Without this file, it is not possible to"
" approximate the bandwidth of a socsk5 server."
" Error: %s", download_url, e
)
continue
bandwidth_checked = True
bandwidth = socks5.approx_bandwidth()
if bandwidth:
log.debug(
"Approximate bandwidth: %s Mbit/s down", bandwidth
)
else:
log.warning(
"Bandwidth approximation test failed for: '%s:%s'",
socks5.host, socks5.port
)
if bandwidth_checked:
bandwidth_checked = False
last_bandwidth = time.time()
if not repeated:
break
time.sleep(cfg("socks5man", "verify_interval"))
def update_geodb():
version_file = cwd("geodb", ".version")
if not os.path.isfile(version_file):
log.error("No geodb version file '%s' is missing", version_file)
return
with open(version_file, "rb") as fp:
current_version = fp.read()
try:
latest_version = urllib2.urlopen(cfg("geodb", "geodb_md5_url")).read()
except urllib2.URLError as e:
log.error("Error retrieving latest geodb version hash: %s", e)
return
if current_version == latest_version:
log.info("GeoIP database at latest version")
return
extracted = cwd("geodb", "extracted")
renamed = None
if os.path.exists(extracted):
renamed = cwd("geodb", "old-extracted")
os.rename(extracted, renamed)
try:
url = cfg("geodb", "geodb_url")
log.info("Downloading latest version: '%s'", url)
mmdbtar = urllib2.urlopen(url).read()
except urllib2.URLError as e:
log.error(
"Failed to download new mmdb tar. Is the URL correct? %s", e
)
if renamed:
log.error("Restoring old version..")
os.rename(renamed, extracted)
return
tarpath = cwd("geodb", "geodblite.tar.gz")
with open(tarpath, "wb") as fw:
fw.write(mmdbtar)
os.mkdir(extracted)
unpack_mmdb(tarpath, cwd("geodb", "extracted", "geodblite.mmdb"))
log.info("Version update complete")
if renamed:
log.debug("Removing old version")
shutil.rmtree(renamed)
log.info("Updating geo IP information for all existing servers")
GeoInfo.georeader = None
for socks5 in db.list_socks5():
log.debug(
"Updating server: '%s'. Current country: %s",
socks5.host, socks5.country
)
ip = socks5.host
if not is_ipv4(ip):
ip = get_ipv4_hostname(ip)
geoinfo = GeoInfo.ipv4info(ip)
old = (socks5.country, socks5.country_code, socks5.city)
new = (geoinfo["country"], geoinfo["country_code"], geoinfo["city"])
if old == new:
log.debug("Geo IP info unchanged")
continue
log.debug(
"Geo IP info changed. New country=%s, country_code=%s, city=%s",
geoinfo["country"], geoinfo["country_code"], geoinfo["city"]
)
db.update_geoinfo(
socks5.id, country=geoinfo["country"],
country_code=geoinfo["country_code"], city=geoinfo["city"]
)
