def test_read_from_cache(self): create_cwd(cwd()) Config._cache = {} assert cfg("operationality", "ip_api") == "http://api.ipify.org" assert "operationality" in Config._cache Config._cache["operationality"]["ip_api"] = "http://example.com" assert cfg("operationality", "ip_api") == "http://example.com"
def verify(self): """ Test if this socks5 can be connected to and retrieve its own IP through the configured IP api. Automatically updates the 'operational' value in the database :returns: True if server is operational, false otherwise :rtype: bool """ operational = False ip = self.host if not is_ipv4(ip): ip = get_ipv4_hostname(ip) response = get_over_socks5( cfg("operationality", "ip_api"), self.host, self.port, username=self.username, password=self.password, timeout=cfg("operationality", "timeout") ) if response: if ip == response: operational = True # If a private ip is used, the api response will not match with # the configured host or its ip. There was however a response, # therefore we still mark it as operational elif is_reserved_ipv4(ip) and is_ipv4(response): operational = True db.set_operational(self.id, operational) return operational
def measure_connection_time(self): """ Measure the time it takes to connect to the specified connection test URL in the config. Result is automatically stored in the database :returns: An approximate connection time in seconds :rtype: float """ s = socks.socksocket() s.set_proxy(socks.SOCKS5, self.host, self.port, username=self.username, password=self.password) s.settimeout(cfg("connection_time", "timeout")) start = time.time() try: s.connect((cfg("connection_time", "hostname"), cfg("connection_time", "port"))) s.close() # socket.error, socks.ProxyError except Exception as e: log.error("Error connecting in connection time test: %s", e) connect_time = None else: connect_time = time.time() - start db.set_connect_time(self.id, connect_time) return connect_time
def measure_connection_time(self): """ Measure the time it takes to connect to the specified connection test URL in the config. Result is automatically stored in the database :returns: An approximate connection time in seconds :rtype: float """ s = socks.socksocket() s.set_proxy( socks.SOCKS5, self.host, self.port, username=self.username, password=self.password ) s.settimeout(cfg("connection_time", "timeout")) start = time.time() try: s.connect(( cfg("connection_time", "hostname"), cfg("connection_time", "port") )) s.close() except (socks.ProxyError, socket.error) as e: log.error("Error connecting in connection time test: %s", e) connect_time = None else: connect_time = time.time() - start db.set_connect_time(self.id, connect_time) return connect_time
def test_invalid_conf(self): create_cwd(cwd()) Config._cache = {} with open(cwd("conf", "socks5man.conf"), "wb") as fw: fw.write(os.urandom(512)) with pytest.raises(Socks5ConfigError): cfg("socks5man", "verify_interval")
def test_ip_api(self): """Verify that the default ip api returns an actual ip""" create_cwd(cwd()) res = urllib2.urlopen(cfg("operationality", "ip_api"), timeout=cfg("operationality", "timeout")) assert res.getcode() == 200 assert re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", res.read())
def test_invalid_conf(self): create_cwd(cwd()) Config._cache = {} with open(cwd("conf", "socks5man.conf"), "w") as fw: fw.write("socks5man to dominate them all") with pytest.raises(Socks5ConfigError): cfg("socks5man", "verify_interval")
def test_download_url(self): """Verify that the url used to measure an approximate bandwidth is still available""" create_cwd(cwd()) res = urllib2.urlopen(cfg("bandwidth", "download_url"), timeout=cfg("bandwidth", "timeout")) assert res.getcode() == 200 assert len(res.read()) > 0
def test_ip_api(self): """Verify that the default ip api returns an actual ip""" create_cwd(cwd()) res = urllib2.urlopen( cfg("operationality", "ip_api"), timeout=cfg("operationality", "timeout") ) assert res.getcode() == 200 assert re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", res.read())
def test_measure_time_host(self): """Verify that the default connection measurement still accepts connections""" create_cwd(cwd()) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(cfg("connection_time", "timeout")) s.connect((cfg("connection_time", "hostname"), cfg("connection_time", "port"))) s.close()
def test_measure_time_host(self): """Verify that the default connection measurement still accepts connections""" create_cwd(cwd()) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(cfg("connection_time", "timeout")) s.connect(( cfg("connection_time", "hostname"), cfg("connection_time", "port") )) s.close()
def test_download_url(self): """Verify that the url used to measure an approximate bandwidth is still available""" create_cwd(cwd()) res = urllib2.urlopen( cfg("bandwidth", "download_url"), timeout=cfg("bandwidth", "timeout") ) assert res.getcode() == 200 assert len(res.read()) > 0
def approx_bandwidth(self): """ Calculate an approximate Mbit/s download speed using the file specified in the config to download. Automatically updated in the database :returns: An approximate download speed in Mbit/s :rtype: float """ approx_bandwidth = approximate_bandwidth( self.host, self.port, username=self.username, password=self.password, times=cfg("bandwidth", "times"), timeout=cfg("bandwidth", "timeout") ) db.set_approx_bandwidth(self.id, approx_bandwidth) return approx_bandwidth
def approximate_bandwidth(host, port, username=None, password=None, maxfail=1, times=2, timeout=10): """Tries to determine the average download speed in Mbit/s. Higher 'times' values will result in more accurate speeds, but will take a longer time. specified socks5. This value is subtracted from the total time it takes to download a test file. @param maxfail: The maximum amount of times the socks5 is allowed to timeout before the measurement should stop @param times: The amount of times the test file should be downloaded. Optimal amount would be 3-4. """ total = 0 fails = 0 test_url = cfg("bandwidth", "download_url") for t in range(times): start = time.time() response = get_over_socks5(test_url, host, port, username=username, password=password, timeout=timeout) if not response: if fails >= maxfail: return None fails += 1 continue took = time.time() - start # Calculate the approximate Mbit/s try: speed = (len(response) / took) / 1000000 * 8 except ZeroDivisionError: # Can be thrown if the download was instant. To still calculate # a speed, use 0.001 as the time it took speed = (len(response) / 0.001) / 1000000 * 8 # If the used file to measure is smaller than approx 1 MB, # add a small amount as the small size might cause the TCP window # to stay small if len(response) < 1000000: speed += speed * 0.1 total += speed return total / times
def test_geoipdb_hash_url(self): create_cwd(cwd()) res = urllib.request.urlopen(cfg("geodb", "geodb_md5_url")) assert res.getcode() == 200 assert len(res.read()) == 32
def test_invalid_optiontype(self): create_cwd(cwd()) Config._cache = {} Config._conf["operationality"]["ip_api"] = int with pytest.raises(Socks5ConfigError): cfg("socks5man", "verify_interval")
def test_unknown_section(self): create_cwd(cwd()) Config._cache = {} del Config._conf["operationality"] with pytest.raises(Socks5ConfigError): cfg("socks5man", "verify_interval")
def test_missing_conf(self): create_cwd(cwd()) os.remove(cwd("conf", "socks5man.conf")) Config._cache = {} with pytest.raises(Socks5ConfigError): cfg("socks5man", "verify_interval")
def test_cfg_invalid(self): create_cwd(cwd()) with pytest.raises(Socks5ConfigError): cfg("socks5man", "nonexistingkeystuffdogestosti") with pytest.raises(Socks5ConfigError): cfg("doges", "wut")
def test_cfg_values(self): create_cwd(cwd()) assert cfg("socks5man", "verify_interval") == 300 assert cfg("operationality", "ip_api") == "http://api.ipify.org" assert not cfg("bandwidth", "enabled") assert cfg("connection_time", "enabled")
def test_cfg_defaults(self): create_cwd(cwd()) assert isinstance(cfg("socks5man", "verify_interval"), int) assert isinstance(cfg("socks5man", "bandwidth_interval"), int) assert isinstance(cfg("operationality", "ip_api"), (str, basestring)) assert isinstance(cfg("operationality", "timeout"), int) assert isinstance(cfg("connection_time", "enabled"), bool) assert isinstance(cfg("connection_time", "timeout"), int) assert isinstance(cfg("connection_time", "hostname"),(str, basestring)) assert isinstance(cfg("connection_time", "port"), int) assert isinstance(cfg("bandwidth", "enabled"), bool) assert isinstance(cfg("bandwidth", "download_url"), (str, basestring)) assert isinstance(cfg("bandwidth", "times"), int) assert isinstance(cfg("bandwidth", "timeout"), int) assert isinstance(cfg("geodb", "geodb_url"), (str, basestring)) assert isinstance(cfg("geodb", "geodb_md5_url"), (str, basestring))
def test_geoipdb_hash_url(self): create_cwd(cwd()) res = urllib2.urlopen(cfg("geodb", "geodb_md5_url")) assert res.getcode() == 200 assert len(res.read()) == 32
def verify_all(repeated=False, operational=None, unverified=None): last_bandwidth = None bandwidth_checked = False download_verified = False if repeated: log.info("Starting continuous verification") while True: socks5_list = db.list_socks5( operational=operational, unverified=unverified ) log.debug("Verifying %s socks5 servers", len(socks5_list)) for socks5 in socks5_list: socks5 = Socks5(socks5) log.info( "Testing socks5 server: '%s:%s'", socks5.host, socks5.port ) if socks5.verify(): log.info("Operationality check: OK") else: log.warning( "Operationality check (%s:%s): FAILED", socks5.host, socks5.port ) continue if cfg("connection_time", "enabled"): con_time = socks5.measure_connection_time() if con_time: log.debug("Measured connection time: %s", con_time) else: log.warning( "Connection time measurement failed for: '%s:%s'", socks5.host, socks5.port ) continue if cfg("bandwidth", "enabled"): print "BLABLA 2" if last_bandwidth: waited = time.time() - last_bandwidth if waited < cfg("socks5man", "bandwidth_interval"): continue if not download_verified: download_url = cfg("bandwidth", "download_url") try: urllib2.urlopen(download_url, timeout=5) download_verified = True except (socket.error, urllib2.URLError) as e: log.error( "Failed to download speed test file: '%s'. Please" " verify the configured file is still online!" " Without this file, it is not possible to" " approximate the bandwidth of a socsk5 server." " Error: %s", download_url, e ) continue bandwidth_checked = True bandwidth = socks5.approx_bandwidth() if bandwidth: log.debug( "Approximate bandwidth: %s Mbit/s down", bandwidth ) else: log.warning( "Bandwidth approximation test failed for: '%s:%s'", socks5.host, socks5.port ) if bandwidth_checked: bandwidth_checked = False last_bandwidth = time.time() if not repeated: break time.sleep(cfg("socks5man", "verify_interval"))
def update_geodb(): version_file = cwd("geodb", ".version") if not os.path.isfile(version_file): log.error("No geodb version file '%s' is missing", version_file) return with open(version_file, "rb") as fp: current_version = fp.read() try: latest_version = urllib2.urlopen(cfg("geodb", "geodb_md5_url")).read() except urllib2.URLError as e: log.error("Error retrieving latest geodb version hash: %s", e) return if current_version == latest_version: log.info("GeoIP database at latest version") return extracted = cwd("geodb", "extracted") renamed = None if os.path.exists(extracted): renamed = cwd("geodb", "old-extracted") os.rename(extracted, renamed) try: url = cfg("geodb", "geodb_url") log.info("Downloading latest version: '%s'", url) mmdbtar = urllib2.urlopen(url).read() except urllib2.URLError as e: log.error( "Failed to download new mmdb tar. Is the URL correct? %s", e ) if renamed: log.error("Restoring old version..") os.rename(renamed, extracted) return tarpath = cwd("geodb", "geodblite.tar.gz") with open(tarpath, "wb") as fw: fw.write(mmdbtar) os.mkdir(extracted) unpack_mmdb(tarpath, cwd("geodb", "extracted", "geodblite.mmdb")) log.info("Version update complete") if renamed: log.debug("Removing old version") shutil.rmtree(renamed) log.info("Updating geo IP information for all existing servers") GeoInfo.georeader = None for socks5 in db.list_socks5(): log.debug( "Updating server: '%s'. Current country: %s", socks5.host, socks5.country ) ip = socks5.host if not is_ipv4(ip): ip = get_ipv4_hostname(ip) geoinfo = GeoInfo.ipv4info(ip) old = (socks5.country, socks5.country_code, socks5.city) new = (geoinfo["country"], geoinfo["country_code"], geoinfo["city"]) if old == new: log.debug("Geo IP info unchanged") continue log.debug( "Geo IP info changed. New country=%s, country_code=%s, city=%s", geoinfo["country"], geoinfo["country_code"], geoinfo["city"] ) db.update_geoinfo( socks5.id, country=geoinfo["country"], country_code=geoinfo["country_code"], city=geoinfo["city"] )