def fetch_page(exit_desc): """ Fetch check.torproject.org and see if we are using Tor. """ data = None try: f = urllib2.urlopen("https://check.torproject.org", timeout=10).read() data = f.decode('utf-8') except Exception as err: logger.debug("urllib2.urlopen says: %s" % err) if not data: return # This is the string, we are looking for in the response. identifier = "Congratulations. This browser is configured to use Tor." url = exiturl(exit_desc.fingerprint) if not (identifier in data): logger.error("Detected false negative for %s." % url) else: logger.debug("Exit relay %s passed the check test." % url)
def is_cloudflared(exit_fpr): """ Check if site returns a CloudFlare CAPTCHA. """ exit_url = util.exiturl(exit_fpr) logger.debug("Probing exit relay \"%s\"." % exit_url) conn = httplib.HTTPSConnection(DOMAIN, PORT, strict=False) conn.request("GET", "/", headers=collections.OrderedDict(HTTP_HEADERS)) try: response = conn.getresponse() except Exception as err: logger.warning("urlopen() over %s says: %s" % (exit_url, err)) return data = decompress(response.read()) if not data: logger.warning("Did not get any data over %s." % exit_url) return if data and (CAPTCHA_SIGN in data): logger.info("Exit %s sees a CAPTCHA." % exit_url) else: logger.info("Exit %s does not see a CAPTCHA." % exit_url)
def resolve(exit_desc, domain, whitelist): """ Resolve a `domain' and compare it to the `whitelist'. If the domain is not part of the whitelist, an error is logged. """ exit = exiturl(exit_desc.fingerprint) sock = torsocks.torsocket() sock.settimeout(10) # Resolve the domain using Tor's SOCKS extension. try: ipv4 = sock.resolve(domain) except error.SOCKSv5Error as err: logger.debug("Exit relay %s could not resolve IPv4 address for " "\"%s\" because: %s" % (exit, domain, err)) return except socket.timeout as err: logger.debug("Socket over exit relay %s timed out: %s" % (exit, err)) return if ipv4 not in whitelist: logger.critical("Exit relay %s returned unexpected IPv4 address %s " "for domain %s" % (exit, ipv4, domain)) else: logger.debug("IPv4 address of domain %s as expected for %s." % (domain, exit))
def resolve(exit_desc, domain, whitelist): """ Resolve a `domain' and compare it to the `whitelist'. If the domain is not part of the whitelist, an error is logged. """ exit = exiturl(exit_desc.fingerprint) sock = torsocks.torsocket() sock.settimeout(10) # Resolve the domain using Tor's SOCKS extension. try: ipv4 = sock.resolve(domain) except error.SOCKSv5Error as err: log.debug("Exit relay %s could not resolve IPv4 address for " "\"%s\" because: %s" % (exit, domain, err)) return except socket.timeout as err: log.debug("Socket over exit relay %s timed out: %s" % (exit, err)) return except EOFError as err: log.debug("EOF error: %s" % err) return if ipv4 not in whitelist: log.critical("Exit relay %s returned unexpected IPv4 address %s " "for domain %s" % (exit, ipv4, domain)) else: log.debug("IPv4 address of domain %s as expected for %s." % (domain, exit))
def fetch_page(exit_desc): """ Fetch check.torproject.org and see if we are using Tor. """ data = None url = exiturl(exit_desc.fingerprint) try: data = urllib2.urlopen("https://check.torproject.org/api/ip", timeout=10).read() except Exception as err: log.debug("urllib2.urlopen says: %s" % err) return if not data: return try: check_answer = json.loads(data) except ValueError as err: log.warning("Couldn't parse JSON over relay %s: %s" % (url, data)) return check_addr = check_answer["IP"].strip() if not check_answer["IsTor"]: log.error("Found false negative for %s. Desc addr is %s and check " "addr is %s." % (url, exit_desc.address, check_addr)) else: log.debug("Exit relay %s passed the check test." % url)
def fetch_page(exit_desc): expected = "This file is to check if your exit relay has enough file " \ "descriptors to fetch it." exit_url = exiturl(exit_desc.fingerprint) logger.debug("Probing exit relay %s." % exit_url) data = None try: data = urllib2.urlopen("https://people.torproject.org/~phw/check_file", timeout=10).read() except Exception as err: logger.warning("urllib2.urlopen for %s says: %s." % (exit_desc.fingerprint, err)) return if not data: logger.warning("Exit relay %s did not return data." % exit_url) return data = data.strip() if not re.match(expected, data): logger.warning("Got unexpected response from %s: %s." % (exit_url, data)) else: logger.debug("Exit relay %s worked fine." % exit_url)
def fetch_page(exit_desc): """ Fetch check.torproject.org and see if we are using Tor. """ data = None url = exiturl(exit_desc.fingerprint) try: data = urllib.request.urlopen("https://check.torproject.org/api/ip", timeout=10).read() except Exception as err: log.debug("urllib.request.urlopen says: %s" % err) return if not data: return try: check_answer = json.loads(data) except ValueError as err: log.warning("Couldn't parse JSON over relay %s: %s" % (url, data)) return check_addr = check_answer["IP"].strip() if not check_answer["IsTor"]: log.error("Check thinks %s isn't Tor. Desc addr is %s and check " "addr is %s." % (url, exit_desc.address, check_addr)) else: log.debug("Exit relay %s passed the check test." % url)
def fetch_page(exit_desc): """ Fetch check.torproject.org and see if we are using Tor. """ data = None url = exiturl(exit_desc.fingerprint) try: data = urllib2.urlopen("https://check.torproject.org/api/ip", timeout=10).read() except Exception as err: log.debug("urllib2.urlopen says: %s" % err) return if not data: return try: check_answer = json.loads(data) except ValueError as err: log.warning("Couldn't parse JSON over relay %s: %s" % (url, data)) return check_answer["DescPublished"] = exit_desc.published.isoformat() check_answer["Fingerprint"] = exit_desc.fingerprint log.info(json.dumps(check_answer))
def test_dnssec(exit_fpr): """ Test if broken DNSSEC domain can be resolved. """ exit_url = util.exiturl(exit_fpr) sock = torsocks.torsocket() sock.settimeout(10) # Resolve domain using Tor's SOCKS extension. try: ipv4 = sock.resolve(BROKEN_DOMAIN) except error.SOCKSv5Error as err: logger.debug("%s did not resolve broken domain because: %s. Good." % (exit_url, err)) return except socket.timeout as err: logger.debug("Socket over exit relay %s timed out: %s" % (exit_url, err)) return except Exception as err: logger.debug("Could not resolve domain because: %s" % err) return logger.critical("%s resolved domain to %s" % (exit_url, ipv4))
def test_dnssec(exit_fpr): """ Test if broken DNSSEC domain can be resolved. """ exit_url = util.exiturl(exit_fpr) sock = torsocks.torsocket() sock.settimeout(10) # Resolve domain using Tor's SOCKS extension. try: ipv4 = sock.resolve(BROKEN_DOMAIN) except error.SOCKSv5Error as err: log.debug("%s did not resolve broken domain because: %s. Good." % (exit_url, err)) return except socket.timeout as err: log.debug("Socket over exit relay %s timed out: %s" % (exit_url, err)) return except Exception as err: log.debug("Could not resolve domain because: %s" % err) return log.critical("%s resolved domain to %s" % (exit_url, ipv4))
def fetch_page(exit_desc): """ Fetch check.torproject.org and see if we are using Tor. """ data = None try: data = urllib2.urlopen("https://check.torproject.org", timeout=10).read() except Exception as err: logger.debug("urllib2.urlopen says: %s" % err) if not data: return # This is the string, we are looking for in the response. identifier = "Congratulations. This browser is configured to use Tor." url = exiturl(exit_desc.fingerprint) if not (identifier in data): logger.error("Detected false negative for %s." % url) else: logger.debug("Exit relay %s passed the check test." % url)
def my_callback(output, exit_desc, proc_kill): #log.info(output) exit = exiturl(exit_desc.fingerprint) if "BEGIN CERTIFICATE" in output: signature = output.split('BEGIN CERTIFICATE-----\n')[1] signature = signature.split('\n-----END CERTIFICATE')[0] valid = False for domain in domains.iterkeys(): if domains[domain] == signature: valid = True if not valid: log.critical("%s providing invalid signature: %s" % (exit, output)) return True
def run_check(exit_desc): """ Download file and check if its checksum is as expected. """ exiturl = util.exiturl(exit_desc.fingerprint) for url, file_info in check_files.iteritems(): orig_file, orig_digest = file_info logger.debug("Attempting to download <%s> over %s." % (url, exiturl)) data = None request = urllib2.Request(url) request.add_header('User-Agent', test_agent) try: data = urllib2.urlopen(request, timeout=20).read() except Exception as err: logger.warning("urlopen() failed for %s: %s" % (exiturl, err)) continue if not data: logger.warning("No data received from <%s> over %s." % (url, exiturl)) continue file_name = url.split("/")[-1] _, tmp_file = tempfile.mkstemp(prefix="exitmap_%s_%s_" % (exit_desc.fingerprint, file_name)) with open(tmp_file, "wb") as fd: fd.write(data) observed_digest = sha512_file(tmp_file) if (observed_digest != orig_digest) and \ (not files_identical(tmp_file, orig_file)): logger.critical("File \"%s\" differs from reference file \"%s\". " "Downloaded over exit relay %s." % (tmp_file, orig_file, exiturl)) else: logger.debug("File \"%s\" fetched over %s as expected." % (tmp_file, exiturl)) os.remove(tmp_file)
def resolve(exit_fpr, domain, whitelist): """ Resolve a `domain' and compare it to the `whitelist'. If the domain is not part of the whitelist, an error is logged. """ sock = mysocks.socksocket() # Resolve the domain using Tor's SOCKS extension. try: ipv4 = sock.resolve(domain) except mysocks.GeneralProxyError as err: logger.debug("Exit relay %s could not resolve IPv4 address for " "\"%s\" because: %s" % (exiturl(exit_fpr), domain, err)) return if ipv4 not in whitelist: logger.critical("Exit relay %s returned unexpected IPv4 address %s " "for domain %s" % (exiturl(exit_fpr), ipv4, domain)) else: logger.debug("IPv4 address of domain %s as expected for %s." % (domain, exiturl(exit_fpr)))
def resolve(exit_desc): """ Resolve exit relay-specific domain. """ exit_url = util.exiturl(exit_desc.fingerprint) # Prepend the exit relay's fingerprint so we know which relay issued the # DNS request. fingerprint = exit_desc.fingerprint.encode("ascii", "ignore") domain = "%s.%s.%s" % (fingerprint, time.strftime("%Y-%m-%d-%H"), TARGET_DOMAIN) log.debug("Resolving %s over %s." % (domain, exit_url)) sock = torsocks.torsocket() sock.settimeout(10) # Resolve the domain using Tor's SOCKS extension. log.debug("Resolving %s over %s." % (domain, exit_url)) try: ipv4 = sock.resolve(domain) except error.SOCKSv5Error as err: # This is expected because all domains resolve to 127.0.0.1. log.warning("SOCKSv5 error while resolving domain: %s" % err) ipv4 = "0.0.0.0" pass except socket.timeout as err: log.debug("Socket over exit relay %s timed out: %s" % (exit_url, err)) return log.info("Successfully resolved domain over %s to %s." % (exit_url, ipv4)) # Log a CSV including timestamp, exit fingerprint, exit IP address, and the # domain we resolved. timestamp = time.strftime("%Y-%m-%d_%H:%M:%S_%z") content = "%s, %s, %s, %s\n" % (timestamp, fingerprint, exit_desc.address, ipv4) util.dump_to_file(content, fingerprint)
def test_exiturl(self): self.assertEqual(util.exiturl("foo"), "<https://atlas.torproject.or" "g/#details/foo>") self.assertEqual(util.exiturl(4), "<https://atlas.torproject.org/#det" "ails/4>")
def checker(exit_desc, domain, expected_sig): exit = exiturl(exit_desc.fingerprint) tor_sig = get_ssl_signature(domain, 5222) if tor_sig != expected_sig: log.critical("ExitNode %s returned different SSL cert: %s" % (exit, tor_sig))
def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs): global mutex, success_fp_file, malicious_fp_file, fail_fp_file, timeouted_fp client = None tb_proc = None socks_port = 0 fn = None # check whether setup was done correctly and caller is patched if 'socks_port' in kwargs: socks_port = kwargs['socks_port'] else: log.error("no socks port passed.") return if mutex is None: exit("Mutex doesn't exist") # get tb mutex before starting tor browser mutex.acquire() log.debug("mutex %s acquired." % str(mutex)) # module setup client, tb_proc = single_setup(socks_port, run_cmd_over_tor) if client is None: exit('Error: marionette session connection not found') # variable setup exit_url = exiturl(exit_desc.fingerprint) elementExists = True fp = str(exit_desc.fingerprint) works = True log.info('start checking exit %s' % fp) # do the site-specific probing use_cnet = bool(getrandbits(1)) try: if use_cnet: status, fn = cnet(client) else: status, fn = filehippo(client) except InvalidSessionIdException: works = False except UnknownException as uex: if not 'Reached error page' in str(uex): raise else: works = False # if there was some problem with one of the sites, # just try the other one if not works: log.warn(':::::::::: session gone or timeout, 1st try') works = True try: if use_cnet: status, fn = filehippo(client) else: status, fn = cnet(client) except InvalidSessionIdException: works = False except UnknownException as uex: if not 'Reached error page' in str(uex): raise else: works = False if not works: log.warn(':::::::::: session gone or timeout, 2nd try') log.warn('giving up.') # log exitnode to timeout file with open(timeouted_fp, 'a') as f: f.write('%s\n' % fp) sleep(10) cancel_all_downloads_and_exit(client, tb_proc) client = None tb_proc = None teardown_single() return mutex # check return status and log fingerprint to corresponding file. if status == Status.NONIDENT: log.error("!!! HASH MISMATCH: original file hash %s" % sha256sum('test.exe')) with open(malicious_fp_file, 'a') as f: f.write('%s\n' % fp) # we found a binary patched file! # move to safe location rename(fn, os.path.join(bad_dir, os.path.basename(fn) + '.' + fp)) elif status == Status.IDENTICAL: log.info("+++ HASH CORRECT: original file hash %s" % sha256sum('test.exe')) with open(success_fp_file, 'a') as f: f.write('%s\n' % fp) # clean up remove(fn) elif status == Status.NODL: log.warn("@@@@@@@@@@ no file downloaded") with open(fail_fp_file, 'a') as f: f.write('%s\n' % fp) elif status == Status.CRIT: log.warn("critical error, exiting.") cancel_all_downloads_and_exit(client, tb_proc) teardown_single() exit(1) # stop down tor browser and return log.info("Exit %s done." % fp) cancel_all_downloads_and_exit(client, tb_proc) client = None tb_proc = None teardown_single() return mutex
def test_ssh(exit_desc): """ is you or is you not MITMing my ssh? """ exit_fp = exit_desc.fingerprint exit_url = exiturl(exit_fp) log.debug('testing exit %s' % (exit_fp)) fail_count = 0 for host, port in destinations: # construct the tor socket sock = torsocket() sock.settimeout(10) # resolve the ip over tor, like it normally would for a client. try: ipv4 = sock.resolve(host) log.debug("destination %s resolves to: %s" % (host, ipv4)) except SOCKSv5Error as err: log.debug("%s did not resolve broken domain because: %s." % (exit_url, err)) fail_count += 1 continue except socket.timeout as err: log.debug("Socket over exit relay %s timed out: %s" % (exit_url, err)) fail_count += 1 continue except Exception as err: log.debug("Could not resolve domain because: %s" % err) fail_count += 1 continue finally: sock.close() # connect to the actual target sock = torsocket() sock.settimeout(10) address = (ipv4, port) sock.connect(address) # get the over-tor key information try: client = paramiko.transport.Transport(sock) except EOFError as err: log.info('unknown ssh connection error to %s:%s (%s) over exit relay %s: %s' % (host, port, ipv4, exit_fp, err)) fail_count += 1 continue except paramiko.SSHException as err: log.info('ssh exception conneting to %s:%s (%s) over exit relay %s: %s' % (host, port, ipv4, exit_fp, err)) fail_count += 1 continue try: client.start_client() except EOFError as err: log.info('unknown ssh connection error to %s:%s (%s) over exit relay %s: %s' % (host, port, ipv4, exit_fp, err)) fail_count += 1 continue except paramiko.SSHException as err: log.info('ssh connection error to %s:%s (%s) over exit relay %s: %s' % (host, port, ipv4, exit_fp, err)) fail_count += 1 continue tor_version = client.remote_version key = client.get_remote_server_key() client.close() sock.close() tor_key_name = key.get_name() tor_key_base64 = key.get_base64() log.debug('ssh key (tor) name for %s:%s (%s): %s' % (host, port, ipv4, tor_key_name)) log.debug('ssh key (tor) for %s:%s (%s): %s' % (host, port, ipv4, tor_key_base64)) log.debug('ssh version (tor) for %s:%s (%s): %s' % (host, port, ipv4, tor_version)) # do the matching version = details[host]['version'] key_name = details[host]['key_name'] key_base64 = details[host]['key_base64'] if not key_name == tor_key_name: log.critical('tor ssh key name mismatch for %s:%s (%s) over exit relay %s clear wire value: %s, over tor value: %s' % (host, port, ipv4, exit_fp, key_name, tor_key_name)) else: log.debug('tor ssh key name match for %s:%s (%s) over exit relay %s' % (host, port, ipv4, exit_fp)) if not key_base64 == tor_key_base64: log.critical('tor ssh key mismatch for %s:%s (%s) over exit relay %s' % (host, port, ipv4, exit_fp)) log.critical('clear wire key: %s' % (key_base64)) log.critical('clear wire version: %s' % (version)) log.critical('over tor key: %s' % (tor_key_base64)) log.critical('over tor version: %s' % (tor_version)) log.info('atlas link: https://atlas.torproject.org/#details/%s' % (exit_fp)) else: log.debug('tor ssh key match for %s:%s (%s) over exit relay %s' % (host, port, ipv4, exit_fp)) # if EVERY host is unable to be connected to, this could indicate a broken/misconfigured exit if fail_count == len(details): log.warning('exit %s appears to be having issues connecting over ssh. misconfiguration?' % (exit_fp)) log.info('atlas link: https://atlas.torproject.org/#details/%s' % (exit_fp)) if fail_count > 0: log.warning('%s of %s ssh connections have failed over exit %s' % (fail_count, len(details), exit_fp))
def test_exiturl(self): self.assertEqual(util.exiturl("foo"), ("<https://metrics.torproject" ".org/rs.html#details/foo>")) self.assertEqual(util.exiturl(4), ("<https://metrics.torproject.org/" "rs.html#details/4>"))