def fetch_page(exit_desc):
"""
Fetch check.torproject.org and see if we are using Tor.
"""
data = None
try:
f = urllib2.urlopen("https://check.torproject.org",
timeout=10).read()
data = f.decode('utf-8')
except Exception as err:
logger.debug("urllib2.urlopen says: %s" % err)
if not data:
return
# This is the string, we are looking for in the response.
identifier = "Congratulations. This browser is configured to use Tor."
url = exiturl(exit_desc.fingerprint)
if not (identifier in data):
logger.error("Detected false negative for %s." % url)
else:
logger.debug("Exit relay %s passed the check test." % url)
def is_cloudflared(exit_fpr):
"""
Check if site returns a CloudFlare CAPTCHA.
"""
exit_url = util.exiturl(exit_fpr)
logger.debug("Probing exit relay \"%s\"." % exit_url)
conn = httplib.HTTPSConnection(DOMAIN, PORT, strict=False)
conn.request("GET", "/", headers=collections.OrderedDict(HTTP_HEADERS))
try:
response = conn.getresponse()
except Exception as err:
logger.warning("urlopen() over %s says: %s" % (exit_url, err))
return
data = decompress(response.read())
if not data:
logger.warning("Did not get any data over %s." % exit_url)
return
if data and (CAPTCHA_SIGN in data):
logger.info("Exit %s sees a CAPTCHA." % exit_url)
else:
logger.info("Exit %s does not see a CAPTCHA." % exit_url)
def resolve(exit_desc, domain, whitelist):
"""
Resolve a `domain' and compare it to the `whitelist'.
If the domain is not part of the whitelist, an error is logged.
"""
exit = exiturl(exit_desc.fingerprint)
sock = torsocks.torsocket()
sock.settimeout(10)
# Resolve the domain using Tor's SOCKS extension.
try:
ipv4 = sock.resolve(domain)
except error.SOCKSv5Error as err:
logger.debug("Exit relay %s could not resolve IPv4 address for "
"\"%s\" because: %s" % (exit, domain, err))
return
except socket.timeout as err:
logger.debug("Socket over exit relay %s timed out: %s" % (exit, err))
return
if ipv4 not in whitelist:
logger.critical("Exit relay %s returned unexpected IPv4 address %s "
"for domain %s" % (exit, ipv4, domain))
else:
logger.debug("IPv4 address of domain %s as expected for %s." %
(domain, exit))
def resolve(exit_desc, domain, whitelist):
"""
Resolve a `domain' and compare it to the `whitelist'.
If the domain is not part of the whitelist, an error is logged.
"""
exit = exiturl(exit_desc.fingerprint)
sock = torsocks.torsocket()
sock.settimeout(10)
# Resolve the domain using Tor's SOCKS extension.
try:
ipv4 = sock.resolve(domain)
except error.SOCKSv5Error as err:
log.debug("Exit relay %s could not resolve IPv4 address for "
"\"%s\" because: %s" % (exit, domain, err))
return
except socket.timeout as err:
log.debug("Socket over exit relay %s timed out: %s" % (exit, err))
return
except EOFError as err:
log.debug("EOF error: %s" % err)
return
if ipv4 not in whitelist:
log.critical("Exit relay %s returned unexpected IPv4 address %s "
"for domain %s" % (exit, ipv4, domain))
else:
log.debug("IPv4 address of domain %s as expected for %s." %
(domain, exit))
def is_cloudflared(exit_fpr):
"""
Check if site returns a CloudFlare CAPTCHA.
"""
exit_url = util.exiturl(exit_fpr)
logger.debug("Probing exit relay \"%s\"." % exit_url)
conn = httplib.HTTPSConnection(DOMAIN, PORT, strict=False)
conn.request("GET", "/", headers=collections.OrderedDict(HTTP_HEADERS))
try:
response = conn.getresponse()
except Exception as err:
logger.warning("urlopen() over %s says: %s" % (exit_url, err))
return
data = decompress(response.read())
if not data:
logger.warning("Did not get any data over %s." % exit_url)
return
if data and (CAPTCHA_SIGN in data):
logger.info("Exit %s sees a CAPTCHA." % exit_url)
else:
logger.info("Exit %s does not see a CAPTCHA." % exit_url)
def fetch_page(exit_desc):
"""
Fetch check.torproject.org and see if we are using Tor.
"""
data = None
url = exiturl(exit_desc.fingerprint)
try:
data = urllib2.urlopen("https://check.torproject.org/api/ip",
timeout=10).read()
except Exception as err:
log.debug("urllib2.urlopen says: %s" % err)
return
if not data:
return
try:
check_answer = json.loads(data)
except ValueError as err:
log.warning("Couldn't parse JSON over relay %s: %s" % (url, data))
return
check_addr = check_answer["IP"].strip()
if not check_answer["IsTor"]:
log.error("Found false negative for %s. Desc addr is %s and check "
"addr is %s." % (url, exit_desc.address, check_addr))
else:
log.debug("Exit relay %s passed the check test." % url)
def fetch_page(exit_desc):
expected = "This file is to check if your exit relay has enough file " \
"descriptors to fetch it."
exit_url = exiturl(exit_desc.fingerprint)
logger.debug("Probing exit relay %s." % exit_url)
data = None
try:
data = urllib2.urlopen("https://people.torproject.org/~phw/check_file",
timeout=10).read()
except Exception as err:
logger.warning("urllib2.urlopen for %s says: %s." %
(exit_desc.fingerprint, err))
return
if not data:
logger.warning("Exit relay %s did not return data." % exit_url)
return
data = data.strip()
if not re.match(expected, data):
logger.warning("Got unexpected response from %s: %s." %
(exit_url, data))
else:
logger.debug("Exit relay %s worked fine." % exit_url)
def fetch_page(exit_desc):
"""
Fetch check.torproject.org and see if we are using Tor.
"""
data = None
url = exiturl(exit_desc.fingerprint)
try:
data = urllib.request.urlopen("https://check.torproject.org/api/ip",
timeout=10).read()
except Exception as err:
log.debug("urllib.request.urlopen says: %s" % err)
return
if not data:
return
try:
check_answer = json.loads(data)
except ValueError as err:
log.warning("Couldn't parse JSON over relay %s: %s" % (url, data))
return
check_addr = check_answer["IP"].strip()
if not check_answer["IsTor"]:
log.error("Check thinks %s isn't Tor. Desc addr is %s and check "
"addr is %s." % (url, exit_desc.address, check_addr))
else:
log.debug("Exit relay %s passed the check test." % url)
def fetch_page(exit_desc):
"""
Fetch check.torproject.org and see if we are using Tor.
"""
data = None
url = exiturl(exit_desc.fingerprint)
try:
data = urllib2.urlopen("https://check.torproject.org/api/ip",
timeout=10).read()
except Exception as err:
log.debug("urllib2.urlopen says: %s" % err)
return
if not data:
return
try:
check_answer = json.loads(data)
except ValueError as err:
log.warning("Couldn't parse JSON over relay %s: %s" % (url, data))
return
check_answer["DescPublished"] = exit_desc.published.isoformat()
check_answer["Fingerprint"] = exit_desc.fingerprint
log.info(json.dumps(check_answer))
def test_dnssec(exit_fpr):
"""
Test if broken DNSSEC domain can be resolved.
"""
exit_url = util.exiturl(exit_fpr)
sock = torsocks.torsocket()
sock.settimeout(10)
# Resolve domain using Tor's SOCKS extension.
try:
ipv4 = sock.resolve(BROKEN_DOMAIN)
except error.SOCKSv5Error as err:
logger.debug("%s did not resolve broken domain because: %s. Good." %
(exit_url, err))
return
except socket.timeout as err:
logger.debug("Socket over exit relay %s timed out: %s" %
(exit_url, err))
return
except Exception as err:
logger.debug("Could not resolve domain because: %s" % err)
return
logger.critical("%s resolved domain to %s" % (exit_url, ipv4))
def test_dnssec(exit_fpr):
"""
Test if broken DNSSEC domain can be resolved.
"""
exit_url = util.exiturl(exit_fpr)
sock = torsocks.torsocket()
sock.settimeout(10)
# Resolve domain using Tor's SOCKS extension.
try:
ipv4 = sock.resolve(BROKEN_DOMAIN)
except error.SOCKSv5Error as err:
log.debug("%s did not resolve broken domain because: %s. Good." %
(exit_url, err))
return
except socket.timeout as err:
log.debug("Socket over exit relay %s timed out: %s" % (exit_url, err))
return
except Exception as err:
log.debug("Could not resolve domain because: %s" % err)
return
log.critical("%s resolved domain to %s" % (exit_url, ipv4))
def fetch_page(exit_desc):
"""
Fetch check.torproject.org and see if we are using Tor.
"""
data = None
try:
data = urllib2.urlopen("https://check.torproject.org",
timeout=10).read()
except Exception as err:
logger.debug("urllib2.urlopen says: %s" % err)
if not data:
return
# This is the string, we are looking for in the response.
identifier = "Congratulations. This browser is configured to use Tor."
url = exiturl(exit_desc.fingerprint)
if not (identifier in data):
logger.error("Detected false negative for %s." % url)
else:
logger.debug("Exit relay %s passed the check test." % url)
def my_callback(output, exit_desc, proc_kill):
#log.info(output)
exit = exiturl(exit_desc.fingerprint)
if "BEGIN CERTIFICATE" in output:
signature = output.split('BEGIN CERTIFICATE-----\n')[1]
signature = signature.split('\n-----END CERTIFICATE')[0]
valid = False
for domain in domains.iterkeys():
if domains[domain] == signature:
valid = True
if not valid:
log.critical("%s providing invalid signature: %s" % (exit, output))
return True
def run_check(exit_desc):
"""
Download file and check if its checksum is as expected.
"""
exiturl = util.exiturl(exit_desc.fingerprint)
for url, file_info in check_files.iteritems():
orig_file, orig_digest = file_info
logger.debug("Attempting to download <%s> over %s." % (url, exiturl))
data = None
request = urllib2.Request(url)
request.add_header('User-Agent', test_agent)
try:
data = urllib2.urlopen(request, timeout=20).read()
except Exception as err:
logger.warning("urlopen() failed for %s: %s" % (exiturl, err))
continue
if not data:
logger.warning("No data received from <%s> over %s." %
(url, exiturl))
continue
file_name = url.split("/")[-1]
_, tmp_file = tempfile.mkstemp(prefix="exitmap_%s_%s_" %
(exit_desc.fingerprint, file_name))
with open(tmp_file, "wb") as fd:
fd.write(data)
observed_digest = sha512_file(tmp_file)
if (observed_digest != orig_digest) and \
(not files_identical(tmp_file, orig_file)):
logger.critical("File \"%s\" differs from reference file \"%s\". "
"Downloaded over exit relay %s." %
(tmp_file, orig_file, exiturl))
else:
logger.debug("File \"%s\" fetched over %s as expected." %
(tmp_file, exiturl))
os.remove(tmp_file)
def run_check(exit_desc):
"""
Download file and check if its checksum is as expected.
"""
exiturl = util.exiturl(exit_desc.fingerprint)
for url, file_info in check_files.iteritems():
orig_file, orig_digest = file_info
logger.debug("Attempting to download <%s> over %s." % (url, exiturl))
data = None
request = urllib2.Request(url)
request.add_header('User-Agent', test_agent)
try:
data = urllib2.urlopen(request, timeout=20).read()
except Exception as err:
logger.warning("urlopen() failed for %s: %s" % (exiturl, err))
continue
if not data:
logger.warning("No data received from <%s> over %s." %
(url, exiturl))
continue
file_name = url.split("/")[-1]
_, tmp_file = tempfile.mkstemp(prefix="exitmap_%s_%s_" %
(exit_desc.fingerprint, file_name))
with open(tmp_file, "wb") as fd:
fd.write(data)
observed_digest = sha512_file(tmp_file)
if (observed_digest != orig_digest) and \
(not files_identical(tmp_file, orig_file)):
logger.critical("File \"%s\" differs from reference file \"%s\". "
"Downloaded over exit relay %s." %
(tmp_file, orig_file, exiturl))
else:
logger.debug("File \"%s\" fetched over %s as expected." %
(tmp_file, exiturl))
os.remove(tmp_file)
def resolve(exit_fpr, domain, whitelist):
"""
Resolve a `domain' and compare it to the `whitelist'.
If the domain is not part of the whitelist, an error is logged.
"""
sock = mysocks.socksocket()
# Resolve the domain using Tor's SOCKS extension.
try:
ipv4 = sock.resolve(domain)
except mysocks.GeneralProxyError as err:
logger.debug("Exit relay %s could not resolve IPv4 address for "
"\"%s\" because: %s" % (exiturl(exit_fpr), domain, err))
return
if ipv4 not in whitelist:
logger.critical("Exit relay %s returned unexpected IPv4 address %s "
"for domain %s" % (exiturl(exit_fpr), ipv4, domain))
else:
logger.debug("IPv4 address of domain %s as expected for %s." %
(domain, exiturl(exit_fpr)))
def resolve(exit_fpr, domain, whitelist):
"""
Resolve a `domain' and compare it to the `whitelist'.
If the domain is not part of the whitelist, an error is logged.
"""
sock = mysocks.socksocket()
# Resolve the domain using Tor's SOCKS extension.
try:
ipv4 = sock.resolve(domain)
except mysocks.GeneralProxyError as err:
logger.debug("Exit relay %s could not resolve IPv4 address for "
"\"%s\" because: %s" % (exiturl(exit_fpr), domain, err))
return
if ipv4 not in whitelist:
logger.critical("Exit relay %s returned unexpected IPv4 address %s "
"for domain %s" % (exiturl(exit_fpr), ipv4, domain))
else:
logger.debug("IPv4 address of domain %s as expected for %s." %
(domain, exiturl(exit_fpr)))
def resolve(exit_desc):
"""
Resolve exit relay-specific domain.
"""
exit_url = util.exiturl(exit_desc.fingerprint)
# Prepend the exit relay's fingerprint so we know which relay issued the
# DNS request.
fingerprint = exit_desc.fingerprint.encode("ascii", "ignore")
domain = "%s.%s.%s" % (fingerprint,
time.strftime("%Y-%m-%d-%H"),
TARGET_DOMAIN)
log.debug("Resolving %s over %s." % (domain, exit_url))
sock = torsocks.torsocket()
sock.settimeout(10)
# Resolve the domain using Tor's SOCKS extension.
log.debug("Resolving %s over %s." % (domain, exit_url))
try:
ipv4 = sock.resolve(domain)
except error.SOCKSv5Error as err:
# This is expected because all domains resolve to 127.0.0.1.
log.warning("SOCKSv5 error while resolving domain: %s" % err)
ipv4 = "0.0.0.0"
pass
except socket.timeout as err:
log.debug("Socket over exit relay %s timed out: %s" % (exit_url, err))
return
log.info("Successfully resolved domain over %s to %s." % (exit_url, ipv4))
# Log a CSV including timestamp, exit fingerprint, exit IP address, and the
# domain we resolved.
timestamp = time.strftime("%Y-%m-%d_%H:%M:%S_%z")
content = "%s, %s, %s, %s\n" % (timestamp,
fingerprint,
exit_desc.address,
ipv4)
util.dump_to_file(content, fingerprint)
def test_exiturl(self):
self.assertEqual(util.exiturl("foo"), "<https://atlas.torproject.or"
"g/#details/foo>")
self.assertEqual(util.exiturl(4), "<https://atlas.torproject.org/#det"
"ails/4>")
def checker(exit_desc, domain, expected_sig):
exit = exiturl(exit_desc.fingerprint)
tor_sig = get_ssl_signature(domain, 5222)
if tor_sig != expected_sig:
log.critical("ExitNode %s returned different SSL cert: %s" %
(exit, tor_sig))
def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs):
global mutex, success_fp_file, malicious_fp_file, fail_fp_file, timeouted_fp
client = None
tb_proc = None
socks_port = 0
fn = None
# check whether setup was done correctly and caller is patched
if 'socks_port' in kwargs:
socks_port = kwargs['socks_port']
else:
log.error("no socks port passed.")
return
if mutex is None:
exit("Mutex doesn't exist")
# get tb mutex before starting tor browser
mutex.acquire()
log.debug("mutex %s acquired." % str(mutex))
# module setup
client, tb_proc = single_setup(socks_port, run_cmd_over_tor)
if client is None:
exit('Error: marionette session connection not found')
# variable setup
exit_url = exiturl(exit_desc.fingerprint)
elementExists = True
fp = str(exit_desc.fingerprint)
works = True
log.info('start checking exit %s' % fp)
# do the site-specific probing
use_cnet = bool(getrandbits(1))
try:
if use_cnet:
status, fn = cnet(client)
else:
status, fn = filehippo(client)
except InvalidSessionIdException:
works = False
except UnknownException as uex:
if not 'Reached error page' in str(uex):
raise
else:
works = False
# if there was some problem with one of the sites,
# just try the other one
if not works:
log.warn(':::::::::: session gone or timeout, 1st try')
works = True
try:
if use_cnet:
status, fn = filehippo(client)
else:
status, fn = cnet(client)
except InvalidSessionIdException:
works = False
except UnknownException as uex:
if not 'Reached error page' in str(uex):
raise
else:
works = False
if not works:
log.warn(':::::::::: session gone or timeout, 2nd try')
log.warn('giving up.')
# log exitnode to timeout file
with open(timeouted_fp, 'a') as f:
f.write('%s\n' % fp)
sleep(10)
cancel_all_downloads_and_exit(client, tb_proc)
client = None
tb_proc = None
teardown_single()
return mutex
# check return status and log fingerprint to corresponding file.
if status == Status.NONIDENT:
log.error("!!! HASH MISMATCH: original file hash %s" %
sha256sum('test.exe'))
with open(malicious_fp_file, 'a') as f:
f.write('%s\n' % fp)
# we found a binary patched file!
# move to safe location
rename(fn, os.path.join(bad_dir,
os.path.basename(fn) + '.' + fp))
elif status == Status.IDENTICAL:
log.info("+++ HASH CORRECT: original file hash %s" %
sha256sum('test.exe'))
with open(success_fp_file, 'a') as f:
f.write('%s\n' % fp)
# clean up
remove(fn)
elif status == Status.NODL:
log.warn("@@@@@@@@@@ no file downloaded")
with open(fail_fp_file, 'a') as f:
f.write('%s\n' % fp)
elif status == Status.CRIT:
log.warn("critical error, exiting.")
cancel_all_downloads_and_exit(client, tb_proc)
teardown_single()
exit(1)
# stop down tor browser and return
log.info("Exit %s done." % fp)
cancel_all_downloads_and_exit(client, tb_proc)
client = None
tb_proc = None
teardown_single()
return mutex
def test_ssh(exit_desc):
"""
is you or is you not MITMing my ssh?
"""
exit_fp = exit_desc.fingerprint
exit_url = exiturl(exit_fp)
log.debug('testing exit %s' % (exit_fp))
fail_count = 0
for host, port in destinations:
# construct the tor socket
sock = torsocket()
sock.settimeout(10)
# resolve the ip over tor, like it normally would for a client.
try:
ipv4 = sock.resolve(host)
log.debug("destination %s resolves to: %s" % (host, ipv4))
except SOCKSv5Error as err:
log.debug("%s did not resolve broken domain because: %s." % (exit_url, err))
fail_count += 1
continue
except socket.timeout as err:
log.debug("Socket over exit relay %s timed out: %s" % (exit_url, err))
fail_count += 1
continue
except Exception as err:
log.debug("Could not resolve domain because: %s" % err)
fail_count += 1
continue
finally:
sock.close()
# connect to the actual target
sock = torsocket()
sock.settimeout(10)
address = (ipv4, port)
sock.connect(address)
# get the over-tor key information
try:
client = paramiko.transport.Transport(sock)
except EOFError as err:
log.info('unknown ssh connection error to %s:%s (%s) over exit relay %s: %s' % (host, port, ipv4, exit_fp, err))
fail_count += 1
continue
except paramiko.SSHException as err:
log.info('ssh exception conneting to %s:%s (%s) over exit relay %s: %s' % (host, port, ipv4, exit_fp, err))
fail_count += 1
continue
try:
client.start_client()
except EOFError as err:
log.info('unknown ssh connection error to %s:%s (%s) over exit relay %s: %s' % (host, port, ipv4, exit_fp, err))
fail_count += 1
continue
except paramiko.SSHException as err:
log.info('ssh connection error to %s:%s (%s) over exit relay %s: %s' % (host, port, ipv4, exit_fp, err))
fail_count += 1
continue
tor_version = client.remote_version
key = client.get_remote_server_key()
client.close()
sock.close()
tor_key_name = key.get_name()
tor_key_base64 = key.get_base64()
log.debug('ssh key (tor) name for %s:%s (%s): %s' % (host, port, ipv4, tor_key_name))
log.debug('ssh key (tor) for %s:%s (%s): %s' % (host, port, ipv4, tor_key_base64))
log.debug('ssh version (tor) for %s:%s (%s): %s' % (host, port, ipv4, tor_version))
# do the matching
version = details[host]['version']
key_name = details[host]['key_name']
key_base64 = details[host]['key_base64']
if not key_name == tor_key_name:
log.critical('tor ssh key name mismatch for %s:%s (%s) over exit relay %s clear wire value: %s, over tor value: %s' % (host, port, ipv4, exit_fp, key_name, tor_key_name))
else:
log.debug('tor ssh key name match for %s:%s (%s) over exit relay %s' % (host, port, ipv4, exit_fp))
if not key_base64 == tor_key_base64:
log.critical('tor ssh key mismatch for %s:%s (%s) over exit relay %s' % (host, port, ipv4, exit_fp))
log.critical('clear wire key: %s' % (key_base64))
log.critical('clear wire version: %s' % (version))
log.critical('over tor key: %s' % (tor_key_base64))
log.critical('over tor version: %s' % (tor_version))
log.info('atlas link: https://atlas.torproject.org/#details/%s' % (exit_fp))
else:
log.debug('tor ssh key match for %s:%s (%s) over exit relay %s' % (host, port, ipv4, exit_fp))
# if EVERY host is unable to be connected to, this could indicate a broken/misconfigured exit
if fail_count == len(details):
log.warning('exit %s appears to be having issues connecting over ssh. misconfiguration?' % (exit_fp))
log.info('atlas link: https://atlas.torproject.org/#details/%s' % (exit_fp))
if fail_count > 0:
log.warning('%s of %s ssh connections have failed over exit %s' % (fail_count, len(details), exit_fp))
def test_exiturl(self):
self.assertEqual(util.exiturl("foo"), ("<https://metrics.torproject"
".org/rs.html#details/foo>"))
self.assertEqual(util.exiturl(4), ("<https://metrics.torproject.org/"
"rs.html#details/4>"))
def test_exiturl(self):
self.assertEqual(util.exiturl("foo"), "<https://atlas.torproject.or"
"g/#details/foo>")
self.assertEqual(util.exiturl(4), "<https://atlas.torproject.org/#det"
"ails/4>")
def test_exiturl(self):
self.assertEqual(util.exiturl("foo"), ("<https://metrics.torproject"
".org/rs.html#details/foo>"))
self.assertEqual(util.exiturl(4), ("<https://metrics.torproject.org/"
"rs.html#details/4>"))