def setUp(self):
log.debug('PsiphonTest.setUp')
self.report['bootstrapped_success'] = None
self.report['request_success'] = None
self.report['psiphon_found'] = None
self.report['default_configuration'] = True
self.bootstrapped = defer.Deferred()
self.url = self.localOptions['url']
if self.localOptions['url'] != net.GOOGLE_HUMANS[0]:
self.report['default_configuration'] = False
if self.localOptions['expected-body'] != net.GOOGLE_HUMANS[1]:
self.report['default_configuration'] = False
if self.localOptions['psiphonpath']:
self.psiphonpath = self.localOptions['psiphonpath']
else:
# Psiphon is not installable and to run it manually, it has to be
# run from the psiphon directory, so it wouldn't make sense to
# install it in the PATH. For now, we assume that Psiphon sources
# are in the user's home directory.
from os import path, getenv
self.psiphonpath = path.join(
getenv('HOME'), 'psiphon-circumvention-system/pyclient/pyclient')
log.debug('psiphon path: %s' % self.psiphonpath)
def remove_public_relays(state, bridges):
"""
Remove bridges from our bridge list which are also listed as public
relays. This must be called after Tor has fully bootstrapped and we have a
:class:`ooni.lib.txtorcon.TorState` with the
:attr:`ooni.lib.txtorcon.TorState.routers` attribute assigned.
XXX Does state.router.values() have all of the relays in the consensus, or
just the ones we know about so far?
XXX FIXME: There is a problem in that Tor needs a Bridge line to already be
configured in order to bootstrap. However, after bootstrapping, we grab the
microdescriptors of all the relays and check if any of our bridges are
listed as public relays. Because of this, the first bridge does not get
checked for being a relay.
"""
IPs = map(lambda addr: addr.split(':',1)[0], bridges['all'])
both = set(state.routers.values()).intersection(IPs)
if len(both) > 0:
try:
updated = map(lambda node: remove_node_from_list(node), both)
log.debug("Bridges in both: %s" % both)
log.debug("Updated = %s" % updated)
#if not updated:
# defer.returnValue(state)
#else:
# defer.returnValue(state)
return state
except Exception, e:
log.err("Removing public relays %s from bridge list failed:\n%s"
% (both, e))
def outReceived(self, data):
self.stdout += data
# output received, see if we have bootstrapped
if not self.bootstrapped.called and "Connected to proxy on localhost" in self.stdout:
log.debug("Bootstrap Detected")
self.cancelTimer()
self.bootstrapped.callback("bootstrapped")
def addToReport(self,
request,
response=None,
response_body=None,
failure_string=None):
"""
Adds to the report the specified request and response.
Args:
request (dict): A dict describing the request that was made
response (instance): An instance of
:class:twisted.web.client.Response.
Note: headers is our modified True Headers version.
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
def _representHeaders(headers):
represented_headers = {}
for name, value in headers.getAllRawHeaders():
represented_headers[name] = value[0]
return represented_headers
def _representBody(body):
# XXX perhaps add support for decoding gzip in the future.
try:
body = unicode(body, 'ascii')
body = body.replace('\0', '')
except UnicodeDecodeError:
try:
body = unicode(body, 'utf-8')
body = body.replace('\0', '')
except UnicodeDecodeError:
body = base64Dict(body)
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
session = {
'request': {
'headers': _representHeaders(request_headers),
'body': request['body'],
'url': request['url'],
'method': request['method'],
'tor': request['tor']
}
}
if response:
if self.localOptions.get('withoutbody', 0) is 0:
response_body = _representBody(response_body)
else:
response_body = ''
session['response'] = {
'headers': _representHeaders(response.headers),
'body': response_body,
'code': response.code
}
session['failure'] = None
if failure_string:
session['failure'] = failure_string
self.report['requests'].append(session)
def _cbResponse(self, response, request, headers_processor,
body_processor):
if not response:
log.err("Got no response")
return
else:
log.debug("Got response %s" % response)
if str(response.code).startswith('3'):
self.processRedirect(response.headers.getRawHeaders('Location')[0])
# [!] We are passing to the headers_processor the headers dict and
# not the Headers() object
response_headers_dict = list(response.headers.getAllRawHeaders())
if headers_processor:
headers_processor(response_headers_dict)
else:
self.processResponseHeaders(response_headers_dict)
finished = defer.Deferred()
response.deliverBody(BodyReceiver(finished))
finished.addCallback(self._processResponseBody, request, response,
body_processor)
return finished
def addToReport(self, request, response=None, response_body=None,
failure_string=None, previous_response=None):
"""
Adds to the report the specified request and response.
Args:
request (dict): A dict describing the request that was made
response (instance): An instance of
:class:twisted.web.client.Response.
Note: headers is our modified True Headers version.
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
session = {
'request': {
'headers': _representHeaders(request_headers),
'body': request['body'],
'url': request['url'],
'method': request['method'],
'tor': request['tor']
},
'response': None
}
if response:
if (getattr(response, 'request', None) and
getattr(response.request, 'absoluteURI', None)):
session['request']['url'] = response.request.absoluteURI
if self.localOptions.get('withoutbody', 0) is 0:
response_body = representBody(response_body)
else:
response_body = ''
response_headers = _representHeaders(response.headers)
# Attempt to redact the IP address of the probe from the responses
if (config.privacy.includeip is False and probe_ip.address is not None and
(isinstance(response_body, str) or isinstance(response_body, unicode))):
response_body = response_body.replace(probe_ip.address, "[REDACTED]")
for key, value in response_headers.items():
response_headers[key] = value.replace(probe_ip.address,
"[REDACTED]")
session['response'] = {
'headers': response_headers,
'body': response_body,
'code': response.code
}
session['failure'] = None
if failure_string:
session['failure'] = failure_string
self.report['requests'].append(session)
if response and response.previousResponse:
previous_response = response.previousResponse
if previous_response:
self.addToReport(request, previous_response,
response_body=None,
failure_string=None)
def test_cacheobject(self):
"""
This detects the presence of a squid transparent HTTP proxy by sending
a request for cache_object://localhost/info.
The response to this request will usually also contain the squid
version number.
"""
log.debug("Running")
def process_body(body):
if "Access Denied." in body:
self.report['transparent_http_proxy'] = True
else:
self.report['transparent_http_proxy'] = False
log.msg("Testing Squid proxy presence by sending a request for "\
"cache_object")
headers = {}
#headers["Host"] = [self.input]
self.report['trans_http_proxy'] = None
method = "GET"
body = "cache_object://localhost/info"
return self.doRequest(self.localOptions['backend'],
method=method,
body=body,
headers=headers,
body_processor=process_body)
def blocking_call(self):
try:
result = threads.blockingCallFromThread(reactor, os.system, self.run_me)
except:
log.debug("Netalyzr had an error, please see the log file: %s" % self.output_file)
finally:
self.clean_up()
def getX509Name(certificate, get_components=False):
"""Get the DER-encoded form of the Name fields of an X509 certificate.
@param certificate: A :class:`OpenSSL.crypto.X509Name` object.
@param get_components: A boolean. If True, returns a list of tuples of
the (name, value)s of each Name field in the
:param:`certificate`. If False, returns the DER
encoded form of the Name fields of the
:param:`certificate`.
"""
x509_name = None
try:
assert isinstance(certificate, crypto.X509Name), \
"getX509Name takes OpenSSL.crypto.X509Name as first argument!"
x509_name = crypto.X509Name(certificate)
except AssertionError as ae:
log.err(ae)
except Exception as exc:
log.exception(exc)
if not x509_name is None:
if not get_components:
return x509_name.der()
else:
return x509_name.get_components()
else:
log.debug("getX509Name: got None for ivar x509_name")
def start_director(self):
log.debug("Starting director")
d = self.director.start()
d.addCallback(self.director_started)
d.addErrback(self.director_startup_failed)
d.addBoth(lambda _: self.status_poller.notify())
def addToReport(self,
request,
response=None,
response_body=None,
failure_string=None):
"""
Adds to the report the specified request and response.
Args:
request (dict): A dict describing the request that was made
response (instance): An instance of
:class:twisted.web.client.Response.
Note: headers is our modified True Headers version.
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
session = {
'request': {
'headers': _representHeaders(request_headers),
'body': request['body'],
'url': request['url'],
'method': request['method'],
'tor': request['tor']
},
'response': None
}
if response:
if self.localOptions.get('withoutbody', 0) is 0:
response_body = representBody(response_body)
else:
response_body = ''
# Attempt to redact the IP address of the probe from the responses
if (config.privacy.includeip is False
and config.probe_ip.address is not None
and (isinstance(response_body, str)
or isinstance(response_body, unicode))):
response_body = response_body.replace(config.probe_ip.address,
"[REDACTED]")
if (getattr(response, 'request', None)
and getattr(response.request, 'absoluteURI', None)):
session['request']['url'] = response.request.absoluteURI
session['response'] = {
'headers': _representHeaders(response.headers),
'body': response_body,
'code': response.code
}
session['failure'] = None
if failure_string:
session['failure'] = failure_string
self.report['requests'].append(session)
if response and response.previousResponse:
self.addToReport(request,
response.previousResponse,
response_body=None,
failure_string=None)
def doRequest(noreason):
log.debug("Doing HTTP request via Lantern (127.0.0.1:8787) for %s" % self.url)
request = agent.request("GET", self.url)
request.addCallback(readBody)
request.addCallback(addResultToReport)
request.addCallback(self.processDirector.close)
return request
def outReceived(self, data):
self.stdout += data
# output received, see if we have bootstrapped
if not self.bootstrapped.called and "client (http) proxy at" in self.stdout:
log.debug("Bootstrap Detected")
self.cancelTimer()
self.bootstrapped.callback("bootstrapped")
def run_rst_tests(self, packet_hash):
"""Check for RST injection and bisect if found.
Args:
packet_hash (str): the unique hash of the responses to
test.
"""
word_list, bisected = self.hr_sent_wordlist[packet_hash]
packet_list = self.responses[packet_hash]
results = []
detected = False
for detector in [RstSeqData, DataSeqRst, DataSeqChange, RstAckChange]:
test = detector(packet_list)
if test.detected:
detected = True
results.append(test.name)
# Send bisected or update tested list
if detected and not bisected:
log.debug("RST blocking detected: Bisecting.")
self.identified_rst.update(results)
if len(word_list) > 1:
lists = self.bisect(word_list)
for lst in lists:
self.to_send.append(lst)
# A single item list means that we have found a blocked item
elif len(word_list) == 1:
log.debug("The following term was blocked:" " {0}".format(word_list[0]))
if word_list[0] not in self.blocked_words:
self.blocked_words.update([word_list[0]])
else:
self.tested_words.update(word_list)
def process_headers(headers):
"""
Checks if any of the headers that squid is known to add match the
squid regexp.
We are looking for something that looks like this:
via: 1.0 cache_server:3128 (squid/2.6.STABLE21)
x-cache: MISS from cache_server
x-cache-lookup: MISS from cache_server:3128
"""
squid_headers = {
'via': r'.* \((squid.*)\)',
'x-cache': r'MISS from (\w+)',
'x-cache-lookup': r'MISS from (\w+:?\d+?)'
}
self.report['transparent_http_proxy'] = False
for key in squid_headers.keys():
if key in headers:
log.debug("Found %s in headers" % key)
m = re.search(squid_headers[key], headers[key])
if m:
log.msg("Detected the presence of squid transparent"\
" HTTP Proxy")
self.report['transparent_http_proxy'] = True
def measurementFailed(self, failure, measurement):
log.debug("Failed doing measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.failedMeasurements += 1
measurement.result = failure
return measurement
def gotResponse(message):
log.debug(dns_type + " Lookup successful")
log.debug(str(message))
addrs = []
answers = []
if dns_server:
msg = message.answers
else:
msg = message[0]
for answer in msg:
if answer.type is dnsType:
if dnsType is dns.SOA:
addr = (answer.name.name, answer.payload.serial)
elif dnsType in [dns.NS, dns.PTR]:
addr = answer.payload.name.name
elif dnsType is dns.A:
addr = answer.payload.dottedQuad()
else:
addr = None
addrs.append(addr)
answers.append(representAnswer(answer))
DNSTest.addToReport(self,
query,
resolver=dns_server,
query_type=dns_type,
answers=answers,
addrs=addrs)
return addrs
def lookup(self, include_ip=None, include_asn=None, include_country=None):
if self._state == IN_PROGRESS:
yield self._looking_up
elif self._last_lookup < time.time() - self._expire_in:
self.address = None
if self.address:
self.resolveGeodata(include_ip, include_asn, include_country)
defer.returnValue(self.address)
else:
self._state = IN_PROGRESS
try:
yield self.askTor()
log.msg("Found your IP via Tor")
self.resolveGeodata(include_ip, include_asn, include_country)
self._looking_up.callback(self.address)
defer.returnValue(self.address)
except errors.TorStateNotFound:
log.debug("Tor is not running. Skipping IP lookup via Tor.")
except Exception:
log.msg("Unable to lookup the probe IP via Tor.")
try:
yield self.askGeoIPService()
log.msg("Found your IP via a GeoIP service")
self.resolveGeodata(include_ip, include_asn, include_country)
self._looking_up.callback(self.address)
defer.returnValue(self.address)
except Exception as exc:
log.msg("Unable to lookup the probe IP via GeoIPService")
self._looking_up.errback(defer.failure.Failure(exc))
raise
def _cbResponse(self, response, headers_processor, body_processor):
log.debug("Got response %s" % response)
if not response:
self.report['response'] = None
log.err("We got an empty response")
return
self.response['headers'] = list(response.headers.getAllRawHeaders())
self.response['code'] = response.code
self.response['length'] = response.length
self.response['version'] = response.length
if str(self.response['code']).startswith('3'):
self.processRedirect(response.headers.getRawHeaders('Location')[0])
if headers_processor:
headers_processor(self.response['headers'])
else:
self.processResponseHeaders(self.response['headers'])
finished = defer.Deferred()
response.deliverBody(BodyReceiver(finished))
finished.addCallback(self._processResponseBody, body_processor)
return finished
def sendPayload(self, payload):
d1 = defer.Deferred()
def closeConnection(proto):
self.report['sent'].append(proto.sent_data)
self.report['received'].append(proto.received_data)
proto.transport.loseConnection()
log.debug("Closing connection")
d1.callback(proto.received_data)
def timedOut(proto):
self.report['failure'] = 'tcp_timed_out_error'
proto.transport.loseConnection()
def errback(failure):
self.report['failure'] = failureToString(failure)
d1.errback(failure)
def connected(proto):
log.debug("Connected to %s:%s" % (self.address, self.port))
proto.report = self.report
proto.deferred = d1
proto.sendPayload(payload)
if self.timeout:
# XXX-Twisted this logic should probably go inside of the protocol
reactor.callLater(self.timeout, closeConnection, proto)
point = TCP4ClientEndpoint(reactor, self.address, self.port)
log.debug("Connecting to %s:%s" % (self.address, self.port))
d2 = point.connect(TCPSenderFactory())
d2.addCallback(connected)
d2.addErrback(errback)
return d1
def setUp(self):
log.debug("Setting up HTTPTest")
try:
import OpenSSL
except:
log.err("Warning! pyOpenSSL is not installed. https websites will"
"not work")
from twisted.web.client import Agent
from twisted.internet import reactor
self.agent = Agent(reactor)
if self.followRedirects:
try:
from twisted.web.client import RedirectAgent
self.agent = RedirectAgent(self.agent)
except:
log.err("Warning! You are running an old version of twisted"\
"(<= 10.1). I will not be able to follow redirects."\
"This may make the testing less precise.")
self.report['errors'].append("Could not import RedirectAgent")
self.request = {}
self.response = {}
log.debug("Finished test setup")
def inputProcessor(self, filename=None):
"""
You may replace this with your own custom input processor. It takes as
input a file descriptor so remember to close it when you are done.
This can be useful when you have some input data that is in a certain
format and you want to set the input attribute of the test to something
that you will be able to properly process.
For example you may wish to have an input processor that will allow you
to ignore comments in files. This can be easily achieved like so:
fp = open(filename)
for x in fp.xreadlines():
if x.startswith("#"):
continue
yield x.strip()
fp.close()
Other fun stuff is also possible.
"""
log.debug("Running default input processor")
if filename:
fp = open(filename)
for x in fp.xreadlines():
yield x.strip()
fp.close()
else:
pass
def createDeck(global_options, url=None):
from ooni.nettest import NetTestLoader
from ooni.deck import Deck, nettest_to_path
if url:
log.msg("Creating deck for: %s" % (url))
if global_options['no-yamloo']:
log.msg("Will not write to a yamloo report file")
deck = Deck(bouncer=global_options['bouncer'],
no_collector=global_options['no-collector'])
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'], global_options)
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
if url is not None:
args = ('-u', url)
else:
args = tuple()
if any(global_options['subargs']):
args = global_options['subargs'] + args
net_test_loader = NetTestLoader(args,
test_file=test_file,
annotations=global_options['annotations'])
if global_options['collector']:
net_test_loader.collector = \
CollectorClient(global_options['collector'])
deck.insert(net_test_loader)
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
sys.exit(2)
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
sys.exit(3)
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
sys.exit(4)
except errors.HTTPSCollectorUnsupported:
log.err("HTTPS collectors require a twisted version of at least 14.0.2.")
sys.exit(6)
except errors.InsecureBackend:
log.err("Attempting to report to an insecure collector.")
log.err("To enable reporting to insecure collector set the "
"advanced->insecure_backend option to true in "
"your ooniprobe.conf file.")
sys.exit(7)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
sys.exit(5)
return deck
def doRequest(noreason):
log.debug("Doing HTTP request via Lantern (127.0.0.1:8787) for %s" % self.url)
request = agent.request("GET", self.url)
request.addCallback(readBody)
request.addCallback(addResultToReport)
request.addCallback(self.processDirector.close)
return request
def validate_fields(fields):
log.debug("Report fields are: %s" % fields)
# check report version
if fields['test_version'] not in valid_test_versions:
raise InvalidReportField('test_version')
# check report CC
#XXX: confirm what value we use for default CC and whether
# or not we should support > 2 character CC
if fields['probe_cc'] is None:
fields['probe_cc'] = default_probe_cc
if not re.match('[A-Z\?]{2,4}', fields['probe_cc'].upper()):
raise InvalidReportField('probe_cc')
# check report ASN
if fields['probe_asn'] is None:
fields['probe_asn'] = 'AS0'
if not re.match('^AS[0-9]{1,10}', fields['probe_asn'].upper()):
raise InvalidReportField('probe_asn')
# check report timestamp
try:
datetime_ts = datetime.fromtimestamp(fields['start_time'])
datetime_str = timestamp(datetime_ts)
except InvalidTimestampFormat:
raise InvalidReportField('start_time')
# check report IP
try:
IPAddress(fields['probe_ip'])
except ValueError:
raise InvalidReportField('probe_ip')
def lookup(self):
try:
yield self.askTor()
log.msg("Found your IP via Tor %s" % self.address)
defer.returnValue(self.address)
except errors.TorStateNotFound:
log.debug("Tor is not running. Skipping IP lookup via Tor.")
except Exception:
log.msg("Unable to lookup the probe IP via Tor.")
try:
yield self.askTraceroute()
log.msg("Found your IP via Traceroute %s" % self.address)
defer.returnValue(self.address)
except errors.InsufficientPrivileges:
log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
except:
log.msg("Unable to lookup the probe IP via traceroute")
try:
yield self.askGeoIPService()
log.msg("Found your IP via a GeoIP service: %s" % self.address)
defer.returnValue(self.address)
except Exception, e:
log.msg("Unable to lookup the probe IP via GeoIPService")
raise e
def _test_http_request(self):
http_blocked = True
for dc_id, address in TELEGRAM_DCS:
if http_blocked == False:
break
for port in [80, 443]:
url = 'http://{}:{}'.format(address, port)
try:
response = yield self.doRequest(url, 'POST')
except Exception as exc:
failure_string = failureToString(defer.failure.Failure(exc))
log.err("Failed to connect to {}: {}".format(url, failure_string))
continue
log.debug("Got back status code {}".format(response.code))
log.debug("{}".format(response.body))
if response.code == 501:
http_blocked = False
break
if http_blocked == True:
self.report['telegram_http_blocking'] = True
log.msg("Telegram servers are BLOCKED based on HTTP")
else:
self.report['telegram_http_blocking'] = False
log.msg("Telegram servers are not blocked based on HTTP")
def setUp(self):
if (not self.localOptions['testresolvers']
and not self.localOptions['testresolver']):
self.test_resolvers = []
with open('/etc/resolv.conf') as f:
for line in f:
if line.startswith('nameserver'):
self.test_resolvers.append(line.split(' ')[1].strip())
self.report['test_resolvers'] = self.test_resolvers
elif self.localOptions['testresolvers']:
test_resolvers_file = self.localOptions['testresolvers']
elif self.localOptions['testresolver']:
self.test_resolvers = [self.localOptions['testresolver']]
try:
with open(test_resolvers_file) as f:
self.test_resolvers = [
x.split('#')[0].strip() for x in f.readlines()
]
self.report['test_resolvers'] = self.test_resolvers
f.close()
except IOError as e:
log.exception(e)
raise usage.UsageError("Invalid test resolvers file")
except NameError:
log.debug("No test resolver file configured")
dns_ip, dns_port = self.localOptions['backend'].split(':')
self.control_dns_server = (str(dns_ip), int(dns_port))
self.report['control_resolver'] = "%s:%d" % self.control_dns_server
def process_headers(headers):
"""
Checks if any of the headers that squid is known to add match the
squid regexp.
We are looking for something that looks like this:
via: 1.0 cache_server:3128 (squid/2.6.STABLE21)
x-cache: MISS from cache_server
x-cache-lookup: MISS from cache_server:3128
"""
squid_headers = {'via': r'.* \((squid.*)\)',
'x-cache': r'MISS from (\w+)',
'x-cache-lookup': r'MISS from (\w+:?\d+?)'
}
self.report['transparent_http_proxy'] = False
for key in squid_headers.keys():
if key in headers:
log.debug("Found %s in headers" % key)
m = re.search(squid_headers[key], headers[key])
if m:
log.msg("Detected the presence of squid transparent"\
" HTTP Proxy")
self.report['transparent_http_proxy'] = True
def runLantern(self):
paths = filter(os.path.exists, [
os.path.join(os.path.expanduser(x), self.command[0])
for x in getenv('PATH').split(':')
])
log.debug("Spawning Lantern")
reactor.spawnProcess(self.processDirector, paths[0], self.command)
def _current_step_data(self):
step_idx, mutation_idx = self.factory.mutation
log.debug("Mutating %s %s" % (step_idx, mutation_idx))
mutated_step = daphn3Mutate(self.steps,
step_idx, mutation_idx)
log.debug("Mutated packet into %s" % mutated_step)
return mutated_step[self.current_step].values()[0]
def closeConnection(proto):
self.report["sent"].append(proto.sent_data)
self.report["received"].append(proto.received_data)
proto.transport.loseConnection()
scapySender.stopListening()
log.debug("Closing connection")
d1.callback(proto.received_data)
def nextMutation(self):
log.debug("Moving onto next mutation")
# [step_idx, mutation_idx]
c_step_idx, c_mutation_idx = self.factory.mutation
log.debug("[%s]: c_step_idx: %s | c_mutation_idx: %s" % (self.role,
c_step_idx, c_mutation_idx))
if c_step_idx >= (len(self.steps) - 1):
log.err("No censorship fingerprint bisected.")
log.err("Givinig up.")
self.transport.loseConnection()
return
# This means we have mutated all bytes in the step
# we should proceed to mutating the next step.
log.debug("steps: %s | %s" % (self.steps, self.steps[c_step_idx]))
if c_mutation_idx >= (len(self.steps[c_step_idx].values()[0]) - 1):
log.debug("Finished mutating step")
# increase step
self.factory.mutation[0] += 1
# reset mutation idx
self.factory.mutation[1] = 0
else:
log.debug("Mutating next byte in step")
# increase mutation index
self.factory.mutation[1] += 1
def sendPayload(self, payload):
d1 = defer.Deferred()
def closeConnection(proto):
self.report['sent'].append(proto.sent_data)
self.report['received'].append(proto.received_data)
proto.transport.loseConnection()
log.debug("Closing connection")
d1.callback(proto.received_data)
def timedOut(proto):
self.report['failure'] = 'tcp_timed_out_error'
proto.transport.loseConnection()
def errback(failure):
self.report['failure'] = failureToString(failure)
d1.errback(failure)
def connected(proto):
log.debug("Connected to %s:%s" % (self.address, self.port))
proto.report = self.report
proto.deferred = d1
proto.sendPayload(payload)
if self.timeout:
# XXX-Twisted this logic should probably go inside of the protocol
reactor.callLater(self.timeout, closeConnection, proto)
point = TCP4ClientEndpoint(reactor, self.address, self.port)
log.debug("Connecting to %s:%s" % (self.address, self.port))
d2 = point.connect(TCPSenderFactory())
d2.addCallback(connected)
d2.addErrback(errback)
return d1
def _processResponseBody(self, data):
log.debug("HTTPTest._processResponseBody")
self.response['body'] = data
#self.result['response'] = self.response
self.processResponseBody(data)
def process_a_answers(self, message, resolver_address):
log.msg("Processing A answers for %s" % resolver_address)
log.debug("These are the answers I got %s" % message.answers)
all_a = []
a_a = []
for answer in message.answers:
if answer.type is 1:
# A type query
r = answer.payload.dottedQuad()
self.report['a_lookups'][resolver_address] = r
a_a.append(r)
lookup = str(answer.payload)
all_a.append(lookup)
if resolver_address == 'control':
self.report['control_server'] = self.localOptions['backend']
self.report['control_lookup'] = all_a
self.control_a_lookups = a_a
else:
self.test_a_lookups[resolver_address] = a_a
self.report['test_lookups'][resolver_address] = all_a
log.msg("Done")
def checkAllTasksDone(self):
log.debug("Checking all tasks for completion %s == %s" % (self.doneTasks, self.tasks))
if self.completedScheduling and self.doneTasks == self.tasks:
if self.allTasksDone.called:
log.err("allTasksDone was already called. This is probably a bug.")
else:
self.allTasksDone.callback(self.doneTasks)
def remove_public_relays(state, bridges):
"""
Remove bridges from our bridge list which are also listed as public
relays. This must be called after Tor has fully bootstrapped and we have a
:class:`ooni.lib.txtorcon.TorState` with the
:attr:`ooni.lib.txtorcon.TorState.routers` attribute assigned.
XXX Does state.router.values() have all of the relays in the consensus, or
just the ones we know about so far?
XXX FIXME: There is a problem in that Tor needs a Bridge line to already be
configured in order to bootstrap. However, after bootstrapping, we grab the
microdescriptors of all the relays and check if any of our bridges are
listed as public relays. Because of this, the first bridge does not get
checked for being a relay.
"""
IPs = map(lambda addr: addr.split(':',1)[0], bridges['all'])
both = set(state.routers.values()).intersection(IPs)
if len(both) > 0:
try:
updated = map(lambda node: remove_node_from_list(node), both)
log.debug("Bridges in both: %s" % both)
log.debug("Updated = %s" % updated)
#if not updated:
# defer.returnValue(state)
#else:
# defer.returnValue(state)
return state
except Exception, e:
log.err("Removing public relays %s from bridge list failed:\n%s"
% (both, e))
def nextMutation(self):
log.debug("Moving onto next mutation")
# [step_idx, mutation_idx]
c_step_idx, c_mutation_idx = self.factory.mutation
log.debug("[%s]: c_step_idx: %s | c_mutation_idx: %s" %
(self.role, c_step_idx, c_mutation_idx))
if c_step_idx >= (len(self.steps) - 1):
log.err("No censorship fingerprint bisected.")
log.err("Givinig up.")
self.transport.loseConnection()
return
# This means we have mutated all bytes in the step
# we should proceed to mutating the next step.
log.debug("steps: %s | %s" % (self.steps, self.steps[c_step_idx]))
if c_mutation_idx >= (len(self.steps[c_step_idx].values()[0]) - 1):
log.debug("Finished mutating step")
# increase step
self.factory.mutation[0] += 1
# reset mutation idx
self.factory.mutation[1] = 0
else:
log.debug("Mutating next byte in step")
# increase mutation index
self.factory.mutation[1] += 1
def _setUp(self):
log.debug("Setting up HTTPTest")
try:
import OpenSSL
except:
log.err("Warning! pyOpenSSL is not installed. https websites will"
"not work")
self.control_agent = Agent(reactor,
sockshost="127.0.0.1",
socksport=config.advanced.tor_socksport)
sockshost, socksport = (None, None)
if self.localOptions['socksproxy']:
self.report['socksproxy'] = self.localOptions['socksproxy']
sockshost, socksport = self.localOptions['socksproxy'].split(':')
socksport = int(socksport)
self.agent = Agent(reactor, sockshost=sockshost, socksport=socksport)
if self.followRedirects:
try:
from twisted.web.client import RedirectAgent
self.control_agent = RedirectAgent(self.control_agent)
self.agent = RedirectAgent(self.agent)
except:
log.err("Warning! You are running an old version of twisted"\
"(<= 10.1). I will not be able to follow redirects."\
"This may make the testing less precise.")
self.report['errors'].append("Could not import RedirectAgent")
self.processInputs()
log.debug("Finished test setup")
def outReceived(self, data):
self.stdout += data
# output received, see if we have bootstrapped
if not self.bootstrapped.called and "Connected to proxy on localhost" in self.stdout:
log.debug("Bootstrap Detected")
self.cancelTimer()
self.bootstrapped.callback("bootstrapped")
def addToReport(self, request, response=None, response_body=None, failure_string=None):
"""
Adds to the report the specified request and response.
Args:
request (dict): A dict describing the request that was made
response (instance): An instance of
:class:twisted.web.client.Response.
Note: headers is our modified True Headers version.
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
request_response = {
'request': {
'headers': list(request_headers.getAllRawHeaders()),
'body': request['body'],
'url': request['url'],
'method': request['method']
}
}
if response:
request_response['response'] = {
'headers': list(response.headers.getAllRawHeaders()),
'body': response_body,
'code': response.code
}
if failure_string:
request_response['failure'] = failure_string
self.report['requests'].append(request_response)
def _processResponseBody(self, response_body, request, response, body_processor):
log.debug("Processing response body")
self.addToReport(request, response, response_body)
if body_processor:
body_processor(response_body)
else:
self.processResponseBody(response_body)
def inputProcessor(self, filename):
"""
You may replace this with your own custom input processor. It takes as
input a file name.
An inputProcessor is an iterator that will yield one item from the file
and takes as argument a filename.
This can be useful when you have some input data that is in a certain
format and you want to set the input attribute of the test to something
that you will be able to properly process.
For example you may wish to have an input processor that will allow you
to ignore comments in files. This can be easily achieved like so::
fp = open(filename)
for x in fp.xreadlines():
if x.startswith("#"):
continue
yield x.strip()
fp.close()
Other fun stuff is also possible.
"""
log.debug("Running default input processor")
with open(filename) as f:
for line in f:
l = line.strip()
# Skip empty lines
if not l:
continue
# Skip comment lines
elif l.startswith('#'):
continue
yield l
def updateProgressMeters(test_filename, input_unit_factory, test_case_number):
"""
Update the progress meters for keeping track of test state.
"""
if not config.state.test_filename:
config.state[test_filename] = Storage()
config.state[test_filename].per_item_average = 2.0
input_unit_idx = float(config.stateDict[test_filename])
input_unit_items = len(input_unit_factory)
test_case_number = float(test_case_number)
total_iterations = input_unit_items * test_case_number
current_iteration = input_unit_idx * test_case_number
log.debug("input_unit_items: %s" % input_unit_items)
log.debug("test_case_number: %s" % test_case_number)
log.debug("Test case number: %s" % test_case_number)
log.debug("Total iterations: %s" % total_iterations)
log.debug("Current iteration: %s" % current_iteration)
def progress():
return (current_iteration / total_iterations) * 100.0
config.state[test_filename].progress = progress
def eta():
return (total_iterations - current_iteration) * config.state[test_filename].per_item_average
config.state[test_filename].eta = eta
config.state[test_filename].input_unit_idx = input_unit_idx
config.state[test_filename].input_unit_items = input_unit_items
def addToReport(self, request, response=None, response_body=None, failure_string=None):
"""
Adds to the report the specified request and response.
Args:
request (dict): A dict describing the request that was made
response (instance): An instance of
:class:twisted.web.client.Response.
Note: headers is our modified True Headers version.
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
request_response = {
'request': {
'headers': list(request_headers.getAllRawHeaders()),
'body': request['body'],
'url': request['url'],
'method': request['method']
}
}
if response:
request_response['response'] = {
'headers': list(response.headers.getAllRawHeaders()),
'body': response_body,
'code': response.code
}
if failure_string:
request_response['failure'] = failure_string
self.report['requests'].append(request_response)
def setUp(self):
if (not self.localOptions['testresolvers'] and
not self.localOptions['testresolver']):
self.test_resolvers = []
with open('/etc/resolv.conf') as f:
for line in f:
if line.startswith('nameserver'):
self.test_resolvers.append(line.split(' ')[1].strip())
self.report['test_resolvers'] = self.test_resolvers
elif self.localOptions['testresolvers']:
test_resolvers_file = self.localOptions['testresolvers']
elif self.localOptions['testresolver']:
self.test_resolvers = [self.localOptions['testresolver']]
try:
with open(test_resolvers_file) as f:
self.test_resolvers = [
x.split('#')[0].strip() for x in f.readlines()]
self.report['test_resolvers'] = self.test_resolvers
f.close()
except IOError as e:
log.exception(e)
raise usage.UsageError("Invalid test resolvers file")
except NameError:
log.debug("No test resolver file configured")
dns_ip, dns_port = self.localOptions['backend'].split(':')
self.control_dns_server = (str(dns_ip), int(dns_port))
self.report['control_resolver'] = "%s:%d" % self.control_dns_server
def resumeTest(test_filename, input_unit_factory):
"""
Returns the an input_unit_factory that is at the index of the previous run of the test
for the specified test_filename.
Args:
test_filename (str): the filename of the test that is being run
including the .py extension.
input_unit_factory (:class:ooni.inputunit.InputUnitFactory): with the
same input of the past run.
Returns:
:class:ooni.inputunit.InputUnitFactory that is at the index of the
previous test run.
"""
try:
idx = config.stateDict[test_filename]
for x in range(idx):
try:
input_unit_factory.next()
except StopIteration:
log.msg("Previous run was complete")
return input_unit_factory
return input_unit_factory
except KeyError:
log.debug("No resume key found for selected test name. It is therefore 0")
config.stateDict[test_filename] = 0
return input_unit_factory
def getX509Name(certificate, get_components=False):
"""Get the DER-encoded form of the Name fields of an X509 certificate.
@param certificate: A :class:`OpenSSL.crypto.X509Name` object.
@param get_components: A boolean. If True, returns a list of tuples of
the (name, value)s of each Name field in the
:param:`certificate`. If False, returns the DER
encoded form of the Name fields of the
:param:`certificate`.
"""
x509_name = None
try:
assert isinstance(certificate, crypto.X509Name), \
"getX509Name takes OpenSSL.crypto.X509Name as first argument!"
x509_name = crypto.X509Name(certificate)
except AssertionError as ae:
log.err(ae)
except Exception as exc:
log.exception(exc)
if not x509_name is None:
if not get_components:
return x509_name.der()
else:
return x509_name.get_components()
else:
log.debug("getX509Name: got None for ivar x509_name")
def inputProcessor(self, filename):
"""
You may replace this with your own custom input processor. It takes as
input a file name.
An inputProcessor is an iterator that will yield one item from the file
and takes as argument a filename.
This can be useful when you have some input data that is in a certain
format and you want to set the input attribute of the test to something
that you will be able to properly process.
For example you may wish to have an input processor that will allow you
to ignore comments in files. This can be easily achieved like so::
fp = open(filename)
for x in fp.xreadlines():
if x.startswith("#"):
continue
yield x.strip()
fp.close()
Other fun stuff is also possible.
"""
log.debug("Running default input processor")
with open(filename) as f:
for line in f:
l = line.strip()
# Skip empty lines
if not l:
continue
# Skip comment lines
elif l.startswith('#'):
continue
yield l
def lookup(self):
try:
yield self.askTor()
log.msg("Found your IP via Tor %s" % self.address)
self.resolveGeodata()
defer.returnValue(self.address)
except errors.TorStateNotFound:
log.debug("Tor is not running. Skipping IP lookup via Tor.")
except Exception:
log.msg("Unable to lookup the probe IP via Tor.")
try:
yield self.askTraceroute()
log.msg("Found your IP via Traceroute %s" % self.address)
self.resolveGeodata()
defer.returnValue(self.address)
except errors.InsufficientPrivileges:
log.debug(
"Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges"
)
except:
log.msg("Unable to lookup the probe IP via traceroute")
try:
yield self.askGeoIPService()
log.msg("Found your IP via a GeoIP service: %s" % self.address)
self.resolveGeodata()
defer.returnValue(self.address)
except Exception, e:
log.msg("Unable to lookup the probe IP via GeoIPService")
raise e
def _test_http_request(self):
http_blocked = True
for dc_id, address in TELEGRAM_DCS:
if http_blocked == False:
break
for port in [80, 443]:
url = 'http://{}:{}'.format(address, port)
try:
response = yield self.doRequest(url, 'POST')
except Exception as exc:
failure_string = failureToString(
defer.failure.Failure(exc))
log.err("Failed to connect to {}: {}".format(
url, failure_string))
continue
log.debug("Got back status code {}".format(response.code))
log.debug("{}".format(response.body))
if response.code == 501:
http_blocked = False
break
if http_blocked == True:
self.report['telegram_http_blocking'] = True
log.msg("Telegram servers are BLOCKED based on HTTP")
else:
self.report['telegram_http_blocking'] = False
log.msg("Telegram servers are not blocked based on HTTP")
def _setUp(self):
super(BaseScapyTest, self)._setUp()
if config.scapyFactory is None:
log.debug("Scapy factory not set, registering it.")
config.scapyFactory = ScapyFactory(config.advanced.interface)
self.report['answer_flags'] = []
if self.localOptions['ipsrc']:
config.checkIPsrc = 0
else:
self.report['answer_flags'].append('ipsrc')
config.checkIPsrc = 1
if self.localOptions['ipid']:
self.report['answer_flags'].append('ipid')
config.checkIPID = 1
else:
config.checkIPID = 0
# XXX we don't support strict matching
# since (from scapy's documentation), some stacks have a bug for which
# the bytes in the IPID are swapped.
# Perhaps in the future we will want to have more fine grained control
# over this.
if self.localOptions['seqack']:
self.report['answer_flags'].append('seqack')
config.check_TCPerror_seqack = 1
else:
config.check_TCPerror_seqack = 0
self.report['sent_packets'] = []
self.report['answered_packets'] = []
def tryInterfaces(self, ifaces):
try:
from scapy.all import sr1 ## we want this check to be blocking
except:
log.msg("This test requires scapy: www.secdev.org/projects/scapy")
raise SystemExit
ifup = {}
while ifaces:
for ifname, ifaddr in ifaces:
log.debug("Currently testing network capabilities of interface"
+ "%s by sending a packet to our address %s"
% (ifname, ifaddr))
try:
pkt = IP(dst=ifaddr)/ICMP()
ans, unans = sr(pkt, iface=ifname, timeout=self.timeout)
except Exception, e:
raise PermissionsError if e.find("Errno 1") else log.err(e)
else:
## xxx i think this logic might be wrong
log.debug("Interface test packet\n%s\n\n%s"
% (pkt.summary(), pkt.show2()))
if ans.summary():
log.info("Received answer for test packet on interface"
+"%s :\n%s" % (ifname, ans.summary()))
ifup.update(ifname, ifaddr)
else:
log.info("Our interface test packet was unanswered:\n%s"
% unans.summary())
def writeReportEntry(self, entry):
log.debug("Writing report with YAML reporter")
self._write('---\n')
if isinstance(entry, Failure):
self._write(entry.value)
else:
self._write(safe_dump(entry))
self._write('...\n')
def task(self):
log.debug("Updating the inputs")
yield probe_ip.lookup()
log.debug("Updating the inputs for country %s" %
probe_ip.geodata['countrycode'])
yield resources.check_for_update(probe_ip.geodata['countrycode'])
yield input_store.update(probe_ip.geodata['countrycode'])
yield probe_ip.resolveGeodata()
def inConnectionLost(self):
"""Monkeypatch inConnectionLost to log failure if the process ends
unexpectedly before OpenVPN bootstraps.
"""
log.debug("inConnectionLost")
if not self.bootstrapped.called:
self.bootstrapped.errback(Exception("openvpn_exited_unexpectedly"))
def connected(proto):
log.debug("Connected to %s:%s" % (self.address, self.port))
proto.report = self.report
proto.deferred = d1
proto.sendPayload(payload)
if self.timeout:
# XXX-Twisted this logic should probably go inside of the protocol
reactor.callLater(self.timeout, closeConnection, proto)
