def notify(self, event):
for handler in self._subscribers:
try:
handler(event)
except Exception as exc:
log.err("Failed to run handler")
log.exception(exc)
def writeReportEntry(self, entry):
log.debug("Writing report with OONIB reporter")
url = self.collectorAddress + '/report/' + self.reportID
if "json" in self.supportedFormats:
serialisation_format = 'json'
else:
serialisation_format = 'yaml'
request = {
'format': serialisation_format,
'content': self.serializeEntry(entry, serialisation_format)
}
log.debug("Updating report with id %s (%s)" % (self.reportID, url))
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(request_json)
try:
yield self.agent.request("POST", str(url),
bodyProducer=bodyProducer)
except Exception as exc:
log.err("Error in writing report entry")
log.exception(exc)
raise errors.OONIBReportUpdateError
def getX509Name(certificate, get_components=False):
"""Get the DER-encoded form of the Name fields of an X509 certificate.
@param certificate: A :class:`OpenSSL.crypto.X509Name` object.
@param get_components: A boolean. If True, returns a list of tuples of
the (name, value)s of each Name field in the
:param:`certificate`. If False, returns the DER
encoded form of the Name fields of the
:param:`certificate`.
"""
x509_name = None
try:
assert isinstance(certificate, crypto.X509Name), \
"getX509Name takes OpenSSL.crypto.X509Name as first argument!"
x509_name = crypto.X509Name(certificate)
except AssertionError as ae:
log.err(ae)
except Exception as exc:
log.exception(exc)
if not x509_name is None:
if not get_components:
return x509_name.der()
else:
return x509_name.get_components()
else:
log.debug("getX509Name: got None for ivar x509_name")
def setUp(self):
if (not self.localOptions['testresolvers'] and \
not self.localOptions['testresolver']):
self.test_resolvers = []
with open('/etc/resolv.conf') as f:
for line in f:
if line.startswith('nameserver'):
self.test_resolvers.append(line.split(' ')[1].strip())
self.report['test_resolvers'] = self.test_resolvers
elif self.localOptions['testresolvers']:
test_resolvers_file = self.localOptions['testresolvers']
elif self.localOptions['testresolver']:
self.test_resolvers = [self.localOptions['testresolver']]
try:
with open(test_resolvers_file) as f:
self.test_resolvers = [x.split('#')[0].strip() for x in f.readlines()]
self.report['test_resolvers'] = self.test_resolvers
f.close()
except IOError, e:
log.exception(e)
raise usage.UsageError("Invalid test resolvers file")
def createReport(self):
"""
Creates a report on the oonib collector.
"""
log.msg("Creating report with OONIB Reporter. Please be patient.")
log.msg("This may take up to 1-2 minutes...")
try:
response = yield self.collector_client.createReport(
self.testDetails
)
except ConnectionRefusedError:
log.err("Connection to reporting backend failed "
"(ConnectionRefusedError)")
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
log.err("Host is not reachable (HostUnreachable error")
raise errors.OONIBReportCreationError
except (errors.OONIBInvalidInputHash,
errors.OONIBInvalidNettestName):
log.err("The specified input or nettests cannot be submitted to "
"this collector.")
log.msg("Try running a different test or try reporting to a "
"different collector.")
raise errors.OONIBReportCreationError
except Exception, e:
log.err("Failed to connect to reporter backend")
log.exception(e)
raise errors.OONIBReportCreationError
def writeReportEntry(self, entry):
log.debug("Writing report with OONIB reporter")
url = self.collectorAddress + '/report/' + self.reportID
if "json" in self.supportedFormats:
serialisation_format = 'json'
else:
serialisation_format = 'yaml'
request = {
'format': serialisation_format,
'content': self.serializeEntry(entry, serialisation_format)
}
log.debug("Updating report with id %s (%s)" % (self.reportID, url))
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(request_json)
try:
yield self.agent.request("POST",
str(url),
bodyProducer=bodyProducer)
except Exception as exc:
log.err("Error in writing report entry")
log.exception(exc)
raise errors.OONIBReportUpdateError
def director_startup_failed(failure):
log.err("Failed to start the director")
r = failure.trap(errors.TorNotRunning,
errors.InvalidOONIBCollectorAddress,
errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper,
errors.CouldNotFindTestCollector)
if isinstance(failure.value, errors.TorNotRunning):
log.err("Tor does not appear to be running")
log.err("Reporting with the collector %s is not possible" %
global_options['collector'])
log.msg("Try with a different collector or disable collector reporting with -n")
elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
log.err("Invalid format for oonib collector address.")
log.msg("Should be in the format http://<collector_address>:<port>")
log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
elif isinstance(failure.value, errors.UnableToLoadDeckInput):
log.err("Unable to fetch the required inputs for the test deck.")
log.msg("Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues")
elif isinstance(failure.value, errors.CouldNotFindTestHelper):
log.err("Unable to obtain the required test helpers.")
log.msg("Try with a different bouncer or check that Tor is running properly.")
elif isinstance(failure.value, errors.CouldNotFindTestCollector):
log.err("Could not find a valid collector.")
log.msg("Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.")
if config.advanced.debug:
log.exception(failure)
reactor.stop()
def run():
options = Options()
try:
options.parseOptions()
except usage.UsageError as error_message:
print "%s: %s" % (sys.argv[0], error_message)
print "%s: Try --help for usage details." % (sys.argv[0])
sys.exit(1)
if not any(options.values()):
print("%s: no command specified" % sys.argv[0])
print options
sys.exit(1)
if options['update-inputs']:
print "Downloading inputs"
try:
yield update.download_inputs()
except Exception as exc:
log.err("failed to download geoip files")
log.exception(exc)
if options['update-geoip']:
print "Downloading geoip files"
try:
yield update.download_geoip()
except Exception as exc:
log.err("failed to download geoip files")
log.exception(exc)
def readmsg(_, channel, queue_object, consumer_tag, counter):
# Wait for a message and decode it.
if counter >= lifetime:
log.msg("Counter")
queue_object.close(LifetimeExceeded())
yield channel.basic_cancel(consumer_tag=consumer_tag)
finished.callback(None)
else:
log.msg("Waiting for message")
try:
ch, method, properties, body = yield queue_object.get()
log.msg("Got message")
data = json.loads(body)
counter += 1
log.msg("Received %d/%d: %s" % (counter, lifetime, data['url'],))
# acknowledge the message
ch.basic_ack(delivery_tag=method.delivery_tag)
d = runTestWithDirector(director=director,
start_tor=start_tor,
global_options=global_options,
url=data['url'].encode('utf8'))
# When the test has been completed, go back to waiting for a message.
d.addCallback(readmsg, channel, queue_object, consumer_tag, counter+1)
except exceptions.AMQPError, v:
log.msg("Error")
log.exception(v)
finished.errback(v)
def notify(self, event):
for handler in self._subscribers:
try:
handler(event)
except Exception as exc:
log.err("Failed to run handler")
log.exception(exc)
def gotError(failure):
log.err("Failed to perform lookup")
log.exception(failure)
failure.trap(gaierror, TimeoutError)
DNSTest.addToReport(self, query, resolver=dns_server,
query_type = 'PTR', failure=failure)
return None
def api_nettest_start(self, request, test_name):
try:
_ = self.director.netTests[test_name]
except KeyError:
raise WebUIError(500, 'Could not find the specified test')
try:
test_options = json.load(request.content)
except ValueError:
raise WebUIError(500, 'Invalid JSON message recevied')
test_options["test_name"] = test_name
deck_data = {"tasks": [{"ooni": test_options}]}
try:
deck = NGDeck()
deck.load(deck_data)
self.run_deck(deck)
except errors.MissingRequiredOption as option_name:
raise WebUIError(
400, 'Missing required option: "{}"'.format(option_name))
except usage.UsageError as ue:
raise WebUIError(400, 'Error in parsing options')
except errors.InsufficientPrivileges:
raise WebUIError(400, 'Insufficient privileges')
except Exception as exc:
log.exception(exc)
raise WebUIError(500, 'Failed to start nettest')
return self.render_json({"status": "started"}, request)
def run():
options = Options()
try:
options.parseOptions()
except usage.UsageError as error_message:
print "%s: %s" % (sys.argv[0], error_message)
print "%s: Try --help for usage details." % (sys.argv[0])
sys.exit(1)
if not any(options.values()):
print("%s: no command specified" % sys.argv[0])
print options
sys.exit(1)
if options['update-inputs']:
print "Downloading inputs"
try:
yield update.download_inputs()
except Exception as exc:
log.err("failed to download geoip files")
log.exception(exc)
if options['update-geoip']:
print "Downloading geoip files"
try:
yield update.download_geoip()
except Exception as exc:
log.err("failed to download geoip files")
log.exception(exc)
def setUp(self):
if (not self.localOptions['testresolvers']
and not self.localOptions['testresolver']):
self.test_resolvers = []
with open('/etc/resolv.conf') as f:
for line in f:
if line.startswith('nameserver'):
self.test_resolvers.append(line.split(' ')[1].strip())
self.report['test_resolvers'] = self.test_resolvers
elif self.localOptions['testresolvers']:
test_resolvers_file = self.localOptions['testresolvers']
elif self.localOptions['testresolver']:
self.test_resolvers = [self.localOptions['testresolver']]
try:
with open(test_resolvers_file) as f:
self.test_resolvers = [
x.split('#')[0].strip() for x in f.readlines()
]
self.report['test_resolvers'] = self.test_resolvers
f.close()
except IOError as e:
log.exception(e)
raise usage.UsageError("Invalid test resolvers file")
except NameError:
log.debug("No test resolver file configured")
dns_ip, dns_port = self.localOptions['backend'].split(':')
self.control_dns_server = (str(dns_ip), int(dns_port))
self.report['control_resolver'] = "%s:%d" % self.control_dns_server
def getX509Name(certificate, get_components=False):
"""Get the DER-encoded form of the Name fields of an X509 certificate.
@param certificate: A :class:`OpenSSL.crypto.X509Name` object.
@param get_components: A boolean. If True, returns a list of tuples of
the (name, value)s of each Name field in the
:param:`certificate`. If False, returns the DER
encoded form of the Name fields of the
:param:`certificate`.
"""
x509_name = None
try:
assert isinstance(certificate, crypto.X509Name), \
"getX509Name takes OpenSSL.crypto.X509Name as first argument!"
x509_name = crypto.X509Name(certificate)
except AssertionError as ae:
log.err(ae)
except Exception as exc:
log.exception(exc)
if not x509_name is None:
if not get_components:
return x509_name.der()
else:
return x509_name.get_components()
else:
log.debug("getX509Name: got None for ivar x509_name")
def runTestCases(test_cases, options, cmd_line_options):
log.debug("Running %s" % test_cases)
log.debug("Options %s" % options)
log.debug("cmd_line_options %s" % dict(cmd_line_options))
test_inputs = options['inputs']
oonib_reporter = OONIBReporter(cmd_line_options)
yaml_reporter = YAMLReporter(cmd_line_options)
if cmd_line_options['collector']:
log.msg("Using remote collector, please be patient while we create the report.")
try:
yield oonib_reporter.createReport(options)
except OONIBReportError:
log.err("Error in creating new report")
log.msg("We will only create reports to a file")
oonib_reporter = None
else:
oonib_reporter = None
yield yaml_reporter.createReport(options)
log.msg("Reporting to file %s" % yaml_reporter._stream.name)
try:
input_unit_factory = InputUnitFactory(test_inputs)
input_unit_factory.inputUnitSize = int(cmd_line_options['parallelism'])
except Exception, e:
log.exception(e)
def createDeck(global_options, url=None):
from ooni.nettest import NetTestLoader
from ooni.deck import Deck, nettest_to_path
if url:
log.msg("Creating deck for: %s" % (url))
if global_options['no-yamloo']:
log.msg("Will not write to a yamloo report file")
deck = Deck(bouncer=global_options['bouncer'],
no_collector=global_options['no-collector'])
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'], global_options)
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
if url is not None:
args = ('-u', url)
else:
args = tuple()
if any(global_options['subargs']):
args = global_options['subargs'] + args
net_test_loader = NetTestLoader(args,
test_file=test_file,
annotations=global_options['annotations'])
if global_options['collector']:
net_test_loader.collector = \
CollectorClient(global_options['collector'])
deck.insert(net_test_loader)
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
sys.exit(2)
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
sys.exit(3)
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
sys.exit(4)
except errors.HTTPSCollectorUnsupported:
log.err("HTTPS collectors require a twisted version of at least 14.0.2.")
sys.exit(6)
except errors.InsecureBackend:
log.err("Attempting to report to an insecure collector.")
log.err("To enable reporting to insecure collector set the "
"advanced->insecure_backend option to true in "
"your ooniprobe.conf file.")
sys.exit(7)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
sys.exit(5)
return deck
def setUp(self):
if (not self.localOptions['testresolvers'] and
not self.localOptions['testresolver']):
self.test_resolvers = []
with open('/etc/resolv.conf') as f:
for line in f:
if line.startswith('nameserver'):
self.test_resolvers.append(line.split(' ')[1].strip())
self.report['test_resolvers'] = self.test_resolvers
elif self.localOptions['testresolvers']:
test_resolvers_file = self.localOptions['testresolvers']
elif self.localOptions['testresolver']:
self.test_resolvers = [self.localOptions['testresolver']]
try:
with open(test_resolvers_file) as f:
self.test_resolvers = [
x.split('#')[0].strip() for x in f.readlines()]
self.report['test_resolvers'] = self.test_resolvers
f.close()
except IOError as e:
log.exception(e)
raise usage.UsageError("Invalid test resolvers file")
except NameError:
log.debug("No test resolver file configured")
dns_ip, dns_port = self.localOptions['backend'].split(':')
self.control_dns_server = (str(dns_ip), int(dns_port))
self.report['control_resolver'] = "%s:%d" % self.control_dns_server
def test_udp_traceroute(self):
"""
Does a traceroute to the destination by sending UDP packets with empty
payloads with TTLs from 1 until max_ttl.
"""
def finished(packets, port):
log.debug("Finished running UDP traceroute test on port %s" % port)
answered, unanswered = packets
self.report['hops_'+str(port)] = []
for snd, rcv in answered:
report = {'ttl': snd.ttl,
'address': rcv.src,
'rtt': rcv.time - snd.time,
'sport': snd[UDP].sport
}
log.debug("%s: %s" % (port, report))
self.report['hops_'+str(port)].append(report)
try:
dl = []
max_ttl, timeout = self.max_ttl_and_timeout()
for port in self.dst_ports:
packets = IP(dst=self.localOptions['backend'],
ttl=(1,max_ttl),id=RandShort())/UDP(dport=port,
sport=self.get_sport('udp'))
d = self.sr(packets, timeout=timeout)
d.addCallback(finished, port)
dl.append(d)
return defer.DeferredList(dl)
except Exception, e:
log.msg("TestStatus: [ FAILED ]")
log.exception(e)
log.msg("TestException: [ %s ]" % (e))
def test_icmp_traceroute(self):
"""
Does a traceroute to the destination by sending ICMP echo request
packets with TTLs from 1 until max_ttl.
"""
def finished(packets):
log.debug("Finished running ICMP traceroute test")
answered, unanswered = packets
self.report['hops'] = []
for snd, rcv in answered:
report = {'ttl': snd.ttl,
'address': rcv.src,
'rtt': rcv.time - snd.time
}
log.debug("%s" % (report))
self.report['hops'].append(report)
try:
dl = []
max_ttl, timeout = self.max_ttl_and_timeout()
packets = IP(dst=self.localOptions['backend'],
ttl=(1,max_ttl), id=RandShort())/ICMP()
d = self.sr(packets, timeout=timeout)
d.addCallback(finished)
return d
except Exception, e:
log.msg("TestStatus: [ FAILED ]")
log.exception(e)
log.msg("TestException: [ %s ]" % (e))
def createReport(self):
"""
Creates a report on the oonib collector.
"""
log.msg("Creating report with OONIB Reporter. Please be patient.")
log.msg("This may take up to 1-2 minutes...")
try:
response = yield self.collector_client.createReport(
self.testDetails)
except ConnectionRefusedError:
log.err("Connection to reporting backend failed "
"(ConnectionRefusedError)")
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
log.err("Host is not reachable (HostUnreachable error")
raise errors.OONIBReportCreationError
except (errors.OONIBInvalidInputHash, errors.OONIBInvalidNettestName):
log.err("The specified input or nettests cannot be submitted to "
"this collector.")
log.msg("Try running a different test or try reporting to a "
"different collector.")
raise errors.OONIBReportCreationError
except Exception, e:
log.err("Failed to connect to reporter backend")
log.exception(e)
raise errors.OONIBReportCreationError
def _tor_startup_failure(self, failure):
log.err("Failed to start tor")
log.exception(failure)
self._reset_tor_state()
self.notify(DirectorEvent("error",
"Failed to start Tor"))
return failure
def readmsg(_, channel, queue_object, consumer_tag, counter):
# Wait for a message and decode it.
if counter >= lifetime:
log.msg("Counter")
queue_object.close(LifetimeExceeded())
yield channel.basic_cancel(consumer_tag=consumer_tag)
finished.callback(None)
else:
log.msg("Waiting for message")
try:
ch, method, properties, body = yield queue_object.get()
log.msg("Got message")
data = json.loads(body)
counter += 1
log.msg("Received %d/%d: %s" % (counter, lifetime, data['url'],))
# acknowledge the message
ch.basic_ack(delivery_tag=method.delivery_tag)
d = runTestWithDirector(director=director,
start_tor=start_tor,
global_options=global_options,
url=data['url'].encode('utf8'))
# When the test has been completed, go back to waiting for a message.
d.addCallback(readmsg, channel, queue_object, consumer_tag, counter+1)
except exceptions.AMQPError, v:
log.msg("Error")
log.exception(v)
finished.errback(v)
def __init__(self, backend_url):
from ooni.utils.txagentwithsocks import Agent
from twisted.internet import reactor
try:
self.agent = Agent(reactor, sockshost="127.0.0.1",
socksport=int(config.advanced.tor_socksport))
except Exception, e:
log.exception(e)
def upload_all(collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
for report_file, value in oonib_report_log.reports_to_upload:
try:
yield upload(report_file, collector, bouncer)
except Exception as exc:
log.exception(exc)
def lookupTestHelpers(self, test_helper_names):
try:
test_helper = yield self.queryBackend('POST', '/bouncer',
query={'test-helpers': test_helper_names})
except Exception, exc:
log.exception(exc)
raise e.CouldNotFindTestHelper
def gotError(failure):
log.exception(failure)
result = {}
result['resolver'] = dns_server
result['query_type'] = 'PTR'
result['query'] = repr(query)
result['error'] = str(failure)
return None
def gotError(failure):
log.exception(failure)
result = {}
result['resolver'] = dns_server
result['query_type'] = 'A'
result['query'] = repr(query)
result['error'] = str(failure)
return None
def upload_all(collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
for report_file, value in oonib_report_log.reports_to_upload:
try:
yield upload(report_file, collector, bouncer)
except Exception as exc:
log.exception(exc)
def __init__(self, backend_url):
from ooni.utils.txagentwithsocks import Agent
from twisted.internet import reactor
try:
self.agent = Agent(reactor,
sockshost="127.0.0.1",
socksport=int(config.advanced.tor_socksport))
except Exception, e:
log.exception(e)
def lookupTestCollector(self, net_tests):
try:
test_collector = yield self.queryBackend(
'POST', '/bouncer/net-tests', query={'net-tests': net_tests})
except Exception as exc:
log.exception(exc)
raise e.CouldNotFindTestCollector
defer.returnValue(test_collector)
def lookupTestCollector(self, net_tests):
try:
test_collector = yield self.queryBackend('POST', '/bouncer/net-tests',
query={'net-tests': net_tests})
except Exception as exc:
log.exception(exc)
raise e.CouldNotFindTestCollector
defer.returnValue(test_collector)
def setUpClass(cls):
try:
answers = yield client.lookupAddress(cls.localOptions["dns-discovery"])
assert len(answers) > 0
assert len(answers[0]) > 0
cls.resolverIp = answers[0][0].payload.dottedQuad()
except Exception as exc:
log.exception(exc)
log.err("Failed to lookup the resolver IP address")
def createReport(self, options):
"""
Creates a report on the oonib collector.
"""
url = self.backend_url + '/report'
try:
test_details = getTestDetails(options)
except Exception, e:
log.exception(e)
def gotError(failure):
log.err("Failed to perform lookup")
log.exception(failure)
failure.trap(gaierror, TimeoutError)
DNSTest.addToReport(self,
query,
resolver=dns_server,
query_type='PTR',
failure=failure)
return None
def setUpClass(cls):
try:
answers = yield client.lookupAddress(
cls.localOptions['dns-discovery'])
assert len(answers) > 0
assert len(answers[0]) > 0
cls.resolverIp = answers[0][0].payload.dottedQuad()
except Exception as exc:
log.exception(exc)
log.err("Failed to lookup the resolver IP address")
def createDeck(global_options, url=None):
if url:
log.msg("Creating deck for: %s" % (url))
if global_options['no-yamloo']:
log.msg("Will not write to a yamloo report file")
deck = Deck(no_collector=global_options['no-collector'])
deck.bouncer = global_options['bouncer']
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
if url is not None:
args = ('-u', url)
else:
args = tuple()
if any(global_options['subargs']):
args = global_options['subargs'] + args
net_test_loader = NetTestLoader(args, test_file=test_file)
if global_options['collector']:
net_test_loader.collector = global_options['collector']
deck.insert(net_test_loader)
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
sys.exit(2)
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
sys.exit(3)
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
sys.exit(4)
except errors.HTTPSCollectorUnsupported:
log.err(
"HTTPS collectors require a twisted version of at least 14.0.2.")
sys.exit(6)
except errors.InsecureCollector:
log.err("Attempting to report to an insecure collector.")
log.err("To enable reporting to insecure collector set the "
"advanced->insecure_collector option to true in "
"your ooniprobe.conf file.")
sys.exit(7)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
sys.exit(5)
return deck
def errback(err, attempts):
# We we will recursively keep trying to perform a request until
# we have reached the retry count.
if attempts < self.retries:
log.err("Lookup failed. Retrying.")
log.exception(err)
attempts += 1
perform_request(attempts)
else:
log.err("Failed. Giving up.")
finished.errback(err)
def __init__(self, cmd_line_options):
self.backend_url = cmd_line_options['collector']
self.report_id = None
from ooni.utils.txagentwithsocks import Agent
from twisted.internet import reactor
try:
self.agent = Agent(reactor, sockshost="127.0.0.1",
socksport=int(config.tor.socks_port))
except Exception, e:
log.exception(e)
def lookupTestHelpers(self, test_helper_names):
try:
test_helper = yield self.queryBackend('POST', '/bouncer/test-helpers',
query={'test-helpers': test_helper_names})
except Exception as exc:
log.exception(exc)
raise e.CouldNotFindTestHelper
if not test_helper:
raise e.CouldNotFindTestHelper
defer.returnValue(test_helper)
def test_probe_site(self):
"""
Sends a GET-requests to specified URL.
Defaults to "https://www.torproject.org".
"""
try:
response = yield self.doRequest(self.url, method="GET",
use_tor=False, headers=self.headers)
except Exception, e:
log.msg("TestStatus: [ FAILED ]")
log.exception(e)
log.msg("TestException: [ %s ]" % e.message)
def lookupTestHelpers(self, test_helper_names):
try:
test_helper = yield self.queryBackend('POST', '/bouncer/test-helpers',
query={'test-helpers': test_helper_names})
except Exception as exc:
log.exception(exc)
raise e.CouldNotFindTestHelper
if not test_helper:
raise e.CouldNotFindTestHelper
defer.returnValue(test_helper)
def remove_log(self, measurement_id):
lock = self._lock_for_report_log(measurement_id)
yield lock.deferUntilLocked()
report_log_file = self._get_report_log_file(measurement_id)
try:
log.debug("Deleting log file")
report_log_file.remove()
except Exception as exc:
log.exception(exc)
lock.unlock()
def remove_log(self, measurement_id):
lock = self._lock_for_report_log(measurement_id)
yield lock.deferUntilLocked()
report_log_file = self._get_report_log_file(measurement_id)
try:
log.debug("Deleting log file")
report_log_file.remove()
except Exception as exc:
log.exception(exc)
lock.unlock()
def director_startup_handled_failures(failure):
log.err("Could not start the director")
failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress,
errors.UnableToLoadDeckInput,
errors.CouldNotFindTestHelper,
errors.CouldNotFindTestCollector, errors.ProbeIPUnknown,
errors.InvalidInputFile, errors.ConfigFileIncoherent)
if isinstance(failure.value, errors.TorNotRunning):
log.err("Tor does not appear to be running")
log.err("Reporting with the collector %s is not possible" %
global_options['collector'])
log.msg(
"Try with a different collector or disable collector reporting with -n"
)
elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
log.err("Invalid format for oonib collector address.")
log.msg(
"Should be in the format http://<collector_address>:<port>")
log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
elif isinstance(failure.value, errors.UnableToLoadDeckInput):
log.err("Unable to fetch the required inputs for the test deck.")
log.msg(
"Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues"
)
elif isinstance(failure.value, errors.CouldNotFindTestHelper):
log.err("Unable to obtain the required test helpers.")
log.msg(
"Try with a different bouncer or check that Tor is running properly."
)
elif isinstance(failure.value, errors.CouldNotFindTestCollector):
log.err("Could not find a valid collector.")
log.msg(
"Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n."
)
elif isinstance(failure.value, errors.ProbeIPUnknown):
log.err("Failed to lookup probe IP address.")
log.msg("Check your internet connection.")
elif isinstance(failure.value, errors.InvalidInputFile):
log.err("Invalid input file \"%s\"" % failure.value)
elif isinstance(failure.value, errors.ConfigFileIncoherent):
log.err("Incoherent config file")
if config.advanced.debug:
log.exception(failure)
def director_startup_handled_failures(failure):
log.err("Could not start the director")
failure.trap(errors.TorNotRunning,
errors.InvalidOONIBCollectorAddress,
errors.UnableToLoadDeckInput,
errors.CouldNotFindTestHelper,
errors.CouldNotFindTestCollector,
errors.ProbeIPUnknown,
errors.InvalidInputFile,
errors.ConfigFileIncoherent,
SystemExit)
if isinstance(failure.value, errors.TorNotRunning):
log.err("Tor does not appear to be running")
log.err("Reporting with a collector is not possible")
log.msg(
"Try with a different collector or disable collector reporting with -n")
elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
log.err("Invalid format for oonib collector address.")
log.msg(
"Should be in the format http://<collector_address>:<port>")
log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
elif isinstance(failure.value, errors.UnableToLoadDeckInput):
log.err("Unable to fetch the required inputs for the test deck.")
log.msg(
"Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues")
elif isinstance(failure.value, errors.CouldNotFindTestHelper):
log.err("Unable to obtain the required test helpers.")
log.msg(
"Try with a different bouncer or check that Tor is running properly.")
elif isinstance(failure.value, errors.CouldNotFindTestCollector):
log.err("Could not find a valid collector.")
log.msg(
"Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.")
elif isinstance(failure.value, errors.ProbeIPUnknown):
log.err("Failed to lookup probe IP address.")
log.msg("Check your internet connection.")
elif isinstance(failure.value, errors.InvalidInputFile):
log.err("Invalid input file \"%s\"" % failure.value)
elif isinstance(failure.value, errors.ConfigFileIncoherent):
log.err("Incoherent config file")
if config.advanced.debug:
log.exception(failure)
def writeReportEntry(self, entry):
if "json" in self.supportedFormats:
serialization_format = 'json'
else:
serialization_format = 'yaml'
log.debug("Updating report with id %s" % (self.reportId))
entry_content = self.serializeEntry(entry, serialization_format)
try:
yield self.collector_client.updateReport(self.reportId,
serialization_format,
entry_content)
except Exception as exc:
log.err("Error in writing report entry")
log.exception(exc)
raise errors.OONIBReportUpdateError
def writeReportEntry(self, entry):
if "json" in self.supportedFormats:
serialization_format = 'json'
else:
serialization_format = 'yaml'
log.debug("Updating report with id %s" % (self.reportId))
entry_content = self.serializeEntry(entry, serialization_format)
try:
yield self.collector_client.updateReport(self.reportId,
serialization_format,
entry_content)
except Exception as exc:
log.err("Error in writing report entry")
log.exception(exc)
raise errors.OONIBReportUpdateError
def createReport(self):
"""
Creates a report on the oonib collector.
"""
# XXX we should probably be setting this inside of the constructor,
# however config.tor.socks_port is not set until Tor is started and the
# reporter is instantiated before Tor is started. We probably want to
# do this with some deferred kung foo or instantiate the reporter after
# tor is started.
from ooni.utils.txagentwithsocks import Agent
from twisted.internet import reactor
try:
self.agent = Agent(reactor, sockshost="127.0.0.1",
socksport=int(config.tor.socks_port))
except Exception, e:
log.exception(e)
def start_tor(self, check_incoherences=False):
""" Starts Tor
Launches a Tor with :param: socks_port :param: control_port
:param: tor_binary set in ooniprobe.conf
"""
if self._tor_state == 'running':
log.debug("Tor is already running")
defer.returnValue(self._tor_state)
elif self._tor_state == 'starting':
log.debug("Tor is starting")
yield self._tor_starting
defer.returnValue(self._tor_state)
log.msg("Starting Tor")
self._tor_state = 'starting'
if check_incoherences:
try:
yield config.check_tor()
except Exception as exc:
self._tor_starting.errback(Failure(exc))
raise exc
if config.advanced.start_tor and config.tor_state is None:
tor_config = get_tor_config()
try:
yield start_tor(tor_config)
self._tor_starting.callback(self._tor_state)
except Exception as exc:
log.err("Failed to start tor")
log.exception(exc)
self._tor_starting.errback(Failure(exc))
elif config.tor.control_port and config.tor_state is None:
try:
yield connect_to_control_port()
self._tor_starting.callback(self._tor_state)
except Exception as exc:
self._tor_starting.errback(Failure(exc))
else:
# This happens when we require tor to not be started and the
# socks port is set.
self._tor_starting.callback(self._tor_state)
def upload_all(collector=None, bouncer=None, upload_incomplete=False):
oonib_report_log = OONIBReportLog()
reports_to_upload = yield oonib_report_log.get_to_upload()
for report_file, value in reports_to_upload:
try:
yield upload(report_file, collector, bouncer,
value['measurement_id'])
except Exception as exc:
log.exception(exc)
if upload_incomplete:
reports_to_upload = yield oonib_report_log.get_incomplete()
for report_file, value in reports_to_upload:
try:
yield upload(report_file, collector, bouncer,
value['measurement_id'])
except Exception as exc:
log.exception(exc)
def createReport(self, options):
"""
Creates a report on the oonib collector.
"""
test_name = options['name']
test_version = options['version']
log.debug("Creating report with OONIB Reporter")
url = self.backend_url + '/report/new'
software_version = '0.0.1'
test_details = yield getTestDetails(options)
content = '---\n'
content += safe_dump(test_details)
content += '...\n'
request = {
'software_name': 'ooniprobe',
'software_version': software_version,
'test_name': test_name,
'test_version': test_version,
'progress': 0,
'content': content
}
log.debug("Creating report via url %s" % url)
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(json.dumps(request))
try:
response = yield self.agent.request("POST",
url,
bodyProducer=bodyProducer)
except ConnectionRefusedError:
log.err(
"Connection to reporting backend failed (ConnectionRefusedError)"
)
raise OONIBReportCreationFailed
except Exception, e:
log.exception(e)
raise OONIBReportCreationFailed
def setUp(self):
if (not self.localOptions['testresolvers'] and \
not self.localOptions['testresolver']):
raise usage.UsageError("You did not specify a testresolver")
elif self.localOptions['testresolvers']:
test_resolvers_file = self.localOptions['testresolvers']
elif self.localOptions['testresolver']:
self.test_resolvers = [self.localOptions['testresolver']]
try:
with open(test_resolvers_file) as f:
self.test_resolvers = [x.split('#')[0].strip() for x in f.readlines()]
self.report['test_resolvers'] = self.test_resolvers
f.close()
except IOError, e:
log.exception(e)
raise usage.UsageError("Invalid test resolvers file")
def run_system_tasks(no_input_store=False):
task_classes = SYSTEM_TASKS[:]
if no_input_store:
log.debug("Not updating the inputs")
try:
task_classes.remove(UpdateInputsAndResources)
except ValueError:
pass
for task_class in task_classes:
task = task_class()
log.debug("Running task {0}".format(task.identifier))
try:
yield task.run()
except DidNotRun:
log.debug("Did not run {0}".format(task.identifier))
except Exception as exc:
log.err("Failed to run task {0}".format(task.identifier))
log.exception(exc)
def test_parasitic_tcp_traceroute(self):
"""
Establishes a TCP stream and send the packets inside of such stream.
Requires the backend to respond with an ACK to our SYN packet.
"""
max_ttl, timeout = self.max_ttl_and_timeout()
sport = self.get_sport()
dport = self.dport
ipid = int(RandShort())
packet = IP(dst=self.localOptions['backend'], ttl=max_ttl,
id=ipid)/TCP(sport=sport, dport=dport,
flags="S", seq=0)
log.msg("Sending SYN towards %s" % dport)
try:
answered, unanswered = yield self.sr(packet, timeout=timeout)
except Exception, e:
log.exception(e)
def list_measurements(compute_size=False, order=None):
measurements = []
measurement_path = FilePath(config.measurements_directory)
if not measurement_path.exists():
return measurements
for measurement_id in measurement_path.listdir():
try:
measurements.append(get_measurement(measurement_id, compute_size))
except Exception as exc:
log.err("Failed to get metadata for measurement {0}".format(measurement_id))
log.exception(exc)
if order is None:
return measurements
if order.lower() in ['asc', 'desc']:
reverse = {'asc': False, 'desc': True}[order.lower()]
measurements.sort(key=operator.itemgetter('test_start_time'),
reverse=reverse)
return measurements
else:
raise ValueError("order must be either 'asc' 'desc' or None")
def createReport(self):
"""
Creates a report on the oonib collector.
"""
log.msg("Creating report with OONIB Reporter. Please be patient.")
log.msg("This may take up to 1-2 minutes...")
try:
response = yield self.collector_client.createReport(
self.testDetails)
except ConnectionRefusedError:
log.err("Connection to reporting backend failed "
"(ConnectionRefusedError)")
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
log.err("Host is not reachable (HostUnreachable error")
raise errors.OONIBReportCreationError
except (errors.OONIBInvalidInputHash, errors.OONIBInvalidNettestName):
log.err("The specified input or nettests cannot be submitted to "
"this collector.")
log.msg("Try running a different test or try reporting to a "
"different collector.")
raise errors.OONIBReportCreationError
except errors.OONIBError:
log.err("Failed to connect to reporter backend")
raise errors.OONIBReportCreationError
except Exception as exc:
log.err("Failed to connect to reporter backend")
log.exception(exc)
raise errors.OONIBReportCreationError
self.reportId = response['report_id'].encode('ascii')
self.backendVersion = response['backend_version']
self.supportedFormats = response.get('supported_formats', ["yaml"])
log.debug("Created report with id %s" % response['report_id'])
defer.returnValue(response['report_id'])
def _failed(self, failure, task):
"""
The has failed to complete, we append it to the end of the task chain
to be re-run once all the currently scheduled tasks have run.
"""
log.err("Task %s has failed %s times" % (task, task.failures))
log.exception(failure)
self._active_tasks.remove(task)
self.failures.append((failure, task))
if task.failures <= self.retries:
log.debug("Rescheduling...")
self._tasks = itertools.chain(self._tasks,
makeIterable(task))
else:
# This fires the errback when the task is done but has failed.
log.err('Permanent failure for %s' % task)
task.done.errback(failure)
self._fillSlots()
self.failed(failure, task)
def test_a_lookup(self):
"""
We perform an A lookup on the DNS test servers for the domains to be
tested and an A lookup on the known good DNS server.
We then compare the results from test_resolvers and that from
control_resolver and see if the match up.
If they match up then no censorship is happening (tampering: false).
If they do not we do a reverse lookup (PTR) on the test_resolvers and
the control resolver for every IP address we got back and check to see
if anyone of them matches the control ones.
If they do then we take not of the fact that censorship is probably not
happening (tampering: reverse-match).
If they do not match then censorship is probably going on (tampering:
true).
"""
log.msg("Doing the test lookups on %s" % self.input)
list_of_ds = []
hostname = self.input
self.report['tampering'] = {}
control_answers = yield self.performALookup(hostname, self.control_dns_server)
if not control_answers:
log.err("Got no response from control DNS server %s," \
" perhaps the DNS resolver is down?" % self.control_dns_server[0])
self.report['tampering'][self.control_dns_server] = 'no_answer'
return
for test_resolver in self.test_resolvers:
log.msg("Testing resolver: %s" % test_resolver)
test_dns_server = (test_resolver, 53)
try:
experiment_answers = yield self.performALookup(hostname, test_dns_server)
except Exception, e:
log.err("Problem performing the DNS lookup")
log.exception(e)
self.report['tampering'][test_resolver] = 'dns_lookup_error'
continue
if not experiment_answers:
log.err("Got no response, perhaps the DNS resolver is down?")
self.report['tampering'][test_resolver] = 'no_answer'
continue
else:
log.debug("Got the following A lookup answers %s from %s" % (experiment_answers, test_resolver))
def lookup_details():
"""
A closure useful for printing test details.
"""
log.msg("test resolver: %s" % test_resolver)
log.msg("experiment answers: %s" % experiment_answers)
log.msg("control answers: %s" % control_answers)
log.debug("Comparing %s with %s" % (experiment_answers, control_answers))
if set(experiment_answers) & set(control_answers):
lookup_details()
log.msg("tampering: false")
self.report['tampering'][test_resolver] = False
else:
log.msg("Trying to do reverse lookup")
experiment_reverse = yield self.performPTRLookup(experiment_answers[0], test_dns_server)
control_reverse = yield self.performPTRLookup(control_answers[0], self.control_dns_server)
if experiment_reverse == control_reverse:
log.msg("Further testing has eliminated false positives")
lookup_details()
log.msg("tampering: reverse_match")
self.report['tampering'][test_resolver] = 'reverse_match'
else:
log.msg("Reverse lookups do not match")
lookup_details()
log.msg("tampering: true")
self.report['tampering'][test_resolver] = True
