def setupGlobalOptions(logging, start_tor, check_incoherences):
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.initialize_ooni_home()
try:
config.read_config_file(check_incoherences=check_incoherences)
except errors.ConfigFileIncoherent:
sys.exit(6)
if global_options['verbose']:
config.advanced.debug = True
if not start_tor:
config.advanced.start_tor = False
if logging:
log.start(global_options['logfile'])
if config.privacy.includepcap or global_options['pcapfile']:
from ooni.utils.net import hasRawSocketPermission
if hasRawSocketPermission():
from ooni.utils.txscapy import ScapyFactory
config.scapyFactory = ScapyFactory(config.advanced.interface)
else:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
global_options['check_incoherences'] = check_incoherences
return global_options
def setupGlobalOptions(logging, start_tor, check_incoherences):
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.initialize_ooni_home()
try:
config.read_config_file(check_incoherences=check_incoherences)
except errors.ConfigFileIncoherent:
sys.exit(6)
if not config.is_initialized():
initializeOoniprobe(global_options)
if global_options['verbose']:
config.advanced.debug = True
if not start_tor:
config.advanced.start_tor = False
if logging:
log.start(global_options['logfile'])
if config.privacy.includepcap or global_options['pcapfile']:
from ooni.utils.net import hasRawSocketPermission
if hasRawSocketPermission():
from ooni.utils.txscapy import ScapyFactory
config.scapyFactory = ScapyFactory(config.advanced.interface)
else:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
global_options['check_incoherences'] = check_incoherences
return global_options
def askTraceroute(self):
"""
Perform a UDP traceroute to determine the probes IP address.
"""
if not hasRawSocketPermission():
raise errors.InsufficientPrivileges
raise NotImplemented
def askTraceroute(self):
"""
Perform a UDP traceroute to determine the probes IP address.
"""
if not hasRawSocketPermission():
raise errors.InsufficientPrivileges
raise NotImplemented
def checkOptions(self):
"""
Call processTest and processOptions methods of each NetTestCase
"""
for klass in self.testClasses:
options = self.usageOptions()
try:
options.parseOptions(self.options)
except usage.UsageError:
tb = sys.exc_info()[2]
raise e.OONIUsageError(self), None, tb
if options:
klass.localOptions = options
# XXX this class all needs to be refactored and this is kind of a
# hack.
self.setTestDetails()
test_instance = klass()
if test_instance.requiresRoot and not hasRawSocketPermission():
raise e.InsufficientPrivileges
if test_instance.requiresTor:
self.requiresTor = True
test_instance.requirements()
test_instance._checkRequiredOptions()
test_instance._checkValidOptions()
def _checkTestClassOptions(self, test_class):
if test_class.requiresRoot and not hasRawSocketPermission():
raise e.InsufficientPrivileges
if test_class.requiresTor:
self.requiresTor = True
self._checkRequiredOptions(test_class)
self._setTestHelpers(test_class)
test_instance = netTestCaseFactory(test_class, self.localOptions)()
test_instance.requirements()
def setUp(self):
super(TestRunDirector, self).setUp()
if not is_internet_connected():
self.skipTest("You must be connected to the internet to run this test")
elif not hasRawSocketPermission():
self.skipTest("You must run this test as root or have the capabilities "
"cap_net_admin,cap_net_raw+eip")
config.tor.socks_port = 9050
config.tor.control_port = None
self.filenames = ['example-input.txt']
with open('example-input.txt', 'w+') as f:
f.write('http://torproject.org/\n')
f.write('http://bridges.torproject.org/\n')
f.write('http://blog.torproject.org/\n')
def setUp(self):
super(TestRunDirector, self).setUp()
if not is_internet_connected():
self.skipTest(
"You must be connected to the internet to run this test")
elif not hasRawSocketPermission():
self.skipTest(
"You must run this test as root or have the capabilities "
"cap_net_admin,cap_net_raw+eip")
config.tor.socks_port = 9050
config.tor.control_port = None
self.filenames = ['example-input.txt']
with open('example-input.txt', 'w+') as f:
f.write('http://torproject.org/\n')
f.write('http://bridges.torproject.org/\n')
f.write('http://blog.torproject.org/\n')
def test_sniffing_activated(self):
if not hasRawSocketPermission():
self.skipTest("You must run this test as root or have the "
"capabilities cap_net_admin,cap_net_raw+eip")
self.skipTest("Not properly set packet capture?")
filename = os.path.abspath('test_report.pcap')
self.filenames.append(filename)
conf_file = os.path.abspath('fake_config.conf')
with open(conf_file, 'w') as cfg:
cfg.writelines(config_includepcap)
self.filenames.append(conf_file)
def verify_function(_):
assert os.path.exists(filename)
self.assertGreater(os.stat(filename).st_size, 0)
yield self.run_helper('blocking/http_requests',
['-f', 'example-input.txt'],
verify_function, ooni_args=['-f', conf_file])
config.scapyFactory.connectionLost('')
def test_sniffing_activated(self):
if not hasRawSocketPermission():
self.skipTest("You must run this test as root or have the "
"capabilities cap_net_admin,cap_net_raw+eip")
self.skipTest("Not properly set packet capture?")
filename = os.path.abspath('test_report.pcap')
self.filenames.append(filename)
conf_file = os.path.abspath('fake_config.conf')
with open(conf_file, 'w') as cfg:
cfg.writelines(config_includepcap)
self.filenames.append(conf_file)
def verify_function(_):
assert os.path.exists(filename)
self.assertGreater(os.stat(filename).st_size, 0)
yield self.run_helper('blocking/http_requests',
['-f', 'example-input.txt'],
verify_function,
ooni_args=['-f', conf_file])
config.scapyFactory.connectionLost('')
class BaseScapyTest(NetTestCase):
"""
The report of a test run with scapy looks like this:
report:
sent_packets: [
{
'raw_packet': BASE64Encoding of packet,
'summary': 'IP / TCP 192.168.2.66:ftp_data > 8.8.8.8:http S'
}
]
answered_packets: []
"""
name = "Base Scapy Test"
version = 0.1
requiresRoot = not hasRawSocketPermission()
baseFlags = [
['ipsrc', 's',
'Does *not* check if IP src and ICMP IP citation '
'matches when processing answers'],
['seqack', 'k',
'Check if TCP sequence number and ACK match in the '
'ICMP citation when processing answers'],
['ipid', 'i', 'Check if the IPID matches when processing answers']]
def _setUp(self):
super(BaseScapyTest, self)._setUp()
if config.scapyFactory is None:
log.debug("Scapy factory not set, registering it.")
config.scapyFactory = ScapyFactory(config.advanced.interface)
self.report['answer_flags'] = []
if self.localOptions['ipsrc']:
config.checkIPsrc = 0
else:
self.report['answer_flags'].append('ipsrc')
config.checkIPsrc = 1
if self.localOptions['ipid']:
self.report['answer_flags'].append('ipid')
config.checkIPID = 1
else:
config.checkIPID = 0
# XXX we don't support strict matching
# since (from scapy's documentation), some stacks have a bug for which
# the bytes in the IPID are swapped.
# Perhaps in the future we will want to have more fine grained control
# over this.
if self.localOptions['seqack']:
self.report['answer_flags'].append('seqack')
config.check_TCPerror_seqack = 1
else:
config.check_TCPerror_seqack = 0
self.report['sent_packets'] = []
self.report['answered_packets'] = []
def finishedSendReceive(self, packets):
"""
This gets called when all packets have been sent and received.
"""
answered, unanswered = packets
for snd, rcv in answered:
log.debug("Writing report for scapy test")
sent_packet = snd
received_packet = rcv
if not config.privacy.includeip:
log.debug("Detected you would not like to "
"include your ip in the report")
log.debug(
"Stripping source and destination IPs from the reports")
sent_packet.src = '127.0.0.1'
received_packet.dst = '127.0.0.1'
self.report['sent_packets'].append(representPacket(sent_packet))
self.report['answered_packets'].append(representPacket(received_packet))
return packets
def sr(self, packets, timeout=None, *arg, **kw):
"""
Wrapper around scapy.sendrecv.sr for sending and receiving of packets
at layer 3.
"""
scapySender = ScapySender(timeout=timeout)
config.scapyFactory.registerProtocol(scapySender)
log.debug("Using sending with hash %s" % scapySender.__hash__)
d = scapySender.startSending(packets)
d.addCallback(self.finishedSendReceive)
return d
def sr1(self, packets, *arg, **kw):
def done(packets):
"""
We do this so that the returned value is only the one packet that
we expected a response for, identical to the scapy implementation
of sr1.
"""
try:
return packets[0][0][1]
except IndexError:
log.err("Got no response...")
return packets
scapySender = ScapySender()
scapySender.expected_answers = 1
config.scapyFactory.registerProtocol(scapySender)
log.debug("Running sr1")
d = scapySender.startSending(packets)
log.debug("Started to send")
d.addCallback(self.finishedSendReceive)
d.addCallback(done)
return d
def send(self, packets, *arg, **kw):
"""
Wrapper around scapy.sendrecv.send for sending of packets at layer 3
"""
scapySender = ScapySender()
config.scapyFactory.registerProtocol(scapySender)
scapySender.startSending(packets)
scapySender.stopSending()
for sent_packet in packets:
self.report['sent_packets'].append(representPacket(sent_packet))
def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.initialize_ooni_home()
try:
config.read_config_file(check_incoherences=check_incoherences)
except errors.ConfigFileIncoherent:
sys.exit(6)
if global_options['verbose']:
config.advanced.debug = True
if not start_tor:
config.advanced.start_tor = False
if logging:
log.start(global_options['logfile'])
if config.privacy.includepcap:
if hasRawSocketPermission():
from ooni.utils.txscapy import ScapyFactory
config.scapyFactory = ScapyFactory(config.advanced.interface)
else:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
director = Director()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.getNetTests().items():
print "* %s (%s/%s)" % (net_test['name'], net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
if global_options.get('annotations') is not None:
annotations = {}
for annotation in global_options["annotations"].split(","):
pair = annotation.split(":")
if len(pair) == 2:
key = pair[0].strip()
value = pair[1].strip()
annotations[key] = value
else:
log.err("Invalid annotation: %s" % annotation)
sys.exit(1)
global_options["annotations"] = annotations
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
else:
start_tor = True
deck = Deck(no_collector=global_options['no-collector'])
deck.bouncer = global_options['bouncer']
if global_options['collector']:
start_tor |= True
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
net_test_loader = NetTestLoader(global_options['subargs'],
test_file=test_file)
if global_options['collector']:
net_test_loader.collector = global_options['collector']
deck.insert(net_test_loader)
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
sys.exit(2)
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
sys.exit(3)
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
sys.exit(4)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
sys.exit(5)
start_tor |= deck.requiresTor
d = director.start(start_tor=start_tor,
check_incoherences=check_incoherences)
def setup_nettest(_):
try:
return deck.setup()
except errors.UnableToLoadDeckInput as error:
return defer.failure.Failure(error)
def director_startup_handled_failures(failure):
log.err("Could not start the director")
failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress,
errors.UnableToLoadDeckInput,
errors.CouldNotFindTestHelper,
errors.CouldNotFindTestCollector, errors.ProbeIPUnknown,
errors.InvalidInputFile, errors.ConfigFileIncoherent)
if isinstance(failure.value, errors.TorNotRunning):
log.err("Tor does not appear to be running")
log.err("Reporting with the collector %s is not possible" %
global_options['collector'])
log.msg(
"Try with a different collector or disable collector reporting with -n"
)
elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
log.err("Invalid format for oonib collector address.")
log.msg(
"Should be in the format http://<collector_address>:<port>")
log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
elif isinstance(failure.value, errors.UnableToLoadDeckInput):
log.err("Unable to fetch the required inputs for the test deck.")
log.msg(
"Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues"
)
elif isinstance(failure.value, errors.CouldNotFindTestHelper):
log.err("Unable to obtain the required test helpers.")
log.msg(
"Try with a different bouncer or check that Tor is running properly."
)
elif isinstance(failure.value, errors.CouldNotFindTestCollector):
log.err("Could not find a valid collector.")
log.msg(
"Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n."
)
elif isinstance(failure.value, errors.ProbeIPUnknown):
log.err("Failed to lookup probe IP address.")
log.msg("Check your internet connection.")
elif isinstance(failure.value, errors.InvalidInputFile):
log.err("Invalid input file \"%s\"" % failure.value)
elif isinstance(failure.value, errors.ConfigFileIncoherent):
log.err("Incoherent config file")
if config.advanced.debug:
log.exception(failure)
def director_startup_other_failures(failure):
log.err("An unhandled exception occurred while starting the director!")
log.exception(failure)
# Wait until director has started up (including bootstrapping Tor)
# before adding tests
def post_director_start(_):
for net_test_loader in deck.netTestLoaders:
# Decks can specify different collectors
# for each net test, so that each NetTest
# may be paired with a test_helper and its collector
# However, a user can override this behavior by
# specifying a collector from the command-line (-c).
# If a collector is not specified in the deck, or the
# deck is a singleton, the default collector set in
# ooniprobe.conf will be used
collector = None
if not global_options['no-collector']:
if global_options['collector']:
collector = global_options['collector']
elif 'collector' in config.reports \
and config.reports['collector']:
collector = config.reports['collector']
elif net_test_loader.collector:
collector = net_test_loader.collector
if collector and collector.startswith('httpo:') \
and (not (config.tor_state or config.tor.socks_port)):
raise errors.TorNotRunning
test_details = net_test_loader.testDetails
test_details['annotations'] = global_options['annotations']
director.startNetTest(net_test_loader,
global_options['reportfile'], collector)
return director.allTestsDone
def start():
d.addCallback(setup_nettest)
d.addCallback(post_director_start)
d.addErrback(director_startup_handled_failures)
d.addErrback(director_startup_other_failures)
return d
return start()
def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.initialize_ooni_home()
try:
config.read_config_file(check_incoherences=check_incoherences)
except errors.ConfigFileIncoherent:
sys.exit(6)
if global_options['verbose']:
config.advanced.debug = True
if not start_tor:
config.advanced.start_tor = False
if logging:
log.start(global_options['logfile'])
if config.privacy.includepcap:
if hasRawSocketPermission():
from ooni.utils.txscapy import ScapyFactory
config.scapyFactory = ScapyFactory(config.advanced.interface)
else:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
director = Director()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.getNetTests().items():
print "* %s (%s/%s)" % (net_test['name'],
net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
if global_options.get('annotations') is not None:
annotations = {}
for annotation in global_options["annotations"].split(","):
pair = annotation.split(":")
if len(pair) == 2:
key = pair[0].strip()
value = pair[1].strip()
annotations[key] = value
else:
log.err("Invalid annotation: %s" % annotation)
sys.exit(1)
global_options["annotations"] = annotations
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
else:
start_tor = True
deck = Deck(no_collector=global_options['no-collector'])
deck.bouncer = global_options['bouncer']
if global_options['collector']:
start_tor |= True
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
net_test_loader = NetTestLoader(global_options['subargs'],
test_file=test_file)
if global_options['collector']:
net_test_loader.collector = global_options['collector']
deck.insert(net_test_loader)
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
sys.exit(2)
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
sys.exit(3)
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
sys.exit(4)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
sys.exit(5)
start_tor |= deck.requiresTor
d = director.start(start_tor=start_tor,
check_incoherences=check_incoherences)
def setup_nettest(_):
try:
return deck.setup()
except errors.UnableToLoadDeckInput as error:
return defer.failure.Failure(error)
def director_startup_handled_failures(failure):
log.err("Could not start the director")
failure.trap(errors.TorNotRunning,
errors.InvalidOONIBCollectorAddress,
errors.UnableToLoadDeckInput,
errors.CouldNotFindTestHelper,
errors.CouldNotFindTestCollector,
errors.ProbeIPUnknown,
errors.InvalidInputFile,
errors.ConfigFileIncoherent)
if isinstance(failure.value, errors.TorNotRunning):
log.err("Tor does not appear to be running")
log.err("Reporting with the collector %s is not possible" %
global_options['collector'])
log.msg(
"Try with a different collector or disable collector reporting with -n")
elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
log.err("Invalid format for oonib collector address.")
log.msg(
"Should be in the format http://<collector_address>:<port>")
log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
elif isinstance(failure.value, errors.UnableToLoadDeckInput):
log.err("Unable to fetch the required inputs for the test deck.")
log.msg(
"Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues")
elif isinstance(failure.value, errors.CouldNotFindTestHelper):
log.err("Unable to obtain the required test helpers.")
log.msg(
"Try with a different bouncer or check that Tor is running properly.")
elif isinstance(failure.value, errors.CouldNotFindTestCollector):
log.err("Could not find a valid collector.")
log.msg(
"Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.")
elif isinstance(failure.value, errors.ProbeIPUnknown):
log.err("Failed to lookup probe IP address.")
log.msg("Check your internet connection.")
elif isinstance(failure.value, errors.InvalidInputFile):
log.err("Invalid input file \"%s\"" % failure.value)
elif isinstance(failure.value, errors.ConfigFileIncoherent):
log.err("Incoherent config file")
if config.advanced.debug:
log.exception(failure)
def director_startup_other_failures(failure):
log.err("An unhandled exception occurred while starting the director!")
log.exception(failure)
# Wait until director has started up (including bootstrapping Tor)
# before adding tests
def post_director_start(_):
for net_test_loader in deck.netTestLoaders:
# Decks can specify different collectors
# for each net test, so that each NetTest
# may be paired with a test_helper and its collector
# However, a user can override this behavior by
# specifying a collector from the command-line (-c).
# If a collector is not specified in the deck, or the
# deck is a singleton, the default collector set in
# ooniprobe.conf will be used
collector = None
if not global_options['no-collector']:
if global_options['collector']:
collector = global_options['collector']
elif 'collector' in config.reports \
and config.reports['collector']:
collector = config.reports['collector']
elif net_test_loader.collector:
collector = net_test_loader.collector
if collector and collector.startswith('httpo:') \
and (not (config.tor_state or config.tor.socks_port)):
raise errors.TorNotRunning
test_details = net_test_loader.testDetails
test_details['annotations'] = global_options['annotations']
director.startNetTest(net_test_loader,
global_options['reportfile'],
collector)
return director.allTestsDone
def start():
d.addCallback(setup_nettest)
d.addCallback(post_director_start)
d.addErrback(director_startup_handled_failures)
d.addErrback(director_startup_other_failures)
return d
return start()
